From e5786f530f9555469c01435f86039b06aa53feba Mon Sep 17 00:00:00 2001 From: Peter Breitenlohner Date: Mon, 20 Apr 2009 18:13:48 +0000 Subject: new build system: build icu libs and xetex plus misc updates git-svn-id: svn://tug.org/texlive/trunk@12759 c570f23f-e606-0410-a88d-b1316a301751 --- Build/source/libs/icu/icu-xetex/common/Makefile.in | 203 + Build/source/libs/icu/icu-xetex/common/bmpset.cpp | 723 +++ Build/source/libs/icu/icu-xetex/common/bmpset.h | 161 + Build/source/libs/icu/icu-xetex/common/brkeng.cpp | 290 ++ Build/source/libs/icu/icu-xetex/common/brkeng.h | 292 ++ Build/source/libs/icu/icu-xetex/common/brkiter.cpp | 454 ++ Build/source/libs/icu/icu-xetex/common/caniter.cpp | 611 +++ .../source/libs/icu/icu-xetex/common/chariter.cpp | 96 + Build/source/libs/icu/icu-xetex/common/charstr.h | 88 + Build/source/libs/icu/icu-xetex/common/cmemory.c | 124 + Build/source/libs/icu/icu-xetex/common/cmemory.h | 82 + Build/source/libs/icu/icu-xetex/common/common.rc | 108 + .../source/libs/icu/icu-xetex/common/common.vcproj | 2588 ++++++++++ Build/source/libs/icu/icu-xetex/common/cpputils.h | 84 + Build/source/libs/icu/icu-xetex/common/cstring.c | 328 ++ Build/source/libs/icu/icu-xetex/common/cstring.h | 120 + Build/source/libs/icu/icu-xetex/common/cwchar.c | 53 + Build/source/libs/icu/icu-xetex/common/cwchar.h | 56 + Build/source/libs/icu/icu-xetex/common/dictbe.cpp | 426 ++ Build/source/libs/icu/icu-xetex/common/dictbe.h | 193 + Build/source/libs/icu/icu-xetex/common/hash.h | 207 + Build/source/libs/icu/icu-xetex/common/icucfg.h.in | 61 + Build/source/libs/icu/icu-xetex/common/localsvc.h | 25 + .../source/libs/icu/icu-xetex/common/locbased.cpp | 46 + Build/source/libs/icu/icu-xetex/common/locbased.h | 97 + Build/source/libs/icu/icu-xetex/common/locid.cpp | 1330 ++++++ Build/source/libs/icu/icu-xetex/common/locmap.c | 893 ++++ Build/source/libs/icu/icu-xetex/common/locmap.h | 37 + Build/source/libs/icu/icu-xetex/common/locutil.cpp | 267 ++ Build/source/libs/icu/icu-xetex/common/locutil.h | 37 + Build/source/libs/icu/icu-xetex/common/msvcres.h | 20 + Build/source/libs/icu/icu-xetex/common/mutex.h | 85 + Build/source/libs/icu/icu-xetex/common/normlzr.cpp | 611 +++ .../source/libs/icu/icu-xetex/common/parsepos.cpp | 21 + .../source/libs/icu/icu-xetex/common/propname.cpp | 758 +++ Build/source/libs/icu/icu-xetex/common/propname.h | 515 ++ Build/source/libs/icu/icu-xetex/common/punycode.c | 580 +++ Build/source/libs/icu/icu-xetex/common/punycode.h | 118 + Build/source/libs/icu/icu-xetex/common/putil.c | 1675 +++++++ Build/source/libs/icu/icu-xetex/common/putilimp.h | 277 ++ Build/source/libs/icu/icu-xetex/common/rbbi.cpp | 1832 ++++++++ Build/source/libs/icu/icu-xetex/common/rbbicst.pl | 453 ++ .../source/libs/icu/icu-xetex/common/rbbidata.cpp | 444 ++ Build/source/libs/icu/icu-xetex/common/rbbidata.h | 193 + .../source/libs/icu/icu-xetex/common/rbbinode.cpp | 355 ++ Build/source/libs/icu/icu-xetex/common/rbbinode.h | 118 + Build/source/libs/icu/icu-xetex/common/rbbirb.cpp | 312 ++ Build/source/libs/icu/icu-xetex/common/rbbirb.h | 211 + Build/source/libs/icu/icu-xetex/common/rbbirpt.h | 275 ++ Build/source/libs/icu/icu-xetex/common/rbbirpt.txt | 315 ++ .../source/libs/icu/icu-xetex/common/rbbiscan.cpp | 1193 +++++ Build/source/libs/icu/icu-xetex/common/rbbiscan.h | 160 + .../source/libs/icu/icu-xetex/common/rbbisetb.cpp | 684 +++ Build/source/libs/icu/icu-xetex/common/rbbisetb.h | 130 + .../source/libs/icu/icu-xetex/common/rbbistbl.cpp | 269 ++ .../source/libs/icu/icu-xetex/common/rbbitblb.cpp | 1232 +++++ Build/source/libs/icu/icu-xetex/common/rbbitblb.h | 127 + Build/source/libs/icu/icu-xetex/common/resbund.cpp | 396 ++ .../libs/icu/icu-xetex/common/resbund_cnv.cpp | 55 + .../source/libs/icu/icu-xetex/common/ruleiter.cpp | 160 + Build/source/libs/icu/icu-xetex/common/ruleiter.h | 232 + .../source/libs/icu/icu-xetex/common/schriter.cpp | 115 + Build/source/libs/icu/icu-xetex/common/serv.cpp | 974 ++++ Build/source/libs/icu/icu-xetex/common/serv.h | 996 ++++ Build/source/libs/icu/icu-xetex/common/servlk.cpp | 187 + Build/source/libs/icu/icu-xetex/common/servlkf.cpp | 151 + Build/source/libs/icu/icu-xetex/common/servloc.h | 550 +++ Build/source/libs/icu/icu-xetex/common/servls.cpp | 297 ++ .../source/libs/icu/icu-xetex/common/servnotf.cpp | 118 + Build/source/libs/icu/icu-xetex/common/servnotf.h | 124 + Build/source/libs/icu/icu-xetex/common/servrbf.cpp | 94 + .../source/libs/icu/icu-xetex/common/servslkf.cpp | 122 + Build/source/libs/icu/icu-xetex/common/sprpimpl.h | 129 + .../source/libs/icu/icu-xetex/common/triedict.cpp | 1407 ++++++ Build/source/libs/icu/icu-xetex/common/triedict.h | 346 ++ Build/source/libs/icu/icu-xetex/common/uarrsort.c | 236 + Build/source/libs/icu/icu-xetex/common/uarrsort.h | 84 + Build/source/libs/icu/icu-xetex/common/uassert.h | 32 + Build/source/libs/icu/icu-xetex/common/ubidi.c | 2224 +++++++++ .../source/libs/icu/icu-xetex/common/ubidi_props.c | 520 ++ .../source/libs/icu/icu-xetex/common/ubidi_props.h | 160 + .../libs/icu/icu-xetex/common/ubidi_props_data.c | 549 +++ Build/source/libs/icu/icu-xetex/common/ubidiimp.h | 385 ++ Build/source/libs/icu/icu-xetex/common/ubidiln.c | 1346 ++++++ Build/source/libs/icu/icu-xetex/common/ubidiwrt.c | 630 +++ Build/source/libs/icu/icu-xetex/common/ubrk.cpp | 293 ++ Build/source/libs/icu/icu-xetex/common/ubrkimpl.h | 13 + Build/source/libs/icu/icu-xetex/common/ucase.c | 1599 +++++++ Build/source/libs/icu/icu-xetex/common/ucase.h | 376 ++ .../libs/icu/icu-xetex/common/ucase_props_data.c | 601 +++ Build/source/libs/icu/icu-xetex/common/ucasemap.c | 563 +++ Build/source/libs/icu/icu-xetex/common/ucat.c | 76 + Build/source/libs/icu/icu-xetex/common/uchar.c | 979 ++++ .../libs/icu/icu-xetex/common/uchar_props_data.c | 2155 +++++++++ .../source/libs/icu/icu-xetex/common/uchriter.cpp | 363 ++ Build/source/libs/icu/icu-xetex/common/ucln.h | 68 + Build/source/libs/icu/icu-xetex/common/ucln_cmn.c | 74 + Build/source/libs/icu/icu-xetex/common/ucln_cmn.h | 63 + Build/source/libs/icu/icu-xetex/common/ucmndata.c | 300 ++ Build/source/libs/icu/icu-xetex/common/ucmndata.h | 109 + Build/source/libs/icu/icu-xetex/common/ucnv.c | 2888 ++++++++++++ Build/source/libs/icu/icu-xetex/common/ucnv2022.c | 3463 ++++++++++++++ Build/source/libs/icu/icu-xetex/common/ucnv_bld.c | 1583 +++++++ Build/source/libs/icu/icu-xetex/common/ucnv_bld.h | 281 ++ Build/source/libs/icu/icu-xetex/common/ucnv_cb.c | 259 + Build/source/libs/icu/icu-xetex/common/ucnv_cnv.c | 174 + Build/source/libs/icu/icu-xetex/common/ucnv_cnv.h | 303 ++ Build/source/libs/icu/icu-xetex/common/ucnv_err.c | 396 ++ Build/source/libs/icu/icu-xetex/common/ucnv_ext.c | 1088 +++++ Build/source/libs/icu/icu-xetex/common/ucnv_ext.h | 463 ++ Build/source/libs/icu/icu-xetex/common/ucnv_imp.h | 95 + Build/source/libs/icu/icu-xetex/common/ucnv_io.c | 1341 ++++++ Build/source/libs/icu/icu-xetex/common/ucnv_io.h | 125 + Build/source/libs/icu/icu-xetex/common/ucnv_lmb.c | 1281 +++++ Build/source/libs/icu/icu-xetex/common/ucnv_set.c | 67 + Build/source/libs/icu/icu-xetex/common/ucnv_u16.c | 1372 ++++++ Build/source/libs/icu/icu-xetex/common/ucnv_u32.c | 1250 +++++ Build/source/libs/icu/icu-xetex/common/ucnv_u7.c | 1459 ++++++ Build/source/libs/icu/icu-xetex/common/ucnv_u8.c | 1084 +++++ Build/source/libs/icu/icu-xetex/common/ucnvbocu.c | 1393 ++++++ Build/source/libs/icu/icu-xetex/common/ucnvdisp.c | 86 + Build/source/libs/icu/icu-xetex/common/ucnvhz.c | 591 +++ Build/source/libs/icu/icu-xetex/common/ucnvisci.c | 1469 ++++++ Build/source/libs/icu/icu-xetex/common/ucnvlat1.c | 743 +++ Build/source/libs/icu/icu-xetex/common/ucnvmbcs.c | 4946 ++++++++++++++++++++ Build/source/libs/icu/icu-xetex/common/ucnvmbcs.h | 490 ++ Build/source/libs/icu/icu-xetex/common/ucnvscsu.c | 2015 ++++++++ Build/source/libs/icu/icu-xetex/common/ucol_swp.c | 402 ++ Build/source/libs/icu/icu-xetex/common/ucol_swp.h | 58 + Build/source/libs/icu/icu-xetex/common/udata.c | 1474 ++++++ Build/source/libs/icu/icu-xetex/common/udatamem.c | 160 + Build/source/libs/icu/icu-xetex/common/udatamem.h | 61 + Build/source/libs/icu/icu-xetex/common/udataswp.c | 415 ++ Build/source/libs/icu/icu-xetex/common/udataswp.h | 349 ++ Build/source/libs/icu/icu-xetex/common/uenum.c | 181 + Build/source/libs/icu/icu-xetex/common/uenumimp.h | 153 + Build/source/libs/icu/icu-xetex/common/uhash.c | 1001 ++++ Build/source/libs/icu/icu-xetex/common/uhash.h | 728 +++ .../source/libs/icu/icu-xetex/common/uhash_us.cpp | 68 + Build/source/libs/icu/icu-xetex/common/uidna.cpp | 923 ++++ Build/source/libs/icu/icu-xetex/common/uinit.c | 113 + Build/source/libs/icu/icu-xetex/common/uinvchar.c | 515 ++ Build/source/libs/icu/icu-xetex/common/uinvchar.h | 66 + Build/source/libs/icu/icu-xetex/common/uiter.cpp | 1121 +++++ Build/source/libs/icu/icu-xetex/common/uloc.c | 3046 ++++++++++++ Build/source/libs/icu/icu-xetex/common/ulocimp.h | 24 + Build/source/libs/icu/icu-xetex/common/umapfile.c | 494 ++ Build/source/libs/icu/icu-xetex/common/umapfile.h | 32 + Build/source/libs/icu/icu-xetex/common/umath.c | 24 + Build/source/libs/icu/icu-xetex/common/umutex.c | 636 +++ Build/source/libs/icu/icu-xetex/common/umutex.h | 143 + Build/source/libs/icu/icu-xetex/common/unames.c | 2083 +++++++++ .../libs/icu/icu-xetex/common/unicode/brkiter.h | 557 +++ .../libs/icu/icu-xetex/common/unicode/caniter.h | 201 + .../libs/icu/icu-xetex/common/unicode/chariter.h | 716 +++ .../libs/icu/icu-xetex/common/unicode/dbbi.h | 41 + .../libs/icu/icu-xetex/common/unicode/docmain.h | 202 + .../libs/icu/icu-xetex/common/unicode/locid.h | 765 +++ .../libs/icu/icu-xetex/common/unicode/normlzr.h | 823 ++++ .../libs/icu/icu-xetex/common/unicode/parseerr.h | 92 + .../libs/icu/icu-xetex/common/unicode/parsepos.h | 230 + .../icu/icu-xetex/common/unicode/platform.h.in | 313 ++ .../libs/icu/icu-xetex/common/unicode/ppalmos.h | 273 ++ .../libs/icu/icu-xetex/common/unicode/putil.h | 184 + .../libs/icu/icu-xetex/common/unicode/pwin32.h | 311 ++ .../libs/icu/icu-xetex/common/unicode/rbbi.h | 700 +++ .../source/libs/icu/icu-xetex/common/unicode/rep.h | 259 + .../libs/icu/icu-xetex/common/unicode/resbund.h | 485 ++ .../libs/icu/icu-xetex/common/unicode/schriter.h | 187 + .../libs/icu/icu-xetex/common/unicode/strenum.h | 271 ++ .../libs/icu/icu-xetex/common/unicode/symtable.h | 112 + .../libs/icu/icu-xetex/common/unicode/ubidi.h | 1999 ++++++++ .../libs/icu/icu-xetex/common/unicode/ubrk.h | 482 ++ .../libs/icu/icu-xetex/common/unicode/ucasemap.h | 395 ++ .../libs/icu/icu-xetex/common/unicode/ucat.h | 158 + .../libs/icu/icu-xetex/common/unicode/uchar.h | 3013 ++++++++++++ .../libs/icu/icu-xetex/common/unicode/uchriter.h | 381 ++ .../libs/icu/icu-xetex/common/unicode/uclean.h | 267 ++ .../libs/icu/icu-xetex/common/unicode/ucnv.h | 1954 ++++++++ .../libs/icu/icu-xetex/common/unicode/ucnv_cb.h | 162 + .../libs/icu/icu-xetex/common/unicode/ucnv_err.h | 456 ++ .../libs/icu/icu-xetex/common/unicode/uconfig.h | 215 + .../libs/icu/icu-xetex/common/unicode/udata.h | 389 ++ .../libs/icu/icu-xetex/common/unicode/udeprctd.h | 50 + .../libs/icu/icu-xetex/common/unicode/udraft.h | 186 + .../libs/icu/icu-xetex/common/unicode/uenum.h | 134 + .../libs/icu/icu-xetex/common/unicode/uidna.h | 312 ++ .../libs/icu/icu-xetex/common/unicode/uintrnal.h | 180 + .../libs/icu/icu-xetex/common/unicode/uiter.h | 707 +++ .../libs/icu/icu-xetex/common/unicode/uloc.h | 931 ++++ .../libs/icu/icu-xetex/common/unicode/umachine.h | 338 ++ .../libs/icu/icu-xetex/common/unicode/umisc.h | 60 + .../libs/icu/icu-xetex/common/unicode/unifilt.h | 127 + .../libs/icu/icu-xetex/common/unicode/unifunct.h | 125 + .../libs/icu/icu-xetex/common/unicode/unimatch.h | 163 + .../libs/icu/icu-xetex/common/unicode/uniset.h | 1508 ++++++ .../libs/icu/icu-xetex/common/unicode/unistr.h | 4149 ++++++++++++++++ .../libs/icu/icu-xetex/common/unicode/unorm.h | 576 +++ .../libs/icu/icu-xetex/common/unicode/uobject.h | 308 ++ .../libs/icu/icu-xetex/common/unicode/uobslete.h | 32 + .../libs/icu/icu-xetex/common/unicode/urename.h | 1730 +++++++ .../libs/icu/icu-xetex/common/unicode/urep.h | 155 + .../libs/icu/icu-xetex/common/unicode/ures.h | 871 ++++ .../libs/icu/icu-xetex/common/unicode/uscript.h | 238 + .../libs/icu/icu-xetex/common/unicode/uset.h | 1055 +++++ .../libs/icu/icu-xetex/common/unicode/usetiter.h | 318 ++ .../libs/icu/icu-xetex/common/unicode/ushape.h | 263 ++ .../libs/icu/icu-xetex/common/unicode/usprep.h | 156 + .../libs/icu/icu-xetex/common/unicode/ustring.h | 1478 ++++++ .../libs/icu/icu-xetex/common/unicode/usystem.h | 46 + .../libs/icu/icu-xetex/common/unicode/utext.h | 1558 ++++++ .../source/libs/icu/icu-xetex/common/unicode/utf.h | 227 + .../libs/icu/icu-xetex/common/unicode/utf16.h | 605 +++ .../libs/icu/icu-xetex/common/unicode/utf32.h | 23 + .../libs/icu/icu-xetex/common/unicode/utf8.h | 652 +++ .../libs/icu/icu-xetex/common/unicode/utf_old.h | 1171 +++++ .../libs/icu/icu-xetex/common/unicode/utrace.h | 358 ++ .../libs/icu/icu-xetex/common/unicode/utypes.h | 787 ++++ .../libs/icu/icu-xetex/common/unicode/uversion.h | 266 ++ Build/source/libs/icu/icu-xetex/common/unifilt.cpp | 68 + .../source/libs/icu/icu-xetex/common/unifunct.cpp | 26 + Build/source/libs/icu/icu-xetex/common/uniset.cpp | 2145 +++++++++ .../libs/icu/icu-xetex/common/uniset_props.cpp | 1394 ++++++ .../libs/icu/icu-xetex/common/unisetspan.cpp | 1508 ++++++ .../source/libs/icu/icu-xetex/common/unisetspan.h | 155 + Build/source/libs/icu/icu-xetex/common/unistr.cpp | 1385 ++++++ .../libs/icu/icu-xetex/common/unistr_case.cpp | 246 + .../libs/icu/icu-xetex/common/unistr_cnv.cpp | 372 ++ .../libs/icu/icu-xetex/common/unistr_props.cpp | 72 + Build/source/libs/icu/icu-xetex/common/unorm.cpp | 4099 ++++++++++++++++ Build/source/libs/icu/icu-xetex/common/unorm_it.c | 643 +++ Build/source/libs/icu/icu-xetex/common/unorm_it.h | 148 + .../libs/icu/icu-xetex/common/unorm_props_data.c | 3092 ++++++++++++ .../source/libs/icu/icu-xetex/common/unormcmp.cpp | 690 +++ Build/source/libs/icu/icu-xetex/common/unormimp.h | 792 ++++ Build/source/libs/icu/icu-xetex/common/uobject.cpp | 114 + Build/source/libs/icu/icu-xetex/common/uprops.c | 545 +++ Build/source/libs/icu/icu-xetex/common/uprops.h | 426 ++ Build/source/libs/icu/icu-xetex/common/ures_cnv.c | 76 + Build/source/libs/icu/icu-xetex/common/uresbund.c | 2629 +++++++++++ Build/source/libs/icu/icu-xetex/common/uresdata.c | 1095 +++++ Build/source/libs/icu/icu-xetex/common/uresdata.h | 233 + Build/source/libs/icu/icu-xetex/common/uresimp.h | 243 + Build/source/libs/icu/icu-xetex/common/usc_impl.c | 361 ++ Build/source/libs/icu/icu-xetex/common/usc_impl.h | 149 + Build/source/libs/icu/icu-xetex/common/uscript.c | 112 + Build/source/libs/icu/icu-xetex/common/uset.cpp | 634 +++ Build/source/libs/icu/icu-xetex/common/uset_imp.h | 56 + .../libs/icu/icu-xetex/common/uset_props.cpp | 136 + .../source/libs/icu/icu-xetex/common/usetiter.cpp | 150 + Build/source/libs/icu/icu-xetex/common/ushape.c | 1198 +++++ Build/source/libs/icu/icu-xetex/common/usprep.cpp | 926 ++++ Build/source/libs/icu/icu-xetex/common/ustack.cpp | 61 + Build/source/libs/icu/icu-xetex/common/ustr_cnv.c | 230 + Build/source/libs/icu/icu-xetex/common/ustr_cnv.h | 42 + Build/source/libs/icu/icu-xetex/common/ustr_imp.h | 211 + Build/source/libs/icu/icu-xetex/common/ustr_wcs.c | 522 +++ Build/source/libs/icu/icu-xetex/common/ustrcase.c | 981 ++++ .../source/libs/icu/icu-xetex/common/ustrenum.cpp | 323 ++ Build/source/libs/icu/icu-xetex/common/ustrenum.h | 98 + Build/source/libs/icu/icu-xetex/common/ustrfmt.c | 57 + Build/source/libs/icu/icu-xetex/common/ustrfmt.h | 17 + Build/source/libs/icu/icu-xetex/common/ustring.c | 1490 ++++++ Build/source/libs/icu/icu-xetex/common/ustrtrns.c | 1181 +++++ Build/source/libs/icu/icu-xetex/common/utext.cpp | 2822 +++++++++++ Build/source/libs/icu/icu-xetex/common/utf_impl.c | 364 ++ Build/source/libs/icu/icu-xetex/common/util.cpp | 443 ++ Build/source/libs/icu/icu-xetex/common/util.h | 261 ++ .../libs/icu/icu-xetex/common/util_props.cpp | 214 + Build/source/libs/icu/icu-xetex/common/utrace.c | 486 ++ Build/source/libs/icu/icu-xetex/common/utracimp.h | 384 ++ Build/source/libs/icu/icu-xetex/common/utrie.c | 1210 +++++ Build/source/libs/icu/icu-xetex/common/utrie.h | 792 ++++ Build/source/libs/icu/icu-xetex/common/utypes.c | 199 + Build/source/libs/icu/icu-xetex/common/uvector.cpp | 472 ++ Build/source/libs/icu/icu-xetex/common/uvector.h | 408 ++ .../source/libs/icu/icu-xetex/common/uvectr32.cpp | 273 ++ Build/source/libs/icu/icu-xetex/common/uvectr32.h | 292 ++ Build/source/libs/icu/icu-xetex/common/wintz.c | 688 +++ Build/source/libs/icu/icu-xetex/common/wintz.h | 37 + 280 files changed, 167181 insertions(+) create mode 100644 Build/source/libs/icu/icu-xetex/common/Makefile.in create mode 100644 Build/source/libs/icu/icu-xetex/common/bmpset.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/bmpset.h create mode 100644 Build/source/libs/icu/icu-xetex/common/brkeng.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/brkeng.h create mode 100644 Build/source/libs/icu/icu-xetex/common/brkiter.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/caniter.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/chariter.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/charstr.h create mode 100644 Build/source/libs/icu/icu-xetex/common/cmemory.c create mode 100644 Build/source/libs/icu/icu-xetex/common/cmemory.h create mode 100644 Build/source/libs/icu/icu-xetex/common/common.rc create mode 100644 Build/source/libs/icu/icu-xetex/common/common.vcproj create mode 100644 Build/source/libs/icu/icu-xetex/common/cpputils.h create mode 100644 Build/source/libs/icu/icu-xetex/common/cstring.c create mode 100644 Build/source/libs/icu/icu-xetex/common/cstring.h create mode 100644 Build/source/libs/icu/icu-xetex/common/cwchar.c create mode 100644 Build/source/libs/icu/icu-xetex/common/cwchar.h create mode 100644 Build/source/libs/icu/icu-xetex/common/dictbe.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/dictbe.h create mode 100644 Build/source/libs/icu/icu-xetex/common/hash.h create mode 100644 Build/source/libs/icu/icu-xetex/common/icucfg.h.in create mode 100644 Build/source/libs/icu/icu-xetex/common/localsvc.h create mode 100644 Build/source/libs/icu/icu-xetex/common/locbased.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/locbased.h create mode 100644 Build/source/libs/icu/icu-xetex/common/locid.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/locmap.c create mode 100644 Build/source/libs/icu/icu-xetex/common/locmap.h create mode 100644 Build/source/libs/icu/icu-xetex/common/locutil.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/locutil.h create mode 100644 Build/source/libs/icu/icu-xetex/common/msvcres.h create mode 100644 Build/source/libs/icu/icu-xetex/common/mutex.h create mode 100644 Build/source/libs/icu/icu-xetex/common/normlzr.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/parsepos.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/propname.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/propname.h create mode 100644 Build/source/libs/icu/icu-xetex/common/punycode.c create mode 100644 Build/source/libs/icu/icu-xetex/common/punycode.h create mode 100644 Build/source/libs/icu/icu-xetex/common/putil.c create mode 100644 Build/source/libs/icu/icu-xetex/common/putilimp.h create mode 100644 Build/source/libs/icu/icu-xetex/common/rbbi.cpp create mode 100755 Build/source/libs/icu/icu-xetex/common/rbbicst.pl create mode 100644 Build/source/libs/icu/icu-xetex/common/rbbidata.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/rbbidata.h create mode 100644 Build/source/libs/icu/icu-xetex/common/rbbinode.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/rbbinode.h create mode 100644 Build/source/libs/icu/icu-xetex/common/rbbirb.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/rbbirb.h create mode 100644 Build/source/libs/icu/icu-xetex/common/rbbirpt.h create mode 100644 Build/source/libs/icu/icu-xetex/common/rbbirpt.txt create mode 100644 Build/source/libs/icu/icu-xetex/common/rbbiscan.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/rbbiscan.h create mode 100644 Build/source/libs/icu/icu-xetex/common/rbbisetb.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/rbbisetb.h create mode 100644 Build/source/libs/icu/icu-xetex/common/rbbistbl.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/rbbitblb.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/rbbitblb.h create mode 100644 Build/source/libs/icu/icu-xetex/common/resbund.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/resbund_cnv.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/ruleiter.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/ruleiter.h create mode 100644 Build/source/libs/icu/icu-xetex/common/schriter.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/serv.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/serv.h create mode 100644 Build/source/libs/icu/icu-xetex/common/servlk.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/servlkf.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/servloc.h create mode 100644 Build/source/libs/icu/icu-xetex/common/servls.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/servnotf.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/servnotf.h create mode 100644 Build/source/libs/icu/icu-xetex/common/servrbf.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/servslkf.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/sprpimpl.h create mode 100644 Build/source/libs/icu/icu-xetex/common/triedict.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/triedict.h create mode 100644 Build/source/libs/icu/icu-xetex/common/uarrsort.c create mode 100644 Build/source/libs/icu/icu-xetex/common/uarrsort.h create mode 100644 Build/source/libs/icu/icu-xetex/common/uassert.h create mode 100644 Build/source/libs/icu/icu-xetex/common/ubidi.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ubidi_props.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ubidi_props.h create mode 100644 Build/source/libs/icu/icu-xetex/common/ubidi_props_data.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ubidiimp.h create mode 100644 Build/source/libs/icu/icu-xetex/common/ubidiln.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ubidiwrt.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ubrk.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/ubrkimpl.h create mode 100644 Build/source/libs/icu/icu-xetex/common/ucase.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucase.h create mode 100644 Build/source/libs/icu/icu-xetex/common/ucase_props_data.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucasemap.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucat.c create mode 100644 Build/source/libs/icu/icu-xetex/common/uchar.c create mode 100644 Build/source/libs/icu/icu-xetex/common/uchar_props_data.c create mode 100644 Build/source/libs/icu/icu-xetex/common/uchriter.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/ucln.h create mode 100644 Build/source/libs/icu/icu-xetex/common/ucln_cmn.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucln_cmn.h create mode 100644 Build/source/libs/icu/icu-xetex/common/ucmndata.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucmndata.h create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnv.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnv2022.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnv_bld.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnv_bld.h create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnv_cb.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnv_cnv.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnv_cnv.h create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnv_err.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnv_ext.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnv_ext.h create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnv_imp.h create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnv_io.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnv_io.h create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnv_lmb.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnv_set.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnv_u16.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnv_u32.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnv_u7.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnv_u8.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnvbocu.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnvdisp.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnvhz.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnvisci.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnvlat1.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnvmbcs.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnvmbcs.h create mode 100644 Build/source/libs/icu/icu-xetex/common/ucnvscsu.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucol_swp.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ucol_swp.h create mode 100644 Build/source/libs/icu/icu-xetex/common/udata.c create mode 100644 Build/source/libs/icu/icu-xetex/common/udatamem.c create mode 100644 Build/source/libs/icu/icu-xetex/common/udatamem.h create mode 100644 Build/source/libs/icu/icu-xetex/common/udataswp.c create mode 100644 Build/source/libs/icu/icu-xetex/common/udataswp.h create mode 100644 Build/source/libs/icu/icu-xetex/common/uenum.c create mode 100644 Build/source/libs/icu/icu-xetex/common/uenumimp.h create mode 100644 Build/source/libs/icu/icu-xetex/common/uhash.c create mode 100644 Build/source/libs/icu/icu-xetex/common/uhash.h create mode 100644 Build/source/libs/icu/icu-xetex/common/uhash_us.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/uidna.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/uinit.c create mode 100644 Build/source/libs/icu/icu-xetex/common/uinvchar.c create mode 100644 Build/source/libs/icu/icu-xetex/common/uinvchar.h create mode 100644 Build/source/libs/icu/icu-xetex/common/uiter.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/uloc.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ulocimp.h create mode 100644 Build/source/libs/icu/icu-xetex/common/umapfile.c create mode 100644 Build/source/libs/icu/icu-xetex/common/umapfile.h create mode 100644 Build/source/libs/icu/icu-xetex/common/umath.c create mode 100644 Build/source/libs/icu/icu-xetex/common/umutex.c create mode 100644 Build/source/libs/icu/icu-xetex/common/umutex.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unames.c create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/brkiter.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/caniter.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/chariter.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/dbbi.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/docmain.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/locid.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/normlzr.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/parseerr.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/parsepos.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/platform.h.in create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/ppalmos.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/putil.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/pwin32.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/rbbi.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/rep.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/resbund.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/schriter.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/strenum.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/symtable.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/ubidi.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/ubrk.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/ucasemap.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/ucat.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/uchar.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/uchriter.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/uclean.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/ucnv.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/ucnv_cb.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/ucnv_err.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/uconfig.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/udata.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/udeprctd.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/udraft.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/uenum.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/uidna.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/uintrnal.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/uiter.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/uloc.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/umachine.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/umisc.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/unifilt.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/unifunct.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/unimatch.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/uniset.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/unistr.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/unorm.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/uobject.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/uobslete.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/urename.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/urep.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/ures.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/uscript.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/uset.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/usetiter.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/ushape.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/usprep.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/ustring.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/usystem.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/utext.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/utf.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/utf16.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/utf32.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/utf8.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/utf_old.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/utrace.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/utypes.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unicode/uversion.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unifilt.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/unifunct.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/uniset.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/uniset_props.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/unisetspan.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/unisetspan.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unistr.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/unistr_case.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/unistr_cnv.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/unistr_props.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/unorm.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/unorm_it.c create mode 100644 Build/source/libs/icu/icu-xetex/common/unorm_it.h create mode 100644 Build/source/libs/icu/icu-xetex/common/unorm_props_data.c create mode 100644 Build/source/libs/icu/icu-xetex/common/unormcmp.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/unormimp.h create mode 100644 Build/source/libs/icu/icu-xetex/common/uobject.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/uprops.c create mode 100644 Build/source/libs/icu/icu-xetex/common/uprops.h create mode 100644 Build/source/libs/icu/icu-xetex/common/ures_cnv.c create mode 100644 Build/source/libs/icu/icu-xetex/common/uresbund.c create mode 100644 Build/source/libs/icu/icu-xetex/common/uresdata.c create mode 100644 Build/source/libs/icu/icu-xetex/common/uresdata.h create mode 100644 Build/source/libs/icu/icu-xetex/common/uresimp.h create mode 100644 Build/source/libs/icu/icu-xetex/common/usc_impl.c create mode 100644 Build/source/libs/icu/icu-xetex/common/usc_impl.h create mode 100644 Build/source/libs/icu/icu-xetex/common/uscript.c create mode 100644 Build/source/libs/icu/icu-xetex/common/uset.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/uset_imp.h create mode 100644 Build/source/libs/icu/icu-xetex/common/uset_props.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/usetiter.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/ushape.c create mode 100644 Build/source/libs/icu/icu-xetex/common/usprep.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/ustack.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/ustr_cnv.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ustr_cnv.h create mode 100644 Build/source/libs/icu/icu-xetex/common/ustr_imp.h create mode 100644 Build/source/libs/icu/icu-xetex/common/ustr_wcs.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ustrcase.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ustrenum.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/ustrenum.h create mode 100644 Build/source/libs/icu/icu-xetex/common/ustrfmt.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ustrfmt.h create mode 100644 Build/source/libs/icu/icu-xetex/common/ustring.c create mode 100644 Build/source/libs/icu/icu-xetex/common/ustrtrns.c create mode 100644 Build/source/libs/icu/icu-xetex/common/utext.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/utf_impl.c create mode 100644 Build/source/libs/icu/icu-xetex/common/util.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/util.h create mode 100644 Build/source/libs/icu/icu-xetex/common/util_props.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/utrace.c create mode 100644 Build/source/libs/icu/icu-xetex/common/utracimp.h create mode 100644 Build/source/libs/icu/icu-xetex/common/utrie.c create mode 100644 Build/source/libs/icu/icu-xetex/common/utrie.h create mode 100644 Build/source/libs/icu/icu-xetex/common/utypes.c create mode 100644 Build/source/libs/icu/icu-xetex/common/uvector.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/uvector.h create mode 100644 Build/source/libs/icu/icu-xetex/common/uvectr32.cpp create mode 100644 Build/source/libs/icu/icu-xetex/common/uvectr32.h create mode 100644 Build/source/libs/icu/icu-xetex/common/wintz.c create mode 100644 Build/source/libs/icu/icu-xetex/common/wintz.h (limited to 'Build/source/libs/icu/icu-xetex/common') diff --git a/Build/source/libs/icu/icu-xetex/common/Makefile.in b/Build/source/libs/icu/icu-xetex/common/Makefile.in new file mode 100644 index 00000000000..6efb25907d9 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/Makefile.in @@ -0,0 +1,203 @@ +#****************************************************************************** +# +# Copyright (C) 1999-2007, International Business Machines +# Corporation and others. All Rights Reserved. +# +#****************************************************************************** +## Makefile.in for ICU - icuuc.so +## Stephen F. Booth + +## Source directory information +srcdir = @srcdir@ +top_srcdir = @top_srcdir@ + +top_builddir = .. + +## All the flags and other definitions are included here. +include $(top_builddir)/icudefs.mk + +## Build directory information +subdir = common + +# for service hook +LOCALSVC_CPP=localsvc.cpp +SVC_HOOK_INC=$(top_builddir)/common/svchook.mk + +## Extra files to remove for 'make clean' +CLEANFILES = *~ $(DEPS) $(IMPORT_LIB) $(MIDDLE_IMPORT_LIB) $(FINAL_IMPORT_LIB) $(SVC_HOOK_INC) + +## Target information + +TARGET_STUBNAME=$(COMMON_STUBNAME) + +ifneq ($(ENABLE_STATIC),) +TARGET = $(LIBDIR)/$(LIBSICU)$(TARGET_STUBNAME)$(ICULIBSUFFIX).$(A) +endif + +ifneq ($(ENABLE_SHARED),) +SO_TARGET = $(LIBDIR)/$(LIBICU)$(TARGET_STUBNAME)$(ICULIBSUFFIX).$(SO) +ALL_SO_TARGETS = $(SO_TARGET) $(MIDDLE_SO_TARGET) $(FINAL_SO_TARGET) $(SHARED_OBJECT) + +ifeq ($(ENABLE_SO_VERSION_DATA),1) +SO_VERSION_DATA = common.res +endif + +ifeq ($(OS390BATCH),1) +BATCH_TARGET = $(BATCH_COMMON_TARGET) +BATCH_LIBS = $(BATCH_LIBICUDT) -lm +endif # OS390BATCH + +endif # ENABLE_SHARED + +ALL_TARGETS = $(TARGET) $(ALL_SO_TARGETS) $(BATCH_TARGET) + +DYNAMICCPPFLAGS = $(SHAREDLIBCPPFLAGS) +DYNAMICCFLAGS = $(SHAREDLIBCFLAGS) +DYNAMICCXXFLAGS = $(SHAREDLIBCXXFLAGS) +CFLAGS += $(LIBCFLAGS) +CXXFLAGS += $(LIBCXXFLAGS) + +ifneq ($(top_builddir),$(top_srcdir)) +CPPFLAGS += -I$(top_builddir)/common +endif +CPPFLAGS += -I$(srcdir) -I$(top_srcdir)/i18n $(LIBCPPFLAGS) +DEFS += -DU_COMMON_IMPLEMENTATION +LDFLAGS += $(LDFLAGSICUUC) + +# $(LIBICUDT) is either stub data or the real DLL common data. +LIBS = $(LIBICUDT) $(DEFAULT_LIBS) + +OBJECTS = putil.o umath.o utypes.o uinvchar.o umutex.o ucln_cmn.o uinit.o uobject.o cmemory.o \ +udata.o ucmndata.o udatamem.o umapfile.o udataswp.o ucol_swp.o utrace.o \ +uhash.o uhash_us.o uenum.o ustrenum.o uvector.o ustack.o uvectr32.o \ +ucnv.o ucnv_bld.o ucnv_cnv.o ucnv_io.o ucnv_cb.o ucnv_err.o ucnvlat1.o \ +ucnv_u7.o ucnv_u8.o ucnv_u16.o ucnv_u32.o ucnvscsu.o ucnvbocu.o \ +ucnv_ext.o ucnvmbcs.o ucnv2022.o ucnvhz.o ucnv_lmb.o ucnvisci.o ucnvdisp.o ucnv_set.o \ +uresbund.o ures_cnv.o uresdata.o resbund.o resbund_cnv.o \ +ucat.o locmap.o uloc.o locid.o locutil.o \ +ustr_cnv.o unistr_cnv.o unistr.o unistr_case.o unistr_props.o \ +utf_impl.o ustring.o ustrcase.o ucasemap.o cstring.o ustrfmt.o ustrtrns.o ustr_wcs.o utext.o \ +normlzr.o unorm.o unormcmp.o unorm_it.o chariter.o schriter.o uchriter.o uiter.o \ +uchar.o uprops.o ucase.o propname.o ubidi_props.o ubidi.o ubidiwrt.o ubidiln.o ushape.o \ +uscript.o usc_impl.o unames.o \ +utrie.o bmpset.o unisetspan.o uset_props.o uniset_props.o uset.o uniset.o usetiter.o ruleiter.o caniter.o unifilt.o unifunct.o \ +uarrsort.o brkiter.o ubrk.o brkeng.o dictbe.o triedict.o \ +rbbi.o rbbidata.o rbbinode.o rbbirb.o rbbiscan.o rbbisetb.o rbbistbl.o rbbitblb.o \ +serv.o servnotf.o servls.o servlk.o servlkf.o servrbf.o servslkf.o \ +uidna.o usprep.o punycode.o \ +util.o util_props.o parsepos.o locbased.o cwchar.o wintz.o + +## Header files to install +HEADERS = $(srcdir)/unicode/*.h unicode/*.h + +STATIC_OBJECTS = $(OBJECTS:.o=.$(STATIC_O)) + +DEPS = $(OBJECTS:.o=.d) + +-include Makefile.local + +-include $(SVC_HOOK_INC) + + +## List of phony targets +.PHONY : all all-local install install-local clean clean-local \ +distclean distclean-local install-library install-headers dist \ +dist-local check check-local + +## Clear suffix list +.SUFFIXES : + +## List of standard targets +all: all-local +install: install-local +clean: clean-local +distclean : distclean-local +dist: dist-local +check: all check-local + +all-local: $(ALL_TARGETS) unicode/platform.h + +install-local: install-headers install-library + +install-library: all-local + $(MKINSTALLDIRS) $(DESTDIR)$(libdir) +ifneq ($(ENABLE_STATIC),) + $(INSTALL-L) $(TARGET) $(DESTDIR)$(libdir) +endif +ifneq ($(ENABLE_SHARED),) + $(INSTALL-L) $(FINAL_SO_TARGET) $(DESTDIR)$(libdir) +ifneq ($(FINAL_SO_TARGET),$(SO_TARGET)) + cd $(DESTDIR)$(libdir) && $(RM) $(notdir $(SO_TARGET)) && ln -s $(notdir $(FINAL_SO_TARGET)) $(notdir $(SO_TARGET)) +ifneq ($(FINAL_SO_TARGET),$(MIDDLE_SO_TARGET)) + cd $(DESTDIR)$(libdir) && $(RM) $(notdir $(MIDDLE_SO_TARGET)) && ln -s $(notdir $(FINAL_SO_TARGET)) $(notdir $(MIDDLE_SO_TARGET)) +endif +endif +ifneq ($(IMPORT_LIB_EXT),) + $(INSTALL-L) $(FINAL_IMPORT_LIB) $(DESTDIR)$(libdir) +ifneq ($(IMPORT_LIB),$(FINAL_IMPORT_LIB)) + cd $(DESTDIR)$(libdir) && $(RM) $(notdir $(IMPORT_LIB)) && ln -s $(notdir $(FINAL_IMPORT_LIB)) $(notdir $(IMPORT_LIB)) +endif +ifneq ($(MIDDLE_IMPORT_LIB),$(FINAL_IMPORT_LIB)) + cd $(DESTDIR)$(libdir) && $(RM) $(notdir $(MIDDLE_IMPORT_LIB)) && ln -s $(notdir $(FINAL_IMPORT_LIB)) $(notdir $(MIDDLE_IMPORT_LIB)) +endif +endif +endif + +$(SVC_HOOK_INC): + @echo generating $@ + @-test -f $(top_srcdir)/common/$(LOCALSVC_CPP) && ( echo "have $(LOCALSVC_CPP) - U_LOCAL_SERVICE_HOOK=1" ; \ + echo 'CPPFLAGS +=-DU_LOCAL_SERVICE_HOOK=1' > $@ ; \ + echo 'OBJECTS += $(LOCALSVC_CPP:%.cpp=%.o)' >> $@ \ + ) ; true + @echo "# Autogenerated by Makefile" >> $@ + +install-headers: + $(MKINSTALLDIRS) $(DESTDIR)$(includedir)/unicode + @for file in $(HEADERS); do \ + echo "$(INSTALL_DATA) $$file $(DESTDIR)$(includedir)/unicode"; \ + $(INSTALL_DATA) $$file $(DESTDIR)$(includedir)/unicode || exit; \ + done + +dist-local: + +clean-local: + test -z "$(CLEANFILES)" || $(RMV) $(CLEANFILES) + $(RMV) $(OBJECTS) $(STATIC_OBJECTS) $(ALL_TARGETS) $(SO_VERSION_DATA) + +distclean-local: clean-local + $(RMV) Makefile icucfg.h unicode/platform.h $(SVC_HOOK_INC) + +check-local: + +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status $(SVC_HOOK_INC) + cd $(top_builddir) \ + && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= $(SHELL) ./config.status + +unicode/platform.h: $(srcdir)/unicode/platform.h.in $(top_builddir)/config.status + cd $(top_builddir) \ + && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= $(SHELL) ./config.status + +ifneq ($(ENABLE_STATIC),) +$(TARGET): $(STATIC_OBJECTS) + $(AR) $(ARFLAGS) $(AR_OUTOPT)$@ $^ + $(RANLIB) $@ +endif + +ifneq ($(ENABLE_SHARED),) +$(SHARED_OBJECT): $(OBJECTS) $(SO_VERSION_DATA) + $(SHLIB.cc) $(LD_SONAME) $(OUTOPT)$@ $^ $(LIBS) + +ifeq ($(OS390BATCH),1) +$(BATCH_TARGET):$(OBJECTS) + $(SHLIB.cc) $(LD_SONAME) $(OUTOPT)$@ $^ $(BATCH_LIBS) +endif # OS390BATCH +endif # ENABLE_SHARED + +ifeq (,$(MAKECMDGOALS)) +-include $(DEPS) +else +ifneq ($(patsubst %clean,,$(MAKECMDGOALS)),) +-include $(DEPS) +endif +endif + diff --git a/Build/source/libs/icu/icu-xetex/common/bmpset.cpp b/Build/source/libs/icu/icu-xetex/common/bmpset.cpp new file mode 100644 index 00000000000..24b3545f2dd --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/bmpset.cpp @@ -0,0 +1,723 @@ +/* +****************************************************************************** +* +* Copyright (C) 2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: bmpset.cpp +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2007jan29 +* created by: Markus W. Scherer +*/ + +#include "unicode/utypes.h" +#include "unicode/uniset.h" +#include "cmemory.h" +#include "bmpset.h" + +U_NAMESPACE_BEGIN + +BMPSet::BMPSet(const int32_t *parentList, int32_t parentListLength) : + list(parentList), listLength(parentListLength) { + uprv_memset(asciiBytes, 0, sizeof(asciiBytes)); + uprv_memset(table7FF, 0, sizeof(table7FF)); + uprv_memset(bmpBlockBits, 0, sizeof(bmpBlockBits)); + + /* + * Set the list indexes for binary searches for + * U+0800, U+1000, U+2000, .., U+F000, U+10000. + * U+0800 is the first 3-byte-UTF-8 code point. Lower code points are + * looked up in the bit tables. + * The last pair of indexes is for finding supplementary code points. + */ + list4kStarts[0]=findCodePoint(0x800, 0, listLength-1); + int32_t i; + for(i=1; i<=0x10; ++i) { + list4kStarts[i]=findCodePoint(i<<12, list4kStarts[i-1], listLength-1); + } + list4kStarts[0x11]=listLength-1; + + initBits(); + overrideIllegal(); +} + +BMPSet::BMPSet(const BMPSet &otherBMPSet, const int32_t *newParentList, int32_t newParentListLength) : + list(newParentList), listLength(newParentListLength) { + uprv_memcpy(asciiBytes, otherBMPSet.asciiBytes, sizeof(asciiBytes)); + uprv_memcpy(table7FF, otherBMPSet.table7FF, sizeof(table7FF)); + uprv_memcpy(bmpBlockBits, otherBMPSet.bmpBlockBits, sizeof(bmpBlockBits)); + uprv_memcpy(list4kStarts, otherBMPSet.list4kStarts, sizeof(list4kStarts)); +} + +BMPSet::~BMPSet() { +} + +/* + * Set bits in a bit rectangle in "vertical" bit organization. + * start>6; + int32_t trail=start&0x3f; + + // Set one bit indicating an all-one block. + uint32_t bits=(uint32_t)1<>6; + int32_t limitTrail=limit&0x3f; + + if(lead==limitLead) { + // Partial vertical bit column. + while(trail0) { + do { + table[trail++]|=bits; + } while(trail<64); + ++lead; + } + if(lead=0x80) { + break; + } + do { + asciiBytes[start++]=1; + } while(start0x800) { + start=0x800; + break; + } + + start=list[listIndex++]; + if(listIndex0x10000) { + limit=0x10000; + } + + if(start>=6; + bmpBlockBits[start&0x3f]|=0x10001<<(start>>6); + start=(start+1)<<6; // Round up to the next block boundary. + minStart=start; // Ignore further ranges in this block. + } + if(start>6, limit>>6); + } + + if(limit&0x3f) { + // Mixed-value block of 64 code points. + limit>>=6; + bmpBlockBits[limit&0x3f]|=0x10001<<(limit>>6); + limit=(limit+1)<<6; // Round up to the next block boundary. + minStart=limit; // Ignore further ranges in this block. + } + } + } + + if(limit==0x10000) { + break; + } + + start=list[listIndex++]; + if(listIndex= hi || c >= list[hi-1]) + return hi; + // invariant: c >= list[lo] + // invariant: c < list[hi] + for (;;) { + int32_t i = (lo + hi) >> 1; + if (i == lo) { + break; // Found! + } else if (c < list[i]) { + hi = i; + } else { + lo = i; + } + } + return hi; +} + +UBool +BMPSet::contains(UChar32 c) const { + if((uint32_t)c<=0x7f) { + return (UBool)asciiBytes[c]; + } else if((uint32_t)c<=0x7ff) { + return (UBool)((table7FF[c&0x3f]&((uint32_t)1<<(c>>6)))!=0); + } else if((uint32_t)c<0xd800 || (c>=0xe000 && c<=0xffff)) { + int lead=c>>12; + uint32_t twoBits=(bmpBlockBits[(c>>6)&0x3f]>>lead)&0x10001; + if(twoBits<=1) { + // All 64 code points with the same bits 15..6 + // are either in the set or not. + return (UBool)twoBits; + } else { + // Look up the code point in its 4k block of code points. + return containsSlow(c, list4kStarts[lead], list4kStarts[lead+1]); + } + } else if((uint32_t)c<=0x10ffff) { + // surrogate or supplementary code point + return containsSlow(c, list4kStarts[0xd], list4kStarts[0x11]); + } else { + // Out-of-range code points get FALSE, consistent with long-standing + // behavior of UnicodeSet::contains(c). + return FALSE; + } +} + +/* + * Check for sufficient length for trail unit for each surrogate pair. + * Handle single surrogates as surrogate code points as usual in ICU. + */ +const UChar * +BMPSet::span(const UChar *s, const UChar *limit, USetSpanCondition spanCondition) const { + UChar c, c2; + + if(spanCondition) { + // span + do { + c=*s; + if(c<=0x7f) { + if(!asciiBytes[c]) { + break; + } + } else if(c<=0x7ff) { + if((table7FF[c&0x3f]&((uint32_t)1<<(c>>6)))==0) { + break; + } + } else if(c<0xd800 || c>=0xe000) { + int lead=c>>12; + uint32_t twoBits=(bmpBlockBits[(c>>6)&0x3f]>>lead)&0x10001; + if(twoBits<=1) { + // All 64 code points with the same bits 15..6 + // are either in the set or not. + if(twoBits==0) { + break; + } + } else { + // Look up the code point in its 4k block of code points. + if(!containsSlow(c, list4kStarts[lead], list4kStarts[lead+1])) { + break; + } + } + } else if(c>=0xdc00 || (s+1)==limit || (c2=s[1])<0xdc00 || c2>=0xe000) { + // surrogate code point + if(!containsSlow(c, list4kStarts[0xd], list4kStarts[0xe])) { + break; + } + } else { + // surrogate pair + if(!containsSlow(U16_GET_SUPPLEMENTARY(c, c2), list4kStarts[0x10], list4kStarts[0x11])) { + break; + } + ++s; + } + } while(++s>6)))!=0) { + break; + } + } else if(c<0xd800 || c>=0xe000) { + int lead=c>>12; + uint32_t twoBits=(bmpBlockBits[(c>>6)&0x3f]>>lead)&0x10001; + if(twoBits<=1) { + // All 64 code points with the same bits 15..6 + // are either in the set or not. + if(twoBits!=0) { + break; + } + } else { + // Look up the code point in its 4k block of code points. + if(containsSlow(c, list4kStarts[lead], list4kStarts[lead+1])) { + break; + } + } + } else if(c>=0xdc00 || (s+1)==limit || (c2=s[1])<0xdc00 || c2>=0xe000) { + // surrogate code point + if(containsSlow(c, list4kStarts[0xd], list4kStarts[0xe])) { + break; + } + } else { + // surrogate pair + if(containsSlow(U16_GET_SUPPLEMENTARY(c, c2), list4kStarts[0x10], list4kStarts[0x11])) { + break; + } + ++s; + } + } while(++s>6)))==0) { + break; + } + } else if(c<0xd800 || c>=0xe000) { + int lead=c>>12; + uint32_t twoBits=(bmpBlockBits[(c>>6)&0x3f]>>lead)&0x10001; + if(twoBits<=1) { + // All 64 code points with the same bits 15..6 + // are either in the set or not. + if(twoBits==0) { + break; + } + } else { + // Look up the code point in its 4k block of code points. + if(!containsSlow(c, list4kStarts[lead], list4kStarts[lead+1])) { + break; + } + } + } else if(c<0xdc00 || s==limit || (c2=*(limit-1))<0xd800 || c2>=0xdc00) { + // surrogate code point + if(!containsSlow(c, list4kStarts[0xd], list4kStarts[0xe])) { + break; + } + } else { + // surrogate pair + if(!containsSlow(U16_GET_SUPPLEMENTARY(c2, c), list4kStarts[0x10], list4kStarts[0x11])) { + break; + } + --limit; + } + if(s==limit) { + return s; + } + } + } else { + // span not + for(;;) { + c=*(--limit); + if(c<=0x7f) { + if(asciiBytes[c]) { + break; + } + } else if(c<=0x7ff) { + if((table7FF[c&0x3f]&((uint32_t)1<<(c>>6)))!=0) { + break; + } + } else if(c<0xd800 || c>=0xe000) { + int lead=c>>12; + uint32_t twoBits=(bmpBlockBits[(c>>6)&0x3f]>>lead)&0x10001; + if(twoBits<=1) { + // All 64 code points with the same bits 15..6 + // are either in the set or not. + if(twoBits!=0) { + break; + } + } else { + // Look up the code point in its 4k block of code points. + if(containsSlow(c, list4kStarts[lead], list4kStarts[lead+1])) { + break; + } + } + } else if(c<0xdc00 || s==limit || (c2=*(limit-1))<0xd800 || c2>=0xdc00) { + // surrogate code point + if(containsSlow(c, list4kStarts[0xd], list4kStarts[0xe])) { + break; + } + } else { + // surrogate pair + if(containsSlow(U16_GET_SUPPLEMENTARY(c2, c), list4kStarts[0x10], list4kStarts[0x11])) { + break; + } + --limit; + } + if(s==limit) { + return s; + } + } + } + return limit+1; +} + +/* + * Precheck for sufficient trail bytes at end of string only once per span. + * Check validity. + */ +const uint8_t * +BMPSet::spanUTF8(const uint8_t *s, int32_t length, USetSpanCondition spanCondition) const { + const uint8_t *limit=s+length; + uint8_t b=*s; + if((int8_t)b>=0) { + // Initial all-ASCII span. + if(spanCondition) { + do { + if(!asciiBytes[b] || ++s==limit) { + return s; + } + b=*s; + } while((int8_t)b>=0); + } else { + do { + if(asciiBytes[b] || ++s==limit) { + return s; + } + b=*s; + } while((int8_t)b>=0); + } + length=(int32_t)(limit-s); + } + + if(spanCondition!=USET_SPAN_NOT_CONTAINED) { + spanCondition=USET_SPAN_CONTAINED; // Pin to 0/1 values. + } + + const uint8_t *limit0=limit; + + /* + * Make sure that the last 1/2/3/4-byte sequence before limit is complete + * or runs into a lead byte. + * In the span loop compare s with limit only once + * per multi-byte character. + * + * Give a trailing illegal sequence the same value as the result of contains(FFFD), + * including it if that is part of the span, otherwise set limit0 to before + * the truncated sequence. + */ + b=*(limit-1); + if((int8_t)b<0) { + // b>=0x80: lead or trail byte + if(b<0xc0) { + // single trail byte, check for preceding 3- or 4-byte lead byte + if(length>=2 && (b=*(limit-2))>=0xe0) { + limit-=2; + if(asciiBytes[0x80]!=spanCondition) { + limit0=limit; + } + } else if(b<0xc0 && b>=0x80 && length>=3 && (b=*(limit-3))>=0xf0) { + // 4-byte lead byte with only two trail bytes + limit-=3; + if(asciiBytes[0x80]!=spanCondition) { + limit0=limit; + } + } + } else { + // lead byte with no trail bytes + --limit; + if(asciiBytes[0x80]!=spanCondition) { + limit0=limit; + } + } + } + + uint8_t t1, t2, t3; + + while(s=0xe0) { + if(b<0xf0) { + if( /* handle U+0000..U+FFFF inline */ + (t1=(uint8_t)(s[0]-0x80)) <= 0x3f && + (t2=(uint8_t)(s[1]-0x80)) <= 0x3f + ) { + b&=0xf; + uint32_t twoBits=(bmpBlockBits[t1]>>b)&0x10001; + if(twoBits<=1) { + // All 64 code points with this lead byte and middle trail byte + // are either in the set or not. + if(twoBits!=(uint32_t)spanCondition) { + return s-1; + } + } else { + // Look up the code point in its 4k block of code points. + UChar32 c=(b<<12)|(t1<<6)|t2; + if(containsSlow(c, list4kStarts[b], list4kStarts[b+1]) != spanCondition) { + return s-1; + } + } + s+=2; + continue; + } + } else if( /* handle U+10000..U+10FFFF inline */ + (t1=(uint8_t)(s[0]-0x80)) <= 0x3f && + (t2=(uint8_t)(s[1]-0x80)) <= 0x3f && + (t3=(uint8_t)(s[2]-0x80)) <= 0x3f + ) { + // Give an illegal sequence the same value as the result of contains(FFFD). + UChar32 c=((UChar32)(b-0xf0)<<18)|((UChar32)t1<<12)|(t2<<6)|t3; + if( ( (0x10000<=c && c<=0x10ffff) ? + containsSlow(c, list4kStarts[0x10], list4kStarts[0x11]) : + asciiBytes[0x80] + ) != spanCondition + ) { + return s-1; + } + s+=3; + continue; + } + } else /* 0xc0<=b<0xe0 */ { + if( /* handle U+0000..U+07FF inline */ + (t1=(uint8_t)(*s-0x80)) <= 0x3f + ) { + if(((table7FF[t1]&((uint32_t)1<<(b&0x1f)))!=0) != spanCondition) { + return s-1; + } + ++s; + continue; + } + } + + // Give an illegal sequence the same value as the result of contains(FFFD). + // Handle each byte of an illegal sequence separately to simplify the code; + // no need to optimize error handling. + if(asciiBytes[0x80]!=spanCondition) { + return s-1; + } + } + + return limit0; +} + +/* + * While going backwards through UTF-8 optimize only for ASCII. + * Unlike UTF-16, UTF-8 is not forward-backward symmetrical, that is, it is not + * possible to tell from the last byte in a multi-byte sequence how many + * preceding bytes there should be. Therefore, going backwards through UTF-8 + * is much harder than going forward. + */ +int32_t +BMPSet::spanBackUTF8(const uint8_t *s, int32_t length, USetSpanCondition spanCondition) const { + if(spanCondition!=USET_SPAN_NOT_CONTAINED) { + spanCondition=USET_SPAN_CONTAINED; // Pin to 0/1 values. + } + + uint8_t b; + + do { + b=s[--length]; + if((int8_t)b>=0) { + // ASCII sub-span + if(spanCondition) { + do { + if(!asciiBytes[b]) { + return length+1; + } else if(length==0) { + return 0; + } + b=s[--length]; + } while((int8_t)b>=0); + } else { + do { + if(asciiBytes[b]) { + return length+1; + } else if(length==0) { + return 0; + } + b=s[--length]; + } while((int8_t)b>=0); + } + } + + int32_t prev=length; + UChar32 c; + if(b<0xc0) { + // trail byte: collect a multi-byte character + c=utf8_prevCharSafeBody(s, 0, &length, b, -1); + if(c<0) { + c=0xfffd; + } + } else { + // lead byte in last-trail position + c=0xfffd; + } + // c is a valid code point, not ASCII, not a surrogate + if(c<=0x7ff) { + if(((table7FF[c&0x3f]&((uint32_t)1<<(c>>6)))!=0) != spanCondition) { + return prev+1; + } + } else if(c<=0xffff) { + int lead=c>>12; + uint32_t twoBits=(bmpBlockBits[(c>>6)&0x3f]>>lead)&0x10001; + if(twoBits<=1) { + // All 64 code points with the same bits 15..6 + // are either in the set or not. + if(twoBits!=(uint32_t)spanCondition) { + return prev+1; + } + } else { + // Look up the code point in its 4k block of code points. + if(containsSlow(c, list4kStarts[lead], list4kStarts[lead+1]) != spanCondition) { + return prev+1; + } + } + } else { + if(containsSlow(c, list4kStarts[0x10], list4kStarts[0x11]) != spanCondition) { + return prev+1; + } + } + } while(length>0); + return 0; +} + +U_NAMESPACE_END diff --git a/Build/source/libs/icu/icu-xetex/common/bmpset.h b/Build/source/libs/icu/icu-xetex/common/bmpset.h new file mode 100644 index 00000000000..d9e08eaa66f --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/bmpset.h @@ -0,0 +1,161 @@ +/* +****************************************************************************** +* +* Copyright (C) 2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: bmpset.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2007jan29 +* created by: Markus W. Scherer +*/ + +#ifndef __BMPSET_H__ +#define __BMPSET_H__ + +#include "unicode/utypes.h" +#include "unicode/uniset.h" + +U_NAMESPACE_BEGIN + +/* + * Helper class for frozen UnicodeSets, implements contains() and span() + * optimized for BMP code points. Structured to be UTF-8-friendly. + * + * ASCII: Look up bytes. + * 2-byte characters: Bits organized vertically. + * 3-byte characters: Use zero/one/mixed data per 64-block in U+0000..U+FFFF, + * with mixed for illegal ranges. + * Supplementary characters: Call contains() on the parent set. + */ +class BMPSet : public UMemory { +public: + BMPSet(const int32_t *parentList, int32_t parentListLength); + BMPSet(const BMPSet &otherBMPSet, const int32_t *newParentList, int32_t newParentListLength); + virtual ~BMPSet(); + + virtual UBool contains(UChar32 c) const; + + /* + * Span the initial substring for which each character c has spanCondition==contains(c). + * It must be s0 and spanCondition==0 or 1. + * @return The string pointer which limits the span. + */ + const uint8_t *spanUTF8(const uint8_t *s, int32_t length, USetSpanCondition spanCondition) const; + + /* + * Span the trailing substring for which each character c has spanCondition==contains(c). + * It must be length>0 and spanCondition==0 or 1. + * @return The start of the span. + */ + int32_t spanBackUTF8(const uint8_t *s, int32_t length, USetSpanCondition spanCondition) const; + +private: + void initBits(); + void overrideIllegal(); + + /** + * Same as UnicodeSet::findCodePoint(UChar32 c) const except that the + * binary search is restricted for finding code points in a certain range. + * + * For restricting the search for finding in the range start..end, + * pass in + * lo=findCodePoint(start) and + * hi=findCodePoint(end) + * with 0<=lo<=hi= 0 && breakType < (int32_t)(sizeof(fHandled)/sizeof(fHandled[0])) + && fHandled[breakType] != 0 && fHandled[breakType]->contains(c)); +} + +int32_t +UnhandledEngine::findBreaks( UText *text, + int32_t startPos, + int32_t endPos, + UBool reverse, + int32_t breakType, + UStack &/*foundBreaks*/ ) const { + if (breakType >= 0 && breakType < (int32_t)(sizeof(fHandled)/sizeof(fHandled[0]))) { + UChar32 c = utext_current32(text); + if (reverse) { + while((int32_t)utext_getNativeIndex(text) > startPos && fHandled[breakType]->contains(c)) { + c = utext_previous32(text); + } + } + else { + while((int32_t)utext_getNativeIndex(text) < endPos && fHandled[breakType]->contains(c)) { + utext_next32(text); // TODO: recast loop to work with post-increment operations. + c = utext_current32(text); + } + } + } + return 0; +} + +void +UnhandledEngine::handleCharacter(UChar32 c, int32_t breakType) { + if (breakType >= 0 && breakType < (int32_t)(sizeof(fHandled)/sizeof(fHandled[0]))) { + if (fHandled[breakType] == 0) { + fHandled[breakType] = new UnicodeSet(); + if (fHandled[breakType] == 0) { + return; + } + } + if (!fHandled[breakType]->contains(c)) { + UErrorCode status = U_ZERO_ERROR; + // Apply the entire script of the character. + int32_t script = u_getIntPropertyValue(c, UCHAR_SCRIPT); + fHandled[breakType]->applyIntPropertyValue(UCHAR_SCRIPT, script, status); + } + } +} + +/* + ****************************************************************** + */ + +ICULanguageBreakFactory::ICULanguageBreakFactory(UErrorCode &/*status*/) { + fEngines = 0; +} + +ICULanguageBreakFactory::~ICULanguageBreakFactory() { + if (fEngines != 0) { + delete fEngines; + } +} + +U_NAMESPACE_END +U_CDECL_BEGIN +static void U_CALLCONV _deleteEngine(void *obj) { + delete (const U_NAMESPACE_QUALIFIER LanguageBreakEngine *) obj; +} +U_CDECL_END +U_NAMESPACE_BEGIN + +const LanguageBreakEngine * +ICULanguageBreakFactory::getEngineFor(UChar32 c, int32_t breakType) { + UBool needsInit; + int32_t i; + const LanguageBreakEngine *lbe = NULL; + UErrorCode status = U_ZERO_ERROR; + + // TODO: The global mutex should not be used. + // The global mutex should only be used for short periods. + // A ICULanguageBreakFactory specific mutex should be used. + umtx_lock(NULL); + needsInit = (UBool)(fEngines == NULL); + if (!needsInit) { + i = fEngines->size(); + while (--i >= 0) { + lbe = (const LanguageBreakEngine *)(fEngines->elementAt(i)); + if (lbe != NULL && lbe->handles(c, breakType)) { + break; + } + lbe = NULL; + } + } + umtx_unlock(NULL); + + if (lbe != NULL) { + return lbe; + } + + if (needsInit) { + UStack *engines = new UStack(_deleteEngine, NULL, status); + if (U_SUCCESS(status) && engines == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + } + else if (U_FAILURE(status)) { + delete engines; + engines = NULL; + } + else { + umtx_lock(NULL); + if (fEngines == NULL) { + fEngines = engines; + engines = NULL; + } + umtx_unlock(NULL); + delete engines; + } + } + + if (fEngines == NULL) { + return NULL; + } + + // We didn't find an engine the first time through, or there was no + // stack. Create an engine. + const LanguageBreakEngine *newlbe = loadEngineFor(c, breakType); + + // Now get the lock, and see if someone else has created it in the + // meantime + umtx_lock(NULL); + i = fEngines->size(); + while (--i >= 0) { + lbe = (const LanguageBreakEngine *)(fEngines->elementAt(i)); + if (lbe != NULL && lbe->handles(c, breakType)) { + break; + } + lbe = NULL; + } + if (lbe == NULL && newlbe != NULL) { + fEngines->push((void *)newlbe, status); + lbe = newlbe; + newlbe = NULL; + } + umtx_unlock(NULL); + + delete newlbe; + + return lbe; +} + +const LanguageBreakEngine * +ICULanguageBreakFactory::loadEngineFor(UChar32 c, int32_t breakType) { + UErrorCode status = U_ZERO_ERROR; + UScriptCode code = uscript_getScript(c, &status); + if (U_SUCCESS(status)) { + const CompactTrieDictionary *dict = loadDictionaryFor(code, breakType); + if (dict != NULL) { + const LanguageBreakEngine *engine = NULL; + switch(code) { + case USCRIPT_THAI: + engine = new ThaiBreakEngine(dict, status); + break; + default: + break; + } + if (engine == NULL) { + delete dict; + } + else if (U_FAILURE(status)) { + delete engine; + engine = NULL; + } + return engine; + } + } + return NULL; +} + +const CompactTrieDictionary * +ICULanguageBreakFactory::loadDictionaryFor(UScriptCode script, int32_t /*breakType*/) { + UErrorCode status = U_ZERO_ERROR; + // Open root from brkitr tree. + char dictnbuff[256]; + char ext[4]={'\0'}; + + UResourceBundle *b = ures_open(U_ICUDATA_BRKITR, "", &status); + b = ures_getByKeyWithFallback(b, "dictionaries", b, &status); + b = ures_getByKeyWithFallback(b, uscript_getShortName(script), b, &status); + int32_t dictnlength = 0; + const UChar *dictfname = ures_getString(b, &dictnlength, &status); + if (U_SUCCESS(status) && (size_t)dictnlength >= sizeof(dictnbuff)) { + dictnlength = 0; + status = U_BUFFER_OVERFLOW_ERROR; + } + if (U_SUCCESS(status) && dictfname) { + UChar* extStart=u_strchr(dictfname, 0x002e); + int len = 0; + if(extStart!=NULL){ + len = extStart-dictfname; + u_UCharsToChars(extStart+1, ext, sizeof(ext)); // nul terminates the buff + u_UCharsToChars(dictfname, dictnbuff, len); + } + dictnbuff[len]=0; // nul terminate + } + ures_close(b); + UDataMemory *file = udata_open(U_ICUDATA_BRKITR, ext, dictnbuff, &status); + if (U_SUCCESS(status)) { + const CompactTrieDictionary *dict = new CompactTrieDictionary( + file, status); + if (U_SUCCESS(status) && dict == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + } + if (U_FAILURE(status)) { + delete dict; + dict = NULL; + } + return dict; + } + return NULL; +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_BREAK_ITERATION */ diff --git a/Build/source/libs/icu/icu-xetex/common/brkeng.h b/Build/source/libs/icu/icu-xetex/common/brkeng.h new file mode 100644 index 00000000000..618b2aeea5f --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/brkeng.h @@ -0,0 +1,292 @@ +/** + ************************************************************************************ + * Copyright (C) 2006-2007, International Business Machines Corporation and others. * + * All Rights Reserved. * + ************************************************************************************ + */ + +#ifndef BRKENG_H +#define BRKENG_H + +#include "unicode/utypes.h" +#include "unicode/uobject.h" +#include "unicode/utext.h" +#include "unicode/uscript.h" + +U_NAMESPACE_BEGIN + +class UnicodeSet; +class UStack; +class CompactTrieDictionary; + +/******************************************************************* + * LanguageBreakEngine + */ + +/** + *

LanguageBreakEngines implement language-specific knowledge for + * finding text boundaries within a run of characters belonging to a + * specific set. The boundaries will be of a specific kind, e.g. word, + * line, etc.

+ * + *

LanguageBreakEngines should normally be implemented so as to + * be shared between threads without locking.

+ */ +class LanguageBreakEngine : public UMemory { + public: + + /** + *

Default constructor.

+ * + */ + LanguageBreakEngine(); + + /** + *

Virtual destructor.

+ */ + virtual ~LanguageBreakEngine(); + + /** + *

Indicate whether this engine handles a particular character for + * a particular kind of break.

+ * + * @param c A character which begins a run that the engine might handle + * @param breakType The type of text break which the caller wants to determine + * @return TRUE if this engine handles the particular character and break + * type. + */ + virtual UBool handles(UChar32 c, int32_t breakType) const = 0; + + /** + *

Find any breaks within a run in the supplied text.

+ * + * @param text A UText representing the text. The + * iterator is left at the end of the run of characters which the engine + * is capable of handling. + * @param startPos The start of the run within the supplied text. + * @param endPos The end of the run within the supplied text. + * @param reverse Whether the caller is looking for breaks in a reverse + * direction. + * @param breakType The type of break desired, or -1. + * @param foundBreaks An allocated C array of the breaks found, if any + * @return The number of breaks found. + */ + virtual int32_t findBreaks( UText *text, + int32_t startPos, + int32_t endPos, + UBool reverse, + int32_t breakType, + UStack &foundBreaks ) const = 0; + +}; + +/******************************************************************* + * LanguageBreakFactory + */ + +/** + *

LanguageBreakFactorys find and return a LanguageBreakEngine + * that can determine breaks for characters in a specific set, if + * such an object can be found.

+ * + *

If a LanguageBreakFactory is to be shared between threads, + * appropriate synchronization must be used; there is none internal + * to the factory.

+ * + *

A LanguageBreakEngine returned by a LanguageBreakFactory can + * normally be shared between threads without synchronization, unless + * the specific subclass of LanguageBreakFactory indicates otherwise.

+ * + *

A LanguageBreakFactory is responsible for deleting any LanguageBreakEngine + * it returns when it itself is deleted, unless the specific subclass of + * LanguageBreakFactory indicates otherwise. Naturally, the factory should + * not be deleted until the LanguageBreakEngines it has returned are no + * longer needed.

+ */ +class LanguageBreakFactory : public UMemory { + public: + + /** + *

Default constructor.

+ * + */ + LanguageBreakFactory(); + + /** + *

Virtual destructor.

+ */ + virtual ~LanguageBreakFactory(); + + /** + *

Find and return a LanguageBreakEngine that can find the desired + * kind of break for the set of characters to which the supplied + * character belongs. It is up to the set of available engines to + * determine what the sets of characters are.

+ * + * @param c A character that begins a run for which a LanguageBreakEngine is + * sought. + * @param breakType The kind of text break for which a LanguageBreakEngine is + * sought. + * @return A LanguageBreakEngine with the desired characteristics, or 0. + */ + virtual const LanguageBreakEngine *getEngineFor(UChar32 c, int32_t breakType) = 0; + +}; + +/******************************************************************* + * UnhandledEngine + */ + +/** + *

UnhandledEngine is a special subclass of LanguageBreakEngine that + * handles characters that no other LanguageBreakEngine is available to + * handle. It is told the character and the type of break; at its + * discretion it may handle more than the specified character (e.g., + * the entire script to which that character belongs.

+ * + *

UnhandledEngines may not be shared between threads without + * external synchronization.

+ */ + +class UnhandledEngine : public LanguageBreakEngine { + private: + + /** + * The sets of characters handled, for each break type + * @internal + */ + + UnicodeSet *fHandled[4]; + + public: + + /** + *

Default constructor.

+ * + */ + UnhandledEngine(UErrorCode &status); + + /** + *

Virtual destructor.

+ */ + virtual ~UnhandledEngine(); + + /** + *

Indicate whether this engine handles a particular character for + * a particular kind of break.

+ * + * @param c A character which begins a run that the engine might handle + * @param breakType The type of text break which the caller wants to determine + * @return TRUE if this engine handles the particular character and break + * type. + */ + virtual UBool handles(UChar32 c, int32_t breakType) const; + + /** + *

Find any breaks within a run in the supplied text.

+ * + * @param text A UText representing the text (TODO: UText). The + * iterator is left at the end of the run of characters which the engine + * is capable of handling. + * @param startPos The start of the run within the supplied text. + * @param endPos The end of the run within the supplied text. + * @param reverse Whether the caller is looking for breaks in a reverse + * direction. + * @param breakType The type of break desired, or -1. + * @param foundBreaks An allocated C array of the breaks found, if any + * @return The number of breaks found. + */ + virtual int32_t findBreaks( UText *text, + int32_t startPos, + int32_t endPos, + UBool reverse, + int32_t breakType, + UStack &foundBreaks ) const; + + /** + *

Tell the engine to handle a particular character and break type.

+ * + * @param c A character which the engine should handle + * @param breakType The type of text break for which the engine should handle c + */ + virtual void handleCharacter(UChar32 c, int32_t breakType); + +}; + +/******************************************************************* + * ICULanguageBreakFactory + */ + +/** + *

ICULanguageBreakFactory is the default LanguageBreakFactory for + * ICU. It creates dictionary-based LanguageBreakEngines from dictionary + * data in the ICU data file.

+ */ +class ICULanguageBreakFactory : public LanguageBreakFactory { + private: + + /** + * The stack of break engines created by this factory + * @internal + */ + + UStack *fEngines; + + public: + + /** + *

Standard constructor.

+ * + */ + ICULanguageBreakFactory(UErrorCode &status); + + /** + *

Virtual destructor.

+ */ + virtual ~ICULanguageBreakFactory(); + + /** + *

Find and return a LanguageBreakEngine that can find the desired + * kind of break for the set of characters to which the supplied + * character belongs. It is up to the set of available engines to + * determine what the sets of characters are.

+ * + * @param c A character that begins a run for which a LanguageBreakEngine is + * sought. + * @param breakType The kind of text break for which a LanguageBreakEngine is + * sought. + * @return A LanguageBreakEngine with the desired characteristics, or 0. + */ + virtual const LanguageBreakEngine *getEngineFor(UChar32 c, int32_t breakType); + + protected: + + /** + *

Create a LanguageBreakEngine for the set of characters to which + * the supplied character belongs, for the specified break type.

+ * + * @param c A character that begins a run for which a LanguageBreakEngine is + * sought. + * @param breakType The kind of text break for which a LanguageBreakEngine is + * sought. + * @return A LanguageBreakEngine with the desired characteristics, or 0. + */ + virtual const LanguageBreakEngine *loadEngineFor(UChar32 c, int32_t breakType); + + /** + *

Create a CompactTrieDictionary for the specified script and break type.

+ * + * @param script An ISO 15924 script code that identifies the dictionary to be + * created. + * @param breakType The kind of text break for which a dictionary is + * sought. + * @return A CompactTrieDictionary with the desired characteristics, or 0. + */ + virtual const CompactTrieDictionary *loadDictionaryFor(UScriptCode script, int32_t breakType); + +}; + +U_NAMESPACE_END + + /* BRKENG_H */ +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/brkiter.cpp b/Build/source/libs/icu/icu-xetex/common/brkiter.cpp new file mode 100644 index 00000000000..3bfc13adee2 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/brkiter.cpp @@ -0,0 +1,454 @@ +/* +******************************************************************************* +* Copyright (C) 1997-2007, International Business Machines Corporation and * +* others. All Rights Reserved. * +******************************************************************************* +* +* File TXTBDRY.CPP +* +* Modification History: +* +* Date Name Description +* 02/18/97 aliu Converted from OpenClass. Added DONE. +* 01/13/2000 helena Added UErrorCode parameter to createXXXInstance methods. +***************************************************************************************** +*/ + +// ***************************************************************************** +// This file was generated from the java source file BreakIterator.java +// ***************************************************************************** + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_BREAK_ITERATION + +#include "unicode/rbbi.h" +#include "unicode/brkiter.h" +#include "unicode/udata.h" +#include "unicode/ures.h" +#include "unicode/ustring.h" +#include "ucln_cmn.h" +#include "cstring.h" +#include "umutex.h" +#include "servloc.h" +#include "locbased.h" +#include "uresimp.h" +#include "uassert.h" +#include "ubrkimpl.h" + +// ***************************************************************************** +// class BreakIterator +// This class implements methods for finding the location of boundaries in text. +// Instances of BreakIterator maintain a current position and scan over text +// returning the index of characters where boundaries occur. +// ***************************************************************************** + +U_NAMESPACE_BEGIN + +// ------------------------------------- + +BreakIterator* +BreakIterator::buildInstance(const Locale& loc, const char *type, int32_t kind, UErrorCode &status) +{ + char fnbuff[256]; + char ext[4]={'\0'}; + char actualLocale[ULOC_FULLNAME_CAPACITY]; + int32_t size; + const UChar* brkfname = NULL; + UResourceBundle brkRulesStack; + UResourceBundle brkNameStack; + UResourceBundle *brkRules = &brkRulesStack; + UResourceBundle *brkName = &brkNameStack; + RuleBasedBreakIterator *result = NULL; + + if (U_FAILURE(status)) + return NULL; + + ures_initStackObject(brkRules); + ures_initStackObject(brkName); + + // Get the locale + UResourceBundle *b = ures_open(U_ICUDATA_BRKITR, loc.getName(), &status); + /* this is a hack for now. Should be fixed when the data is fetched from + brk_index.txt */ + if(status==U_USING_DEFAULT_WARNING){ + status=U_ZERO_ERROR; + ures_openFillIn(b, U_ICUDATA_BRKITR, "", &status); + } + + // Get the "boundaries" array. + if (U_SUCCESS(status)) { + brkRules = ures_getByKeyWithFallback(b, "boundaries", brkRules, &status); + // Get the string object naming the rules file + brkName = ures_getByKeyWithFallback(brkRules, type, brkName, &status); + // Get the actual string + brkfname = ures_getString(brkName, &size, &status); + U_ASSERT((size_t)size=sizeof(fnbuff)) { + size=0; + if (U_SUCCESS(status)) { + status = U_BUFFER_OVERFLOW_ERROR; + } + } + + // Use the string if we found it + if (U_SUCCESS(status) && brkfname) { + uprv_strncpy(actualLocale, + ures_getLocale(brkName, &status), + sizeof(actualLocale)/sizeof(actualLocale[0])); + + UChar* extStart=u_strchr(brkfname, 0x002e); + int len = 0; + if(extStart!=NULL){ + len = extStart-brkfname; + u_UCharsToChars(extStart+1, ext, sizeof(ext)); // nul terminates the buff + u_UCharsToChars(brkfname, fnbuff, len); + } + fnbuff[len]=0; // nul terminate + } + } + + ures_close(brkRules); + ures_close(brkName); + + UDataMemory* file = udata_open(U_ICUDATA_BRKITR, ext, fnbuff, &status); + if (U_FAILURE(status)) { + ures_close(b); + return NULL; + } + + // Create a RuleBasedBreakIterator + result = new RuleBasedBreakIterator(file, status); + + // If there is a result, set the valid locale and actual locale, and the kind + if (U_SUCCESS(status) && result != NULL) { + U_LOCALE_BASED(locBased, *(BreakIterator*)result); + locBased.setLocaleIDs(ures_getLocaleByType(b, ULOC_VALID_LOCALE, &status), actualLocale); + result->setBreakType(kind); + } + + ures_close(b); + + if (U_FAILURE(status) && result != NULL) { // Sometimes redundant check, but simple + delete result; + return NULL; + } + + if (result == NULL) { + udata_close(file); + if (U_SUCCESS(status)) { + status = U_MEMORY_ALLOCATION_ERROR; + } + } + + return result; +} + +// Creates a break iterator for word breaks. +BreakIterator* U_EXPORT2 +BreakIterator::createWordInstance(const Locale& key, UErrorCode& status) +{ + return createInstance(key, UBRK_WORD, status); +} + +// ------------------------------------- + +// Creates a break iterator for line breaks. +BreakIterator* U_EXPORT2 +BreakIterator::createLineInstance(const Locale& key, UErrorCode& status) +{ + return createInstance(key, UBRK_LINE, status); +} + +// ------------------------------------- + +// Creates a break iterator for character breaks. +BreakIterator* U_EXPORT2 +BreakIterator::createCharacterInstance(const Locale& key, UErrorCode& status) +{ + return createInstance(key, UBRK_CHARACTER, status); +} + +// ------------------------------------- + +// Creates a break iterator for sentence breaks. +BreakIterator* U_EXPORT2 +BreakIterator::createSentenceInstance(const Locale& key, UErrorCode& status) +{ + return createInstance(key, UBRK_SENTENCE, status); +} + +// ------------------------------------- + +// Creates a break iterator for title casing breaks. +BreakIterator* U_EXPORT2 +BreakIterator::createTitleInstance(const Locale& key, UErrorCode& status) +{ + return createInstance(key, UBRK_TITLE, status); +} + +// ------------------------------------- + +// Gets all the available locales that has localized text boundary data. +const Locale* U_EXPORT2 +BreakIterator::getAvailableLocales(int32_t& count) +{ + return Locale::getAvailableLocales(count); +} + +// ------------------------------------- +// Gets the objectLocale display name in the default locale language. +UnicodeString& U_EXPORT2 +BreakIterator::getDisplayName(const Locale& objectLocale, + UnicodeString& name) +{ + return objectLocale.getDisplayName(name); +} + +// ------------------------------------- +// Gets the objectLocale display name in the displayLocale language. +UnicodeString& U_EXPORT2 +BreakIterator::getDisplayName(const Locale& objectLocale, + const Locale& displayLocale, + UnicodeString& name) +{ + return objectLocale.getDisplayName(displayLocale, name); +} + +// ------------------------------------------ +// +// Default constructor and destructor +// +//------------------------------------------- + +BreakIterator::BreakIterator() +{ + fBufferClone = FALSE; + *validLocale = *actualLocale = 0; +} + +BreakIterator::~BreakIterator() +{ +} + +// ------------------------------------------ +// +// Registration +// +//------------------------------------------- +#if !UCONFIG_NO_SERVICE + +// ------------------------------------- + +class ICUBreakIteratorFactory : public ICUResourceBundleFactory { +protected: + virtual UObject* handleCreate(const Locale& loc, int32_t kind, const ICUService* /*service*/, UErrorCode& status) const { + return BreakIterator::makeInstance(loc, kind, status); + } +}; + +// ------------------------------------- + +class ICUBreakIteratorService : public ICULocaleService { +public: + ICUBreakIteratorService() + : ICULocaleService(UNICODE_STRING("Break Iterator", 14)) + { + UErrorCode status = U_ZERO_ERROR; + registerFactory(new ICUBreakIteratorFactory(), status); + } + + virtual UObject* cloneInstance(UObject* instance) const { + return ((BreakIterator*)instance)->clone(); + } + + virtual UObject* handleDefault(const ICUServiceKey& key, UnicodeString* /*actualID*/, UErrorCode& status) const { + LocaleKey& lkey = (LocaleKey&)key; + int32_t kind = lkey.kind(); + Locale loc; + lkey.currentLocale(loc); + return BreakIterator::makeInstance(loc, kind, status); + } + + virtual UBool isDefault() const { + return countFactories() == 1; + } +}; + +// ------------------------------------- + +U_NAMESPACE_END + +// defined in ucln_cmn.h + +static U_NAMESPACE_QUALIFIER ICULocaleService* gService = NULL; + +/** + * Release all static memory held by breakiterator. + */ +U_CDECL_BEGIN +static UBool U_CALLCONV breakiterator_cleanup(void) { +#if !UCONFIG_NO_SERVICE + if (gService) { + delete gService; + gService = NULL; + } +#endif + return TRUE; +} +U_CDECL_END +U_NAMESPACE_BEGIN + +static ICULocaleService* +getService(void) +{ + UBool needsInit; + UMTX_CHECK(NULL, (UBool)(gService == NULL), needsInit); + + if (needsInit) { + ICULocaleService *tService = new ICUBreakIteratorService(); + umtx_lock(NULL); + if (gService == NULL) { + gService = tService; + tService = NULL; + ucln_common_registerCleanup(UCLN_COMMON_BREAKITERATOR, breakiterator_cleanup); + } + umtx_unlock(NULL); + delete tService; + } + return gService; +} + +// ------------------------------------- + +static inline UBool +hasService(void) +{ + UBool retVal; + UMTX_CHECK(NULL, gService != NULL, retVal); + return retVal; +} + +// ------------------------------------- + +URegistryKey U_EXPORT2 +BreakIterator::registerInstance(BreakIterator* toAdopt, const Locale& locale, UBreakIteratorType kind, UErrorCode& status) +{ + return getService()->registerInstance(toAdopt, locale, kind, status); +} + +// ------------------------------------- + +UBool U_EXPORT2 +BreakIterator::unregister(URegistryKey key, UErrorCode& status) +{ + if (U_SUCCESS(status)) { + if (hasService()) { + return gService->unregister(key, status); + } + status = U_ILLEGAL_ARGUMENT_ERROR; + } + return FALSE; +} + +// ------------------------------------- + +StringEnumeration* U_EXPORT2 +BreakIterator::getAvailableLocales(void) +{ + return getService()->getAvailableLocales(); +} +#endif /* UCONFIG_NO_SERVICE */ + +// ------------------------------------- + +BreakIterator* +BreakIterator::createInstance(const Locale& loc, int32_t kind, UErrorCode& status) +{ + if (U_FAILURE(status)) { + return NULL; + } + + u_init(&status); +#if !UCONFIG_NO_SERVICE + if (hasService()) { + Locale actualLoc(""); + BreakIterator *result = (BreakIterator*)gService->get(loc, kind, &actualLoc, status); + // TODO: The way the service code works in ICU 2.8 is that if + // there is a real registered break iterator, the actualLoc + // will be populated, but if the handleDefault path is taken + // (because nothing is registered that can handle the + // requested locale) then the actualLoc comes back empty. In + // that case, the returned object already has its actual/valid + // locale data populated (by makeInstance, which is what + // handleDefault calls), so we don't touch it. YES, A COMMENT + // THIS LONG is a sign of bad code -- so the action item is to + // revisit this in ICU 3.0 and clean it up/fix it/remove it. + if (U_SUCCESS(status) && (result != NULL) && *actualLoc.getName() != 0) { + U_LOCALE_BASED(locBased, *result); + locBased.setLocaleIDs(actualLoc.getName(), actualLoc.getName()); + } + return result; + } + else +#endif + { + return makeInstance(loc, kind, status); + } +} + +// ------------------------------------- + +BreakIterator* +BreakIterator::makeInstance(const Locale& loc, int32_t kind, UErrorCode& status) +{ + + if (U_FAILURE(status)) { + return NULL; + } + + BreakIterator *result = NULL; + switch (kind) { + case UBRK_CHARACTER: + result = BreakIterator::buildInstance(loc, "grapheme", kind, status); + break; + case UBRK_WORD: + result = BreakIterator::buildInstance(loc, "word", kind, status); + break; + case UBRK_LINE: + result = BreakIterator::buildInstance(loc, "line", kind, status); + break; + case UBRK_SENTENCE: + result = BreakIterator::buildInstance(loc, "sentence", kind, status); + break; + case UBRK_TITLE: + result = BreakIterator::buildInstance(loc, "title", kind, status); + break; + default: + status = U_ILLEGAL_ARGUMENT_ERROR; + } + + if (U_FAILURE(status)) { + return NULL; + } + + return result; +} + +Locale +BreakIterator::getLocale(ULocDataLocaleType type, UErrorCode& status) const { + U_LOCALE_BASED(locBased, *this); + return locBased.getLocale(type, status); +} + +const char * +BreakIterator::getLocaleID(ULocDataLocaleType type, UErrorCode& status) const { + U_LOCALE_BASED(locBased, *this); + return locBased.getLocaleID(type, status); +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_BREAK_ITERATION */ + +//eof diff --git a/Build/source/libs/icu/icu-xetex/common/caniter.cpp b/Build/source/libs/icu/icu-xetex/common/caniter.cpp new file mode 100644 index 00000000000..04d48ba8fce --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/caniter.cpp @@ -0,0 +1,611 @@ +/* + ***************************************************************************** + * Copyright (C) 1996-2006, International Business Machines Corporation and * + * others. All Rights Reserved. * + ***************************************************************************** + */ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_NORMALIZATION + +#include "unicode/uset.h" +#include "unicode/ustring.h" +#include "hash.h" +#include "unormimp.h" +#include "unicode/caniter.h" +#include "unicode/normlzr.h" +#include "unicode/uchar.h" +#include "cmemory.h" + +/** + * This class allows one to iterate through all the strings that are canonically equivalent to a given + * string. For example, here are some sample results: +Results for: {LATIN CAPITAL LETTER A WITH RING ABOVE}{LATIN SMALL LETTER D}{COMBINING DOT ABOVE}{COMBINING CEDILLA} +1: \u0041\u030A\u0064\u0307\u0327 + = {LATIN CAPITAL LETTER A}{COMBINING RING ABOVE}{LATIN SMALL LETTER D}{COMBINING DOT ABOVE}{COMBINING CEDILLA} +2: \u0041\u030A\u0064\u0327\u0307 + = {LATIN CAPITAL LETTER A}{COMBINING RING ABOVE}{LATIN SMALL LETTER D}{COMBINING CEDILLA}{COMBINING DOT ABOVE} +3: \u0041\u030A\u1E0B\u0327 + = {LATIN CAPITAL LETTER A}{COMBINING RING ABOVE}{LATIN SMALL LETTER D WITH DOT ABOVE}{COMBINING CEDILLA} +4: \u0041\u030A\u1E11\u0307 + = {LATIN CAPITAL LETTER A}{COMBINING RING ABOVE}{LATIN SMALL LETTER D WITH CEDILLA}{COMBINING DOT ABOVE} +5: \u00C5\u0064\u0307\u0327 + = {LATIN CAPITAL LETTER A WITH RING ABOVE}{LATIN SMALL LETTER D}{COMBINING DOT ABOVE}{COMBINING CEDILLA} +6: \u00C5\u0064\u0327\u0307 + = {LATIN CAPITAL LETTER A WITH RING ABOVE}{LATIN SMALL LETTER D}{COMBINING CEDILLA}{COMBINING DOT ABOVE} +7: \u00C5\u1E0B\u0327 + = {LATIN CAPITAL LETTER A WITH RING ABOVE}{LATIN SMALL LETTER D WITH DOT ABOVE}{COMBINING CEDILLA} +8: \u00C5\u1E11\u0307 + = {LATIN CAPITAL LETTER A WITH RING ABOVE}{LATIN SMALL LETTER D WITH CEDILLA}{COMBINING DOT ABOVE} +9: \u212B\u0064\u0307\u0327 + = {ANGSTROM SIGN}{LATIN SMALL LETTER D}{COMBINING DOT ABOVE}{COMBINING CEDILLA} +10: \u212B\u0064\u0327\u0307 + = {ANGSTROM SIGN}{LATIN SMALL LETTER D}{COMBINING CEDILLA}{COMBINING DOT ABOVE} +11: \u212B\u1E0B\u0327 + = {ANGSTROM SIGN}{LATIN SMALL LETTER D WITH DOT ABOVE}{COMBINING CEDILLA} +12: \u212B\u1E11\u0307 + = {ANGSTROM SIGN}{LATIN SMALL LETTER D WITH CEDILLA}{COMBINING DOT ABOVE} + *
Note: the code is intended for use with small strings, and is not suitable for larger ones, + * since it has not been optimized for that situation. + *@author M. Davis + *@draft + */ + +// public + +U_NAMESPACE_BEGIN + +// TODO: add boilerplate methods. + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(CanonicalIterator) + +/** + *@param source string to get results for + */ +CanonicalIterator::CanonicalIterator(const UnicodeString &sourceStr, UErrorCode &status) : + pieces(NULL), + pieces_length(0), + pieces_lengths(NULL), + current(NULL), + current_length(0) +{ + if(U_SUCCESS(status)) { + setSource(sourceStr, status); + } +} + +CanonicalIterator::~CanonicalIterator() { + cleanPieces(); +} + +void CanonicalIterator::cleanPieces() { + int32_t i = 0; + if(pieces != NULL) { + for(i = 0; i < pieces_length; i++) { + if(pieces[i] != NULL) { + delete[] pieces[i]; + } + } + uprv_free(pieces); + pieces = NULL; + pieces_length = 0; + } + if(pieces_lengths != NULL) { + uprv_free(pieces_lengths); + pieces_lengths = NULL; + } + if(current != NULL) { + uprv_free(current); + current = NULL; + current_length = 0; + } +} + +/** + *@return gets the source: NOTE: it is the NFD form of source + */ +UnicodeString CanonicalIterator::getSource() { + return source; +} + +/** + * Resets the iterator so that one can start again from the beginning. + */ +void CanonicalIterator::reset() { + done = FALSE; + for (int i = 0; i < current_length; ++i) { + current[i] = 0; + } +} + +/** + *@return the next string that is canonically equivalent. The value null is returned when + * the iteration is done. + */ +UnicodeString CanonicalIterator::next() { + int32_t i = 0; + + if (done) { + buffer.setToBogus(); + return buffer; + } + + // delete old contents + buffer.remove(); + + // construct return value + + for (i = 0; i < pieces_length; ++i) { + buffer.append(pieces[i][current[i]]); + } + //String result = buffer.toString(); // not needed + + // find next value for next time + + for (i = current_length - 1; ; --i) { + if (i < 0) { + done = TRUE; + break; + } + current[i]++; + if (current[i] < pieces_lengths[i]) break; // got sequence + current[i] = 0; + } + return buffer; +} + +/** + *@param set the source string to iterate against. This allows the same iterator to be used + * while changing the source string, saving object creation. + */ +void CanonicalIterator::setSource(const UnicodeString &newSource, UErrorCode &status) { + int32_t list_length = 0; + UChar32 cp = 0; + int32_t start = 0; + int32_t i = 0; + UnicodeString *list = NULL; + + Normalizer::normalize(newSource, UNORM_NFD, 0, source, status); + if(U_FAILURE(status)) { + return; + } + done = FALSE; + + cleanPieces(); + + // catch degenerate case + if (newSource.length() == 0) { + pieces = (UnicodeString **)uprv_malloc(sizeof(UnicodeString *)); + pieces_lengths = (int32_t*)uprv_malloc(1 * sizeof(int32_t)); + pieces_length = 1; + current = (int32_t*)uprv_malloc(1 * sizeof(int32_t)); + current_length = 1; + if (pieces == NULL || pieces_lengths == NULL || current == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + goto CleanPartialInitialization; + } + current[0] = 0; + pieces[0] = new UnicodeString[1]; + pieces_lengths[0] = 1; + if (pieces[0] == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + goto CleanPartialInitialization; + } + return; + } + + + list = new UnicodeString[source.length()]; + if (list == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + goto CleanPartialInitialization; + } + + // i should initialy be the number of code units at the + // start of the string + i = UTF16_CHAR_LENGTH(source.char32At(0)); + //int32_t i = 1; + // find the segments + // This code iterates through the source string and + // extracts segments that end up on a codepoint that + // doesn't start any decompositions. (Analysis is done + // on the NFD form - see above). + for (; i < source.length(); i += UTF16_CHAR_LENGTH(cp)) { + cp = source.char32At(i); + if (unorm_isCanonSafeStart(cp)) { + source.extract(start, i-start, list[list_length++]); // add up to i + start = i; + } + } + source.extract(start, i-start, list[list_length++]); // add last one + + + // allocate the arrays, and find the strings that are CE to each segment + pieces = (UnicodeString **)uprv_malloc(list_length * sizeof(UnicodeString *)); + pieces_length = list_length; + pieces_lengths = (int32_t*)uprv_malloc(list_length * sizeof(int32_t)); + current = (int32_t*)uprv_malloc(list_length * sizeof(int32_t)); + current_length = list_length; + if (pieces == NULL || pieces_lengths == NULL || current == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + goto CleanPartialInitialization; + } + + for (i = 0; i < current_length; i++) { + current[i] = 0; + } + // for each segment, get all the combinations that can produce + // it after NFD normalization + for (i = 0; i < pieces_length; ++i) { + //if (PROGRESS) printf("SEGMENT\n"); + pieces[i] = getEquivalents(list[i], pieces_lengths[i], status); + } + + delete[] list; + return; +// Common section to cleanup all local variables and reset object variables. +CleanPartialInitialization: + if (list != NULL) { + delete[] list; + } + cleanPieces(); +} + +/** + * Dumb recursive implementation of permutation. + * TODO: optimize + * @param source the string to find permutations for + * @return the results in a set. + */ +void U_EXPORT2 CanonicalIterator::permute(UnicodeString &source, UBool skipZeros, Hashtable *result, UErrorCode &status) { + if(U_FAILURE(status)) { + return; + } + //if (PROGRESS) printf("Permute: %s\n", UToS(Tr(source))); + int32_t i = 0; + + // optimization: + // if zero or one character, just return a set with it + // we check for length < 2 to keep from counting code points all the time + if (source.length() <= 2 && source.countChar32() <= 1) { + UnicodeString *toPut = new UnicodeString(source); + /* test for NULL */ + if (toPut == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + result->put(source, toPut, status); + return; + } + + // otherwise iterate through the string, and recursively permute all the other characters + UChar32 cp; + Hashtable subpermute(status); + if(U_FAILURE(status)) { + return; + } + subpermute.setValueDeleter(uhash_deleteUnicodeString); + + for (i = 0; i < source.length(); i += UTF16_CHAR_LENGTH(cp)) { + cp = source.char32At(i); + const UHashElement *ne = NULL; + int32_t el = -1; + UnicodeString subPermuteString = source; + + // optimization: + // if the character is canonical combining class zero, + // don't permute it + if (skipZeros && i != 0 && u_getCombiningClass(cp) == 0) { + //System.out.println("Skipping " + Utility.hex(UTF16.valueOf(source, i))); + continue; + } + + subpermute.removeAll(); + + // see what the permutations of the characters before and after this one are + //Hashtable *subpermute = permute(source.substring(0,i) + source.substring(i + UTF16.getCharCount(cp))); + permute(subPermuteString.replace(i, UTF16_CHAR_LENGTH(cp), NULL, 0), skipZeros, &subpermute, status); + /* Test for buffer overflows */ + if(U_FAILURE(status)) { + return; + } + // The upper replace is destructive. The question is do we have to make a copy, or we don't care about the contents + // of source at this point. + + // prefix this character to all of them + ne = subpermute.nextElement(el); + while (ne != NULL) { + UnicodeString *permRes = (UnicodeString *)(ne->value.pointer); + UnicodeString *chStr = new UnicodeString(cp); + //test for NULL + if (chStr == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + chStr->append(*permRes); //*((UnicodeString *)(ne->value.pointer)); + //if (PROGRESS) printf(" Piece: %s\n", UToS(*chStr)); + result->put(*chStr, chStr, status); + ne = subpermute.nextElement(el); + } + } + //return result; +} + +// privates + +// we have a segment, in NFD. Find all the strings that are canonically equivalent to it. +UnicodeString* CanonicalIterator::getEquivalents(const UnicodeString &segment, int32_t &result_len, UErrorCode &status) { + Hashtable result(status); + Hashtable permutations(status); + Hashtable basic(status); + if (U_FAILURE(status)) { + return 0; + } + result.setValueDeleter(uhash_deleteUnicodeString); + permutations.setValueDeleter(uhash_deleteUnicodeString); + basic.setValueDeleter(uhash_deleteUnicodeString); + + UChar USeg[256]; + int32_t segLen = segment.extract(USeg, 256, status); + getEquivalents2(&basic, USeg, segLen, status); + + // now get all the permutations + // add only the ones that are canonically equivalent + // TODO: optimize by not permuting any class zero. + + const UHashElement *ne = NULL; + int32_t el = -1; + //Iterator it = basic.iterator(); + ne = basic.nextElement(el); + //while (it.hasNext()) + while (ne != NULL) { + //String item = (String) it.next(); + UnicodeString item = *((UnicodeString *)(ne->value.pointer)); + + permutations.removeAll(); + permute(item, CANITER_SKIP_ZEROES, &permutations, status); + const UHashElement *ne2 = NULL; + int32_t el2 = -1; + //Iterator it2 = permutations.iterator(); + ne2 = permutations.nextElement(el2); + //while (it2.hasNext()) + while (ne2 != NULL) { + //String possible = (String) it2.next(); + //UnicodeString *possible = new UnicodeString(*((UnicodeString *)(ne2->value.pointer))); + UnicodeString possible(*((UnicodeString *)(ne2->value.pointer))); + UnicodeString attempt; + Normalizer::normalize(possible, UNORM_NFD, 0, attempt, status); + + // TODO: check if operator == is semanticaly the same as attempt.equals(segment) + if (attempt==segment) { + //if (PROGRESS) printf("Adding Permutation: %s\n", UToS(Tr(*possible))); + // TODO: use the hashtable just to catch duplicates - store strings directly (somehow). + result.put(possible, new UnicodeString(possible), status); //add(possible); + } else { + //if (PROGRESS) printf("-Skipping Permutation: %s\n", UToS(Tr(*possible))); + } + + ne2 = permutations.nextElement(el2); + } + ne = basic.nextElement(el); + } + + /* Test for buffer overflows */ + if(U_FAILURE(status)) { + return 0; + } + // convert into a String[] to clean up storage + //String[] finalResult = new String[result.size()]; + UnicodeString *finalResult = NULL; + int32_t resultCount; + if((resultCount = result.count())) { + finalResult = new UnicodeString[resultCount]; + if (finalResult == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + } + else { + status = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + //result.toArray(finalResult); + result_len = 0; + el = -1; + ne = result.nextElement(el); + while(ne != NULL) { + finalResult[result_len++] = *((UnicodeString *)(ne->value.pointer)); + ne = result.nextElement(el); + } + + + return finalResult; +} + +Hashtable *CanonicalIterator::getEquivalents2(Hashtable *fillinResult, const UChar *segment, int32_t segLen, UErrorCode &status) { + + if (U_FAILURE(status)) { + return NULL; + } + + //if (PROGRESS) printf("Adding: %s\n", UToS(Tr(segment))); + + UnicodeString toPut(segment, segLen); + + fillinResult->put(toPut, new UnicodeString(toPut), status); + + USerializedSet starts; + + // cycle through all the characters + UChar32 cp, end = 0; + int32_t i = 0, j; + for (i = 0; i < segLen; i += UTF16_CHAR_LENGTH(cp)) { + // see if any character is at the start of some decomposition + UTF_GET_CHAR(segment, 0, i, segLen, cp); + if (!unorm_getCanonStartSet(cp, &starts)) { + continue; + } + // if so, see which decompositions match + for(j = 0, cp = end+1; cp <= end || uset_getSerializedRange(&starts, j++, &cp, &end); ++cp) { + Hashtable remainder(status); + remainder.setValueDeleter(uhash_deleteUnicodeString); + if (extract(&remainder, cp, segment, segLen, i, status) == NULL) { + continue; + } + + // there were some matches, so add all the possibilities to the set. + UnicodeString prefix(segment, i); + prefix += cp; + + int32_t el = -1; + const UHashElement *ne = remainder.nextElement(el); + while (ne != NULL) { + UnicodeString item = *((UnicodeString *)(ne->value.pointer)); + UnicodeString *toAdd = new UnicodeString(prefix); + /* test for NULL */ + if (toAdd == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + *toAdd += item; + fillinResult->put(*toAdd, toAdd, status); + + //if (PROGRESS) printf("Adding: %s\n", UToS(Tr(*toAdd))); + + ne = remainder.nextElement(el); + } + } + } + + /* Test for buffer overflows */ + if(U_FAILURE(status)) { + return NULL; + } + return fillinResult; +} + +/** + * See if the decomposition of cp2 is at segment starting at segmentPos + * (with canonical rearrangment!) + * If so, take the remainder, and return the equivalents + */ +Hashtable *CanonicalIterator::extract(Hashtable *fillinResult, UChar32 comp, const UChar *segment, int32_t segLen, int32_t segmentPos, UErrorCode &status) { +//Hashtable *CanonicalIterator::extract(UChar32 comp, const UnicodeString &segment, int32_t segLen, int32_t segmentPos, UErrorCode &status) { + //if (PROGRESS) printf(" extract: %s, ", UToS(Tr(UnicodeString(comp)))); + //if (PROGRESS) printf("%s, %i\n", UToS(Tr(segment)), segmentPos); + + if (U_FAILURE(status)) { + return NULL; + } + + const int32_t bufSize = 256; + int32_t bufLen = 0; + UChar temp[bufSize]; + + int32_t inputLen = 0, decompLen; + UChar stackBuffer[4]; + const UChar *decomp; + + U16_APPEND_UNSAFE(temp, inputLen, comp); + decomp = unorm_getCanonicalDecomposition(comp, stackBuffer, &decompLen); + if(decomp == NULL) { + /* copy temp */ + stackBuffer[0] = temp[0]; + if(inputLen > 1) { + stackBuffer[1] = temp[1]; + } + decomp = stackBuffer; + decompLen = inputLen; + } + + UChar *buff = temp+inputLen; + + // See if it matches the start of segment (at segmentPos) + UBool ok = FALSE; + UChar32 cp; + int32_t decompPos = 0; + UChar32 decompCp; + UTF_NEXT_CHAR(decomp, decompPos, decompLen, decompCp); + + int32_t i; + UBool overflow = FALSE; + + i = segmentPos; + while(i < segLen) { + UTF_NEXT_CHAR(segment, i, segLen, cp); + + if (cp == decompCp) { // if equal, eat another cp from decomp + + //if (PROGRESS) printf(" matches: %s\n", UToS(Tr(UnicodeString(cp)))); + + if (decompPos == decompLen) { // done, have all decomp characters! + //u_strcat(buff+bufLen, segment+i); + uprv_memcpy(buff+bufLen, segment+i, (segLen-i)*sizeof(UChar)); + bufLen+=segLen-i; + + ok = TRUE; + break; + } + UTF_NEXT_CHAR(decomp, decompPos, decompLen, decompCp); + } else { + //if (PROGRESS) printf(" buffer: %s\n", UToS(Tr(UnicodeString(cp)))); + + // brute force approach + + U16_APPEND(buff, bufLen, bufSize, cp, overflow); + + if(overflow) { + /* + * ### TODO handle buffer overflow + * The buffer is large, but an overflow may still happen with + * unusual input (many combining marks?). + * Reallocate buffer and continue. + * markus 20020929 + */ + + overflow = FALSE; + } + + /* TODO: optimize + // since we know that the classes are monotonically increasing, after zero + // e.g. 0 5 7 9 0 3 + // we can do an optimization + // there are only a few cases that work: zero, less, same, greater + // if both classes are the same, we fail + // if the decomp class < the segment class, we fail + + segClass = getClass(cp); + if (decompClass <= segClass) return null; + */ + } + } + if (!ok) + return NULL; // we failed, characters left over + + //if (PROGRESS) printf("Matches\n"); + + if (bufLen == 0) { + fillinResult->put(UnicodeString(), new UnicodeString(), status); + return fillinResult; // succeed, but no remainder + } + + // brute force approach + // check to make sure result is canonically equivalent + int32_t tempLen = inputLen + bufLen; + + UChar trial[bufSize]; + unorm_decompose(trial, bufSize, temp, tempLen, FALSE, 0, &status); + + if(U_FAILURE(status) + || uprv_memcmp(segment+segmentPos, trial, (segLen - segmentPos)*sizeof(UChar)) != 0) + { + return NULL; + } + + return getEquivalents2(fillinResult, buff, bufLen, status); +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_NORMALIZATION */ diff --git a/Build/source/libs/icu/icu-xetex/common/chariter.cpp b/Build/source/libs/icu/icu-xetex/common/chariter.cpp new file mode 100644 index 00000000000..a598bd65888 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/chariter.cpp @@ -0,0 +1,96 @@ +/* +********************************************************************** +* Copyright (C) 1999-2004, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +*/ + +#include "unicode/chariter.h" + +U_NAMESPACE_BEGIN + +ForwardCharacterIterator::~ForwardCharacterIterator() {} +ForwardCharacterIterator::ForwardCharacterIterator() +: UObject() +{} +ForwardCharacterIterator::ForwardCharacterIterator(const ForwardCharacterIterator &other) +: UObject(other) +{} + + +CharacterIterator::CharacterIterator() +: textLength(0), pos(0), begin(0), end(0) { +} + +CharacterIterator::CharacterIterator(int32_t length) +: textLength(length), pos(0), begin(0), end(length) { + if(textLength < 0) { + textLength = end = 0; + } +} + +CharacterIterator::CharacterIterator(int32_t length, int32_t position) +: textLength(length), pos(position), begin(0), end(length) { + if(textLength < 0) { + textLength = end = 0; + } + if(pos < 0) { + pos = 0; + } else if(pos > end) { + pos = end; + } +} + +CharacterIterator::CharacterIterator(int32_t length, int32_t textBegin, int32_t textEnd, int32_t position) +: textLength(length), pos(position), begin(textBegin), end(textEnd) { + if(textLength < 0) { + textLength = 0; + } + if(begin < 0) { + begin = 0; + } else if(begin > textLength) { + begin = textLength; + } + if(end < begin) { + end = begin; + } else if(end > textLength) { + end = textLength; + } + if(pos < begin) { + pos = begin; + } else if(pos > end) { + pos = end; + } +} + +CharacterIterator::CharacterIterator(const CharacterIterator &that) : +ForwardCharacterIterator(that), +textLength(that.textLength), pos(that.pos), begin(that.begin), end(that.end) +{ +} + +CharacterIterator & +CharacterIterator::operator=(const CharacterIterator &that) { + ForwardCharacterIterator::operator=(that); + textLength = that.textLength; + pos = that.pos; + begin = that.begin; + end = that.end; + return *this; +} + +// implementing first[32]PostInc() directly in a subclass should be faster +// but these implementations make subclassing a little easier +UChar +CharacterIterator::firstPostInc(void) { + setToStart(); + return nextPostInc(); +} + +UChar32 +CharacterIterator::first32PostInc(void) { + setToStart(); + return next32PostInc(); +} + +U_NAMESPACE_END diff --git a/Build/source/libs/icu/icu-xetex/common/charstr.h b/Build/source/libs/icu/icu-xetex/common/charstr.h new file mode 100644 index 00000000000..3bb11cbad35 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/charstr.h @@ -0,0 +1,88 @@ +/* +********************************************************************** +* Copyright (c) 2001-2004, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 11/19/2001 aliu Creation. +********************************************************************** +*/ + +#ifndef CHARSTRING_H +#define CHARSTRING_H + +#include "unicode/utypes.h" +#include "unicode/uobject.h" +#include "unicode/unistr.h" +#include "cmemory.h" + +//-------------------------------------------------------------------- +// class CharString +// +// This is a tiny wrapper class that is used internally to make a +// UnicodeString look like a const char*. It can be allocated on the +// stack. It only creates a heap buffer if it needs to. +//-------------------------------------------------------------------- + +U_NAMESPACE_BEGIN + +class U_COMMON_API CharString : public UMemory { +public: + +#if !UCONFIG_NO_CONVERSION + // Constructor + // @param str The unicode string to be converted to char * + // @param codepage The char * code page. "" for invariant conversion. + // NULL for default code page. +// inline CharString(const UnicodeString& str, const char *codepage); +#endif + + inline CharString(const UnicodeString& str); + inline ~CharString(); + inline operator const char*() const { return ptr; } + +private: + char buf[128]; + char* ptr; + + CharString(const CharString &other); // forbid copying of this class + CharString &operator=(const CharString &other); // forbid copying of this class +}; + +#if !UCONFIG_NO_CONVERSION + +// PLEASE DON'T USE THIS FUNCTION. +// We don't want the static dependency on conversion or the performance hit that comes from a codepage conversion. +/* +inline CharString::CharString(const UnicodeString& str, const char *codepage) { + int32_t len; + ptr = buf; + len = str.extract(0, 0x7FFFFFFF, buf ,sizeof(buf)-1, codepage); + if (len >= (int32_t)(sizeof(buf)-1)) { + ptr = (char *)uprv_malloc(len+1); + str.extract(0, 0x7FFFFFFF, ptr, len+1, codepage); + } +}*/ + +#endif + +inline CharString::CharString(const UnicodeString& str) { + int32_t len; + ptr = buf; + len = str.extract(0, 0x7FFFFFFF, buf, (int32_t)(sizeof(buf)-1), US_INV); + if (len >= (int32_t)(sizeof(buf)-1)) { + ptr = (char *)uprv_malloc(len+1); + str.extract(0, 0x7FFFFFFF, ptr, len+1, US_INV); + } +} + +inline CharString::~CharString() { + if (ptr != buf) { + uprv_free(ptr); + } +} + +U_NAMESPACE_END + +#endif +//eof diff --git a/Build/source/libs/icu/icu-xetex/common/cmemory.c b/Build/source/libs/icu/icu-xetex/common/cmemory.c new file mode 100644 index 00000000000..0f93f36a63d --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/cmemory.c @@ -0,0 +1,124 @@ +/* +****************************************************************************** +* +* Copyright (C) 2002-2003, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* +* File cmemory.c ICU Heap allocation. +* All ICU heap allocation, both for C and C++ new of ICU +* class types, comes through these functions. +* +* If you have a need to replace ICU allocation, this is the +* place to do it. +* +* Note that uprv_malloc(0) returns a non-NULL pointer, and +* that a subsequent free of that pointer value is a NOP. +* +****************************************************************************** +*/ +#include "unicode/uclean.h" +#include "cmemory.h" +#include + +/* uprv_malloc(0) returns a pointer to this read-only data. */ +static const int32_t zeroMem[] = {0, 0, 0, 0, 0, 0}; + +/* Function Pointers for user-supplied heap functions */ +static const void *pContext; +static UMemAllocFn *pAlloc; +static UMemReallocFn *pRealloc; +static UMemFreeFn *pFree; + +/* Flag indicating whether any heap allocations have happened. + * Used to prevent changing out the heap functions after allocations have been made */ +static UBool gHeapInUse; + +U_CAPI void * U_EXPORT2 +uprv_malloc(size_t s) { + if (s > 0) { + gHeapInUse = TRUE; + if (pAlloc) { + return (*pAlloc)(pContext, s); + } else { + return malloc(s); + } + } else { + return (void *)zeroMem; + } +} + +U_CAPI void * U_EXPORT2 +uprv_realloc(void * buffer, size_t size) { + if (buffer == zeroMem) { + return uprv_malloc(size); + } else if (size == 0) { + if (pFree) { + (*pFree)(pContext, buffer); + } else { + free(buffer); + } + return (void *)zeroMem; + } else { + gHeapInUse = TRUE; + if (pRealloc) { + return (*pRealloc)(pContext, buffer, size); + } else { + return realloc(buffer, size); + } + } +} + +U_CAPI void U_EXPORT2 +uprv_free(void *buffer) { + if (buffer != zeroMem) { + if (pFree) { + (*pFree)(pContext, buffer); + } else { + free(buffer); + } + } +} + +U_CAPI void U_EXPORT2 +u_setMemoryFunctions(const void *context, UMemAllocFn *a, UMemReallocFn *r, UMemFreeFn *f, UErrorCode *status) +{ + if (U_FAILURE(*status)) { + return; + } + if (a==NULL || r==NULL || f==NULL) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + if (gHeapInUse) { + *status = U_INVALID_STATE_ERROR; + return; + } + pContext = context; + pAlloc = a; + pRealloc = r; + pFree = f; +} + + +U_CFUNC UBool cmemory_cleanup(void) { + pContext = NULL; + pAlloc = NULL; + pRealloc = NULL; + pFree = NULL; + gHeapInUse = FALSE; + return TRUE; +} + + +/* + * gHeapInUse + * Return True if ICU has allocated any memory. + * Used by u_SetMutexFunctions() and similar to verify that ICU has not + * been used, that it is in a pristine initial state. + */ +U_CFUNC UBool cmemory_inUse() { + return gHeapInUse; +} + diff --git a/Build/source/libs/icu/icu-xetex/common/cmemory.h b/Build/source/libs/icu/icu-xetex/common/cmemory.h new file mode 100644 index 00000000000..da65d7cebf4 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/cmemory.h @@ -0,0 +1,82 @@ +/* +****************************************************************************** +* +* Copyright (C) 1997-2003, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* +* File CMEMORY.H +* +* Contains stdlib.h/string.h memory functions +* +* @author Bertrand A. Damiba +* +* Modification History: +* +* Date Name Description +* 6/20/98 Bertrand Created. +* 05/03/99 stephen Changed from functions to macros. +* +****************************************************************************** +*/ + +#ifndef CMEMORY_H +#define CMEMORY_H + +#include "unicode/utypes.h" +#include + + +#define uprv_memcpy(dst, src, size) U_STANDARD_CPP_NAMESPACE memcpy(dst, src, size) +#define uprv_memmove(dst, src, size) U_STANDARD_CPP_NAMESPACE memmove(dst, src, size) +#define uprv_memset(buffer, mark, size) U_STANDARD_CPP_NAMESPACE memset(buffer, mark, size) +#define uprv_memcmp(buffer1, buffer2, size) U_STANDARD_CPP_NAMESPACE memcmp(buffer1, buffer2,size) + +U_CAPI void * U_EXPORT2 +uprv_malloc(size_t s); + +U_CAPI void * U_EXPORT2 +uprv_realloc(void *mem, size_t size); + +U_CAPI void U_EXPORT2 +uprv_free(void *mem); + +/** + * This should align the memory properly on any machine. + * This is very useful for the safeClone functions. + */ +typedef union { + long t1; + double t2; + void *t3; +} UAlignedMemory; + +/** + * Get the amount of bytes that a pointer is off by from + * the previous aligned pointer + */ +#define U_ALIGNMENT_OFFSET(ptr) (((size_t)ptr) & (sizeof(UAlignedMemory) - 1)) + +/** + * Get the amount of bytes to add to a pointer + * in order to get the next aligned address + */ +#define U_ALIGNMENT_OFFSET_UP(ptr) (sizeof(UAlignedMemory) - U_ALIGNMENT_OFFSET(ptr)) + +/** + * Indicate whether the ICU allocation functions have been used. + * This is used to determine whether ICU is in an initial, unused state. + */ +U_CFUNC UBool +cmemory_inUse(void); + +/** + * Heap clean up function, called from u_cleanup() + * Clears any user heap functions from u_setMemoryFunctions() + * Does NOT deallocate any remaining allocated memory. + */ +U_CFUNC UBool +cmemory_cleanup(void); + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/common.rc b/Build/source/libs/icu/icu-xetex/common/common.rc new file mode 100644 index 00000000000..331e5423dcc --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/common.rc @@ -0,0 +1,108 @@ +// Do not edit with Microsoft Developer Studio Resource Editor. +// It will permanently substitute version numbers that are intended to be +// picked up by the pre-processor during each build. +// Copyright (c) 2001-2005 International Business Machines +// Corporation and others. All Rights Reserved. +// +#include "msvcres.h" + +#define APSTUDIO_READONLY_SYMBOLS +///////////////////////////////////////////////////////////////////////////// +// +// Generated from the TEXTINCLUDE 2 resource. +// +#include +///////////////////////////////////////////////////////////////////////////// +#undef APSTUDIO_READONLY_SYMBOLS + +///////////////////////////////////////////////////////////////////////////// +// + +LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL +#pragma code_page(1252) + +#ifdef APSTUDIO_INVOKED +///////////////////////////////////////////////////////////////////////////// +// +// TEXTINCLUDE +// + +1 TEXTINCLUDE +BEGIN + "msvcres.h\0" +END + +2 TEXTINCLUDE +BEGIN + "#include \0" +END + +3 TEXTINCLUDE +BEGIN + "\r\n" + "\0" +END + +#endif // APSTUDIO_INVOKED + + +///////////////////////////////////////////////////////////////////////////// +// +// Version +// +#define STR(s) #s +#define CommaVersionString(a, b, c, d) STR(a) ", " STR(b) ", " STR(c) ", " STR(d) "\0" + +VS_VERSION_INFO VERSIONINFO + FILEVERSION U_ICU_VERSION_MAJOR_NUM, U_ICU_VERSION_MINOR_NUM, U_ICU_VERSION_PATCHLEVEL_NUM, 0 + PRODUCTVERSION U_ICU_VERSION_MAJOR_NUM, U_ICU_VERSION_MINOR_NUM, U_ICU_VERSION_PATCHLEVEL_NUM, 0 + FILEFLAGSMASK 0x3fL +#ifdef _DEBUG + FILEFLAGS 0x1L +#else + FILEFLAGS 0x0L +#endif + FILEOS VOS__WINDOWS32 + FILETYPE VFT_DLL + FILESUBTYPE 0x0L +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "00000000" + BEGIN + VALUE "Comments", ICU_WEBSITE "\0" + VALUE "CompanyName", "IBM Corporation and others\0" + VALUE "FileDescription", "IBM ICU Common DLL\0" + VALUE "FileVersion", CommaVersionString(U_ICU_VERSION_MAJOR_NUM, U_ICU_VERSION_MINOR_NUM, U_ICU_VERSION_PATCHLEVEL_NUM, 0) + VALUE "LegalCopyright", U_COPYRIGHT_STRING "\0" +#ifdef _DEBUG + VALUE "OriginalFilename", "icuuc" U_ICU_VERSION_SHORT "d.dll\0") +#else + VALUE "OriginalFilename", "icuuc" U_ICU_VERSION_SHORT ".dll\0") +#endif + VALUE "PrivateBuild", "\0" + VALUE "ProductName", "International Components for Unicode\0" + VALUE "ProductVersion", CommaVersionString(U_ICU_VERSION_MAJOR_NUM, U_ICU_VERSION_MINOR_NUM, U_ICU_VERSION_PATCHLEVEL_NUM, 0) + VALUE "SpecialBuild", "\0" + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x000, 0000 + END +END + +///////////////////////////////////////////////////////////////////////////// + + + +#ifndef APSTUDIO_INVOKED +///////////////////////////////////////////////////////////////////////////// +// +// Generated from the TEXTINCLUDE 3 resource. +// + + +///////////////////////////////////////////////////////////////////////////// +#endif // not APSTUDIO_INVOKED + diff --git a/Build/source/libs/icu/icu-xetex/common/common.vcproj b/Build/source/libs/icu/icu-xetex/common/common.vcproj new file mode 100644 index 00000000000..d8d6510e581 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/common.vcproj @@ -0,0 +1,2588 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Build/source/libs/icu/icu-xetex/common/cpputils.h b/Build/source/libs/icu/icu-xetex/common/cpputils.h new file mode 100644 index 00000000000..c1b439c7ff8 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/cpputils.h @@ -0,0 +1,84 @@ +/* +****************************************************************************** +* +* Copyright (C) 1997-2006, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: cpputils.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +*/ + +#ifndef CPPUTILS_H +#define CPPUTILS_H + +#include "unicode/utypes.h" +#include "unicode/unistr.h" +#include "cmemory.h" + +/*==========================================================================*/ +/* Array copy utility functions */ +/*==========================================================================*/ + +static +inline void uprv_arrayCopy(const double* src, double* dst, int32_t count) +{ uprv_memcpy(dst, src, (size_t)(count * sizeof(*src))); } + +static +inline void uprv_arrayCopy(const double* src, int32_t srcStart, + double* dst, int32_t dstStart, int32_t count) +{ uprv_memcpy(dst+dstStart, src+srcStart, (size_t)(count * sizeof(*src))); } + +static +inline void uprv_arrayCopy(const int8_t* src, int8_t* dst, int32_t count) + { uprv_memcpy(dst, src, (size_t)(count * sizeof(*src))); } + +static +inline void uprv_arrayCopy(const int8_t* src, int32_t srcStart, + int8_t* dst, int32_t dstStart, int32_t count) +{ uprv_memcpy(dst+dstStart, src+srcStart, (size_t)(count * sizeof(*src))); } + +static +inline void uprv_arrayCopy(const int16_t* src, int16_t* dst, int32_t count) +{ uprv_memcpy(dst, src, (size_t)(count * sizeof(*src))); } + +static +inline void uprv_arrayCopy(const int16_t* src, int32_t srcStart, + int16_t* dst, int32_t dstStart, int32_t count) +{ uprv_memcpy(dst+dstStart, src+srcStart, (size_t)(count * sizeof(*src))); } + +static +inline void uprv_arrayCopy(const int32_t* src, int32_t* dst, int32_t count) +{ uprv_memcpy(dst, src, (size_t)(count * sizeof(*src))); } + +static +inline void uprv_arrayCopy(const int32_t* src, int32_t srcStart, + int32_t* dst, int32_t dstStart, int32_t count) +{ uprv_memcpy(dst+dstStart, src+srcStart, (size_t)(count * sizeof(*src))); } + +static +inline void +uprv_arrayCopy(const UChar *src, int32_t srcStart, + UChar *dst, int32_t dstStart, int32_t count) +{ uprv_memcpy(dst+dstStart, src+srcStart, (size_t)(count * sizeof(*src))); } + +/** + * Copy an array of UnicodeString OBJECTS (not pointers). + * @internal + */ +static inline void +uprv_arrayCopy(const U_NAMESPACE_QUALIFIER UnicodeString *src, U_NAMESPACE_QUALIFIER UnicodeString *dst, int32_t count) +{ while(count-- > 0) *dst++ = *src++; } + +/** + * Copy an array of UnicodeString OBJECTS (not pointers). + * @internal + */ +static inline void +uprv_arrayCopy(const U_NAMESPACE_QUALIFIER UnicodeString *src, int32_t srcStart, + U_NAMESPACE_QUALIFIER UnicodeString *dst, int32_t dstStart, int32_t count) +{ uprv_arrayCopy(src+srcStart, dst+dstStart, count); } + +#endif /* _CPPUTILS */ diff --git a/Build/source/libs/icu/icu-xetex/common/cstring.c b/Build/source/libs/icu/icu-xetex/common/cstring.c new file mode 100644 index 00000000000..a5b2c6bbccd --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/cstring.c @@ -0,0 +1,328 @@ +/* +****************************************************************************** +* +* Copyright (C) 1997-2003, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* +* File CSTRING.C +* +* @author Helena Shih +* +* Modification History: +* +* Date Name Description +* 6/18/98 hshih Created +* 09/08/98 stephen Added include for ctype, for Mac Port +* 11/15/99 helena Integrated S/390 IEEE changes. +****************************************************************************** +*/ + + + +#include +#include +#include "unicode/utypes.h" +#include "cmemory.h" +#include "cstring.h" +#include "uassert.h" + +/* + * We hardcode case conversion for invariant characters to match our expectation + * and the compiler execution charset. + * This prevents problems on systems + * - with non-default casing behavior, like Turkish system locales where + * tolower('I') maps to dotless i and toupper('i') maps to dotted I + * - where there are no lowercase Latin characters at all, or using different + * codes (some old EBCDIC codepages) + * + * This works because the compiler usually runs on a platform where the execution + * charset includes all of the invariant characters at their expected + * code positions, so that the char * string literals in ICU code match + * the char literals here. + * + * Note that the set of lowercase Latin letters is discontiguous in EBCDIC + * and the set of uppercase Latin letters is discontiguous as well. + */ + +U_CAPI char U_EXPORT2 +uprv_toupper(char c) { +#if U_CHARSET_FAMILY==U_EBCDIC_FAMILY + if(('a'<=c && c<='i') || ('j'<=c && c<='r') || ('s'<=c && c<='z')) { + c=(char)(c+('A'-'a')); + } +#else + if('a'<=c && c<='z') { + c=(char)(c+('A'-'a')); + } +#endif + return c; +} + + +#if 0 +/* + * Commented out because cstring.h defines uprv_tolower() to be + * the same as either uprv_asciitolower() or uprv_ebcdictolower() + * to reduce the amount of code to cover with tests. + * + * Note that this uprv_tolower() definition is likely to work for most + * charset families, not just ASCII and EBCDIC, because its #else branch + * is written generically. + */ +U_CAPI char U_EXPORT2 +uprv_tolower(char c) { +#if U_CHARSET_FAMILY==U_EBCDIC_FAMILY + if(('A'<=c && c<='I') || ('J'<=c && c<='R') || ('S'<=c && c<='Z')) { + c=(char)(c+('a'-'A')); + } +#else + if('A'<=c && c<='Z') { + c=(char)(c+('a'-'A')); + } +#endif + return c; +} +#endif + +U_CAPI char U_EXPORT2 +uprv_asciitolower(char c) { + if(0x41<=c && c<=0x5a) { + c=(char)(c+0x20); + } + return c; +} + +U_CAPI char U_EXPORT2 +uprv_ebcdictolower(char c) { + if( (0xc1<=(uint8_t)c && (uint8_t)c<=0xc9) || + (0xd1<=(uint8_t)c && (uint8_t)c<=0xd9) || + (0xe2<=(uint8_t)c && (uint8_t)c<=0xe9) + ) { + c=(char)(c-0x40); + } + return c; +} + + +U_CAPI char* U_EXPORT2 +T_CString_toLowerCase(char* str) +{ + char* origPtr = str; + + if (str) { + do + *str = (char)uprv_tolower(*str); + while (*(str++)); + } + + return origPtr; +} + +U_CAPI char* U_EXPORT2 +T_CString_toUpperCase(char* str) +{ + char* origPtr = str; + + if (str) { + do + *str = (char)uprv_toupper(*str); + while (*(str++)); + } + + return origPtr; +} + +/* + * Takes a int32_t and fills in a char* string with that number "radix"-based. + * Does not handle negative values (makes an empty string for them). + * Writes at most 12 chars ("-2147483647" plus NUL). + * Returns the length of the string (not including the NUL). + */ +U_CAPI int32_t U_EXPORT2 +T_CString_integerToString(char* buffer, int32_t v, int32_t radix) +{ + char tbuf[30]; + int32_t tbx = sizeof(tbuf); + uint8_t digit; + int32_t length = 0; + uint32_t uval; + + U_ASSERT(radix>=2 && radix<=16); + uval = (uint32_t) v; + if(v<0 && radix == 10) { + /* Only in base 10 do we conside numbers to be signed. */ + uval = (uint32_t)(-v); + buffer[length++] = '-'; + } + + tbx = sizeof(tbuf)-1; + tbuf[tbx] = 0; /* We are generating the digits backwards. Null term the end. */ + do { + digit = (uint8_t)(uval % radix); + tbuf[--tbx] = (char)(T_CString_itosOffset(digit)); + uval = uval / radix; + } while (uval != 0); + + /* copy converted number into user buffer */ + uprv_strcpy(buffer+length, tbuf+tbx); + length += sizeof(tbuf) - tbx -1; + return length; +} + + + +/* + * Takes a int64_t and fills in a char* string with that number "radix"-based. + * Writes at most 21: chars ("-9223372036854775807" plus NUL). + * Returns the length of the string, not including the terminating NULL. + */ +U_CAPI int32_t U_EXPORT2 +T_CString_int64ToString(char* buffer, int64_t v, uint32_t radix) +{ + char tbuf[30]; + int32_t tbx = sizeof(tbuf); + uint8_t digit; + int32_t length = 0; + uint64_t uval; + + U_ASSERT(radix>=2 && radix<=16); + uval = (uint64_t) v; + if(v<0 && radix == 10) { + /* Only in base 10 do we conside numbers to be signed. */ + uval = (uint64_t)(-v); + buffer[length++] = '-'; + } + + tbx = sizeof(tbuf)-1; + tbuf[tbx] = 0; /* We are generating the digits backwards. Null term the end. */ + do { + digit = (uint8_t)(uval % radix); + tbuf[--tbx] = (char)(T_CString_itosOffset(digit)); + uval = uval / radix; + } while (uval != 0); + + /* copy converted number into user buffer */ + uprv_strcpy(buffer+length, tbuf+tbx); + length += sizeof(tbuf) - tbx -1; + return length; +} + + +U_CAPI int32_t U_EXPORT2 +T_CString_stringToInteger(const char *integerString, int32_t radix) +{ + char *end; + return uprv_strtoul(integerString, &end, radix); + +} + +U_CAPI int U_EXPORT2 +T_CString_stricmp(const char *str1, const char *str2) { + if(str1==NULL) { + if(str2==NULL) { + return 0; + } else { + return -1; + } + } else if(str2==NULL) { + return 1; + } else { + /* compare non-NULL strings lexically with lowercase */ + int rc; + unsigned char c1, c2; + + for(;;) { + c1=(unsigned char)*str1; + c2=(unsigned char)*str2; + if(c1==0) { + if(c2==0) { + return 0; + } else { + return -1; + } + } else if(c2==0) { + return 1; + } else { + /* compare non-zero characters with lowercase */ + rc=(int)(unsigned char)uprv_tolower(c1)-(int)(unsigned char)uprv_tolower(c2); + if(rc!=0) { + return rc; + } + } + ++str1; + ++str2; + } + } +} + +U_CAPI int U_EXPORT2 +T_CString_strnicmp(const char *str1, const char *str2, uint32_t n) { + if(str1==NULL) { + if(str2==NULL) { + return 0; + } else { + return -1; + } + } else if(str2==NULL) { + return 1; + } else { + /* compare non-NULL strings lexically with lowercase */ + int rc; + unsigned char c1, c2; + + for(; n--;) { + c1=(unsigned char)*str1; + c2=(unsigned char)*str2; + if(c1==0) { + if(c2==0) { + return 0; + } else { + return -1; + } + } else if(c2==0) { + return 1; + } else { + /* compare non-zero characters with lowercase */ + rc=(int)(unsigned char)uprv_tolower(c1)-(int)(unsigned char)uprv_tolower(c2); + if(rc!=0) { + return rc; + } + } + ++str1; + ++str2; + } + } + + return 0; +} + +U_CAPI char* U_EXPORT2 +uprv_strdup(const char *src) { + size_t len = uprv_strlen(src) + 1; + char *dup = (char *) uprv_malloc(len); + + if (dup) { + uprv_memcpy(dup, src, len); + } + + return dup; +} + +U_CAPI char* U_EXPORT2 +uprv_strndup(const char *src, int32_t n) { + char *dup; + + if(n < 0) { + dup = uprv_strdup(src); + } else { + dup = (char*)uprv_malloc(n+1); + if (dup) { + uprv_memcpy(dup, src, n); + dup[n] = 0; + } + } + + return dup; +} diff --git a/Build/source/libs/icu/icu-xetex/common/cstring.h b/Build/source/libs/icu/icu-xetex/common/cstring.h new file mode 100644 index 00000000000..6d2fd0e8159 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/cstring.h @@ -0,0 +1,120 @@ +/* +****************************************************************************** +* +* Copyright (C) 1997-2005, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* +* File CSTRING.H +* +* Contains CString interface +* +* @author Helena Shih +* +* Modification History: +* +* Date Name Description +* 6/17/98 hshih Created. +* 05/03/99 stephen Changed from functions to macros. +* 06/14/99 stephen Added icu_strncat, icu_strncmp, icu_tolower +* +****************************************************************************** +*/ + +#ifndef CSTRING_H +#define CSTRING_H 1 + +#include "unicode/utypes.h" +#include +#include +#include + +#define uprv_strcpy(dst, src) U_STANDARD_CPP_NAMESPACE strcpy(dst, src) +#define uprv_strncpy(dst, src, size) U_STANDARD_CPP_NAMESPACE strncpy(dst, src, size) +#define uprv_strlen(str) U_STANDARD_CPP_NAMESPACE strlen(str) +#define uprv_strcmp(s1, s2) U_STANDARD_CPP_NAMESPACE strcmp(s1, s2) +#define uprv_strncmp(s1, s2, n) U_STANDARD_CPP_NAMESPACE strncmp(s1, s2, n) +#define uprv_strcat(dst, src) U_STANDARD_CPP_NAMESPACE strcat(dst, src) +#define uprv_strncat(dst, src, n) U_STANDARD_CPP_NAMESPACE strncat(dst, src, n) +#define uprv_strchr(s, c) U_STANDARD_CPP_NAMESPACE strchr(s, c) +#define uprv_strstr(s, c) U_STANDARD_CPP_NAMESPACE strstr(s, c) +#define uprv_strrchr(s, c) U_STANDARD_CPP_NAMESPACE strrchr(s, c) + +U_CAPI char U_EXPORT2 +uprv_toupper(char c); + + +U_CAPI char U_EXPORT2 +uprv_asciitolower(char c); + +U_CAPI char U_EXPORT2 +uprv_ebcdictolower(char c); + +#if U_CHARSET_FAMILY==U_ASCII_FAMILY +# define uprv_tolower uprv_asciitolower +#elif U_CHARSET_FAMILY==U_EBCDIC_FAMILY +# define uprv_tolower uprv_ebcdictolower +#else +# error U_CHARSET_FAMILY is not valid +#endif + +#define uprv_strtod(source, end) U_STANDARD_CPP_NAMESPACE strtod(source, end) +#define uprv_strtoul(str, end, base) U_STANDARD_CPP_NAMESPACE strtoul(str, end, base) +#define uprv_strtol(str, end, base) U_STANDARD_CPP_NAMESPACE strtol(str, end, base) +#ifdef U_WINDOWS +# if defined(__BORLANDC__) +# define uprv_stricmp(str1, str2) U_STANDARD_CPP_NAMESPACE stricmp(str1, str2) +# define uprv_strnicmp(str1, str2, n) U_STANDARD_CPP_NAMESPACE strnicmp(str1, str2, n) +# else +# define uprv_stricmp(str1, str2) U_STANDARD_CPP_NAMESPACE _stricmp(str1, str2) +# define uprv_strnicmp(str1, str2, n) U_STANDARD_CPP_NAMESPACE _strnicmp(str1, str2, n) +# endif +#elif defined(POSIX) +# define uprv_stricmp(str1, str2) U_STANDARD_CPP_NAMESPACE strcasecmp(str1, str2) +# define uprv_strnicmp(str1, str2, n) U_STANDARD_CPP_NAMESPACE strncasecmp(str1, str2, n) +#else +# define uprv_stricmp(str1, str2) T_CString_stricmp(str1, str2) +# define uprv_strnicmp(str1, str2, n) T_CString_strnicmp(str1, str2, n) +#endif + +/* Conversion from a digit to the character with radix base from 2-19 */ +/* May need to use U_UPPER_ORDINAL*/ +#define T_CString_itosOffset(a) ((a)<=9?('0'+(a)):('A'+(a)-10)) + +U_CAPI char* U_EXPORT2 +uprv_strdup(const char *src); + +/** + * uprv_malloc n+1 bytes, and copy n bytes from src into the new string. + * Terminate with a null at offset n. If n is -1, works like uprv_strdup + * @param src + * @param n length of the input string, not including null. + * @return new string (owned by caller, use uprv_free to free). + * @internal + */ +U_CAPI char* U_EXPORT2 +uprv_strndup(const char *src, int32_t n); + +U_CAPI char* U_EXPORT2 +T_CString_toLowerCase(char* str); + +U_CAPI char* U_EXPORT2 +T_CString_toUpperCase(char* str); + +U_CAPI int32_t U_EXPORT2 +T_CString_integerToString(char *buffer, int32_t n, int32_t radix); + +U_CAPI int32_t U_EXPORT2 +T_CString_int64ToString(char *buffer, int64_t n, uint32_t radix); + +U_CAPI int32_t U_EXPORT2 +T_CString_stringToInteger(const char *integerString, int32_t radix); + +U_CAPI int U_EXPORT2 +T_CString_stricmp(const char *str1, const char *str2); + +U_CAPI int U_EXPORT2 +T_CString_strnicmp(const char *str1, const char *str2, uint32_t n); + +#endif /* ! CSTRING_H */ diff --git a/Build/source/libs/icu/icu-xetex/common/cwchar.c b/Build/source/libs/icu/icu-xetex/common/cwchar.c new file mode 100644 index 00000000000..78bb8c59dbb --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/cwchar.c @@ -0,0 +1,53 @@ +/* +****************************************************************************** +* +* Copyright (C) 2001, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: cwchar.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2001may25 +* created by: Markus W. Scherer +*/ + +#include "unicode/utypes.h" + +#if !U_HAVE_WCSCPY + +#include "cwchar.h" + +U_CAPI wchar_t *uprv_wcscat(wchar_t *dst, const wchar_t *src) { + wchar_t *start=dst; + while(*dst!=0) { + ++dst; + } + while((*dst=*src)!=0) { + ++dst; + ++src; + } + return start; +} + +U_CAPI wchar_t *uprv_wcscpy(wchar_t *dst, const wchar_t *src) { + wchar_t *start=dst; + while((*dst=*src)!=0) { + ++dst; + ++src; + } + return start; +} + +U_CAPI size_t uprv_wcslen(const wchar_t *src) { + const wchar_t *start=src; + while(*src!=0) { + ++src; + } + return src-start; +} + +#endif + diff --git a/Build/source/libs/icu/icu-xetex/common/cwchar.h b/Build/source/libs/icu/icu-xetex/common/cwchar.h new file mode 100644 index 00000000000..2ab36c032e0 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/cwchar.h @@ -0,0 +1,56 @@ +/* +****************************************************************************** +* +* Copyright (C) 2001, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: cwchar.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2001may25 +* created by: Markus W. Scherer +* +* This file contains ICU-internal definitions of wchar_t operations. +* These definitions were moved here from cstring.h so that fewer +* ICU implementation files include wchar.h. +*/ + +#ifndef __CWCHAR_H__ +#define __CWCHAR_H__ + +#include +#include +#include "unicode/utypes.h" + +/* Do this after utypes.h so that we have U_HAVE_WCHAR_H . */ +#if U_HAVE_WCHAR_H +# include +#endif + +/*===========================================================================*/ +/* Wide-character functions */ +/*===========================================================================*/ + +/* The following are not available on all systems, defined in wchar.h or string.h. */ +#if U_HAVE_WCSCPY +# define uprv_wcscpy wcscpy +# define uprv_wcscat wcscat +# define uprv_wcslen wcslen +#else +U_CAPI wchar_t* U_EXPORT2 +uprv_wcscpy(wchar_t *dst, const wchar_t *src); +U_CAPI wchar_t* U_EXPORT2 +uprv_wcscat(wchar_t *dst, const wchar_t *src); +U_CAPI size_t U_EXPORT2 +uprv_wcslen(const wchar_t *src); +#endif + +/* The following are part of the ANSI C standard, defined in stdlib.h . */ +#define uprv_wcstombs(mbstr, wcstr, count) U_STANDARD_CPP_NAMESPACE wcstombs(mbstr, wcstr, count) +#define uprv_mbstowcs(wcstr, mbstr, count) U_STANDARD_CPP_NAMESPACE mbstowcs(wcstr, mbstr, count) + + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/dictbe.cpp b/Build/source/libs/icu/icu-xetex/common/dictbe.cpp new file mode 100644 index 00000000000..a779727ff47 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/dictbe.cpp @@ -0,0 +1,426 @@ +/** + ******************************************************************************* + * Copyright (C) 2006, International Business Machines Corporation and others. * + * All Rights Reserved. * + ******************************************************************************* + */ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_BREAK_ITERATION + +#include "brkeng.h" +#include "dictbe.h" +#include "unicode/uniset.h" +#include "unicode/chariter.h" +#include "unicode/ubrk.h" +#include "uvector.h" +#include "triedict.h" + +U_NAMESPACE_BEGIN + +/* + ****************************************************************** + */ + +/*DictionaryBreakEngine::DictionaryBreakEngine() { + fTypes = 0; +}*/ + +DictionaryBreakEngine::DictionaryBreakEngine(uint32_t breakTypes) { + fTypes = breakTypes; +} + +DictionaryBreakEngine::~DictionaryBreakEngine() { +} + +UBool +DictionaryBreakEngine::handles(UChar32 c, int32_t breakType) const { + return (breakType >= 0 && breakType < 32 && (((uint32_t)1 << breakType) & fTypes) + && fSet.contains(c)); +} + +int32_t +DictionaryBreakEngine::findBreaks( UText *text, + int32_t startPos, + int32_t endPos, + UBool reverse, + int32_t breakType, + UStack &foundBreaks ) const { + int32_t result = 0; + + // Find the span of characters included in the set. + int32_t start = (int32_t)utext_getNativeIndex(text); + int32_t current; + int32_t rangeStart; + int32_t rangeEnd; + UChar32 c = utext_current32(text); + if (reverse) { + UBool isDict = fSet.contains(c); + while((current = (int32_t)utext_getNativeIndex(text)) > startPos && isDict) { + c = utext_previous32(text); + isDict = fSet.contains(c); + } + rangeStart = (current < startPos) ? startPos : current+(isDict ? 0 : 1); + rangeEnd = start + 1; + } + else { + while((current = (int32_t)utext_getNativeIndex(text)) < endPos && fSet.contains(c)) { + utext_next32(text); // TODO: recast loop for postincrement + c = utext_current32(text); + } + rangeStart = start; + rangeEnd = current; + } + if (breakType >= 0 && breakType < 32 && (((uint32_t)1 << breakType) & fTypes)) { + result = divideUpDictionaryRange(text, rangeStart, rangeEnd, foundBreaks); + utext_setNativeIndex(text, current); + } + + return result; +} + +void +DictionaryBreakEngine::setCharacters( const UnicodeSet &set ) { + fSet = set; + // Compact for caching + fSet.compact(); +} + +/*void +DictionaryBreakEngine::setBreakTypes( uint32_t breakTypes ) { + fTypes = breakTypes; +}*/ + +/* + ****************************************************************** + */ + + +// Helper class for improving readability of the Thai word break +// algorithm. The implementation is completely inline. + +// List size, limited by the maximum number of words in the dictionary +// that form a nested sequence. +#define POSSIBLE_WORD_LIST_MAX 20 + +class PossibleWord { + private: + // list of word candidate lengths, in increasing length order + int32_t lengths[POSSIBLE_WORD_LIST_MAX]; + int count; // Count of candidates + int32_t prefix; // The longest match with a dictionary word + int32_t offset; // Offset in the text of these candidates + int mark; // The preferred candidate's offset + int current; // The candidate we're currently looking at + + public: + PossibleWord(); + ~PossibleWord(); + + // Fill the list of candidates if needed, select the longest, and return the number found + int candidates( UText *text, const TrieWordDictionary *dict, int32_t rangeEnd ); + + // Select the currently marked candidate, point after it in the text, and invalidate self + int32_t acceptMarked( UText *text ); + + // Back up from the current candidate to the next shorter one; return TRUE if that exists + // and point the text after it + UBool backUp( UText *text ); + + // Return the longest prefix this candidate location shares with a dictionary word + int32_t longestPrefix(); + + // Mark the current candidate as the one we like + void markCurrent(); +}; + +inline +PossibleWord::PossibleWord() { + offset = -1; +} + +inline +PossibleWord::~PossibleWord() { +} + +inline int +PossibleWord::candidates( UText *text, const TrieWordDictionary *dict, int32_t rangeEnd ) { + // TODO: If getIndex is too slow, use offset < 0 and add discardAll() + int32_t start = (int32_t)utext_getNativeIndex(text); + if (start != offset) { + offset = start; + prefix = dict->matches(text, rangeEnd-start, lengths, count, sizeof(lengths)/sizeof(lengths[0])); + // Dictionary leaves text after longest prefix, not longest word. Back up. + if (count <= 0) { + utext_setNativeIndex(text, start); + } + } + if (count > 0) { + utext_setNativeIndex(text, start+lengths[count-1]); + } + current = count-1; + mark = current; + return count; +} + +inline int32_t +PossibleWord::acceptMarked( UText *text ) { + utext_setNativeIndex(text, offset + lengths[mark]); + return lengths[mark]; +} + +inline UBool +PossibleWord::backUp( UText *text ) { + if (current > 0) { + utext_setNativeIndex(text, offset + lengths[--current]); + return TRUE; + } + return FALSE; +} + +inline int32_t +PossibleWord::longestPrefix() { + return prefix; +} + +inline void +PossibleWord::markCurrent() { + mark = current; +} + +// How many words in a row are "good enough"? +#define THAI_LOOKAHEAD 3 + +// Will not combine a non-word with a preceding dictionary word longer than this +#define THAI_ROOT_COMBINE_THRESHOLD 3 + +// Will not combine a non-word that shares at least this much prefix with a +// dictionary word, with a preceding word +#define THAI_PREFIX_COMBINE_THRESHOLD 3 + +// Ellision character +#define THAI_PAIYANNOI 0x0E2F + +// Repeat character +#define THAI_MAIYAMOK 0x0E46 + +// Minimum word size +#define THAI_MIN_WORD 2 + +// Minimum number of characters for two words +#define THAI_MIN_WORD_SPAN (THAI_MIN_WORD * 2) + +ThaiBreakEngine::ThaiBreakEngine(const TrieWordDictionary *adoptDictionary, UErrorCode &status) + : DictionaryBreakEngine((1< 1) { + // If we're already at the end of the range, we're done + if ((int32_t)utext_getNativeIndex(text) >= rangeEnd) { + goto foundBest; + } + do { + int wordsMatched = 1; + if (words[(wordsFound+1)%THAI_LOOKAHEAD].candidates(text, fDictionary, rangeEnd) > 0) { + if (wordsMatched < 2) { + // Followed by another dictionary word; mark first word as a good candidate + words[wordsFound%THAI_LOOKAHEAD].markCurrent(); + wordsMatched = 2; + } + + // If we're already at the end of the range, we're done + if ((int32_t)utext_getNativeIndex(text) >= rangeEnd) { + goto foundBest; + } + + // See if any of the possible second words is followed by a third word + do { + // If we find a third word, stop right away + if (words[(wordsFound+2)%THAI_LOOKAHEAD].candidates(text, fDictionary, rangeEnd)) { + words[wordsFound%THAI_LOOKAHEAD].markCurrent(); + goto foundBest; + } + } + while (words[(wordsFound+1)%THAI_LOOKAHEAD].backUp(text)); + } + } + while (words[wordsFound%THAI_LOOKAHEAD].backUp(text)); +foundBest: + wordLength = words[wordsFound%THAI_LOOKAHEAD].acceptMarked(text); + wordsFound += 1; + } + + // We come here after having either found a word or not. We look ahead to the + // next word. If it's not a dictionary word, we will combine it withe the word we + // just found (if there is one), but only if the preceding word does not exceed + // the threshold. + // The text iterator should now be positioned at the end of the word we found. + if ((int32_t)utext_getNativeIndex(text) < rangeEnd && wordLength < THAI_ROOT_COMBINE_THRESHOLD) { + // if it is a dictionary word, do nothing. If it isn't, then if there is + // no preceding word, or the non-word shares less than the minimum threshold + // of characters with a dictionary word, then scan to resynchronize + if (words[wordsFound%THAI_LOOKAHEAD].candidates(text, fDictionary, rangeEnd) <= 0 + && (wordLength == 0 + || words[wordsFound%THAI_LOOKAHEAD].longestPrefix() < THAI_PREFIX_COMBINE_THRESHOLD)) { + // Look for a plausible word boundary + //TODO: This section will need a rework for UText. + int32_t remaining = rangeEnd - (current+wordLength); + UChar32 pc = utext_current32(text); + int32_t chars = 0; + for (;;) { + utext_next32(text); + uc = utext_current32(text); + // TODO: Here we're counting on the fact that the SA languages are all + // in the BMP. This should get fixed with the UText rework. + chars += 1; + if (--remaining <= 0) { + break; + } + if (fEndWordSet.contains(pc) && fBeginWordSet.contains(uc)) { + // Maybe. See if it's in the dictionary. + // NOTE: In the original Apple code, checked that the next + // two characters after uc were not 0x0E4C THANTHAKHAT before + // checking the dictionary. That is just a performance filter, + // but it's not clear it's faster than checking the trie. + int candidates = words[(wordsFound+1)%THAI_LOOKAHEAD].candidates(text, fDictionary, rangeEnd); + utext_setNativeIndex(text, current+wordLength+chars); + if (candidates > 0) { + break; + } + } + pc = uc; + } + + // Bump the word count if there wasn't already one + if (wordLength <= 0) { + wordsFound += 1; + } + + // Update the length with the passed-over characters + wordLength += chars; + } + else { + // Back up to where we were for next iteration + utext_setNativeIndex(text, current+wordLength); + } + } + + // Never stop before a combining mark. + int32_t currPos; + while ((currPos = (int32_t)utext_getNativeIndex(text)) < rangeEnd && fMarkSet.contains(utext_current32(text))) { + utext_next32(text); + wordLength += (int32_t)utext_getNativeIndex(text) - currPos; + } + + // Look ahead for possible suffixes if a dictionary word does not follow. + // We do this in code rather than using a rule so that the heuristic + // resynch continues to function. For example, one of the suffix characters + // could be a typo in the middle of a word. + if ((int32_t)utext_getNativeIndex(text) < rangeEnd && wordLength > 0) { + if (words[wordsFound%THAI_LOOKAHEAD].candidates(text, fDictionary, rangeEnd) <= 0 + && fSuffixSet.contains(uc = utext_current32(text))) { + if (uc == THAI_PAIYANNOI) { + if (!fSuffixSet.contains(utext_previous32(text))) { + // Skip over previous end and PAIYANNOI + utext_next32(text); + utext_next32(text); + wordLength += 1; // Add PAIYANNOI to word + uc = utext_current32(text); // Fetch next character + } + else { + // Restore prior position + utext_next32(text); + } + } + if (uc == THAI_MAIYAMOK) { + if (utext_previous32(text) != THAI_MAIYAMOK) { + // Skip over previous end and MAIYAMOK + utext_next32(text); + utext_next32(text); + wordLength += 1; // Add MAIYAMOK to word + } + else { + // Restore prior position + utext_next32(text); + } + } + } + else { + utext_setNativeIndex(text, current+wordLength); + } + } + + // Did we find a word on this iteration? If so, push it on the break stack + if (wordLength > 0) { + foundBreaks.push((current+wordLength), status); + } + } + + // Don't return a break for the end of the dictionary range if there is one there. + if (foundBreaks.peeki() >= rangeEnd) { + (void) foundBreaks.popi(); + wordsFound -= 1; + } + + return wordsFound; +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_BREAK_ITERATION */ diff --git a/Build/source/libs/icu/icu-xetex/common/dictbe.h b/Build/source/libs/icu/icu-xetex/common/dictbe.h new file mode 100644 index 00000000000..d6f8b1462d2 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/dictbe.h @@ -0,0 +1,193 @@ +/** + ******************************************************************************* + * Copyright (C) 2006, International Business Machines Corporation and others. * + * All Rights Reserved. * + ******************************************************************************* + */ + +#ifndef DICTBE_H +#define DICTBE_H + +#include "unicode/utypes.h" +#include "unicode/uniset.h" +#include "unicode/utext.h" + +#include "brkeng.h" + +U_NAMESPACE_BEGIN + +class TrieWordDictionary; + +/******************************************************************* + * DictionaryBreakEngine + */ + +/** + *

DictionaryBreakEngine is a kind of LanguageBreakEngine that uses a + * dictionary to determine language-specific breaks.

+ * + *

After it is constructed a DictionaryBreakEngine may be shared between + * threads without synchronization.

+ */ +class DictionaryBreakEngine : public LanguageBreakEngine { + private: + /** + * The set of characters handled by this engine + * @internal + */ + + UnicodeSet fSet; + + /** + * The set of break types handled by this engine + * @internal + */ + + uint32_t fTypes; + + /** + *

Default constructor.

+ * + */ + DictionaryBreakEngine(); + + public: + + /** + *

Constructor setting the break types handled.

+ * + * @param breakTypes A bitmap of types handled by the engine. + */ + DictionaryBreakEngine( uint32_t breakTypes ); + + /** + *

Virtual destructor.

+ */ + virtual ~DictionaryBreakEngine(); + + /** + *

Indicate whether this engine handles a particular character for + * a particular kind of break.

+ * + * @param c A character which begins a run that the engine might handle + * @param breakType The type of text break which the caller wants to determine + * @return TRUE if this engine handles the particular character and break + * type. + */ + virtual UBool handles( UChar32 c, int32_t breakType ) const; + + /** + *

Find any breaks within a run in the supplied text.

+ * + * @param text A UText representing the text. The + * iterator is left at the end of the run of characters which the engine + * is capable of handling. + * @param startPos The start of the run within the supplied text. + * @param endPos The end of the run within the supplied text. + * @param reverse Whether the caller is looking for breaks in a reverse + * direction. + * @param breakType The type of break desired, or -1. + * @param foundBreaks An allocated C array of the breaks found, if any + * @return The number of breaks found. + */ + virtual int32_t findBreaks( UText *text, + int32_t startPos, + int32_t endPos, + UBool reverse, + int32_t breakType, + UStack &foundBreaks ) const; + + protected: + + /** + *

Set the character set handled by this engine.

+ * + * @param set A UnicodeSet of the set of characters handled by the engine + */ + virtual void setCharacters( const UnicodeSet &set ); + + /** + *

Set the break types handled by this engine.

+ * + * @param breakTypes A bitmap of types handled by the engine. + */ +// virtual void setBreakTypes( uint32_t breakTypes ); + + /** + *

Divide up a range of known dictionary characters.

+ * + * @param text A UText representing the text + * @param rangeStart The start of the range of dictionary characters + * @param rangeEnd The end of the range of dictionary characters + * @param foundBreaks Output of C array of int32_t break positions, or 0 + * @return The number of breaks found + */ + virtual int32_t divideUpDictionaryRange( UText *text, + int32_t rangeStart, + int32_t rangeEnd, + UStack &foundBreaks ) const = 0; + +}; + +/******************************************************************* + * ThaiBreakEngine + */ + +/** + *

ThaiBreakEngine is a kind of DictionaryBreakEngine that uses a + * TrieWordDictionary and heuristics to determine Thai-specific breaks.

+ * + *

After it is constructed a ThaiBreakEngine may be shared between + * threads without synchronization.

+ */ +class ThaiBreakEngine : public DictionaryBreakEngine { + private: + /** + * The set of characters handled by this engine + * @internal + */ + + UnicodeSet fThaiWordSet; + UnicodeSet fEndWordSet; + UnicodeSet fBeginWordSet; + UnicodeSet fSuffixSet; + UnicodeSet fMarkSet; + const TrieWordDictionary *fDictionary; + + public: + + /** + *

Default constructor.

+ * + * @param adoptDictionary A TrieWordDictionary to adopt. Deleted when the + * engine is deleted. + */ + ThaiBreakEngine(const TrieWordDictionary *adoptDictionary, UErrorCode &status); + + /** + *

Virtual destructor.

+ */ + virtual ~ThaiBreakEngine(); + + protected: + /** + *

Divide up a range of known dictionary characters.

+ * + * @param text A UText representing the text + * @param rangeStart The start of the range of dictionary characters + * @param rangeEnd The end of the range of dictionary characters + * @param foundBreaks Output of C array of int32_t break positions, or 0 + * @return The number of breaks found + */ + virtual int32_t divideUpDictionaryRange( UText *text, + int32_t rangeStart, + int32_t rangeEnd, + UStack &foundBreaks ) const; + +}; + + +U_NAMESPACE_END + + /* DICTBE_H */ +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/hash.h b/Build/source/libs/icu/icu-xetex/common/hash.h new file mode 100644 index 00000000000..375499bf1b8 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/hash.h @@ -0,0 +1,207 @@ +/* +****************************************************************************** +* Copyright (C) 1997-2006, International Business Machines +* Corporation and others. All Rights Reserved. +****************************************************************************** +* Date Name Description +* 03/28/00 aliu Creation. +****************************************************************************** +*/ + +#ifndef HASH_H +#define HASH_H + +#include "unicode/unistr.h" +#include "unicode/uobject.h" +#include "uhash.h" + +U_NAMESPACE_BEGIN + +/** + * Hashtable is a thin C++ wrapper around UHashtable, a general-purpose void* + * hashtable implemented in C. Hashtable is designed to be idiomatic and + * easy-to-use in C++. + * + * Hashtable is an INTERNAL CLASS. + */ +class U_COMMON_API Hashtable : public UMemory { + UHashtable* hash; + UHashtable hashObj; + + inline void init(UHashFunction *keyHash, UKeyComparator *keyComp, UValueComparator *valueComp, UErrorCode& status); + +public: + /** + * Construct a hashtable + * @param ignoreKeyCase If true, keys are case insensitive. + * @param status Error code + */ + Hashtable(UBool ignoreKeyCase, UErrorCode& status); + + /** + * Construct a hashtable + * @param keyComp Compartor for comparing the keys + * @param valueComp Compartor for comparing the values + * @param status Error code + */ + Hashtable(UKeyComparator *keyComp, UValueComparator *valueComp, UErrorCode& status); + + /** + * Construct a hashtable + * @param status Error code + */ + Hashtable(UErrorCode& status); + + /** + * Construct a hashtable, _disregarding any error_. Use this constructor + * with caution. + */ + Hashtable(); + + /** + * Non-virtual destructor; make this virtual if Hashtable is subclassed + * in the future. + */ + ~Hashtable(); + + UObjectDeleter *setValueDeleter(UObjectDeleter *fn); + + int32_t count() const; + + void* put(const UnicodeString& key, void* value, UErrorCode& status); + + int32_t puti(const UnicodeString& key, int32_t value, UErrorCode& status); + + void* get(const UnicodeString& key) const; + + int32_t geti(const UnicodeString& key) const; + + void* remove(const UnicodeString& key); + + int32_t removei(const UnicodeString& key); + + void removeAll(void); + + const UHashElement* find(const UnicodeString& key) const; + + const UHashElement* nextElement(int32_t& pos) const; + + UKeyComparator* setKeyCompartor(UKeyComparator*keyComp); + + UValueComparator* setValueCompartor(UValueComparator* valueComp); + + UBool equals(const Hashtable& that) const; +private: + Hashtable(const Hashtable &other); // forbid copying of this class + Hashtable &operator=(const Hashtable &other); // forbid copying of this class +}; + +/********************************************************************* + * Implementation + ********************************************************************/ + +inline void Hashtable::init(UHashFunction *keyHash, UKeyComparator *keyComp, + UValueComparator *valueComp, UErrorCode& status) { + if (U_FAILURE(status)) { + return; + } + uhash_init(&hashObj, keyHash, keyComp, valueComp, &status); + if (U_SUCCESS(status)) { + hash = &hashObj; + uhash_setKeyDeleter(hash, uhash_deleteUnicodeString); + } +} + +inline Hashtable::Hashtable(UKeyComparator *keyComp, UValueComparator *valueComp, + UErrorCode& status) : hash(0) { + init( uhash_hashUnicodeString, keyComp, valueComp, status); +} +inline Hashtable::Hashtable(UBool ignoreKeyCase, UErrorCode& status) + : hash(0) +{ + init(ignoreKeyCase ? uhash_hashCaselessUnicodeString + : uhash_hashUnicodeString, + ignoreKeyCase ? uhash_compareCaselessUnicodeString + : uhash_compareUnicodeString, + NULL, + status); +} + +inline Hashtable::Hashtable(UErrorCode& status) + : hash(0) +{ + init(uhash_hashUnicodeString, uhash_compareUnicodeString, NULL, status); +} + +inline Hashtable::Hashtable() + : hash(0) +{ + UErrorCode status = U_ZERO_ERROR; + init(uhash_hashUnicodeString, uhash_compareUnicodeString, NULL, status); +} + +inline Hashtable::~Hashtable() { + if (hash != NULL) { + uhash_close(hash); + } +} + +inline UObjectDeleter *Hashtable::setValueDeleter(UObjectDeleter *fn) { + return uhash_setValueDeleter(hash, fn); +} + +inline int32_t Hashtable::count() const { + return uhash_count(hash); +} + +inline void* Hashtable::put(const UnicodeString& key, void* value, UErrorCode& status) { + return uhash_put(hash, new UnicodeString(key), value, &status); +} + +inline int32_t Hashtable::puti(const UnicodeString& key, int32_t value, UErrorCode& status) { + return uhash_puti(hash, new UnicodeString(key), value, &status); +} + +inline void* Hashtable::get(const UnicodeString& key) const { + return uhash_get(hash, &key); +} + +inline int32_t Hashtable::geti(const UnicodeString& key) const { + return uhash_geti(hash, &key); +} + +inline void* Hashtable::remove(const UnicodeString& key) { + return uhash_remove(hash, &key); +} + +inline int32_t Hashtable::removei(const UnicodeString& key) { + return uhash_removei(hash, &key); +} + +inline const UHashElement* Hashtable::find(const UnicodeString& key) const { + return uhash_find(hash, &key); +} + +inline const UHashElement* Hashtable::nextElement(int32_t& pos) const { + return uhash_nextElement(hash, &pos); +} + +inline void Hashtable::removeAll(void) { + uhash_removeAll(hash); +} + +inline UKeyComparator* Hashtable::setKeyCompartor(UKeyComparator*keyComp){ + return uhash_setKeyComparator(hash, keyComp); +} + +inline UValueComparator* Hashtable::setValueCompartor(UValueComparator* valueComp){ + return uhash_setValueComparator(hash, valueComp); +} + +inline UBool Hashtable::equals(const Hashtable& that)const{ + return uhash_equals(hash, that.hash); +} +U_NAMESPACE_END + +#endif + diff --git a/Build/source/libs/icu/icu-xetex/common/icucfg.h.in b/Build/source/libs/icu/icu-xetex/common/icucfg.h.in new file mode 100644 index 00000000000..98fc0ff9373 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/icucfg.h.in @@ -0,0 +1,61 @@ +/* common/icucfg.h.in. Generated automatically from configure.in by autoheader. */ + +/* Define if you have the ANSI C header files. */ +#undef STDC_HEADERS + +/* Define if your processor stores words with the most significant + byte first (like Motorola and SPARC, unlike Intel and VAX). */ +#undef WORDS_BIGENDIAN + +/* Copyright (c) 1999-2000, International Business Machines Corporation and + others. All Rights Reserved. */ +/* Define to signed char if not in */ +#undef int8_t + +/* Define to unsigned char if not in */ +#undef uint8_t + +/* Define to signed short if not in */ +#undef int16_t + +/* Define to unsigned short if not in */ +#undef uint16_t + +/* Define to signed long if not in */ +#undef int32_t + +/* Define to unsigned long if not in */ +#undef uint32_t + +/* Define to signed char if not in */ +#undef bool_t + +/* Define if your system has */ +#undef HAVE_WCHAR_H + +/* Define to the size of wchar_t */ +#undef SIZEOF_WCHAR_T + +/* Define if you have the header file. */ +#undef HAVE_INTTYPES_H + +/* Define if you have the cma library (-lcma). */ +#undef HAVE_LIBCMA + +/* Define if you have the dl library (-ldl). */ +#undef HAVE_LIBDL + +/* Define if you have the dld library (-ldld). */ +#undef HAVE_LIBDLD + +/* Define if you have the m library (-lm). */ +#undef HAVE_LIBM + +/* Define if you have the pthread library (-lpthread). */ +#undef HAVE_LIBPTHREAD + +/* Define if you have the pthreads library (-lpthreads). */ +#undef HAVE_LIBPTHREADS + +/* Define if you have the wcs library (-lwcs). */ +#undef HAVE_LIBWCS diff --git a/Build/source/libs/icu/icu-xetex/common/localsvc.h b/Build/source/libs/icu/icu-xetex/common/localsvc.h new file mode 100644 index 00000000000..67e5a8455ae --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/localsvc.h @@ -0,0 +1,25 @@ +/* +*************************************************************************** +* Copyright (C) 2006 International Business Machines Corporation * +* and others. All rights reserved. * +*************************************************************************** +*/ + +#ifndef LOCALSVC_H +#define LOCALSVC_H + +#include "unicode/utypes.h" + +#if U_LOCAL_SERVICE_HOOK +/** + * Prototype for user-supplied service hook. This function is expected to return + * a type of factory object specific to the requested service. + * + * @param what service-specific string identifying the specific user hook + * @param status error status + * @return a service-specific hook, or NULL on failure. + */ +U_CAPI void* uprv_svc_hook(const char *what, UErrorCode *status); +#endif + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/locbased.cpp b/Build/source/libs/icu/icu-xetex/common/locbased.cpp new file mode 100644 index 00000000000..e96b9f79f3d --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/locbased.cpp @@ -0,0 +1,46 @@ +/* +********************************************************************** +* Copyright (c) 2004, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Author: Alan Liu +* Created: January 16 2004 +* Since: ICU 2.8 +********************************************************************** +*/ +#include "locbased.h" +#include "cstring.h" + +U_NAMESPACE_BEGIN + +Locale LocaleBased::getLocale(ULocDataLocaleType type, UErrorCode& status) const { + const char* id = getLocaleID(type, status); + return Locale((id != 0) ? id : ""); +} + +const char* LocaleBased::getLocaleID(ULocDataLocaleType type, UErrorCode& status) const { + if (U_FAILURE(status)) { + return NULL; + } + + switch(type) { + case ULOC_VALID_LOCALE: + return valid; + case ULOC_ACTUAL_LOCALE: + return actual; + default: + status = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } +} + +void LocaleBased::setLocaleIDs(const char* validID, const char* actualID) { + if (validID != 0) { + uprv_strcpy(valid, validID); + } + if (actualID != 0) { + uprv_strcpy(actual, actualID); + } +} + +U_NAMESPACE_END diff --git a/Build/source/libs/icu/icu-xetex/common/locbased.h b/Build/source/libs/icu/icu-xetex/common/locbased.h new file mode 100644 index 00000000000..366b15109e8 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/locbased.h @@ -0,0 +1,97 @@ +/* +********************************************************************** +* Copyright (c) 2004, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Author: Alan Liu +* Created: January 16 2004 +* Since: ICU 2.8 +********************************************************************** +*/ +#ifndef LOCBASED_H +#define LOCBASED_H + +#include "unicode/locid.h" +#include "unicode/uobject.h" + +/** + * Macro to declare a locale LocaleBased wrapper object for the given + * object, which must have two members named `validLocale' and + * `actualLocale'. + */ +#define U_LOCALE_BASED(varname, objname) \ + LocaleBased varname((objname).validLocale, (objname).actualLocale); + +U_NAMESPACE_BEGIN + +/** + * A utility class that unifies the implementation of getLocale() by + * various ICU services. This class is likely to be removed in the + * ICU 3.0 time frame in favor of an integrated approach with the + * services framework. + * @since ICU 2.8 + */ +class U_COMMON_API LocaleBased : public UMemory { + + public: + + /** + * Construct a LocaleBased wrapper around the two pointers. These + * will be aliased for the lifetime of this object. + */ + inline LocaleBased(char* validAlias, char* actualAlias); + + /** + * Construct a LocaleBased wrapper around the two const pointers. + * These will be aliased for the lifetime of this object. + */ + inline LocaleBased(const char* validAlias, const char* actualAlias); + + /** + * Return locale meta-data for the service object wrapped by this + * object. Either the valid or the actual locale may be + * retrieved. + * @param type either ULOC_VALID_LOCALE or ULOC_ACTUAL_LOCALE + * @param status input-output error code + * @return the indicated locale + */ + Locale getLocale(ULocDataLocaleType type, UErrorCode& status) const; + + /** + * Return the locale ID for the service object wrapped by this + * object. Either the valid or the actual locale may be + * retrieved. + * @param type either ULOC_VALID_LOCALE or ULOC_ACTUAL_LOCALE + * @param status input-output error code + * @return the indicated locale ID + */ + const char* getLocaleID(ULocDataLocaleType type, UErrorCode& status) const; + + /** + * Set the locale meta-data for the service object wrapped by this + * object. If either parameter is zero, it is ignored. + * @param valid the ID of the valid locale + * @param actual the ID of the actual locale + */ + void setLocaleIDs(const char* valid, const char* actual); + + private: + + char* valid; + + char* actual; +}; + +inline LocaleBased::LocaleBased(char* validAlias, char* actualAlias) : + valid(validAlias), actual(actualAlias) { +} + +inline LocaleBased::LocaleBased(const char* validAlias, + const char* actualAlias) : + // ugh: cast away const + valid((char*)validAlias), actual((char*)actualAlias) { +} + +U_NAMESPACE_END + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/locid.cpp b/Build/source/libs/icu/icu-xetex/common/locid.cpp new file mode 100644 index 00000000000..848a1d7430a --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/locid.cpp @@ -0,0 +1,1330 @@ +/* + ********************************************************************** + * Copyright (C) 1997-2007, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** +* +* File locid.cpp +* +* Created by: Richard Gillam +* +* Modification History: +* +* Date Name Description +* 02/11/97 aliu Changed gLocPath to fgDataDirectory and added +* methods to get and set it. +* 04/02/97 aliu Made operator!= inline; fixed return value +* of getName(). +* 04/15/97 aliu Cleanup for AIX/Win32. +* 04/24/97 aliu Numerous changes per code review. +* 08/18/98 stephen Changed getDisplayName() +* Added SIMPLIFIED_CHINESE, TRADITIONAL_CHINESE +* Added getISOCountries(), getISOLanguages(), +* getLanguagesForCountry() +* 03/16/99 bertrand rehaul. +* 07/21/99 stephen Added U_CFUNC setDefault +* 11/09/99 weiv Added const char * getName() const; +* 04/12/00 srl removing unicodestring api's and cached hash code +* 08/10/01 grhoten Change the static Locales to accessor functions +****************************************************************************** +*/ + + +#include "unicode/locid.h" +#include "unicode/uloc.h" +#include "umutex.h" +#include "uassert.h" +#include "cmemory.h" +#include "cstring.h" +#include "uhash.h" +#include "ucln_cmn.h" + +#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) + +static U_NAMESPACE_QUALIFIER Locale* availableLocaleList = NULL; +static int32_t availableLocaleListCount; +typedef enum ELocalePos { + eENGLISH, + eFRENCH, + eGERMAN, + eITALIAN, + eJAPANESE, + eKOREAN, + eCHINESE, + + eFRANCE, + eGERMANY, + eITALY, + eJAPAN, + eKOREA, + eCHINA, /* Alias for PRC */ + eTAIWAN, + eUK, + eUS, + eCANADA, + eCANADA_FRENCH, + + + //eDEFAULT, + eMAX_LOCALES +} ELocalePos; + +U_CFUNC int32_t locale_getKeywords(const char *localeID, + char prev, + char *keywords, int32_t keywordCapacity, + char *values, int32_t valuesCapacity, int32_t *valLen, + UBool valuesToo, + UErrorCode *status); + +static U_NAMESPACE_QUALIFIER Locale *gLocaleCache = NULL; +static U_NAMESPACE_QUALIFIER Locale *gDefaultLocale = NULL; +static UHashtable *gDefaultLocalesHashT = NULL; + +U_CDECL_BEGIN +// +// Deleter function for Locales owned by the default Locale hash table/ +// +static void U_CALLCONV +deleteLocale(void *obj) { + delete (U_NAMESPACE_QUALIFIER Locale *) obj; +} + +static UBool U_CALLCONV locale_cleanup(void) +{ + U_NAMESPACE_USE + + if (availableLocaleList) { + delete []availableLocaleList; + availableLocaleList = NULL; + } + availableLocaleListCount = 0; + + if (gLocaleCache) { + delete [] gLocaleCache; + gLocaleCache = NULL; + } + + if (gDefaultLocalesHashT) { + uhash_close(gDefaultLocalesHashT); // Automatically deletes all elements, using deleter func. + gDefaultLocalesHashT = NULL; + } + else if (gDefaultLocale) { + // The cache wasn't created, and only one default locale was created. + delete gDefaultLocale; + } + gDefaultLocale = NULL; + + return TRUE; +} +U_CDECL_END + +U_NAMESPACE_BEGIN +// +// locale_set_default_internal. +// +void locale_set_default_internal(const char *id) +{ + UErrorCode status = U_ZERO_ERROR; + UBool canonicalize = FALSE; + + // If given a NULL string for the locale id, grab the default + // name from the system. + // (Different from most other locale APIs, where a null name means use + // the current ICU default locale.) + if (id == NULL) { + umtx_lock(NULL); + id = uprv_getDefaultLocaleID(); + umtx_unlock(NULL); + canonicalize = TRUE; // always canonicalize host ID + } + + // put the locale id into a canonical form, + // in preparation for looking up this locale in the hash table of + // already-created locale objects. + // + status = U_ZERO_ERROR; + char localeNameBuf[512]; + + if (canonicalize) { + uloc_canonicalize(id, localeNameBuf, sizeof(localeNameBuf)-1, &status); + } else { + uloc_getName(id, localeNameBuf, sizeof(localeNameBuf)-1, &status); + } + localeNameBuf[sizeof(localeNameBuf)-1] = 0; // Force null termination in event of + // a long name filling the buffer. + // (long names are truncated.) + + // Lazy creation of the hash table itself, if needed. + UBool isOnlyLocale; + UMTX_CHECK(NULL, (gDefaultLocale == NULL), isOnlyLocale); + if (isOnlyLocale) { + // We haven't seen this locale id before. + // Create a new Locale object for it. + Locale *newFirstDefault = new Locale(Locale::eBOGUS); + if (newFirstDefault == NULL) { + // No way to report errors from here. + return; + } + newFirstDefault->init(localeNameBuf, FALSE); + umtx_lock(NULL); + if (gDefaultLocale == NULL) { + gDefaultLocale = newFirstDefault; // Assignment to gDefaultLocale must happen inside mutex + newFirstDefault = NULL; + ucln_common_registerCleanup(UCLN_COMMON_LOCALE, locale_cleanup); + } + // Else some other thread raced us through here, and set the new Locale. + // Use the hash table next. + umtx_unlock(NULL); + if (newFirstDefault == NULL) { + // We were successful in setting the locale, and we were the first one to set it. + return; + } + // else start using the hash table. + } + + // Lazy creation of the hash table itself, if needed. + UBool hashTableNeedsInit; + UMTX_CHECK(NULL, (gDefaultLocalesHashT == NULL), hashTableNeedsInit); + if (hashTableNeedsInit) { + status = U_ZERO_ERROR; + UHashtable *tHashTable = uhash_open(uhash_hashChars, uhash_compareChars, NULL, &status); + if (U_FAILURE(status)) { + return; + } + uhash_setValueDeleter(tHashTable, deleteLocale); + umtx_lock(NULL); + if (gDefaultLocalesHashT == NULL) { + gDefaultLocalesHashT = tHashTable; + ucln_common_registerCleanup(UCLN_COMMON_LOCALE, locale_cleanup); + } else { + uhash_close(tHashTable); + hashTableNeedsInit = FALSE; + } + umtx_unlock(NULL); + } + + // Hash table lookup, key is the locale full name + umtx_lock(NULL); + Locale *newDefault = (Locale *)uhash_get(gDefaultLocalesHashT, localeNameBuf); + if (newDefault != NULL) { + // We have the requested locale in the hash table already. + // Just set it as default. Inside the mutex lock, for those troublesome processors. + gDefaultLocale = newDefault; + umtx_unlock(NULL); + } else { + umtx_unlock(NULL); + // We haven't seen this locale id before. + // Create a new Locale object for it. + newDefault = new Locale(Locale::eBOGUS); + if (newDefault == NULL) { + // No way to report errors from here. + return; + } + newDefault->init(localeNameBuf, FALSE); + + // Add newly created Locale to the hash table of default Locales + const char *key = newDefault->getName(); + U_ASSERT(uprv_strcmp(key, localeNameBuf) == 0); + umtx_lock(NULL); + Locale *hashTableVal = (Locale *)uhash_get(gDefaultLocalesHashT, key); + if (hashTableVal == NULL) { + if (hashTableNeedsInit) { + // This is the second request to set the locale. + // Cache the first one. + uhash_put(gDefaultLocalesHashT, (void *)gDefaultLocale->getName(), gDefaultLocale, &status); + } + uhash_put(gDefaultLocalesHashT, (void *)key, newDefault, &status); + gDefaultLocale = newDefault; + // ignore errors from hash table insert. (Couldn't do anything anyway) + // We can still set the default Locale, + // it just wont be cached, and will eventually leak. + } else { + // Some other thread raced us through here, and got the new Locale + // into the hash table before us. Use that one. + gDefaultLocale = hashTableVal; // Assignment to gDefaultLocale must happen inside mutex + delete newDefault; + } + umtx_unlock(NULL); + } +} +U_NAMESPACE_END + +/* sfb 07/21/99 */ +U_CFUNC void +locale_set_default(const char *id) +{ + U_NAMESPACE_USE + locale_set_default_internal(id); +} +/* end */ + +U_CFUNC const char * +locale_get_default(void) +{ + U_NAMESPACE_USE + + return Locale::getDefault().getName(); +} + + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(Locale) + +/*Character separating the posix id fields*/ +// '_' +// In the platform codepage. +#define SEP_CHAR '_' + +Locale::~Locale() +{ + /*if fullName is on the heap, we free it*/ + if (fullName != fullNameBuffer) + { + uprv_free(fullName); + fullName = NULL; + } + if (baseName && baseName != baseNameBuffer) { + uprv_free(baseName); + baseName = NULL; + } +} + +Locale::Locale() + : UObject(), fullName(fullNameBuffer), baseName(NULL) +{ + init(NULL, FALSE); +} + +/* + * Internal constructor to allow construction of a locale object with + * NO side effects. (Default constructor tries to get + * the default locale.) + */ +Locale::Locale(Locale::ELocaleType) + : UObject(), fullName(fullNameBuffer), baseName(NULL) +{ + setToBogus(); +} + + +Locale::Locale( const char * newLanguage, + const char * newCountry, + const char * newVariant, + const char * newKeywords) + : UObject(), fullName(fullNameBuffer), baseName(NULL) +{ + if( (newLanguage==NULL) && (newCountry == NULL) && (newVariant == NULL) ) + { + init(NULL, FALSE); /* shortcut */ + } + else + { + char togo_stack[ULOC_FULLNAME_CAPACITY]; + char *togo; + char *togo_heap = NULL; + int32_t size = 0; + int32_t lsize = 0; + int32_t csize = 0; + int32_t vsize = 0; + int32_t ksize = 0; + char *p; + + // Calculate the size of the resulting string. + + // Language + if ( newLanguage != NULL ) + { + lsize = (int32_t)uprv_strlen(newLanguage); + size = lsize; + } + + // _Country + if ( newCountry != NULL ) + { + csize = (int32_t)uprv_strlen(newCountry); + size += csize; + } + + // _Variant + if ( newVariant != NULL ) + { + // remove leading _'s + while(newVariant[0] == SEP_CHAR) + { + newVariant++; + } + + // remove trailing _'s + vsize = (int32_t)uprv_strlen(newVariant); + while( (vsize>1) && (newVariant[vsize-1] == SEP_CHAR) ) + { + vsize--; + } + } + + if( vsize > 0 ) + { + size += vsize; + } + + // Separator rules: + if ( vsize > 0 ) + { + size += 2; // at least: __v + } + else if ( csize > 0 ) + { + size += 1; // at least: _v + } + + if ( newKeywords != NULL) + { + ksize = (int32_t)uprv_strlen(newKeywords); + size += ksize + 1; + } + + + // NOW we have the full locale string.. + + /*if the whole string is longer than our internal limit, we need + to go to the heap for temporary buffers*/ + if (size >= ULOC_FULLNAME_CAPACITY) + { + togo_heap = (char *)uprv_malloc(sizeof(char)*(size+1)); + togo = togo_heap; + } + else + { + togo = togo_stack; + } + + togo[0] = 0; + + // Now, copy it back. + p = togo; + if ( lsize != 0 ) + { + uprv_strcpy(p, newLanguage); + p += lsize; + } + + if ( ( vsize != 0 ) || (csize != 0) ) // at least: __v + { // ^ + *p++ = SEP_CHAR; + } + + if ( csize != 0 ) + { + uprv_strcpy(p, newCountry); + p += csize; + } + + if ( vsize != 0) + { + *p++ = SEP_CHAR; // at least: __v + + uprv_strncpy(p, newVariant, vsize); // Must use strncpy because + p += vsize; // of trimming (above). + *p = 0; // terminate + } + + if ( ksize != 0) + { + if (uprv_strchr(newKeywords, '=')) { + *p++ = '@'; /* keyword parsing */ + } + else { + *p++ = '_'; /* Variant parsing with a script */ + if ( vsize == 0) { + *p++ = '_'; /* No country found */ + } + } + uprv_strcpy(p, newKeywords); + p += ksize; + } + + // Parse it, because for example 'language' might really be a complete + // string. + init(togo, FALSE); + + if (togo_heap) { + uprv_free(togo_heap); + } + } +} + +Locale::Locale(const Locale &other) + : UObject(other), fullName(fullNameBuffer), baseName(NULL) +{ + *this = other; +} + +Locale &Locale::operator=(const Locale &other) +{ + if (this == &other) { + return *this; + } + + if (&other == NULL) { + this->setToBogus(); + return *this; + } + + /* Free our current storage */ + if(fullName != fullNameBuffer) { + uprv_free(fullName); + fullName = fullNameBuffer; + } + + /* Allocate the full name if necessary */ + if(other.fullName != other.fullNameBuffer) { + fullName = (char *)uprv_malloc(sizeof(char)*(uprv_strlen(other.fullName)+1)); + } + /* Copy the full name */ + uprv_strcpy(fullName, other.fullName); + + /* baseName is the cached result of getBaseName. if 'other' has a + baseName and it fits in baseNameBuffer, then copy it. otherwise set + it to NULL, and let the user lazy-create it (in getBaseName) if they + want it. */ + if(baseName && baseName != baseNameBuffer) { + uprv_free(baseName); + } + baseName = NULL; + + if(other.baseName == other.baseNameBuffer) { + uprv_strcpy(baseNameBuffer, other.baseNameBuffer); + baseName = baseNameBuffer; + } + + /* Copy the language and country fields */ + uprv_strcpy(language, other.language); + uprv_strcpy(script, other.script); + uprv_strcpy(country, other.country); + + /* The variantBegin is an offset into fullName, just copy it */ + variantBegin = other.variantBegin; + fIsBogus = other.fIsBogus; + return *this; +} + +Locale * +Locale::clone() const { + return new Locale(*this); +} + +UBool +Locale::operator==( const Locale& other) const +{ + return (uprv_strcmp(other.fullName, fullName) == 0); +} + +/*This function initializes a Locale from a C locale ID*/ +Locale& Locale::init(const char* localeID, UBool canonicalize) +{ + fIsBogus = FALSE; + /* Free our current storage */ + if(fullName != fullNameBuffer) { + uprv_free(fullName); + fullName = fullNameBuffer; + } + + if(baseName && baseName != baseNameBuffer) { + uprv_free(baseName); + baseName = NULL; + } + + // not a loop: + // just an easy way to have a common error-exit + // without goto and without another function + do { + char *separator; + char *field[5] = {0}; + int32_t fieldLen[5] = {0}; + int32_t fieldIdx; + int32_t variantField; + int32_t length; + UErrorCode err; + + if(localeID == NULL) { + // not an error, just set the default locale + return *this = getDefault(); + } + + /* preset all fields to empty */ + language[0] = script[0] = country[0] = 0; + + // "canonicalize" the locale ID to ICU/Java format + err = U_ZERO_ERROR; + length = canonicalize ? + uloc_canonicalize(localeID, fullName, sizeof(fullNameBuffer), &err) : + uloc_getName(localeID, fullName, sizeof(fullNameBuffer), &err); + + if(err == U_BUFFER_OVERFLOW_ERROR || length >= (int32_t)sizeof(fullNameBuffer)) { + /*Go to heap for the fullName if necessary*/ + fullName = (char *)uprv_malloc(sizeof(char)*(length + 1)); + if(fullName == 0) { + fullName = fullNameBuffer; + break; // error: out of memory + } + err = U_ZERO_ERROR; + length = canonicalize ? + uloc_canonicalize(localeID, fullName, length+1, &err) : + uloc_getName(localeID, fullName, length+1, &err); + } + if(U_FAILURE(err) || err == U_STRING_NOT_TERMINATED_WARNING) { + /* should never occur */ + break; + } + + variantBegin = length; + + /* after uloc_getName/canonicalize() we know that only '_' are separators */ + separator = field[0] = fullName; + fieldIdx = 1; + while ((separator = uprv_strchr(field[fieldIdx-1], SEP_CHAR)) && fieldIdx < (int32_t)(sizeof(field)/sizeof(field[0]))-1) { + field[fieldIdx] = separator + 1; + fieldLen[fieldIdx-1] = (int32_t)(separator - field[fieldIdx-1]); + fieldIdx++; + } + // variant may contain @foo or .foo POSIX cruft; remove it + separator = uprv_strchr(field[fieldIdx-1], '@'); + char* sep2 = uprv_strchr(field[fieldIdx-1], '.'); + if (separator!=NULL || sep2!=NULL) { + if (separator==NULL || (sep2!=NULL && separator > sep2)) { + separator = sep2; + } + fieldLen[fieldIdx-1] = (int32_t)(separator - field[fieldIdx-1]); + } else { + fieldLen[fieldIdx-1] = length - (int32_t)(field[fieldIdx-1] - fullName); + } + + if (fieldLen[0] >= (int32_t)(sizeof(language)) + || (fieldLen[1] == 4 && fieldLen[2] >= (int32_t)(sizeof(country))) + || (fieldLen[1] != 4 && fieldLen[1] >= (int32_t)(sizeof(country)))) + { + break; // error: one of the fields is too long + } + + variantField = 2; /* Usually the 2nd one, except when a script is used. */ + if (fieldLen[0] > 0) { + /* We have a language */ + uprv_memcpy(language, fullName, fieldLen[0]); + language[fieldLen[0]] = 0; + } + if (fieldLen[1] == 4) { + /* We have at least a script */ + uprv_memcpy(script, field[1], fieldLen[1]); + script[fieldLen[1]] = 0; + variantField = 3; + if (fieldLen[2] > 0) { + /* We have a country */ + uprv_memcpy(country, field[2], fieldLen[2]); + country[fieldLen[2]] = 0; + } + } + else if (fieldLen[1] > 0) { + /* We have a country and no script */ + uprv_memcpy(country, field[1], fieldLen[1]); + country[fieldLen[1]] = 0; + } + if (variantField > 0 && fieldLen[variantField] > 0) { + /* We have a variant */ + variantBegin = (int32_t)(field[variantField] - fullName); + } + + // successful end of init() + return *this; + } while(0); /*loop doesn't iterate*/ + + // when an error occurs, then set this object to "bogus" (there is no UErrorCode here) + setToBogus(); + + return *this; +} + +int32_t +Locale::hashCode() const +{ + UHashTok hashKey; + hashKey.pointer = fullName; + return uhash_hashChars(hashKey); +} + +void +Locale::setToBogus() { + /* Free our current storage */ + if(fullName != fullNameBuffer) { + uprv_free(fullName); + fullName = fullNameBuffer; + } + *fullNameBuffer = 0; + *language = 0; + *script = 0; + *country = 0; + fIsBogus = TRUE; +} + +const Locale& U_EXPORT2 +Locale::getDefault() +{ + const Locale *retLocale; + UMTX_CHECK(NULL, gDefaultLocale, retLocale); + if (retLocale == NULL) { + locale_set_default_internal(NULL); + umtx_lock(NULL); + // Need a mutex in case some other thread set a new + // default inbetween when we set and when we get the new default. For + // processors with weak memory coherency, we might not otherwise see all + // of the newly created new default locale. + retLocale = gDefaultLocale; + umtx_unlock(NULL); + } + return *retLocale; +} + + + +void U_EXPORT2 +Locale::setDefault( const Locale& newLocale, + UErrorCode& status) +{ + if (U_FAILURE(status)) { + return; + } + + /* Set the default from the full name string of the supplied locale. + * This is a convenient way to access the default locale caching mechanisms. + */ + const char *localeID = newLocale.getName(); + locale_set_default_internal(localeID); +} + +Locale U_EXPORT2 +Locale::createFromName (const char *name) +{ + if (name) { + Locale l(""); + l.init(name, FALSE); + return l; + } + else { + return getDefault(); + } +} + +Locale U_EXPORT2 +Locale::createCanonical(const char* name) { + Locale loc(""); + loc.init(name, TRUE); + return loc; +} + +const char * +Locale::getISO3Language() const +{ + return uloc_getISO3Language(fullName); +} + + +const char * +Locale::getISO3Country() const +{ + return uloc_getISO3Country(fullName); +} + +/** + * Return the LCID value as specified in the "LocaleID" resource for this + * locale. The LocaleID must be expressed as a hexadecimal number, from + * one to four digits. If the LocaleID resource is not present, or is + * in an incorrect format, 0 is returned. The LocaleID is for use in + * Windows (it is an LCID), but is available on all platforms. + */ +uint32_t +Locale::getLCID() const +{ + return uloc_getLCID(fullName); +} + +UnicodeString& +Locale::getDisplayLanguage(UnicodeString& dispLang) const +{ + return this->getDisplayLanguage(getDefault(), dispLang); +} + +/*We cannot make any assumptions on the size of the output display strings +* Yet, since we are calling through to a C API, we need to set limits on +* buffer size. For all the following getDisplay functions we first attempt +* to fill up a stack allocated buffer. If it is to small we heap allocated +* the exact buffer we need copy it to the UnicodeString and delete it*/ + +UnicodeString& +Locale::getDisplayLanguage(const Locale &displayLocale, + UnicodeString &result) const { + UChar *buffer; + UErrorCode errorCode=U_ZERO_ERROR; + int32_t length; + + buffer=result.getBuffer(ULOC_FULLNAME_CAPACITY); + if(buffer==0) { + result.truncate(0); + return result; + } + + length=uloc_getDisplayLanguage(fullName, displayLocale.fullName, + buffer, result.getCapacity(), + &errorCode); + result.releaseBuffer(U_SUCCESS(errorCode) ? length : 0); + + if(errorCode==U_BUFFER_OVERFLOW_ERROR) { + buffer=result.getBuffer(length); + if(buffer==0) { + result.truncate(0); + return result; + } + errorCode=U_ZERO_ERROR; + length=uloc_getDisplayLanguage(fullName, displayLocale.fullName, + buffer, result.getCapacity(), + &errorCode); + result.releaseBuffer(U_SUCCESS(errorCode) ? length : 0); + } + + return result; +} + +UnicodeString& +Locale::getDisplayScript(UnicodeString& dispScript) const +{ + return this->getDisplayScript(getDefault(), dispScript); +} + +UnicodeString& +Locale::getDisplayScript(const Locale &displayLocale, + UnicodeString &result) const { + UChar *buffer; + UErrorCode errorCode=U_ZERO_ERROR; + int32_t length; + + buffer=result.getBuffer(ULOC_FULLNAME_CAPACITY); + if(buffer==0) { + result.truncate(0); + return result; + } + + length=uloc_getDisplayScript(fullName, displayLocale.fullName, + buffer, result.getCapacity(), + &errorCode); + result.releaseBuffer(U_SUCCESS(errorCode) ? length : 0); + + if(errorCode==U_BUFFER_OVERFLOW_ERROR) { + buffer=result.getBuffer(length); + if(buffer==0) { + result.truncate(0); + return result; + } + errorCode=U_ZERO_ERROR; + length=uloc_getDisplayScript(fullName, displayLocale.fullName, + buffer, result.getCapacity(), + &errorCode); + result.releaseBuffer(U_SUCCESS(errorCode) ? length : 0); + } + + return result; +} + +UnicodeString& +Locale::getDisplayCountry(UnicodeString& dispCntry) const +{ + return this->getDisplayCountry(getDefault(), dispCntry); +} + +UnicodeString& +Locale::getDisplayCountry(const Locale &displayLocale, + UnicodeString &result) const { + UChar *buffer; + UErrorCode errorCode=U_ZERO_ERROR; + int32_t length; + + buffer=result.getBuffer(ULOC_FULLNAME_CAPACITY); + if(buffer==0) { + result.truncate(0); + return result; + } + + length=uloc_getDisplayCountry(fullName, displayLocale.fullName, + buffer, result.getCapacity(), + &errorCode); + result.releaseBuffer(U_SUCCESS(errorCode) ? length : 0); + + if(errorCode==U_BUFFER_OVERFLOW_ERROR) { + buffer=result.getBuffer(length); + if(buffer==0) { + result.truncate(0); + return result; + } + errorCode=U_ZERO_ERROR; + length=uloc_getDisplayCountry(fullName, displayLocale.fullName, + buffer, result.getCapacity(), + &errorCode); + result.releaseBuffer(U_SUCCESS(errorCode) ? length : 0); + } + + return result; +} + +UnicodeString& +Locale::getDisplayVariant(UnicodeString& dispVar) const +{ + return this->getDisplayVariant(getDefault(), dispVar); +} + +UnicodeString& +Locale::getDisplayVariant(const Locale &displayLocale, + UnicodeString &result) const { + UChar *buffer; + UErrorCode errorCode=U_ZERO_ERROR; + int32_t length; + + buffer=result.getBuffer(ULOC_FULLNAME_CAPACITY); + if(buffer==0) { + result.truncate(0); + return result; + } + + length=uloc_getDisplayVariant(fullName, displayLocale.fullName, + buffer, result.getCapacity(), + &errorCode); + result.releaseBuffer(U_SUCCESS(errorCode) ? length : 0); + + if(errorCode==U_BUFFER_OVERFLOW_ERROR) { + buffer=result.getBuffer(length); + if(buffer==0) { + result.truncate(0); + return result; + } + errorCode=U_ZERO_ERROR; + length=uloc_getDisplayVariant(fullName, displayLocale.fullName, + buffer, result.getCapacity(), + &errorCode); + result.releaseBuffer(U_SUCCESS(errorCode) ? length : 0); + } + + return result; +} + +UnicodeString& +Locale::getDisplayName( UnicodeString& name ) const +{ + return this->getDisplayName(getDefault(), name); +} + +UnicodeString& +Locale::getDisplayName(const Locale &displayLocale, + UnicodeString &result) const { + UChar *buffer; + UErrorCode errorCode=U_ZERO_ERROR; + int32_t length; + + buffer=result.getBuffer(ULOC_FULLNAME_CAPACITY); + if(buffer==0) { + result.truncate(0); + return result; + } + + length=uloc_getDisplayName(fullName, displayLocale.fullName, + buffer, result.getCapacity(), + &errorCode); + result.releaseBuffer(U_SUCCESS(errorCode) ? length : 0); + + if(errorCode==U_BUFFER_OVERFLOW_ERROR) { + buffer=result.getBuffer(length); + if(buffer==0) { + result.truncate(0); + return result; + } + errorCode=U_ZERO_ERROR; + length=uloc_getDisplayName(fullName, displayLocale.fullName, + buffer, result.getCapacity(), + &errorCode); + result.releaseBuffer(U_SUCCESS(errorCode) ? length : 0); + } + + return result; +} +const Locale* U_EXPORT2 +Locale::getAvailableLocales(int32_t& count) +{ + // for now, there is a hardcoded list, so just walk through that list and set it up. + UBool needInit; + UMTX_CHECK(NULL, availableLocaleList == NULL, needInit); + + if (needInit) { + int32_t locCount = uloc_countAvailable(); + Locale *newLocaleList = 0; + if(locCount) { + newLocaleList = new Locale[locCount]; + } + if (newLocaleList == NULL) { + count = 0; + return NULL; + } + + count = locCount; + + while(--locCount >= 0) { + newLocaleList[locCount].setFromPOSIXID(uloc_getAvailable(locCount)); + } + + umtx_lock(NULL); + if(availableLocaleList == 0) { + availableLocaleListCount = count; + availableLocaleList = newLocaleList; + newLocaleList = NULL; + ucln_common_registerCleanup(UCLN_COMMON_LOCALE, locale_cleanup); + } + umtx_unlock(NULL); + delete []newLocaleList; + } + count = availableLocaleListCount; + return availableLocaleList; +} + +const char* const* U_EXPORT2 Locale::getISOCountries() +{ + return uloc_getISOCountries(); +} + +const char* const* U_EXPORT2 Locale::getISOLanguages() +{ + return uloc_getISOLanguages(); +} + +// Set the locale's data based on a posix id. +void Locale::setFromPOSIXID(const char *posixID) +{ + init(posixID, TRUE); +} + +const Locale & U_EXPORT2 +Locale::getEnglish(void) +{ + return getLocale(eENGLISH); +} + +const Locale & U_EXPORT2 +Locale::getFrench(void) +{ + return getLocale(eFRENCH); +} + +const Locale & U_EXPORT2 +Locale::getGerman(void) +{ + return getLocale(eGERMAN); +} + +const Locale & U_EXPORT2 +Locale::getItalian(void) +{ + return getLocale(eITALIAN); +} + +const Locale & U_EXPORT2 +Locale::getJapanese(void) +{ + return getLocale(eJAPANESE); +} + +const Locale & U_EXPORT2 +Locale::getKorean(void) +{ + return getLocale(eKOREAN); +} + +const Locale & U_EXPORT2 +Locale::getChinese(void) +{ + return getLocale(eCHINESE); +} + +const Locale & U_EXPORT2 +Locale::getSimplifiedChinese(void) +{ + return getLocale(eCHINA); +} + +const Locale & U_EXPORT2 +Locale::getTraditionalChinese(void) +{ + return getLocale(eTAIWAN); +} + + +const Locale & U_EXPORT2 +Locale::getFrance(void) +{ + return getLocale(eFRANCE); +} + +const Locale & U_EXPORT2 +Locale::getGermany(void) +{ + return getLocale(eGERMANY); +} + +const Locale & U_EXPORT2 +Locale::getItaly(void) +{ + return getLocale(eITALY); +} + +const Locale & U_EXPORT2 +Locale::getJapan(void) +{ + return getLocale(eJAPAN); +} + +const Locale & U_EXPORT2 +Locale::getKorea(void) +{ + return getLocale(eKOREA); +} + +const Locale & U_EXPORT2 +Locale::getChina(void) +{ + return getLocale(eCHINA); +} + +const Locale & U_EXPORT2 +Locale::getPRC(void) +{ + return getLocale(eCHINA); +} + +const Locale & U_EXPORT2 +Locale::getTaiwan(void) +{ + return getLocale(eTAIWAN); +} + +const Locale & U_EXPORT2 +Locale::getUK(void) +{ + return getLocale(eUK); +} + +const Locale & U_EXPORT2 +Locale::getUS(void) +{ + return getLocale(eUS); +} + +const Locale & U_EXPORT2 +Locale::getCanada(void) +{ + return getLocale(eCANADA); +} + +const Locale & U_EXPORT2 +Locale::getCanadaFrench(void) +{ + return getLocale(eCANADA_FRENCH); +} + +const Locale & +Locale::getLocale(int locid) +{ + Locale *localeCache = getLocaleCache(); + U_ASSERT((locid < eMAX_LOCALES)&&(locid>=0)); + if (localeCache == NULL) { + // Failure allocating the locale cache. + // The best we can do is return a NULL reference. + locid = 0; + } + return localeCache[locid]; /*operating on NULL*/ +} + +/* +This function is defined this way in order to get around static +initialization and static destruction. + */ +Locale * +Locale::getLocaleCache(void) +{ + umtx_lock(NULL); + UBool needInit = (gLocaleCache == NULL); + umtx_unlock(NULL); + + if (needInit) { + Locale *tLocaleCache = new Locale[(int)eMAX_LOCALES]; + if (tLocaleCache == NULL) { + return NULL; + } + tLocaleCache[eENGLISH] = Locale("en"); + tLocaleCache[eFRENCH] = Locale("fr"); + tLocaleCache[eGERMAN] = Locale("de"); + tLocaleCache[eITALIAN] = Locale("it"); + tLocaleCache[eJAPANESE] = Locale("ja"); + tLocaleCache[eKOREAN] = Locale("ko"); + tLocaleCache[eCHINESE] = Locale("zh"); + tLocaleCache[eFRANCE] = Locale("fr", "FR"); + tLocaleCache[eGERMANY] = Locale("de", "DE"); + tLocaleCache[eITALY] = Locale("it", "IT"); + tLocaleCache[eJAPAN] = Locale("ja", "JP"); + tLocaleCache[eKOREA] = Locale("ko", "KR"); + tLocaleCache[eCHINA] = Locale("zh", "CN"); + tLocaleCache[eTAIWAN] = Locale("zh", "TW"); + tLocaleCache[eUK] = Locale("en", "GB"); + tLocaleCache[eUS] = Locale("en", "US"); + tLocaleCache[eCANADA] = Locale("en", "CA"); + tLocaleCache[eCANADA_FRENCH] = Locale("fr", "CA"); + + umtx_lock(NULL); + if (gLocaleCache == NULL) { + gLocaleCache = tLocaleCache; + tLocaleCache = NULL; + ucln_common_registerCleanup(UCLN_COMMON_LOCALE, locale_cleanup); + } + umtx_unlock(NULL); + if (tLocaleCache) { + delete [] tLocaleCache; // Fancy array delete will destruct each member. + } + } + return gLocaleCache; +} + +class KeywordEnumeration : public StringEnumeration { +private: + char *keywords; + char *current; + int32_t length; + UnicodeString currUSKey; + static const char fgClassID;/* Warning this is used beyond the typical RTTI usage. */ + +public: + static UClassID U_EXPORT2 getStaticClassID(void) { return (UClassID)&fgClassID; } + virtual UClassID getDynamicClassID(void) const { return getStaticClassID(); } +public: + KeywordEnumeration(const char *keys, int32_t keywordLen, int32_t currentIndex, UErrorCode &status) + : keywords((char *)&fgClassID), current((char *)&fgClassID), length(0) { + if(U_SUCCESS(status) && keywordLen != 0) { + if(keys == NULL || keywordLen < 0) { + status = U_ILLEGAL_ARGUMENT_ERROR; + } else { + keywords = (char *)uprv_malloc(keywordLen+1); + if (keywords == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + } + else { + uprv_memcpy(keywords, keys, keywordLen); + keywords[keywordLen] = 0; + current = keywords + currentIndex; + length = keywordLen; + } + } + } + } + + virtual ~KeywordEnumeration() { + uprv_free(keywords); + } + + virtual StringEnumeration * clone() const + { + UErrorCode status = U_ZERO_ERROR; + return new KeywordEnumeration(keywords, length, (int32_t)(current - keywords), status); + } + + virtual int32_t count(UErrorCode &/*status*/) const { + char *kw = keywords; + int32_t result = 0; + while(*kw) { + result++; + kw += uprv_strlen(kw)+1; + } + return result; + } + + virtual const char* next(int32_t* resultLength, UErrorCode& status) { + const char* result; + int32_t len; + if(U_SUCCESS(status) && *current != 0) { + result = current; + len = (int32_t)uprv_strlen(current); + current += len+1; + if(resultLength != NULL) { + *resultLength = len; + } + } else { + if(resultLength != NULL) { + *resultLength = 0; + } + result = NULL; + } + return result; + } + + virtual const UnicodeString* snext(UErrorCode& status) { + int32_t resultLength = 0; + const char *s = next(&resultLength, status); + return setChars(s, resultLength, status); + } + + virtual void reset(UErrorCode& /*status*/) { + current = keywords; + } +}; + +const char KeywordEnumeration::fgClassID = '\0'; + +StringEnumeration * +Locale::createKeywords(UErrorCode &status) const +{ + char keywords[256]; + int32_t keywordCapacity = 256; + StringEnumeration *result = NULL; + + const char* variantStart = uprv_strchr(fullName, '@'); + const char* assignment = uprv_strchr(fullName, '='); + if(variantStart) { + if(assignment > variantStart) { + int32_t keyLen = locale_getKeywords(variantStart+1, '@', keywords, keywordCapacity, NULL, 0, NULL, FALSE, &status); + if(keyLen) { + result = new KeywordEnumeration(keywords, keyLen, 0, status); + } + } else { + status = U_INVALID_FORMAT_ERROR; + } + } + return result; +} + +int32_t +Locale::getKeywordValue(const char* keywordName, char *buffer, int32_t bufLen, UErrorCode &status) const +{ + return uloc_getKeywordValue(fullName, keywordName, buffer, bufLen, &status); +} + +const char * +Locale::getBaseName() const +{ + // lazy init + UErrorCode status = U_ZERO_ERROR; + // semantically const + if(baseName == 0) { + ((Locale *)this)->baseName = ((Locale *)this)->baseNameBuffer; + int32_t baseNameSize = uloc_getBaseName(fullName, baseName, ULOC_FULLNAME_CAPACITY, &status); + if(baseNameSize >= ULOC_FULLNAME_CAPACITY) { + ((Locale *)this)->baseName = (char *)uprv_malloc(sizeof(char) * baseNameSize + 1); + uloc_getBaseName(fullName, baseName, baseNameSize+1, &status); + } + baseName[baseNameSize] = 0; + } + return baseName; +} + + +//eof +U_NAMESPACE_END diff --git a/Build/source/libs/icu/icu-xetex/common/locmap.c b/Build/source/libs/icu/icu-xetex/common/locmap.c new file mode 100644 index 00000000000..802c6c38c17 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/locmap.c @@ -0,0 +1,893 @@ +/* + ********************************************************************** + * Copyright (C) 1996-2006, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + * + * Provides functionality for mapping between + * LCID and Posix IDs or ICU locale to codepage + * + * Note: All classes and code in this file are + * intended for internal use only. + * + * Methods of interest: + * unsigned long convertToLCID(const char*); + * const char* convertToPosix(unsigned long); + * + * Kathleen Wilson, 4/30/96 + * + * Date Name Description + * 3/11/97 aliu Fixed off-by-one bug in assignment operator. Added + * setId() method and safety check against + * MAX_ID_LENGTH. + * 04/23/99 stephen Added C wrapper for convertToPosix. + * 09/18/00 george Removed the memory leaks. + * 08/23/01 george Convert to C + */ + +#include "locmap.h" +#include "cstring.h" + +/* + * Note: + * The mapping from Win32 locale ID numbers to POSIX locale strings should + * be the faster one. + * + * Many LCID values come from winnt.h + * Some also come from http://www.microsoft.com/globaldev/reference/lcid-all.mspx + */ + +/* +//////////////////////////////////////////////// +// +// Internal Classes for LCID <--> POSIX Mapping +// +///////////////////////////////////////////////// +*/ + +typedef struct ILcidPosixElement +{ + const uint32_t hostID; + const char * const posixID; +} ILcidPosixElement; + +typedef struct ILcidPosixMap +{ + const uint32_t numRegions; + const struct ILcidPosixElement* const regionMaps; +} ILcidPosixMap; + + +/* +///////////////////////////////////////////////// +// +// Easy macros to make the LCID <--> POSIX Mapping +// +///////////////////////////////////////////////// +*/ + +/* + The standard one language/one country mapping for LCID. + The first element must be the language, and the following + elements are the language with the country. + */ +#define ILCID_POSIX_ELEMENT_ARRAY(hostID, languageID, posixID) \ +static const ILcidPosixElement languageID[] = { \ + {LANGUAGE_LCID(hostID), #languageID}, /* parent locale */ \ + {hostID, #posixID}, \ +}; + +/* + Create the map for the posixID. This macro supposes that the language string + name is the same as the global variable name, and that the first element + in the ILcidPosixElement is just the language. + */ +#define ILCID_POSIX_MAP(_posixID) \ + {sizeof(_posixID)/sizeof(ILcidPosixElement), _posixID} + +/* +//////////////////////////////////////////// +// +// Create the table of LCID to POSIX Mapping +// None of it should be dynamically created. +// +// Keep static locale variables inside the function so that +// it can be created properly during static init. +// +//////////////////////////////////////////// +*/ + +ILCID_POSIX_ELEMENT_ARRAY(0x0436, af, af_ZA) + +static const ILcidPosixElement ar[] = { + {0x01, "ar"}, + {0x3801, "ar_AE"}, + {0x3c01, "ar_BH"}, + {0x1401, "ar_DZ"}, + {0x0c01, "ar_EG"}, + {0x0801, "ar_IQ"}, + {0x2c01, "ar_JO"}, + {0x3401, "ar_KW"}, + {0x3001, "ar_LB"}, + {0x1001, "ar_LY"}, + {0x1801, "ar_MA"}, + {0x2001, "ar_OM"}, + {0x4001, "ar_QA"}, + {0x0401, "ar_SA"}, + {0x2801, "ar_SY"}, + {0x1c01, "ar_TN"}, + {0x2401, "ar_YE"} +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x044d, as, as_IN) +ILCID_POSIX_ELEMENT_ARRAY(0x045e, am, am_ET) +ILCID_POSIX_ELEMENT_ARRAY(0x047a, arn,arn_CL) + +static const ILcidPosixElement az[] = { + {0x2c, "az"}, + {0x082c, "az_Cyrl_AZ"}, /* Cyrillic based */ + {0x082c, "az_Cyrl"}, /* Cyrillic based */ + {0x042c, "az_Latn_AZ"}, /* Latin based */ + {0x042c, "az_Latn"}, /* Latin based */ + {0x042c, "az_AZ"} /* Latin based */ +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x046d, ba, ba_RU) +ILCID_POSIX_ELEMENT_ARRAY(0x0423, be, be_BY) + +static const ILcidPosixElement ber[] = { + {0x5f, "ber"}, + {0x045f, "ber_Arab_DZ"}, + {0x045f, "ber_Arab"}, + {0x085f, "ber_Latn_DZ"}, + {0x085f, "ber_Latn"} +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x0402, bg, bg_BG) + +static const ILcidPosixElement bn[] = { + {0x45, "bn"}, + {0x0845, "bn_BD"}, + {0x0445, "bn_IN"} +}; + +static const ILcidPosixElement bo[] = { + {0x51, "bo"}, + {0x0851, "bo_BT"}, + {0x0451, "bo_CN"} +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x047e, br, br_FR) +ILCID_POSIX_ELEMENT_ARRAY(0x0403, ca, ca_ES) +ILCID_POSIX_ELEMENT_ARRAY(0x0483, co, co_FR) +ILCID_POSIX_ELEMENT_ARRAY(0x045c, chr,chr_US) + +/* Declared as cs_CZ to get around compiler errors on z/OS, which defines cs as a function */ +static const ILcidPosixElement cs_CZ[] = { + {0x05, "cs"}, + {0x0405, "cs_CZ"}, +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x0452, cy, cy_GB) +ILCID_POSIX_ELEMENT_ARRAY(0x0406, da, da_DK) + +static const ILcidPosixElement de[] = { + {0x07, "de"}, + {0x0c07, "de_AT"}, + {0x0807, "de_CH"}, + {0x0407, "de_DE"}, + {0x1407, "de_LI"}, + {0x1007, "de_LU"}, + {0x10407,"de_DE@collation=phonebook"}, /*This is really de_DE_PHONEBOOK on Windows*/ + {0x10407,"de@collation=phonebook"} /*This is really de_DE_PHONEBOOK on Windows*/ +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x0465, dv, dv_MV) +ILCID_POSIX_ELEMENT_ARRAY(0x0408, el, el_GR) + +static const ILcidPosixElement en[] = { + {0x09, "en"}, + {0x0c09, "en_AU"}, + {0x2809, "en_BZ"}, + {0x1009, "en_CA"}, + {0x0809, "en_GB"}, + {0x1809, "en_IE"}, + {0x4009, "en_IN"}, + {0x2009, "en_JM"}, + {0x4409, "en_MY"}, + {0x1409, "en_NZ"}, + {0x3409, "en_PH"}, + {0x4809, "en_SG"}, + {0x2C09, "en_TT"}, + {0x0409, "en_US"}, + {0x007f, "en_US_POSIX"}, /* duplicate for roundtripping */ + {0x2409, "en_VI"}, /* Virgin Islands AKA Caribbean Islands (en_CB). */ + {0x1c09, "en_ZA"}, + {0x3009, "en_ZW"}, + {0x0409, "en_AS"}, /* Alias for en_US. Leave last. */ + {0x0409, "en_GU"}, /* Alias for en_US. Leave last. */ + {0x0409, "en_MH"}, /* Alias for en_US. Leave last. */ + {0x0409, "en_MP"}, /* Alias for en_US. Leave last. */ + {0x0409, "en_UM"} /* Alias for en_US. Leave last. */ +}; + +static const ILcidPosixElement en_US_POSIX[] = { + {0x007f, "en_US_POSIX"} /* duplicate for roundtripping */ +}; + +static const ILcidPosixElement es[] = { + {0x0a, "es"}, + {0x2c0a, "es_AR"}, + {0x400a, "es_BO"}, + {0x340a, "es_CL"}, + {0x240a, "es_CO"}, + {0x140a, "es_CR"}, + {0x1c0a, "es_DO"}, + {0x300a, "es_EC"}, + {0x0c0a, "es_ES"}, /*Modern sort.*/ + {0x100a, "es_GT"}, + {0x480a, "es_HN"}, + {0x080a, "es_MX"}, + {0x4c0a, "es_NI"}, + {0x180a, "es_PA"}, + {0x280a, "es_PE"}, + {0x500a, "es_PR"}, + {0x3c0a, "es_PY"}, + {0x440a, "es_SV"}, + {0x540a, "es_US"}, + {0x380a, "es_UY"}, + {0x200a, "es_VE"}, + {0x040a, "es_ES@collation=traditional"}, + {0x040a, "es@collation=traditional"} +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x0425, et, et_EE) +ILCID_POSIX_ELEMENT_ARRAY(0x042d, eu, eu_ES) + +/* ISO-639 doesn't distinguish between Persian and Dari.*/ +static const ILcidPosixElement fa[] = { + {0x29, "fa"}, + {0x0429, "fa_IR"}, /* Persian/Farsi (Iran) */ + {0x048c, "fa_AF"} /* Persian/Dari (Afghanistan) */ +}; + +/* duplicate for roundtripping */ +static const ILcidPosixElement fa_AF[] = { + {0x8c, "fa_AF"}, /* Persian/Dari (Afghanistan) */ + {0x048c, "fa_AF"} /* Persian/Dari (Afghanistan) */ +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x040b, fi, fi_FI) +ILCID_POSIX_ELEMENT_ARRAY(0x0464, fil,fil_PH) +ILCID_POSIX_ELEMENT_ARRAY(0x0438, fo, fo_FO) + +static const ILcidPosixElement fr[] = { + {0x0c, "fr"}, + {0x080c, "fr_BE"}, + {0x0c0c, "fr_CA"}, + {0x240c, "fr_CD"}, + {0x100c, "fr_CH"}, + {0x300c, "fr_CI"}, + {0x2c0c, "fr_CM"}, + {0x040c, "fr_FR"}, + {0x3c0c, "fr_HT"}, + {0x140c, "fr_LU"}, + {0x380c, "fr_MA"}, + {0x180c, "fr_MC"}, + {0x340c, "fr_ML"}, + {0x200c, "fr_RE"}, + {0x280c, "fr_SN"} +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x0462, fy, fy_NL) + +/* This LCID is really two different locales.*/ +static const ILcidPosixElement ga[] = { + {0x3c, "ga"}, + {0x3c, "gd"}, + {0x083c, "ga_IE"}, /* Gaelic (Ireland) */ + {0x043c, "gd_GB"} /* Gaelic (Scotland) */ +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x0456, gl, gl_ES) +ILCID_POSIX_ELEMENT_ARRAY(0x0447, gu, gu_IN) +ILCID_POSIX_ELEMENT_ARRAY(0x0474, gn, gn_PY) +ILCID_POSIX_ELEMENT_ARRAY(0x0484, gsw,gsw_FR) +ILCID_POSIX_ELEMENT_ARRAY(0x0468, ha, ha_NG) /* ha_Latn_NG? */ +ILCID_POSIX_ELEMENT_ARRAY(0x0475, haw,haw_US) +ILCID_POSIX_ELEMENT_ARRAY(0x040d, he, he_IL) +ILCID_POSIX_ELEMENT_ARRAY(0x0439, hi, hi_IN) + +/* This LCID is really four different locales.*/ +static const ILcidPosixElement hr[] = { + {0x1a, "hr"}, + {0x141a, "bs_Latn_BA"}, /* Bosnian, Bosnia and Herzegovina */ + {0x141a, "bs_Latn"}, /* Bosnian, Bosnia and Herzegovina */ + {0x141a, "bs_BA"}, /* Bosnian, Bosnia and Herzegovina */ + {0x141a, "bs"}, /* Bosnian */ + {0x201a, "bs_Cyrl_BA"}, /* Bosnian, Bosnia and Herzegovina */ + {0x201a, "bs_Cyrl"}, /* Bosnian, Bosnia and Herzegovina */ + {0x101a, "hr_BA"}, /* Croatian in Bosnia */ + {0x041a, "hr_HR"}, /* Croatian*/ + {0x181a, "sr_Latn_BA"}, /* Serbo-Croatian in Bosnia */ + {0x081a, "sr_Latn_CS"}, /* Serbo-Croatian*/ + {0x081a, "sr_Latn"}, /* It's 0x1a or 0x081a, pick one to make the test program happy. */ + {0x1c1a, "sr_Cyrl_BA"}, /* Serbo-Croatian in Bosnia */ + {0x0c1a, "sr_Cyrl_CS"}, /* Serbian*/ + {0x0c1a, "sr_Cyrl"}, /* It's 0x1a or 0x0c1a, pick one to make the test program happy. */ + {0x0c1a, "sr"} /* In CLDR sr is sr_Cyrl. */ +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x040e, hu, hu_HU) +ILCID_POSIX_ELEMENT_ARRAY(0x042b, hy, hy_AM) +ILCID_POSIX_ELEMENT_ARRAY(0x0421, id, id_ID) +ILCID_POSIX_ELEMENT_ARRAY(0x0470, ig, ig_NG) +ILCID_POSIX_ELEMENT_ARRAY(0x0478, ii, ii_CN) +ILCID_POSIX_ELEMENT_ARRAY(0x040f, is, is_IS) + +static const ILcidPosixElement it[] = { + {0x10, "it"}, + {0x0810, "it_CH"}, + {0x0410, "it_IT"} +}; + +static const ILcidPosixElement iu[] = { + {0x5d, "iu"}, + {0x045d, "iu_Cans_CA"}, + {0x045d, "iu_Cans"}, + {0x085d, "iu_Latn_CA"}, + {0x085d, "iu_Latn"} +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x040d, iw, iw_IL) /*Left in for compatibility*/ +ILCID_POSIX_ELEMENT_ARRAY(0x0411, ja, ja_JP) +ILCID_POSIX_ELEMENT_ARRAY(0x0437, ka, ka_GE) +ILCID_POSIX_ELEMENT_ARRAY(0x043f, kk, kk_KZ) +ILCID_POSIX_ELEMENT_ARRAY(0x046f, kl, kl_GL) +ILCID_POSIX_ELEMENT_ARRAY(0x0453, km, km_KH) +ILCID_POSIX_ELEMENT_ARRAY(0x044b, kn, kn_IN) + +static const ILcidPosixElement ko[] = { + {0x12, "ko"}, + {0x0812, "ko_KP"}, + {0x0412, "ko_KR"} +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x0457, kok, kok_IN) +ILCID_POSIX_ELEMENT_ARRAY(0x0471, kr, kr_NG) + +static const ILcidPosixElement ks[] = { /* We could add PK and CN too */ + {0x60, "ks"}, + {0x0860, "ks_IN"}, /* Documentation doesn't mention script */ + {0x0460, "ks_Arab_IN"} +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x0440, ky, ky_KG) /* Kyrgyz is spoken in Kyrgyzstan */ +ILCID_POSIX_ELEMENT_ARRAY(0x0476, la, la_IT) /* TODO: Verify the country */ +ILCID_POSIX_ELEMENT_ARRAY(0x046e, lb, lb_LU) +ILCID_POSIX_ELEMENT_ARRAY(0x0454, lo, lo_LA) +ILCID_POSIX_ELEMENT_ARRAY(0x0427, lt, lt_LT) +ILCID_POSIX_ELEMENT_ARRAY(0x0426, lv, lv_LV) +ILCID_POSIX_ELEMENT_ARRAY(0x0481, mi, mi_NZ) +ILCID_POSIX_ELEMENT_ARRAY(0x042f, mk, mk_MK) +ILCID_POSIX_ELEMENT_ARRAY(0x044c, ml, ml_IN) + +static const ILcidPosixElement mn[] = { + {0x50, "mn"}, + {0x0850, "mn_CN"}, + {0x0450, "mn_MN"} +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x0458, mni,mni_IN) +ILCID_POSIX_ELEMENT_ARRAY(0x047c, moh,moh_CA) +ILCID_POSIX_ELEMENT_ARRAY(0x044e, mr, mr_IN) + +static const ILcidPosixElement ms[] = { + {0x3e, "ms"}, + {0x083e, "ms_BN"}, /* Brunei Darussalam*/ + {0x043e, "ms_MY"} /* Malaysia*/ +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x043a, mt, mt_MT) +ILCID_POSIX_ELEMENT_ARRAY(0x0455, my, my_MM) + +static const ILcidPosixElement ne[] = { + {0x61, "ne"}, + {0x0861, "ne_IN"}, /* India*/ + {0x0461, "ne_NP"} /* Nepal*/ +}; + +static const ILcidPosixElement nl[] = { + {0x13, "nl"}, + {0x0813, "nl_BE"}, + {0x0413, "nl_NL"} +}; + +/* The "no" locale split into nb and nn. By default in ICU, "no" is nb.*/ +static const ILcidPosixElement no[] = { + {0x14, "nb"}, /* really nb */ + {0x0414, "nb_NO"}, /* really nb_NO. Keep first in the 414 list. */ + {0x0414, "no"}, /* really nb_NO */ + {0x0414, "no_NO"}, /* really nb_NO */ + {0x0814, "nn_NO"}, /* really nn_NO. Keep first in the 814 list. */ + {0x0814, "nn"}, /* It's 0x14 or 0x814, pick one to make the test program happy. */ + {0x0814, "no_NO_NY"}/* really nn_NO */ +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x046c, nso,nso_ZA) /* TODO: Verify the ISO-639 code */ +ILCID_POSIX_ELEMENT_ARRAY(0x0482, oc, oc_FR) +ILCID_POSIX_ELEMENT_ARRAY(0x0472, om, om_ET) /* TODO: Verify the country */ + +/* Declared as or_IN to get around compiler errors*/ +static const ILcidPosixElement or_IN[] = { + {0x48, "or"}, + {0x0448, "or_IN"}, +}; + +static const ILcidPosixElement pa[] = { + {0x46, "pa"}, + {0x0446, "pa_IN"}, + {0x0846, "pa_PK"} +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x0415, pl, pl_PL) +ILCID_POSIX_ELEMENT_ARRAY(0x0463, ps, ps_AF) + +static const ILcidPosixElement pt[] = { + {0x16, "pt"}, + {0x0416, "pt_BR"}, + {0x0816, "pt_PT"} +}; + +static const ILcidPosixElement qu[] = { + {0x6b, "qu"}, + {0x046b, "qu_BO"}, + {0x086b, "qu_EC"}, + {0x0C6b, "qu_PE"} +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x0486, qut, qut_GT) /* qut is an ISO-639-3 code */ +ILCID_POSIX_ELEMENT_ARRAY(0x0417, rm, rm_CH) +ILCID_POSIX_ELEMENT_ARRAY(0x0418, ro, ro_RO) + +static const ILcidPosixElement root[] = { + {0x00, "root"} +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x0419, ru, ru_RU) +ILCID_POSIX_ELEMENT_ARRAY(0x0487, rw, rw_RW) +ILCID_POSIX_ELEMENT_ARRAY(0x044f, sa, sa_IN) +ILCID_POSIX_ELEMENT_ARRAY(0x0485, sah,sah_RU) + +static const ILcidPosixElement sd[] = { + {0x59, "sd"}, + {0x0459, "sd_IN"}, + {0x0859, "sd_PK"} +}; + +static const ILcidPosixElement se[] = { + {0x3b, "se"}, + {0x0c3b, "se_FI"}, + {0x043b, "se_NO"}, + {0x083b, "se_SE"}, + {0x183b, "sma_NO"}, + {0x1c3b, "sma_SE"}, + {0x103b, "smj_NO"}, + {0x143b, "smj_SE"}, + {0x243b, "smn_FI"}, + {0x203b, "sms_FI"}, +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x045b, si, si_LK) +ILCID_POSIX_ELEMENT_ARRAY(0x041b, sk, sk_SK) +ILCID_POSIX_ELEMENT_ARRAY(0x0424, sl, sl_SI) +ILCID_POSIX_ELEMENT_ARRAY(0x0477, so, so_ET) /* TODO: Verify the country */ +ILCID_POSIX_ELEMENT_ARRAY(0x041c, sq, sq_AL) + +static const ILcidPosixElement sv[] = { + {0x1d, "sv"}, + {0x081d, "sv_FI"}, + {0x041d, "sv_SE"} +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x0441, sw, sw_KE) +ILCID_POSIX_ELEMENT_ARRAY(0x045A, syr, syr_SY) +ILCID_POSIX_ELEMENT_ARRAY(0x0449, ta, ta_IN) +ILCID_POSIX_ELEMENT_ARRAY(0x044a, te, te_IN) +ILCID_POSIX_ELEMENT_ARRAY(0x0428, tg, tg_TJ) /* Cyrillic based by default */ +ILCID_POSIX_ELEMENT_ARRAY(0x041e, th, th_TH) + +static const ILcidPosixElement ti[] = { + {0x73, "ti"}, + {0x0873, "ti_ER"}, + {0x0473, "ti_ET"} +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x0442, tk, tk_TM) +ILCID_POSIX_ELEMENT_ARRAY(0x0432, tn, tn_BW) +ILCID_POSIX_ELEMENT_ARRAY(0x041f, tr, tr_TR) +ILCID_POSIX_ELEMENT_ARRAY(0x0444, tt, tt_RU) +ILCID_POSIX_ELEMENT_ARRAY(0x0480, ug, ug_CN) +ILCID_POSIX_ELEMENT_ARRAY(0x0422, uk, uk_UA) + +static const ILcidPosixElement ur[] = { + {0x20, "ur"}, + {0x0820, "ur_IN"}, + {0x0420, "ur_PK"} +}; + +static const ILcidPosixElement uz[] = { + {0x43, "uz"}, + {0x0843, "uz_Cyrl_UZ"}, /* Cyrillic based */ + {0x0843, "uz_Cyrl"}, /* Cyrillic based */ + {0x0843, "uz_UZ"}, /* Cyrillic based */ + {0x0443, "uz_Latn_UZ"}, /* Latin based */ + {0x0443, "uz_Latn"} /* Latin based */ +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x0433, ve, ve_ZA) /* TODO: Verify the country */ +ILCID_POSIX_ELEMENT_ARRAY(0x042a, vi, vi_VN) + +static const ILcidPosixElement wen[] = { + {0x2E, "wen"}, + {0x042E, "wen_DE"}, + {0x042E, "hsb_DE"}, + {0x082E, "dsb_DE"} +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x0488, wo, wo_SN) +ILCID_POSIX_ELEMENT_ARRAY(0x0434, xh, xh_ZA) +ILCID_POSIX_ELEMENT_ARRAY(0x046a, yo, yo_NG) + +static const ILcidPosixElement zh[] = { + {0x04, "zh"}, + {0x0804, "zh_Hans_CN"}, + {0x0804, "zh_Hans"}, + {0x0804, "zh_CN"}, + {0x0c04, "zh_Hant_HK"}, + {0x0c04, "zh_HK"}, + {0x1404, "zh_Hant_MO"}, + {0x1404, "zh_MO"}, + {0x1004, "zh_Hans_SG"}, + {0x1004, "zh_SG"}, + {0x0404, "zh_Hant_TW"}, + {0x0404, "zh_Hant"}, + {0x0404, "zh_TW"}, + {0x30404,"zh_Hant_TW"}, /* Bopomofo order */ + {0x30404,"zh_TW"}, /* Bopomofo order */ + {0x20404,"zh_Hant_TW@collation=stroke"}, + {0x20404,"zh_TW@collation=stroke"}, + {0x20804,"zh_Hans_CN@collation=stroke"}, + {0x20804,"zh_CN@collation=stroke"} +}; + +ILCID_POSIX_ELEMENT_ARRAY(0x0435, zu, zu_ZA) + +/* This must be static and grouped by LCID. */ + +/* non-existent ISO-639-2 codes */ +/* +0x466 Edo +0x467 Fulfulde - Nigeria +0x486 K'iche - Guatemala +0x430 Sutu +*/ +static const ILcidPosixMap gPosixIDmap[] = { + ILCID_POSIX_MAP(af), /* af Afrikaans 0x36 */ + ILCID_POSIX_MAP(am), /* am Amharic 0x5e */ + ILCID_POSIX_MAP(ar), /* ar Arabic 0x01 */ + ILCID_POSIX_MAP(arn), /* arn Araucanian/Mapudungun 0x7a */ + ILCID_POSIX_MAP(as), /* as Assamese 0x4d */ + ILCID_POSIX_MAP(az), /* az Azerbaijani 0x2c */ + ILCID_POSIX_MAP(ba), /* ba Bashkir 0x6d */ + ILCID_POSIX_MAP(be), /* be Belarusian 0x23 */ + ILCID_POSIX_MAP(ber), /* ber Berber/Tamazight 0x5f */ + ILCID_POSIX_MAP(bg), /* bg Bulgarian 0x02 */ + ILCID_POSIX_MAP(bn), /* bn Bengali; Bangla 0x45 */ + ILCID_POSIX_MAP(bo), /* bo Tibetan 0x51 */ + ILCID_POSIX_MAP(br), /* br Breton 0x7e */ + ILCID_POSIX_MAP(ca), /* ca Catalan 0x03 */ + ILCID_POSIX_MAP(chr), /* chr Cherokee 0x5c */ + ILCID_POSIX_MAP(co), /* co Corsican 0x83 */ + ILCID_POSIX_MAP(cs_CZ), /* cs Czech 0x05 */ + ILCID_POSIX_MAP(cy), /* cy Welsh 0x52 */ + ILCID_POSIX_MAP(da), /* da Danish 0x06 */ + ILCID_POSIX_MAP(de), /* de German 0x07 */ + ILCID_POSIX_MAP(dv), /* dv Divehi 0x65 */ + ILCID_POSIX_MAP(el), /* el Greek 0x08 */ + ILCID_POSIX_MAP(en), /* en English 0x09 */ + ILCID_POSIX_MAP(en_US_POSIX), /* invariant 0x7f */ + ILCID_POSIX_MAP(es), /* es Spanish 0x0a */ + ILCID_POSIX_MAP(et), /* et Estonian 0x25 */ + ILCID_POSIX_MAP(eu), /* eu Basque 0x2d */ + ILCID_POSIX_MAP(fa), /* fa Persian/Farsi 0x29 */ + ILCID_POSIX_MAP(fa_AF), /* fa Persian/Dari 0x8c */ + ILCID_POSIX_MAP(fi), /* fi Finnish 0x0b */ + ILCID_POSIX_MAP(fil), /* fil Filipino 0x64 */ + ILCID_POSIX_MAP(fo), /* fo Faroese 0x38 */ + ILCID_POSIX_MAP(fr), /* fr French 0x0c */ + ILCID_POSIX_MAP(fy), /* fy Frisian 0x62 */ + ILCID_POSIX_MAP(ga), /* * Gaelic (Ireland,Scotland) 0x3c */ + ILCID_POSIX_MAP(gl), /* gl Galician 0x56 */ + ILCID_POSIX_MAP(gn), /* gn Guarani 0x74 */ + ILCID_POSIX_MAP(gsw), /* gsw Alemanic/Alsatian/Swiss German 0x84 */ + ILCID_POSIX_MAP(gu), /* gu Gujarati 0x47 */ + ILCID_POSIX_MAP(ha), /* ha Hausa 0x68 */ + ILCID_POSIX_MAP(haw), /* haw Hawaiian 0x75 */ + ILCID_POSIX_MAP(he), /* he Hebrew (formerly iw) 0x0d */ + ILCID_POSIX_MAP(hi), /* hi Hindi 0x39 */ + ILCID_POSIX_MAP(hr), /* * Croatian and others 0x1a */ + ILCID_POSIX_MAP(hu), /* hu Hungarian 0x0e */ + ILCID_POSIX_MAP(hy), /* hy Armenian 0x2b */ + ILCID_POSIX_MAP(id), /* id Indonesian (formerly in) 0x21 */ + ILCID_POSIX_MAP(ig), /* ig Igbo 0x70 */ + ILCID_POSIX_MAP(ii), /* ii Sichuan Yi 0x78 */ + ILCID_POSIX_MAP(is), /* is Icelandic 0x0f */ + ILCID_POSIX_MAP(it), /* it Italian 0x10 */ + ILCID_POSIX_MAP(iu), /* iu Inuktitut 0x5d */ + ILCID_POSIX_MAP(iw), /* iw Hebrew 0x0d */ + ILCID_POSIX_MAP(ja), /* ja Japanese 0x11 */ + ILCID_POSIX_MAP(ka), /* ka Georgian 0x37 */ + ILCID_POSIX_MAP(kk), /* kk Kazakh 0x3f */ + ILCID_POSIX_MAP(kl), /* kl Kalaallisut 0x6f */ + ILCID_POSIX_MAP(km), /* km Khmer 0x53 */ + ILCID_POSIX_MAP(kn), /* kn Kannada 0x4b */ + ILCID_POSIX_MAP(ko), /* ko Korean 0x12 */ + ILCID_POSIX_MAP(kok), /* kok Konkani 0x57 */ + ILCID_POSIX_MAP(kr), /* kr Kanuri 0x71 */ + ILCID_POSIX_MAP(ks), /* ks Kashmiri 0x60 */ + ILCID_POSIX_MAP(ky), /* ky Kyrgyz 0x40 */ + ILCID_POSIX_MAP(lb), /* lb Luxembourgish 0x6e */ + ILCID_POSIX_MAP(la), /* la Latin 0x76 */ + ILCID_POSIX_MAP(lo), /* lo Lao 0x54 */ + ILCID_POSIX_MAP(lt), /* lt Lithuanian 0x27 */ + ILCID_POSIX_MAP(lv), /* lv Latvian, Lettish 0x26 */ + ILCID_POSIX_MAP(mi), /* mi Maori 0x81 */ + ILCID_POSIX_MAP(mk), /* mk Macedonian 0x2f */ + ILCID_POSIX_MAP(ml), /* ml Malayalam 0x4c */ + ILCID_POSIX_MAP(mn), /* mn Mongolian 0x50 */ + ILCID_POSIX_MAP(mni), /* mni Manipuri 0x58 */ + ILCID_POSIX_MAP(moh), /* moh Mohawk 0x7c */ + ILCID_POSIX_MAP(mr), /* mr Marathi 0x4e */ + ILCID_POSIX_MAP(ms), /* ms Malay 0x3e */ + ILCID_POSIX_MAP(mt), /* mt Maltese 0x3a */ + ILCID_POSIX_MAP(my), /* my Burmese 0x55 */ +/* ILCID_POSIX_MAP(nb), // no Norwegian 0x14 */ + ILCID_POSIX_MAP(ne), /* ne Nepali 0x61 */ + ILCID_POSIX_MAP(nl), /* nl Dutch 0x13 */ +/* ILCID_POSIX_MAP(nn), // no Norwegian 0x14 */ + ILCID_POSIX_MAP(no), /* * Norwegian 0x14 */ + ILCID_POSIX_MAP(nso), /* nso Sotho, Northern (Sepedi dialect) 0x6c */ + ILCID_POSIX_MAP(oc), /* oc Occitan 0x82 */ + ILCID_POSIX_MAP(om), /* om Oromo 0x72 */ + ILCID_POSIX_MAP(or_IN), /* or Oriya 0x48 */ + ILCID_POSIX_MAP(pa), /* pa Punjabi 0x46 */ + ILCID_POSIX_MAP(pl), /* pl Polish 0x15 */ + ILCID_POSIX_MAP(ps), /* ps Pashto 0x63 */ + ILCID_POSIX_MAP(pt), /* pt Portuguese 0x16 */ + ILCID_POSIX_MAP(qu), /* qu Quechua 0x6B */ + ILCID_POSIX_MAP(qut), /* qut K'iche 0x86 */ + ILCID_POSIX_MAP(rm), /* rm Raeto-Romance/Romansh 0x17 */ + ILCID_POSIX_MAP(ro), /* ro Romanian 0x18 */ + ILCID_POSIX_MAP(root), /* root 0x00 */ + ILCID_POSIX_MAP(ru), /* ru Russian 0x19 */ + ILCID_POSIX_MAP(rw), /* rw Kinyarwanda 0x87 */ + ILCID_POSIX_MAP(sa), /* sa Sanskrit 0x4f */ + ILCID_POSIX_MAP(sah), /* sah Yakut 0x85 */ + ILCID_POSIX_MAP(sd), /* sd Sindhi 0x59 */ + ILCID_POSIX_MAP(se), /* se Sami 0x3b */ +/* ILCID_POSIX_MAP(sh), // sh Serbo-Croatian 0x1a */ + ILCID_POSIX_MAP(si), /* si Sinhalese 0x5b */ + ILCID_POSIX_MAP(sk), /* sk Slovak 0x1b */ + ILCID_POSIX_MAP(sl), /* sl Slovenian 0x24 */ + ILCID_POSIX_MAP(so), /* so Somali 0x77 */ + ILCID_POSIX_MAP(sq), /* sq Albanian 0x1c */ +/* ILCID_POSIX_MAP(sr), // sr Serbian 0x1a */ + ILCID_POSIX_MAP(sv), /* sv Swedish 0x1d */ + ILCID_POSIX_MAP(sw), /* sw Swahili 0x41 */ + ILCID_POSIX_MAP(syr), /* syr Syriac 0x5A */ + ILCID_POSIX_MAP(ta), /* ta Tamil 0x49 */ + ILCID_POSIX_MAP(te), /* te Telugu 0x4a */ + ILCID_POSIX_MAP(tg), /* tg Tajik 0x28 */ + ILCID_POSIX_MAP(th), /* th Thai 0x1e */ + ILCID_POSIX_MAP(ti), /* ti Tigrigna 0x73 */ + ILCID_POSIX_MAP(tk), /* tk Turkmen 0x42 */ + ILCID_POSIX_MAP(tn), /* tn Tswana 0x32 */ + ILCID_POSIX_MAP(tr), /* tr Turkish 0x1f */ + ILCID_POSIX_MAP(tt), /* tt Tatar 0x44 */ + ILCID_POSIX_MAP(ug), /* ug Uighur 0x80 */ + ILCID_POSIX_MAP(uk), /* uk Ukrainian 0x22 */ + ILCID_POSIX_MAP(ur), /* ur Urdu 0x20 */ + ILCID_POSIX_MAP(uz), /* uz Uzbek 0x43 */ + ILCID_POSIX_MAP(ve), /* ve Venda 0x33 */ + ILCID_POSIX_MAP(vi), /* vi Vietnamese 0x2a */ + ILCID_POSIX_MAP(wen), /* wen Sorbian 0x2e */ + ILCID_POSIX_MAP(wo), /* wo Wolof 0x88 */ + ILCID_POSIX_MAP(xh), /* xh Xhosa 0x34 */ + ILCID_POSIX_MAP(yo), /* yo Yoruba 0x6a */ + ILCID_POSIX_MAP(zh), /* zh Chinese 0x04 */ + ILCID_POSIX_MAP(zu), /* zu Zulu 0x35 */ +}; + +static const uint32_t gLocaleCount = sizeof(gPosixIDmap)/sizeof(ILcidPosixMap); + +/** + * Do not call this function. It is called by hostID. + * The function is not private because this struct must stay as a C struct, + * and this is an internal class. + */ +static int32_t +idCmp(const char* id1, const char* id2) +{ + int32_t diffIdx = 0; + while (*id1 == *id2 && *id1 != 0) { + diffIdx++; + id1++; + id2++; + } + return diffIdx; +} + +/** + * Searches for a Windows LCID + * + * @param posixid the Posix style locale id. + * @param status gets set to U_ILLEGAL_ARGUMENT_ERROR when the Posix ID has + * no equivalent Windows LCID. + * @return the LCID + */ +static uint32_t +getHostID(const ILcidPosixMap *this_0, const char* posixID, UErrorCode* status) +{ + int32_t bestIdx = 0; + int32_t bestIdxDiff = 0; + int32_t posixIDlen = (int32_t)uprv_strlen(posixID); + uint32_t idx; + + for (idx = 0; idx < this_0->numRegions; idx++ ) { + int32_t sameChars = idCmp(posixID, this_0->regionMaps[idx].posixID); + if (sameChars > bestIdxDiff && this_0->regionMaps[idx].posixID[sameChars] == 0) { + if (posixIDlen == sameChars) { + /* Exact match */ + return this_0->regionMaps[idx].hostID; + } + bestIdxDiff = sameChars; + bestIdx = idx; + } + } + /* We asked for something unusual, like en_ZZ, and we try to return the number for the same language. */ + /* We also have to make sure that sid and si and similar string subsets don't match. */ + if ((posixID[bestIdxDiff] == '_' || posixID[bestIdxDiff] == '@') + && this_0->regionMaps[bestIdx].posixID[bestIdxDiff] == 0) + { + *status = U_USING_FALLBACK_WARNING; + return this_0->regionMaps[bestIdx].hostID; + } + + /*no match found */ + *status = U_ILLEGAL_ARGUMENT_ERROR; + return this_0->regionMaps->hostID; +} + +static const char* +getPosixID(const ILcidPosixMap *this_0, uint32_t hostID) +{ + uint32_t i; + for (i = 0; i <= this_0->numRegions; i++) + { + if (this_0->regionMaps[i].hostID == hostID) + { + return this_0->regionMaps[i].posixID; + } + } + + /* If you get here, then no matching region was found, + so return the language id with the wild card region. */ + return this_0->regionMaps[0].posixID; +} + +/* +////////////////////////////////////// +// +// LCID --> POSIX +// +///////////////////////////////////// +*/ + +U_CAPI const char * +uprv_convertToPosix(uint32_t hostid, UErrorCode* status) +{ + uint16_t langID = LANGUAGE_LCID(hostid); + uint32_t index; + + for (index = 0; index < gLocaleCount; index++) + { + if (langID == gPosixIDmap[index].regionMaps->hostID) + { + return getPosixID(&gPosixIDmap[index], hostid); + } + } + + /* no match found */ + *status = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; +} + +/* +////////////////////////////////////// +// +// POSIX --> LCID +// This should only be called from uloc_getLCID. +// The locale ID must be in canonical form. +// langID is separate so that this file doesn't depend on the uloc_* API. +// +///////////////////////////////////// +*/ + +U_CAPI uint32_t +uprv_convertToLCID(const char *langID, const char* posixID, UErrorCode* status) +{ + + uint32_t low = 0; + uint32_t high = gLocaleCount; + uint32_t mid = high; + uint32_t oldmid = 0; + int32_t compVal; + + uint32_t value = 0; + uint32_t fallbackValue = (uint32_t)-1; + UErrorCode myStatus; + uint32_t idx; + + /* Check for incomplete id. */ + if (!langID || !posixID || uprv_strlen(langID) < 2 || uprv_strlen(posixID) < 2) { + return 0; + } + + /*Binary search for the map entry for normal cases */ + + while (high > low) /*binary search*/{ + + mid = (high+low) >> 1; /*Finds median*/ + + if (mid == oldmid) + break; + + compVal = uprv_strcmp(langID, gPosixIDmap[mid].regionMaps->posixID); + if (compVal < 0){ + high = mid; + } + else if (compVal > 0){ + low = mid; + } + else /*we found it*/{ + return getHostID(&gPosixIDmap[mid], posixID, status); + } + oldmid = mid; + } + + /* + * Sometimes we can't do a binary search on posixID because some LCIDs + * go to different locales. We hit one of those special cases. + */ + for (idx = 0; idx < gLocaleCount; idx++ ) { + myStatus = U_ZERO_ERROR; + value = getHostID(&gPosixIDmap[idx], posixID, &myStatus); + if (myStatus == U_ZERO_ERROR) { + return value; + } + else if (myStatus == U_USING_FALLBACK_WARNING) { + fallbackValue = value; + } + } + + if (fallbackValue != (uint32_t)-1) { + *status = U_USING_FALLBACK_WARNING; + return fallbackValue; + } + + /* no match found */ + *status = U_ILLEGAL_ARGUMENT_ERROR; + return 0; /* return international (root) */ +} + diff --git a/Build/source/libs/icu/icu-xetex/common/locmap.h b/Build/source/libs/icu/icu-xetex/common/locmap.h new file mode 100644 index 00000000000..7db06070ec3 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/locmap.h @@ -0,0 +1,37 @@ +/* +****************************************************************************** +* +* Copyright (C) 1996-2004, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* +* File locmap.h : Locale Mapping Classes +* +* +* Created by: Helena Shih +* +* Modification History: +* +* Date Name Description +* 3/11/97 aliu Added setId(). +* 4/20/99 Madhu Added T_convertToPosix() +* 09/18/00 george Removed the memory leaks. +* 08/23/01 george Convert to C +*============================================================================ +*/ + +#ifndef LOCMAP_H +#define LOCMAP_H + +#include "unicode/utypes.h" + +#define LANGUAGE_LCID(hostID) (uint16_t)(0x03FF & hostID) + +U_CAPI const char *uprv_convertToPosix(uint32_t hostid, UErrorCode* status); + +/* Don't call this function directly. Use uloc_getLCID instead. */ +U_CAPI uint32_t uprv_convertToLCID(const char *langID, const char* posixID, UErrorCode* status); + +#endif /* LOCMAP_H */ + diff --git a/Build/source/libs/icu/icu-xetex/common/locutil.cpp b/Build/source/libs/icu/icu-xetex/common/locutil.cpp new file mode 100644 index 00000000000..e70b61628cf --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/locutil.cpp @@ -0,0 +1,267 @@ +/** + ******************************************************************************* + * Copyright (C) 2002-2006, International Business Machines Corporation and * + * others. All Rights Reserved. * + ******************************************************************************* + * + ******************************************************************************* + */ +#include "unicode/utypes.h" + +#if !UCONFIG_NO_SERVICE || !UCONFIG_NO_TRANSLITERATION + +#include "unicode/resbund.h" +#include "cmemory.h" +#include "ustrfmt.h" +#include "locutil.h" +#include "charstr.h" +#include "ucln_cmn.h" +#include "uassert.h" +#include "umutex.h" + +// see LocaleUtility::getAvailableLocaleNames +static U_NAMESPACE_QUALIFIER Hashtable * LocaleUtility_cache = NULL; + +#define UNDERSCORE_CHAR ((UChar)0x005f) +#define AT_SIGN_CHAR ((UChar)64) +#define PERIOD_CHAR ((UChar)46) + +/* + ****************************************************************** + */ + +/** + * Release all static memory held by Locale Utility. + */ +U_CDECL_BEGIN +static UBool U_CALLCONV service_cleanup(void) { + if (LocaleUtility_cache) { + delete LocaleUtility_cache; + LocaleUtility_cache = NULL; + } + return TRUE; +} +U_CDECL_END + +U_NAMESPACE_BEGIN + +UnicodeString& +LocaleUtility::canonicalLocaleString(const UnicodeString* id, UnicodeString& result) +{ + if (id == NULL) { + result.setToBogus(); + } else { + // Fix case only (no other changes) up to the first '@' or '.' or + // end of string, whichever comes first. In 3.0 I changed this to + // stop at first '@' or '.'. It used to run out to the end of + // string. My fix makes the tests pass but is probably + // structurally incorrect. See below. [alan 3.0] + + // TODO: Doug, you might want to revise this... + result = *id; + int32_t i = 0; + int32_t end = result.indexOf(AT_SIGN_CHAR); + int32_t n = result.indexOf(PERIOD_CHAR); + if (n >= 0 && n < end) { + end = n; + } + if (end < 0) { + end = result.length(); + } + n = result.indexOf(UNDERSCORE_CHAR); + if (n < 0) { + n = end; + } + for (; i < n; ++i) { + UChar c = result.charAt(i); + if (c >= 0x0041 && c <= 0x005a) { + c += 0x20; + result.setCharAt(i, c); + } + } + for (n = end; i < n; ++i) { + UChar c = result.charAt(i); + if (c >= 0x0061 && c <= 0x007a) { + c -= 0x20; + result.setCharAt(i, c); + } + } + } + return result; + +#if 0 + // This code does a proper full level 2 canonicalization of id. + // It's nasty to go from UChar to char to char to UChar -- but + // that's what you have to do to use the uloc_canonicalize + // function on UnicodeStrings. + + // I ended up doing the alternate fix (see above) not for + // performance reasons, although performance will certainly be + // better, but because doing a full level 2 canonicalization + // causes some tests to fail. [alan 3.0] + + // TODO: Doug, you might want to revisit this... + result.setToBogus(); + if (id != 0) { + int32_t buflen = id->length() + 8; // space for NUL + char* buf = (char*) uprv_malloc(buflen); + char* canon = (buf == 0) ? 0 : (char*) uprv_malloc(buflen); + if (buf != 0 && canon != 0) { + U_ASSERT(id->extract(0, INT32_MAX, buf, buflen) < buflen); + UErrorCode ec = U_ZERO_ERROR; + uloc_canonicalize(buf, canon, buflen, &ec); + if (U_SUCCESS(ec)) { + result = UnicodeString(canon); + } + } + uprv_free(buf); + uprv_free(canon); + } + return result; +#endif +} + +Locale& +LocaleUtility::initLocaleFromName(const UnicodeString& id, Locale& result) +{ + enum { BUFLEN = 128 }; // larger than ever needed + + if (id.isBogus() || id.length() >= BUFLEN) { + result.setToBogus(); + } else { + /* + * We need to convert from a UnicodeString to char * in order to + * create a Locale. + * + * Problem: Locale ID strings may contain '@' which is a variant + * character and cannot be handled by invariant-character conversion. + * + * Hack: Since ICU code can handle locale IDs with multiple encodings + * of '@' (at least for EBCDIC; it's not known to be a problem for + * ASCII-based systems), + * we use regular invariant-character conversion for everything else + * and manually convert U+0040 into a compiler-char-constant '@'. + * While this compilation-time constant may not match the runtime + * encoding of '@', it should be one of the encodings which ICU + * recognizes. + * + * There should be only at most one '@' in a locale ID. + */ + char buffer[BUFLEN]; + int32_t prev, i; + prev = 0; + for(;;) { + i = id.indexOf((UChar)0x40, prev); + if(i < 0) { + // no @ between prev and the rest of the string + id.extract(prev, INT32_MAX, buffer + prev, BUFLEN - prev, US_INV); + break; // done + } else { + // normal invariant-character conversion for text between @s + id.extract(prev, i - prev, buffer + prev, BUFLEN - prev, US_INV); + // manually "convert" U+0040 at id[i] into '@' at buffer[i] + buffer[i] = '@'; + prev = i + 1; + } + } + result = Locale::createFromName(buffer); + } + return result; +} + +UnicodeString& +LocaleUtility::initNameFromLocale(const Locale& locale, UnicodeString& result) +{ + if (locale.isBogus()) { + result.setToBogus(); + } else { + result.append(UnicodeString(locale.getName(), -1, US_INV)); + } + return result; +} + +const Hashtable* +LocaleUtility::getAvailableLocaleNames(const UnicodeString& bundleID) +{ + // LocaleUtility_cache is a hash-of-hashes. The top-level keys + // are path strings ('bundleID') passed to + // ures_openAvailableLocales. The top-level values are + // second-level hashes. The second-level keys are result strings + // from ures_openAvailableLocales. The second-level values are + // garbage ((void*)1 or other random pointer). + + UErrorCode status = U_ZERO_ERROR; + Hashtable* cache; + umtx_lock(NULL); + cache = LocaleUtility_cache; + umtx_unlock(NULL); + + if (cache == NULL) { + cache = new Hashtable(status); + if (cache == NULL || U_FAILURE(status)) { + return NULL; // catastrophic failure; e.g. out of memory + } + cache->setValueDeleter(uhash_deleteHashtable); + Hashtable* h; // set this to final LocaleUtility_cache value + umtx_lock(NULL); + h = LocaleUtility_cache; + if (h == NULL) { + LocaleUtility_cache = h = cache; + cache = NULL; + ucln_common_registerCleanup(UCLN_COMMON_SERVICE, service_cleanup); + } + umtx_unlock(NULL); + if(cache != NULL) { + delete cache; + } + cache = h; + } + + U_ASSERT(cache != NULL); + + Hashtable* htp; + umtx_lock(NULL); + htp = (Hashtable*) cache->get(bundleID); + umtx_unlock(NULL); + + if (htp == NULL) { + htp = new Hashtable(status); + if (htp && U_SUCCESS(status)) { + CharString cbundleID(bundleID); + const char* path = (const char*) cbundleID; + if (*path == 0) path = NULL; // empty string => NULL + UEnumeration *uenum = ures_openAvailableLocales(path, &status); + for (;;) { + const UChar* id = uenum_unext(uenum, NULL, &status); + if (id == NULL) { + break; + } + htp->put(UnicodeString(id), (void*)htp, status); + } + uenum_close(uenum); + if (U_FAILURE(status)) { + delete htp; + return NULL; + } + umtx_lock(NULL); + cache->put(bundleID, (void*)htp, status); + umtx_unlock(NULL); + } + } + return htp; +} + +UBool +LocaleUtility::isFallbackOf(const UnicodeString& root, const UnicodeString& child) +{ + return child.indexOf(root) == 0 && + (child.length() == root.length() || + child.charAt(root.length()) == UNDERSCORE_CHAR); +} + +U_NAMESPACE_END + +/* !UCONFIG_NO_SERVICE */ +#endif + + diff --git a/Build/source/libs/icu/icu-xetex/common/locutil.h b/Build/source/libs/icu/icu-xetex/common/locutil.h new file mode 100644 index 00000000000..cf64e343d06 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/locutil.h @@ -0,0 +1,37 @@ +/** + ******************************************************************************* + * Copyright (C) 2002-2005, International Business Machines Corporation and * + * others. All Rights Reserved. * + ******************************************************************************* + * + ******************************************************************************* + */ +#ifndef LOCUTIL_H +#define LOCUTIL_H + +#include "unicode/utypes.h" +#include "hash.h" + +#if !UCONFIG_NO_SERVICE || !UCONFIG_NO_TRANSLITERATION + + +U_NAMESPACE_BEGIN + +// temporary utility functions, till I know where to find them +// in header so tests can also access them + +class U_COMMON_API LocaleUtility { +public: + static UnicodeString& canonicalLocaleString(const UnicodeString* id, UnicodeString& result); + static Locale& initLocaleFromName(const UnicodeString& id, Locale& result); + static UnicodeString& initNameFromLocale(const Locale& locale, UnicodeString& result); + static const Hashtable* getAvailableLocaleNames(const UnicodeString& bundleID); + static UBool isFallbackOf(const UnicodeString& root, const UnicodeString& child); +}; + +U_NAMESPACE_END + + +#endif + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/msvcres.h b/Build/source/libs/icu/icu-xetex/common/msvcres.h new file mode 100644 index 00000000000..cad2fe66c41 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/msvcres.h @@ -0,0 +1,20 @@ +//{{NO_DEPENDENCIES}} +// Copyright (c) 2003-2005 International Business Machines +// Corporation and others. All Rights Reserved. +// +// Used by common.rc and other .rc files. +//Do not edit with Microsoft Developer Studio because it will modify this +//header the wrong way. This is here to prevent Visual Studio .NET from +//unnessarily building the resource files when it's not needed. +// + +/* +These are defined before unicode/uversion.h in order to prevent +STLPort's broken stddef.h from being used when rc.exe parses this file. +*/ +#define _STLP_OUTERMOST_HEADER_ID 0 +#define _STLP_WINCE 1 + +#include "unicode/uversion.h" + +#define ICU_WEBSITE "http://ibm.com/software/globalization/icu/" diff --git a/Build/source/libs/icu/icu-xetex/common/mutex.h b/Build/source/libs/icu/icu-xetex/common/mutex.h new file mode 100644 index 00000000000..26aefbf86b5 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/mutex.h @@ -0,0 +1,85 @@ +/* +****************************************************************************** +* +* Copyright (C) 1997-2003, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +*/ +//---------------------------------------------------------------------------- +// File: mutex.h +// +// Lightweight C++ wrapper for umtx_ C mutex functions +// +// Author: Alan Liu 1/31/97 +// History: +// 06/04/97 helena Updated setImplementation as per feedback from 5/21 drop. +// 04/07/1999 srl refocused as a thin wrapper +// +//---------------------------------------------------------------------------- +#ifndef MUTEX_H +#define MUTEX_H + +#include "unicode/utypes.h" +#include "unicode/uobject.h" +#include "umutex.h" + +U_NAMESPACE_BEGIN + +//---------------------------------------------------------------------------- +// Code within that accesses shared static or global data should +// should instantiate a Mutex object while doing so. You should make your own +// private mutex where possible. + +// For example: +// +// UMTX myMutex; +// +// int InitializeMyMutex() +// { +// umtx_init( &myMutex ); +// return 0; +// } +// +// static int initializeMyMutex = InitializeMyMutex(); +// +// void Function(int arg1, int arg2) +// { +// static Object* foo; // Shared read-write object +// Mutex mutex(&myMutex); // or no args for the global lock +// foo->Method(); +// // When 'mutex' goes out of scope and gets destroyed here, the lock is released +// } +// +// Note: Do NOT use the form 'Mutex mutex();' as that merely forward-declares a function +// returning a Mutex. This is a common mistake which silently slips through the +// compiler!! +// + +class U_COMMON_API Mutex : public UMemory { +public: + inline Mutex(UMTX *mutex = NULL); + inline ~Mutex(); + +private: + UMTX *fMutex; + + Mutex(const Mutex &other); // forbid copying of this class + Mutex &operator=(const Mutex &other); // forbid copying of this class +}; + +inline Mutex::Mutex(UMTX *mutex) + : fMutex(mutex) +{ + umtx_lock(fMutex); +} + +inline Mutex::~Mutex() +{ + umtx_unlock(fMutex); +} + +U_NAMESPACE_END + +#endif //_MUTEX_ +//eof diff --git a/Build/source/libs/icu/icu-xetex/common/normlzr.cpp b/Build/source/libs/icu/icu-xetex/common/normlzr.cpp new file mode 100644 index 00000000000..909bd30b27c --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/normlzr.cpp @@ -0,0 +1,611 @@ +/* + ************************************************************************* + * COPYRIGHT: + * Copyright (c) 1996-2005, International Business Machines Corporation and + * others. All Rights Reserved. + ************************************************************************* + */ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_NORMALIZATION + +#include "unicode/unistr.h" +#include "unicode/chariter.h" +#include "unicode/schriter.h" +#include "unicode/uchriter.h" +#include "unicode/uiter.h" +#include "unicode/normlzr.h" +#include "cmemory.h" +#include "unormimp.h" + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(Normalizer) + +//------------------------------------------------------------------------- +// Constructors and other boilerplate +//------------------------------------------------------------------------- + +Normalizer::Normalizer(const UnicodeString& str, UNormalizationMode mode) : + UObject(), fUMode(mode), fOptions(0), + currentIndex(0), nextIndex(0), + buffer(), bufferPos(0) +{ + init(new StringCharacterIterator(str)); +} + +Normalizer::Normalizer(const UChar *str, int32_t length, UNormalizationMode mode) : + UObject(), fUMode(mode), fOptions(0), + currentIndex(0), nextIndex(0), + buffer(), bufferPos(0) +{ + init(new UCharCharacterIterator(str, length)); +} + +Normalizer::Normalizer(const CharacterIterator& iter, UNormalizationMode mode) : + UObject(), fUMode(mode), fOptions(0), + currentIndex(0), nextIndex(0), + buffer(), bufferPos(0) +{ + init(iter.clone()); +} + +Normalizer::Normalizer(const Normalizer ©) : + UObject(copy), fUMode(copy.fUMode), fOptions(copy.fOptions), + currentIndex(copy.currentIndex), nextIndex(copy.nextIndex), + buffer(copy.buffer), bufferPos(copy.bufferPos) +{ + init(((CharacterIterator *)(copy.text->context))->clone()); +} + +static const UChar _NUL=0; + +void +Normalizer::init(CharacterIterator *iter) { + UErrorCode errorCode=U_ZERO_ERROR; + + text=(UCharIterator *)uprv_malloc(sizeof(UCharIterator)); + if(text!=NULL) { + if(unorm_haveData(&errorCode)) { + uiter_setCharacterIterator(text, iter); + } else { + delete iter; + uiter_setCharacterIterator(text, new UCharCharacterIterator(&_NUL, 0)); + } + } else { + delete iter; + } +} + +Normalizer::~Normalizer() +{ + if(text!=NULL) { + delete (CharacterIterator *)text->context; + uprv_free(text); + } +} + +Normalizer* +Normalizer::clone() const +{ + if(this!=0) { + return new Normalizer(*this); + } else { + return 0; + } +} + +/** + * Generates a hash code for this iterator. + */ +int32_t Normalizer::hashCode() const +{ + return ((CharacterIterator *)(text->context))->hashCode() + fUMode + fOptions + buffer.hashCode() + bufferPos + currentIndex + nextIndex; +} + +UBool Normalizer::operator==(const Normalizer& that) const +{ + return + this==&that || + fUMode==that.fUMode && + fOptions==that.fOptions && + *((CharacterIterator *)(text->context))==*((CharacterIterator *)(that.text->context)) && + buffer==that.buffer && + bufferPos==that.bufferPos && + nextIndex==that.nextIndex; +} + +//------------------------------------------------------------------------- +// Static utility methods +//------------------------------------------------------------------------- + +void U_EXPORT2 +Normalizer::normalize(const UnicodeString& source, + UNormalizationMode mode, int32_t options, + UnicodeString& result, + UErrorCode &status) { + if(source.isBogus() || U_FAILURE(status)) { + result.setToBogus(); + if(U_SUCCESS(status)) { + status=U_ILLEGAL_ARGUMENT_ERROR; + } + } else { + UnicodeString localDest; + UnicodeString *dest; + + if(&source!=&result) { + dest=&result; + } else { + // the source and result strings are the same object, use a temporary one + dest=&localDest; + } + + UChar *buffer=dest->getBuffer(source.length()); + int32_t length=unorm_internalNormalize(buffer, dest->getCapacity(), + source.getBuffer(), source.length(), + mode, options, + &status); + dest->releaseBuffer(U_SUCCESS(status) ? length : 0); + if(status==U_BUFFER_OVERFLOW_ERROR) { + status=U_ZERO_ERROR; + buffer=dest->getBuffer(length); + length=unorm_internalNormalize(buffer, dest->getCapacity(), + source.getBuffer(), source.length(), + mode, options, + &status); + dest->releaseBuffer(U_SUCCESS(status) ? length : 0); + } + + if(dest==&localDest) { + result=*dest; + } + if(U_FAILURE(status)) { + result.setToBogus(); + } + } +} + +void U_EXPORT2 +Normalizer::compose(const UnicodeString& source, + UBool compat, int32_t options, + UnicodeString& result, + UErrorCode &status) { + if(source.isBogus() || U_FAILURE(status)) { + result.setToBogus(); + if(U_SUCCESS(status)) { + status=U_ILLEGAL_ARGUMENT_ERROR; + } + } else { + UnicodeString localDest; + UnicodeString *dest; + + if(&source!=&result) { + dest=&result; + } else { + // the source and result strings are the same object, use a temporary one + dest=&localDest; + } + + UChar *buffer=dest->getBuffer(source.length()); + int32_t length=unorm_compose(buffer, dest->getCapacity(), + source.getBuffer(), source.length(), + compat, options, + &status); + dest->releaseBuffer(U_SUCCESS(status) ? length : 0); + if(status==U_BUFFER_OVERFLOW_ERROR) { + status=U_ZERO_ERROR; + buffer=dest->getBuffer(length); + length=unorm_compose(buffer, dest->getCapacity(), + source.getBuffer(), source.length(), + compat, options, + &status); + dest->releaseBuffer(U_SUCCESS(status) ? length : 0); + } + + if(dest==&localDest) { + result=*dest; + } + if(U_FAILURE(status)) { + result.setToBogus(); + } + } +} + +void U_EXPORT2 +Normalizer::decompose(const UnicodeString& source, + UBool compat, int32_t options, + UnicodeString& result, + UErrorCode &status) { + if(source.isBogus() || U_FAILURE(status)) { + result.setToBogus(); + if(U_SUCCESS(status)) { + status=U_ILLEGAL_ARGUMENT_ERROR; + } + } else { + UnicodeString localDest; + UnicodeString *dest; + + if(&source!=&result) { + dest=&result; + } else { + // the source and result strings are the same object, use a temporary one + dest=&localDest; + } + + UChar *buffer=dest->getBuffer(source.length()); + int32_t length=unorm_decompose(buffer, dest->getCapacity(), + source.getBuffer(), source.length(), + compat, options, + &status); + dest->releaseBuffer(U_SUCCESS(status) ? length : 0); + if(status==U_BUFFER_OVERFLOW_ERROR) { + status=U_ZERO_ERROR; + buffer=dest->getBuffer(length); + length=unorm_decompose(buffer, dest->getCapacity(), + source.getBuffer(), source.length(), + compat, options, + &status); + dest->releaseBuffer(U_SUCCESS(status) ? length : 0); + } + + if(dest==&localDest) { + result=*dest; + } + if(U_FAILURE(status)) { + result.setToBogus(); + } + } +} + +UnicodeString & U_EXPORT2 +Normalizer::concatenate(UnicodeString &left, UnicodeString &right, + UnicodeString &result, + UNormalizationMode mode, int32_t options, + UErrorCode &errorCode) { + if(left.isBogus() || right.isBogus() || U_FAILURE(errorCode)) { + result.setToBogus(); + if(U_SUCCESS(errorCode)) { + errorCode=U_ILLEGAL_ARGUMENT_ERROR; + } + } else { + UnicodeString localDest; + UnicodeString *dest; + + if(&left!=&result && &right!=&result) { + dest=&result; + } else { + // the source and result strings are the same object, use a temporary one + dest=&localDest; + } + + UChar *buffer=dest->getBuffer(left.length()+right.length()); + int32_t length=unorm_concatenate(left.getBuffer(), left.length(), + right.getBuffer(), right.length(), + buffer, dest->getCapacity(), + mode, options, + &errorCode); + dest->releaseBuffer(U_SUCCESS(errorCode) ? length : 0); + if(errorCode==U_BUFFER_OVERFLOW_ERROR) { + errorCode=U_ZERO_ERROR; + buffer=dest->getBuffer(length); + int32_t length=unorm_concatenate(left.getBuffer(), left.length(), + right.getBuffer(), right.length(), + buffer, dest->getCapacity(), + mode, options, + &errorCode); + dest->releaseBuffer(U_SUCCESS(errorCode) ? length : 0); + } + + if(dest==&localDest) { + result=*dest; + } + if(U_FAILURE(errorCode)) { + result.setToBogus(); + } + } + return result; +} + +//------------------------------------------------------------------------- +// Iteration API +//------------------------------------------------------------------------- + +/** + * Return the current character in the normalized text. + */ +UChar32 Normalizer::current() { + if(bufferPos0 || previousNormalize()) { + UChar32 c=buffer.char32At(bufferPos-1); + bufferPos-=UTF_CHAR_LENGTH(c); + return c; + } else { + return DONE; + } +} + +void Normalizer::reset() { + currentIndex=nextIndex=text->move(text, 0, UITER_START); + clearBuffer(); +} + +void +Normalizer::setIndexOnly(int32_t index) { + currentIndex=nextIndex=text->move(text, index, UITER_ZERO); // validates index + clearBuffer(); +} + +/** + * Return the first character in the normalized text-> This resets + * the Normalizer's position to the beginning of the text-> + */ +UChar32 Normalizer::first() { + reset(); + return next(); +} + +/** + * Return the last character in the normalized text-> This resets + * the Normalizer's position to be just before the + * the input text corresponding to that normalized character. + */ +UChar32 Normalizer::last() { + currentIndex=nextIndex=text->move(text, 0, UITER_LIMIT); + clearBuffer(); + return previous(); +} + +/** + * Retrieve the current iteration position in the input text that is + * being normalized. This method is useful in applications such as + * searching, where you need to be able to determine the position in + * the input text that corresponds to a given normalized output character. + *

+ * Note: This method sets the position in the input, while + * {@link #next} and {@link #previous} iterate through characters in the + * output. This means that there is not necessarily a one-to-one + * correspondence between characters returned by next and + * previous and the indices passed to and returned from + * setIndex and {@link #getIndex}. + * + */ +int32_t Normalizer::getIndex() const { + if(bufferPos This is the begin index + * of the CharacterIterator or the start (i.e. 0) of the String + * over which this Normalizer is iterating + */ +int32_t Normalizer::startIndex() const { + return text->getIndex(text, UITER_START); +} + +/** + * Retrieve the index of the end of the input text-> This is the end index + * of the CharacterIterator or the length of the String + * over which this Normalizer is iterating + */ +int32_t Normalizer::endIndex() const { + return text->getIndex(text, UITER_LIMIT); +} + +//------------------------------------------------------------------------- +// Property access methods +//------------------------------------------------------------------------- + +void +Normalizer::setMode(UNormalizationMode newMode) +{ + fUMode = newMode; +} + +UNormalizationMode +Normalizer::getUMode() const +{ + return fUMode; +} + +void +Normalizer::setOption(int32_t option, + UBool value) +{ + if (value) { + fOptions |= option; + } else { + fOptions &= (~option); + } +} + +UBool +Normalizer::getOption(int32_t option) const +{ + return (fOptions & option) != 0; +} + +/** + * Set the input text over which this Normalizer will iterate. + * The iteration position is set to the beginning of the input text-> + */ +void +Normalizer::setText(const UnicodeString& newText, + UErrorCode &status) +{ + if (U_FAILURE(status)) { + return; + } + CharacterIterator *newIter = new StringCharacterIterator(newText); + if (newIter == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + delete (CharacterIterator *)(text->context); + text->context = newIter; + reset(); +} + +/** + * Set the input text over which this Normalizer will iterate. + * The iteration position is set to the beginning of the string. + */ +void +Normalizer::setText(const CharacterIterator& newText, + UErrorCode &status) +{ + if (U_FAILURE(status)) { + return; + } + CharacterIterator *newIter = newText.clone(); + if (newIter == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + delete (CharacterIterator *)(text->context); + text->context = newIter; + reset(); +} + +void +Normalizer::setText(const UChar* newText, + int32_t length, + UErrorCode &status) +{ + if (U_FAILURE(status)) { + return; + } + CharacterIterator *newIter = new UCharCharacterIterator(newText, length); + if (newIter == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + delete (CharacterIterator *)(text->context); + text->context = newIter; + reset(); +} + +/** + * Copies the text under iteration into the UnicodeString referred to by "result". + * @param result Receives a copy of the text under iteration. + */ +void +Normalizer::getText(UnicodeString& result) +{ + ((CharacterIterator *)(text->context))->getText(result); +} + +//------------------------------------------------------------------------- +// Private utility methods +//------------------------------------------------------------------------- + +void Normalizer::clearBuffer() { + buffer.remove(); + bufferPos=0; +} + +UBool +Normalizer::nextNormalize() { + UChar *p; + int32_t length; + UErrorCode errorCode; + + clearBuffer(); + currentIndex=nextIndex; + text->move(text, nextIndex, UITER_ZERO); + if(!text->hasNext(text)) { + return FALSE; + } + + errorCode=U_ZERO_ERROR; + p=buffer.getBuffer(-1); + length=unorm_next(text, p, buffer.getCapacity(), + fUMode, fOptions, + TRUE, 0, + &errorCode); + buffer.releaseBuffer(U_SUCCESS(errorCode) ? length : 0); + if(errorCode==U_BUFFER_OVERFLOW_ERROR) { + errorCode=U_ZERO_ERROR; + text->move(text, nextIndex, UITER_ZERO); + p=buffer.getBuffer(length); + length=unorm_next(text, p, buffer.getCapacity(), + fUMode, fOptions, + TRUE, 0, + &errorCode); + buffer.releaseBuffer(U_SUCCESS(errorCode) ? length : 0); + } + + nextIndex=text->getIndex(text, UITER_CURRENT); + return U_SUCCESS(errorCode) && !buffer.isEmpty(); +} + +UBool +Normalizer::previousNormalize() { + UChar *p; + int32_t length; + UErrorCode errorCode; + + clearBuffer(); + nextIndex=currentIndex; + text->move(text, currentIndex, UITER_ZERO); + if(!text->hasPrevious(text)) { + return FALSE; + } + + errorCode=U_ZERO_ERROR; + p=buffer.getBuffer(-1); + length=unorm_previous(text, p, buffer.getCapacity(), + fUMode, fOptions, + TRUE, 0, + &errorCode); + buffer.releaseBuffer(U_SUCCESS(errorCode) ? length : 0); + if(errorCode==U_BUFFER_OVERFLOW_ERROR) { + errorCode=U_ZERO_ERROR; + text->move(text, currentIndex, UITER_ZERO); + p=buffer.getBuffer(length); + length=unorm_previous(text, p, buffer.getCapacity(), + fUMode, fOptions, + TRUE, 0, + &errorCode); + buffer.releaseBuffer(U_SUCCESS(errorCode) ? length : 0); + } + + bufferPos=buffer.length(); + currentIndex=text->getIndex(text, UITER_CURRENT); + return U_SUCCESS(errorCode) && !buffer.isEmpty(); +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_NORMALIZATION */ diff --git a/Build/source/libs/icu/icu-xetex/common/parsepos.cpp b/Build/source/libs/icu/icu-xetex/common/parsepos.cpp new file mode 100644 index 00000000000..26f88205129 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/parsepos.cpp @@ -0,0 +1,21 @@ +/* +********************************************************************** +* Copyright (C) 2003-2003, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +*/ + +#include "unicode/parsepos.h" + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(ParsePosition) + +ParsePosition::~ParsePosition() {} + +ParsePosition * +ParsePosition::clone() const { + return new ParsePosition(*this); +} + +U_NAMESPACE_END diff --git a/Build/source/libs/icu/icu-xetex/common/propname.cpp b/Build/source/libs/icu/icu-xetex/common/propname.cpp new file mode 100644 index 00000000000..55e354044f3 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/propname.cpp @@ -0,0 +1,758 @@ +/* +********************************************************************** +* Copyright (c) 2002-2006, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Author: Alan Liu +* Created: October 30 2002 +* Since: ICU 2.4 +********************************************************************** +*/ +#include "propname.h" +#include "unicode/uchar.h" +#include "unicode/udata.h" +#include "umutex.h" +#include "cmemory.h" +#include "cstring.h" +#include "ucln_cmn.h" +#include "uarrsort.h" + +U_CDECL_BEGIN + +/** + * Get the next non-ignorable ASCII character from a property name + * and lowercases it. + * @return ((advance count for the name)<<8)|character + */ +static inline int32_t +getASCIIPropertyNameChar(const char *name) { + int32_t i; + char c; + + /* Ignore delimiters '-', '_', and ASCII White_Space */ + for(i=0; + (c=name[i++])==0x2d || c==0x5f || + c==0x20 || (0x09<=c && c<=0x0d); + ) {} + + if(c!=0) { + return (i<<8)|(uint8_t)uprv_asciitolower((char)c); + } else { + return i<<8; + } +} + +/** + * Get the next non-ignorable EBCDIC character from a property name + * and lowercases it. + * @return ((advance count for the name)<<8)|character + */ +static inline int32_t +getEBCDICPropertyNameChar(const char *name) { + int32_t i; + char c; + + /* Ignore delimiters '-', '_', and EBCDIC White_Space */ + for(i=0; + (c=name[i++])==0x60 || c==0x6d || + c==0x40 || c==0x05 || c==0x15 || c==0x25 || c==0x0b || c==0x0c || c==0x0d; + ) {} + + if(c!=0) { + return (i<<8)|(uint8_t)uprv_ebcdictolower((char)c); + } else { + return i<<8; + } +} + +/** + * Unicode property names and property value names are compared "loosely". + * + * UCD.html 4.0.1 says: + * For all property names, property value names, and for property values for + * Enumerated, Binary, or Catalog properties, use the following + * loose matching rule: + * + * LM3. Ignore case, whitespace, underscore ('_'), and hyphens. + * + * This function does just that, for (char *) name strings. + * It is almost identical to ucnv_compareNames() but also ignores + * C0 White_Space characters (U+0009..U+000d, and U+0085 on EBCDIC). + * + * @internal + */ + +U_CAPI int32_t U_EXPORT2 +uprv_compareASCIIPropertyNames(const char *name1, const char *name2) { + int32_t rc, r1, r2; + + for(;;) { + r1=getASCIIPropertyNameChar(name1); + r2=getASCIIPropertyNameChar(name2); + + /* If we reach the ends of both strings then they match */ + if(((r1|r2)&0xff)==0) { + return 0; + } + + /* Compare the lowercased characters */ + if(r1!=r2) { + rc=(r1&0xff)-(r2&0xff); + if(rc!=0) { + return rc; + } + } + + name1+=r1>>8; + name2+=r2>>8; + } +} + +U_CAPI int32_t U_EXPORT2 +uprv_compareEBCDICPropertyNames(const char *name1, const char *name2) { + int32_t rc, r1, r2; + + for(;;) { + r1=getEBCDICPropertyNameChar(name1); + r2=getEBCDICPropertyNameChar(name2); + + /* If we reach the ends of both strings then they match */ + if(((r1|r2)&0xff)==0) { + return 0; + } + + /* Compare the lowercased characters */ + if(r1!=r2) { + rc=(r1&0xff)-(r2&0xff); + if(rc!=0) { + return rc; + } + } + + name1+=r1>>8; + name2+=r2>>8; + } +} + +U_CDECL_END + +U_NAMESPACE_BEGIN + +//---------------------------------------------------------------------- +// PropertyAliases implementation + +const char* +PropertyAliases::chooseNameInGroup(Offset offset, + UPropertyNameChoice choice) const { + int32_t c = choice; + if (!offset || c < 0) { + return NULL; + } + const Offset* p = (const Offset*) getPointer(offset); + while (c-- > 0) { + if (*p++ < 0) return NULL; + } + Offset a = *p; + if (a < 0) a = -a; + return (const char*) getPointerNull(a); +} + +const ValueMap* +PropertyAliases::getValueMap(EnumValue prop) const { + NonContiguousEnumToOffset* e2o = (NonContiguousEnumToOffset*) getPointer(enumToValue_offset); + Offset a = e2o->getOffset(prop); + return (const ValueMap*) (a ? getPointerNull(a) : NULL); +} + +inline const char* +PropertyAliases::getPropertyName(EnumValue prop, + UPropertyNameChoice choice) const { + NonContiguousEnumToOffset* e2n = (NonContiguousEnumToOffset*) getPointer(enumToName_offset); + return chooseNameInGroup(e2n->getOffset(prop), choice); +} + +inline EnumValue +PropertyAliases::getPropertyEnum(const char* alias) const { + NameToEnum* n2e = (NameToEnum*) getPointer(nameToEnum_offset); + return n2e->getEnum(alias, *this); +} + +inline const char* +PropertyAliases::getPropertyValueName(EnumValue prop, + EnumValue value, + UPropertyNameChoice choice) const { + const ValueMap* vm = getValueMap(prop); + if (!vm) return NULL; + Offset a; + if (vm->enumToName_offset) { + a = ((EnumToOffset*) getPointer(vm->enumToName_offset))-> + getOffset(value); + } else { + a = ((NonContiguousEnumToOffset*) getPointer(vm->ncEnumToName_offset))-> + getOffset(value); + } + return chooseNameInGroup(a, choice); +} + +inline EnumValue +PropertyAliases::getPropertyValueEnum(EnumValue prop, + const char* alias) const { + const ValueMap* vm = getValueMap(prop); + if (!vm) return UCHAR_INVALID_CODE; + NameToEnum* n2e = (NameToEnum*) getPointer(vm->nameToEnum_offset); + return n2e->getEnum(alias, *this); +} + +U_NAMESPACE_END +U_NAMESPACE_USE + +//---------------------------------------------------------------------- +// UDataMemory structures + +static const PropertyAliases* PNAME = NULL; +static UDataMemory* UDATA = NULL; + +//---------------------------------------------------------------------- +// UDataMemory loading/unloading + +/** + * udata callback to verify the zone data. + */ +U_CDECL_BEGIN +static UBool U_CALLCONV +isPNameAcceptable(void* /*context*/, + const char* /*type*/, const char* /*name*/, + const UDataInfo* info) { + return + info->size >= sizeof(UDataInfo) && + info->isBigEndian == U_IS_BIG_ENDIAN && + info->charsetFamily == U_CHARSET_FAMILY && + info->dataFormat[0] == PNAME_SIG_0 && + info->dataFormat[1] == PNAME_SIG_1 && + info->dataFormat[2] == PNAME_SIG_2 && + info->dataFormat[3] == PNAME_SIG_3 && + info->formatVersion[0] == PNAME_FORMAT_VERSION; +} + +static UBool U_CALLCONV pname_cleanup(void) { + if (UDATA) { + udata_close(UDATA); + UDATA = NULL; + } + PNAME = NULL; + return TRUE; +} +U_CDECL_END + +/** + * Load the property names data. Caller should check that data is + * not loaded BEFORE calling this function. Returns TRUE if the load + * succeeds. + */ +static UBool _load() { + UErrorCode ec = U_ZERO_ERROR; + UDataMemory* data = + udata_openChoice(0, PNAME_DATA_TYPE, PNAME_DATA_NAME, + isPNameAcceptable, 0, &ec); + if (U_SUCCESS(ec)) { + umtx_lock(NULL); + if (UDATA == NULL) { + UDATA = data; + PNAME = (const PropertyAliases*) udata_getMemory(UDATA); + ucln_common_registerCleanup(UCLN_COMMON_PNAME, pname_cleanup); + data = NULL; + } + umtx_unlock(NULL); + } + if (data) { + udata_close(data); + } + return PNAME!=NULL; +} + +/** + * Inline function that expands to code that does a lazy load of the + * property names data. If the data is already loaded, avoids an + * unnecessary function call. If the data is not loaded, call _load() + * to load it, and return TRUE if the load succeeds. + */ +static inline UBool load() { + UBool f; + UMTX_CHECK(NULL, (PNAME!=NULL), f); + return f || _load(); +} + +//---------------------------------------------------------------------- +// Public API implementation + +// The C API is just a thin wrapper. Each function obtains a pointer +// to the singleton PropertyAliases, and calls the appropriate method +// on it. If it cannot obtain a pointer, because valid data is not +// available, then it returns NULL or UCHAR_INVALID_CODE. + +U_CAPI const char* U_EXPORT2 +u_getPropertyName(UProperty property, + UPropertyNameChoice nameChoice) { + return load() ? PNAME->getPropertyName(property, nameChoice) + : NULL; +} + +U_CAPI UProperty U_EXPORT2 +u_getPropertyEnum(const char* alias) { + UProperty p = load() ? (UProperty) PNAME->getPropertyEnum(alias) + : UCHAR_INVALID_CODE; + return p; +} + +U_CAPI const char* U_EXPORT2 +u_getPropertyValueName(UProperty property, + int32_t value, + UPropertyNameChoice nameChoice) { + return load() ? PNAME->getPropertyValueName(property, value, nameChoice) + : NULL; +} + +U_CAPI int32_t U_EXPORT2 +u_getPropertyValueEnum(UProperty property, + const char* alias) { + return load() ? PNAME->getPropertyValueEnum(property, alias) + : (int32_t)UCHAR_INVALID_CODE; +} + +/* data swapping ------------------------------------------------------------ */ + +/* + * Sub-structure-swappers use the temp array (which is as large as the + * actual data) for intermediate storage, + * as well as to indicate if a particular structure has been swapped already. + * The temp array is initially reset to all 0. + * pos is the byte offset of the sub-structure in the inBytes/outBytes/temp arrays. + */ + +int32_t +EnumToOffset::swap(const UDataSwapper *ds, + const uint8_t *inBytes, int32_t length, uint8_t *outBytes, + uint8_t *temp, int32_t pos, + UErrorCode *pErrorCode) { + const EnumToOffset *inMap; + EnumToOffset *outMap, *tempMap; + int32_t size; + + tempMap=(EnumToOffset *)(temp+pos); + if(tempMap->enumStart!=0 || tempMap->enumLimit!=0) { + /* this map was swapped already */ + size=tempMap->getSize(); + return size; + } + + inMap=(const EnumToOffset *)(inBytes+pos); + outMap=(EnumToOffset *)(outBytes+pos); + + tempMap->enumStart=udata_readInt32(ds, inMap->enumStart); + tempMap->enumLimit=udata_readInt32(ds, inMap->enumLimit); + size=tempMap->getSize(); + + if(length>=0) { + if(length<(pos+size)) { + if(length<(int32_t)sizeof(PropertyAliases)) { + udata_printError(ds, "upname_swap(EnumToOffset): too few bytes (%d after header)\n" + " for pnames.icu EnumToOffset{%d..%d} at %d\n", + length, tempMap->enumStart, tempMap->enumLimit, pos); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + } + + /* swap enumStart and enumLimit */ + ds->swapArray32(ds, inMap, 2*sizeof(EnumValue), outMap, pErrorCode); + + /* swap _offsetArray[] */ + ds->swapArray16(ds, inMap->getOffsetArray(), (tempMap->enumLimit-tempMap->enumStart)*sizeof(Offset), + outMap->getOffsetArray(), pErrorCode); + } + + return size; +} + +int32_t +NonContiguousEnumToOffset::swap(const UDataSwapper *ds, + const uint8_t *inBytes, int32_t length, uint8_t *outBytes, + uint8_t *temp, int32_t pos, + UErrorCode *pErrorCode) { + const NonContiguousEnumToOffset *inMap; + NonContiguousEnumToOffset *outMap, *tempMap; + int32_t size; + + tempMap=(NonContiguousEnumToOffset *)(temp+pos); + if(tempMap->count!=0) { + /* this map was swapped already */ + size=tempMap->getSize(); + return size; + } + + inMap=(const NonContiguousEnumToOffset *)(inBytes+pos); + outMap=(NonContiguousEnumToOffset *)(outBytes+pos); + + tempMap->count=udata_readInt32(ds, inMap->count); + size=tempMap->getSize(); + + if(length>=0) { + if(length<(pos+size)) { + if(length<(int32_t)sizeof(PropertyAliases)) { + udata_printError(ds, "upname_swap(NonContiguousEnumToOffset): too few bytes (%d after header)\n" + " for pnames.icu NonContiguousEnumToOffset[%d] at %d\n", + length, tempMap->count, pos); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + } + + /* swap count and _enumArray[] */ + length=(1+tempMap->count)*sizeof(EnumValue); + ds->swapArray32(ds, inMap, length, + outMap, pErrorCode); + + /* swap _offsetArray[] */ + pos+=length; + ds->swapArray16(ds, inBytes+pos, tempMap->count*sizeof(Offset), + outBytes+pos, pErrorCode); + } + + return size; +} + +struct NameAndIndex { + Offset name, index; +}; + +U_CDECL_BEGIN +typedef int32_t U_CALLCONV PropNameCompareFn(const char *name1, const char *name2); + +struct CompareContext { + const char *chars; + PropNameCompareFn *propCompare; +}; + +static int32_t U_CALLCONV +upname_compareRows(const void *context, const void *left, const void *right) { + CompareContext *cmp=(CompareContext *)context; + return cmp->propCompare(cmp->chars+((const NameAndIndex *)left)->name, + cmp->chars+((const NameAndIndex *)right)->name); +} +U_CDECL_END + +int32_t +NameToEnum::swap(const UDataSwapper *ds, + const uint8_t *inBytes, int32_t length, uint8_t *outBytes, + uint8_t *temp, int32_t pos, + UErrorCode *pErrorCode) { + const NameToEnum *inMap; + NameToEnum *outMap, *tempMap; + + const EnumValue *inEnumArray; + EnumValue *outEnumArray; + + const Offset *inNameArray; + Offset *outNameArray; + + NameAndIndex *sortArray; + CompareContext cmp; + + int32_t i, size, oldIndex; + + tempMap=(NameToEnum *)(temp+pos); + if(tempMap->count!=0) { + /* this map was swapped already */ + size=tempMap->getSize(); + return size; + } + + inMap=(const NameToEnum *)(inBytes+pos); + outMap=(NameToEnum *)(outBytes+pos); + + tempMap->count=udata_readInt32(ds, inMap->count); + size=tempMap->getSize(); + + if(length>=0) { + if(length<(pos+size)) { + if(length<(int32_t)sizeof(PropertyAliases)) { + udata_printError(ds, "upname_swap(NameToEnum): too few bytes (%d after header)\n" + " for pnames.icu NameToEnum[%d] at %d\n", + length, tempMap->count, pos); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + } + + /* swap count */ + ds->swapArray32(ds, inMap, 4, outMap, pErrorCode); + + inEnumArray=inMap->getEnumArray(); + outEnumArray=outMap->getEnumArray(); + + inNameArray=(const Offset *)(inEnumArray+tempMap->count); + outNameArray=(Offset *)(outEnumArray+tempMap->count); + + if(ds->inCharset==ds->outCharset) { + /* no need to sort, just swap the enum/name arrays */ + ds->swapArray32(ds, inEnumArray, tempMap->count*4, outEnumArray, pErrorCode); + ds->swapArray16(ds, inNameArray, tempMap->count*2, outNameArray, pErrorCode); + return size; + } + + /* + * The name and enum arrays are sorted by names and must be resorted + * if inCharset!=outCharset. + * We use the corresponding part of the temp array to sort an array + * of pairs of name offsets and sorting indexes. + * Then the sorting indexes are used to permutate-swap the name and enum arrays. + * + * The outBytes must already contain the swapped strings. + */ + sortArray=(NameAndIndex *)tempMap->getEnumArray(); + for(i=0; icount; ++i) { + sortArray[i].name=udata_readInt16(ds, inNameArray[i]); + sortArray[i].index=(Offset)i; + } + + /* + * use a stable sort to avoid shuffling of equal strings, + * which makes testing harder + */ + cmp.chars=(const char *)outBytes; + if (ds->outCharset==U_ASCII_FAMILY) { + cmp.propCompare=uprv_compareASCIIPropertyNames; + } + else { + cmp.propCompare=uprv_compareEBCDICPropertyNames; + } + uprv_sortArray(sortArray, tempMap->count, sizeof(NameAndIndex), + upname_compareRows, &cmp, + TRUE, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + udata_printError(ds, "upname_swap(NameToEnum).uprv_sortArray(%d items) failed\n", + tempMap->count); + return 0; + } + + /* copy/swap/permutate _enumArray[] and _nameArray[] */ + if(inEnumArray!=outEnumArray) { + for(i=0; icount; ++i) { + oldIndex=sortArray[i].index; + ds->swapArray32(ds, inEnumArray+oldIndex, 4, outEnumArray+i, pErrorCode); + ds->swapArray16(ds, inNameArray+oldIndex, 2, outNameArray+i, pErrorCode); + } + } else { + /* + * in-place swapping: need to permutate into a temporary array + * and then copy back to not destroy the data + */ + EnumValue *tempEnumArray; + Offset *oldIndexes; + + /* write name offsets directly from sortArray */ + for(i=0; icount; ++i) { + ds->writeUInt16((uint16_t *)outNameArray+i, (uint16_t)sortArray[i].name); + } + + /* + * compress the oldIndexes into a separate array to make space for tempEnumArray + * the tempMap _nameArray becomes oldIndexes[], getting the index + * values from the 2D sortArray[], + * while sortArray=tempMap _enumArray[] becomes tempEnumArray[] + * this saves us allocating more memory + * + * it works because sizeof(NameAndIndex)<=sizeof(EnumValue) + * and because the nameArray[] can be used for oldIndexes[] + */ + tempEnumArray=(EnumValue *)sortArray; + oldIndexes=(Offset *)(sortArray+tempMap->count); + + /* copy sortArray[].index values into oldIndexes[] */ + for(i=0; icount; ++i) { + oldIndexes[i]=sortArray[i].index; + } + + /* permutate inEnumArray[] into tempEnumArray[] */ + for(i=0; icount; ++i) { + ds->swapArray32(ds, inEnumArray+oldIndexes[i], 4, tempEnumArray+i, pErrorCode); + } + + /* copy tempEnumArray[] to outEnumArray[] */ + uprv_memcpy(outEnumArray, tempEnumArray, tempMap->count*4); + } + } + + return size; +} + +int32_t +PropertyAliases::swap(const UDataSwapper *ds, + const uint8_t *inBytes, int32_t length, uint8_t *outBytes, + UErrorCode *pErrorCode) { + const PropertyAliases *inAliases; + PropertyAliases *outAliases; + PropertyAliases aliases; + + const ValueMap *inValueMaps; + ValueMap *outValueMaps; + ValueMap valueMap; + + uint8_t *temp; + + int32_t i; + + inAliases=(const PropertyAliases *)inBytes; + outAliases=(PropertyAliases *)outBytes; + + /* read the input PropertyAliases - all 16-bit values */ + for(i=0; i<(int32_t)sizeof(PropertyAliases)/2; ++i) { + ((uint16_t *)&aliases)[i]=ds->readUInt16(((const uint16_t *)inBytes)[i]); + } + + if(length>=0) { + if(lengthswapArray16(ds, inAliases, sizeof(PropertyAliases), outAliases, pErrorCode); + + /* swap the name groups */ + ds->swapArray16(ds, inBytes+aliases.nameGroupPool_offset, + aliases.stringPool_offset-aliases.nameGroupPool_offset, + outBytes+aliases.nameGroupPool_offset, pErrorCode); + + /* swap the strings */ + udata_swapInvStringBlock(ds, inBytes+aliases.stringPool_offset, + aliases.total_size-aliases.stringPool_offset, + outBytes+aliases.stringPool_offset, pErrorCode); + + /* + * alloc uint8_t temp[total_size] and reset it + * swap each top-level struct, put at least the count fields into temp + * use subclass-specific swap() functions + * enumerate value maps, for each + * if temp does not have count!=0 yet + * read count, put it into temp + * swap the array(s) + * resort strings in name->enum maps + * swap value maps + */ + temp=(uint8_t *)uprv_malloc(aliases.total_size); + if(temp==NULL) { + udata_printError(ds, "upname_swap(): unable to allocate temp memory (%d bytes)\n", + aliases.total_size); + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return 0; + } + uprv_memset(temp, 0, aliases.total_size); + + /* swap properties->name groups map */ + NonContiguousEnumToOffset::swap(ds, inBytes, length, outBytes, + temp, aliases.enumToName_offset, pErrorCode); + + /* swap name->properties map */ + NameToEnum::swap(ds, inBytes, length, outBytes, + temp, aliases.nameToEnum_offset, pErrorCode); + + /* swap properties->value maps map */ + NonContiguousEnumToOffset::swap(ds, inBytes, length, outBytes, + temp, aliases.enumToValue_offset, pErrorCode); + + /* enumerate all ValueMaps and swap them */ + inValueMaps=(const ValueMap *)(inBytes+aliases.valueMap_offset); + outValueMaps=(ValueMap *)(outBytes+aliases.valueMap_offset); + + for(i=0; iswapArray16(ds, inValueMaps, aliases.valueMap_count*sizeof(ValueMap), + outValueMaps, pErrorCode); + + /* name groups and strings were swapped above */ + + /* release temp */ + uprv_free(temp); + } + + return aliases.total_size; +} + +U_CAPI int32_t U_EXPORT2 +upname_swap(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode) { + const UDataInfo *pInfo; + int32_t headerSize; + + const uint8_t *inBytes; + uint8_t *outBytes; + + /* udata_swapDataHeader checks the arguments */ + headerSize=udata_swapDataHeader(ds, inData, length, outData, pErrorCode); + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + /* check data format and format version */ + pInfo=(const UDataInfo *)((const char *)inData+4); + if(!( + pInfo->dataFormat[0]==0x70 && /* dataFormat="pnam" */ + pInfo->dataFormat[1]==0x6e && + pInfo->dataFormat[2]==0x61 && + pInfo->dataFormat[3]==0x6d && + pInfo->formatVersion[0]==1 + )) { + udata_printError(ds, "upname_swap(): data format %02x.%02x.%02x.%02x (format version %02x) is not recognized as pnames.icu\n", + pInfo->dataFormat[0], pInfo->dataFormat[1], + pInfo->dataFormat[2], pInfo->dataFormat[3], + pInfo->formatVersion[0]); + *pErrorCode=U_UNSUPPORTED_ERROR; + return 0; + } + + inBytes=(const uint8_t *)inData+headerSize; + outBytes=(uint8_t *)outData+headerSize; + + if(length>=0) { + length-=headerSize; + if(length<(int32_t)sizeof(PropertyAliases)) { + udata_printError(ds, "upname_swap(): too few bytes (%d after header) for pnames.icu\n", + length); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + } + + return headerSize+PropertyAliases::swap(ds, inBytes, length, outBytes, pErrorCode); +} + +//eof diff --git a/Build/source/libs/icu/icu-xetex/common/propname.h b/Build/source/libs/icu/icu-xetex/common/propname.h new file mode 100644 index 00000000000..a3a77181a2b --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/propname.h @@ -0,0 +1,515 @@ +/* +********************************************************************** +* Copyright (c) 2002-2004, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Author: Alan Liu +* Created: October 30 2002 +* Since: ICU 2.4 +********************************************************************** +*/ +#ifndef PROPNAME_H +#define PROPNAME_H + +#include "unicode/utypes.h" +#include "unicode/uchar.h" +#include "udataswp.h" +#include "uprops.h" + +/* + * This header defines the in-memory layout of the property names data + * structure representing the UCD data files PropertyAliases.txt and + * PropertyValueAliases.txt. It is used by: + * propname.cpp - reads data + * genpname - creates data + */ + +/* low-level char * property name comparison -------------------------------- */ + +U_CDECL_BEGIN + +/** + * \var uprv_comparePropertyNames + * Unicode property names and property value names are compared "loosely". + * + * UCD.html 4.0.1 says: + * For all property names, property value names, and for property values for + * Enumerated, Binary, or Catalog properties, use the following + * loose matching rule: + * + * LM3. Ignore case, whitespace, underscore ('_'), and hyphens. + * + * This function does just that, for (char *) name strings. + * It is almost identical to ucnv_compareNames() but also ignores + * C0 White_Space characters (U+0009..U+000d, and U+0085 on EBCDIC). + * + * @internal + */ + +U_CAPI int32_t U_EXPORT2 +uprv_compareASCIIPropertyNames(const char *name1, const char *name2); + +U_CAPI int32_t U_EXPORT2 +uprv_compareEBCDICPropertyNames(const char *name1, const char *name2); + +#if U_CHARSET_FAMILY==U_ASCII_FAMILY +# define uprv_comparePropertyNames uprv_compareASCIIPropertyNames +#elif U_CHARSET_FAMILY==U_EBCDIC_FAMILY +# define uprv_comparePropertyNames uprv_compareEBCDICPropertyNames +#else +# error U_CHARSET_FAMILY is not valid +#endif + +U_CDECL_END + +/* UDataMemory structure and signatures ------------------------------------- */ + +#define PNAME_DATA_NAME "pnames" +#define PNAME_DATA_TYPE "icu" + +/* Fields in UDataInfo: */ + +/* PNAME_SIG[] is encoded as numeric literals for compatibility with the HP compiler */ +#define PNAME_SIG_0 ((uint8_t)0x70) /* p */ +#define PNAME_SIG_1 ((uint8_t)0x6E) /* n */ +#define PNAME_SIG_2 ((uint8_t)0x61) /* a */ +#define PNAME_SIG_3 ((uint8_t)0x6D) /* m */ + +#define PNAME_FORMAT_VERSION ((int8_t)1) /* formatVersion[0] */ + +/** + * Swap pnames.icu. See udataswp.h. + * @internal + */ +U_CAPI int32_t U_EXPORT2 +upname_swap(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode); + + +#ifdef XP_CPLUSPLUS + +class Builder; + +U_NAMESPACE_BEGIN + +/** + * An offset from the start of the pnames data to a contained entity. + * This must be a signed value, since negative offsets are used as an + * end-of-list marker. Offsets to actual objects are non-zero. A + * zero offset indicates an absent entry; this corresponds to aliases + * marked "n/a" in the original Unicode data files. + */ +typedef int16_t Offset; /* must be signed */ + +#define MAX_OFFSET 0x7FFF + +/** + * A generic value for a property or property value. Typically an + * enum from uchar.h, but sometimes a non-enum value. It must be + * large enough to accomodate the largest enum value, which as of this + * writing is the largest general category mask. Need not be signed + * but may be. Typically it doesn't matter, since the caller will + * cast it to the proper type before use. Takes the special value + * UCHAR_INVALID_CODE for invalid input. + */ +typedef int32_t EnumValue; + +/* ---------------------------------------------------------------------- */ +/* ValueMap */ + +/** + * For any top-level property that has named values (binary and + * enumerated properties), there is a ValueMap object. This object + * maps from enum values to two other maps. One goes from value enums + * to value names. The other goes from value names to value enums. + * + * The value enum values may be contiguous or disjoint. If they are + * contiguous then the enumToName_offset is nonzero, and the + * ncEnumToName_offset is zero. Vice versa if the value enums are + * disjoint. + * + * There are n of these objects, where n is the number of binary + * properties + the number of enumerated properties. + */ +struct ValueMap { + + /* -- begin pnames data -- */ + /* Enum=>name EnumToOffset / NonContiguousEnumToOffset objects. */ + /* Exactly one of these will be nonzero. */ + Offset enumToName_offset; + Offset ncEnumToName_offset; + + Offset nameToEnum_offset; /* Name=>enum data */ + /* -- end pnames data -- */ +}; + +/* ---------------------------------------------------------------------- */ +/* PropertyAliases class */ + +/** + * A class encapsulating access to the memory-mapped data representing + * property aliases and property value aliases (pnames). The class + * MUST have no v-table and declares certain methods inline -- small + * methods and methods that are called from only one point. + * + * The data members in this class correspond to the in-memory layout + * of the header of the pnames data. + */ +class PropertyAliases { + + /* -- begin pnames data -- */ + /* Enum=>name EnumToOffset object for binary and enumerated */ + /* properties */ + Offset enumToName_offset; + + /* Name=>enum data for binary & enumerated properties */ + Offset nameToEnum_offset; + + /* Enum=>offset EnumToOffset object mapping enumerated properties */ + /* to ValueMap objects */ + Offset enumToValue_offset; + + /* The following are needed by external readers of this data. */ + /* We don't use them ourselves. */ + int16_t total_size; /* size in bytes excluding the udata header */ + Offset valueMap_offset; /* offset to start of array */ + int16_t valueMap_count; /* number of entries */ + Offset nameGroupPool_offset; /* offset to start of array */ + int16_t nameGroupPool_count; /* number of entries (not groups) */ + Offset stringPool_offset; /* offset to start of pool */ + int16_t stringPool_count; /* number of strings (not size in bytes) */ + + /* -- end pnames data -- */ + + friend class ::Builder; + + const ValueMap* getValueMap(EnumValue prop) const; + + const char* chooseNameInGroup(Offset offset, + UPropertyNameChoice choice) const; + + public: + + inline const int8_t* getPointer(Offset o) const { + return ((const int8_t*) this) + o; + } + + inline const int8_t* getPointerNull(Offset o) const { + return o ? getPointer(o) : NULL; + } + + inline const char* getPropertyName(EnumValue prop, + UPropertyNameChoice choice) const; + + inline EnumValue getPropertyEnum(const char* alias) const; + + inline const char* getPropertyValueName(EnumValue prop, EnumValue value, + UPropertyNameChoice choice) const; + + inline EnumValue getPropertyValueEnum(EnumValue prop, + const char* alias) const; + + static int32_t + swap(const UDataSwapper *ds, + const uint8_t *inBytes, int32_t length, uint8_t *outBytes, + UErrorCode *pErrorCode); +}; + +/* ---------------------------------------------------------------------- */ +/* EnumToOffset */ + +/** + * A generic map from enum values to Offsets. The enum values must be + * contiguous, from enumStart to enumLimit. The Offset values may + * point to anything. + */ +class EnumToOffset { + + /* -- begin pnames data -- */ + EnumValue enumStart; + EnumValue enumLimit; + Offset _offsetArray; /* [array of enumLimit-enumStart] */ + /* -- end pnames data -- */ + + friend class ::Builder; + + Offset* getOffsetArray() { + return &_offsetArray; + } + + const Offset* getOffsetArray() const { + return &_offsetArray; + } + + static int32_t getSize(int32_t n) { + return sizeof(EnumToOffset) + sizeof(Offset) * (n - 1); + } + + int32_t getSize() { + return getSize(enumLimit - enumStart); + } + + public: + + Offset getOffset(EnumValue enumProbe) const { + if (enumProbe < enumStart || + enumProbe >= enumLimit) { + return 0; /* not found */ + } + const Offset* p = getOffsetArray(); + return p[enumProbe - enumStart]; + } + + static int32_t + swap(const UDataSwapper *ds, + const uint8_t *inBytes, int32_t length, uint8_t *outBytes, + uint8_t *temp, int32_t pos, + UErrorCode *pErrorCode); +}; + +/* ---------------------------------------------------------------------- */ +/* NonContiguousEnumToOffset */ + +/** + * A generic map from enum values to Offsets. The enum values may be + * disjoint. If they are contiguous, an EnumToOffset should be used + * instead. The Offset values may point to anything. + */ +class NonContiguousEnumToOffset { + + /* -- begin pnames data -- */ + int32_t count; + EnumValue _enumArray; /* [array of count] */ + /* Offset _offsetArray; // [array of count] after enumValue[count-1] */ + /* -- end pnames data -- */ + + friend class ::Builder; + + EnumValue* getEnumArray() { + return &_enumArray; + } + + const EnumValue* getEnumArray() const { + return &_enumArray; + } + + Offset* getOffsetArray() { + return (Offset*) (getEnumArray() + count); + } + + const Offset* getOffsetArray() const { + return (Offset*) (getEnumArray() + count); + } + + static int32_t getSize(int32_t n) { + return sizeof(int32_t) + (sizeof(EnumValue) + sizeof(Offset)) * n; + } + + int32_t getSize() { + return getSize(count); + } + + public: + + Offset getOffset(EnumValue enumProbe) const { + const EnumValue* e = getEnumArray(); + const Offset* p = getOffsetArray(); + /* linear search; binary later if warranted */ + /* (binary is not faster for short lists) */ + for (int32_t i=0; i enumProbe) break; + return p[i]; + } + return 0; /* not found */ + } + + static int32_t + swap(const UDataSwapper *ds, + const uint8_t *inBytes, int32_t length, uint8_t *outBytes, + uint8_t *temp, int32_t pos, + UErrorCode *pErrorCode); +}; + +/* ---------------------------------------------------------------------- */ +/* NameToEnum */ + +/** + * A map from names to enum values. + */ +class NameToEnum { + + /* -- begin pnames data -- */ + int32_t count; /* number of entries */ + EnumValue _enumArray; /* [array of count] EnumValues */ + /* Offset _nameArray; // [array of count] offsets to names */ + /* -- end pnames data -- */ + + friend class ::Builder; + + EnumValue* getEnumArray() { + return &_enumArray; + } + + const EnumValue* getEnumArray() const { + return &_enumArray; + } + + Offset* getNameArray() { + return (Offset*) (getEnumArray() + count); + } + + const Offset* getNameArray() const { + return (Offset*) (getEnumArray() + count); + } + + static int32_t getSize(int32_t n) { + return sizeof(int32_t) + (sizeof(Offset) + sizeof(EnumValue)) * n; + } + + int32_t getSize() { + return getSize(count); + } + + public: + + EnumValue getEnum(const char* alias, const PropertyAliases& data) const { + + const Offset* n = getNameArray(); + const EnumValue* e = getEnumArray(); + + /* linear search; binary later if warranted */ + /* (binary is not faster for short lists) */ + for (int32_t i=0; i 0) continue; + if (c < 0) break; + return e[i]; + } + + return UCHAR_INVALID_CODE; + } + + static int32_t + swap(const UDataSwapper *ds, + const uint8_t *inBytes, int32_t length, uint8_t *outBytes, + uint8_t *temp, int32_t pos, + UErrorCode *pErrorCode); +}; + +/*---------------------------------------------------------------------- + * + * In-memory layout. THIS IS NOT A STANDALONE DOCUMENT. It goes + * together with above C++ declarations and gives an overview. + * + * See above for definitions of Offset and EnumValue. Also, refer to + * above class declarations for the "bottom line" on data layout. + * + * Sizes: + * '*_offset' is an Offset (see above) + * 'count' members are typically int32_t (see above declarations) + * 'enumArray' is an array of EnumValue (see above) + * 'offsetArray' is an array of Offset (see above) + * 'nameArray' is an array of Offset (see above) + * 'enum*' is an EnumValue (see above) + * '*Array [x n]' means that *Array has n elements + * + * References: + * Instead of pointers, this flat data structure contains offsets. + * All offsets are relative to the start of 'header'. A notation + * is used to indicate what structure each offset points to: + * 'foo (>x)' the offset(s) in foo point to structure x + * + * Structures: + * Each structure is assigned a number, except for the header, + * which is called 'header'. The numbers are not contiguous + * for historical reasons. Some structures have sub-parts + * that are denoted with a letter, e.g., "5a". + * + * BEGIN LAYOUT + * ============ + * header: + * enumToName_offset (>0) + * nameToEnum_offset (>2) + * enumToValue_offset (>3) + * (alignment padding build in to header) + * + * The header also contains the following, used by "external readers" + * like ICU4J and icuswap. + * + * // The following are needed by external readers of this data. + * // We don't use them ourselves. + * int16_t total_size; // size in bytes excluding the udata header + * Offset valueMap_offset; // offset to start of array + * int16_t valueMap_count; // number of entries + * Offset nameGroupPool_offset; // offset to start of array + * int16_t nameGroupPool_count; // number of entries (not groups) + * Offset stringPool_offset; // offset to start of pool + * int16_t stringPool_count; // number of strings (not size in bytes) + * + * 0: # NonContiguousEnumToOffset obj for props => name groups + * count + * enumArray [x count] + * offsetArray [x count] (>98) + * + * => pad to next 4-byte boundary + * + * (1: omitted -- no longer used) + * + * 2: # NameToEnum obj for binary & enumerated props + * count + * enumArray [x count] + * nameArray [x count] (>99) + * + * => pad to next 4-byte boundary + * + * 3: # NonContiguousEnumToOffset obj for enumerated props => ValueMaps + * count + * enumArray [x count] + * offsetArray [x count] (>4) + * + * => pad to next 4-byte boundary + * + * 4: # ValueMap array [x one for each enumerated prop i] + * enumToName_offset (>5a +2*i) one of these two is NULL, one is not + * ncEnumToName_offset (>5b +2*i) + * nameToEnums_offset (>6 +2*i) + * + * => pad to next 4-byte boundary + * + * for each enumerated prop (either 5a or 5b): + * + * 5a: # EnumToOffset for enumerated prop's values => name groups + * enumStart + * enumLimit + * offsetArray [x enumLimit - enumStart] (>98) + * + * => pad to next 4-byte boundary + * + * 5b: # NonContiguousEnumToOffset for enumerated prop's values => name groups + * count + * enumArray [x count] + * offsetArray [x count] (>98) + * + * => pad to next 4-byte boundary + * + * 6: # NameToEnum for enumerated prop's values + * count + * enumArray [x count] + * nameArray [x count] (>99) + * + * => pad to next 4-byte boundary + * + * 98: # name group pool {NGP} + * [array of Offset values] (>99) + * + * 99: # string pool {SP} + * [pool of nul-terminated char* strings] + */ +U_NAMESPACE_END + +#endif /* C++ */ + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/punycode.c b/Build/source/libs/icu/icu-xetex/common/punycode.c new file mode 100644 index 00000000000..5376625e71d --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/punycode.c @@ -0,0 +1,580 @@ +/* +******************************************************************************* +* +* Copyright (C) 2002-2003, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: punycode.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2002jan31 +* created by: Markus W. Scherer +*/ + + +/* This ICU code derived from: */ +/* +punycode.c 0.4.0 (2001-Nov-17-Sat) +http://www.cs.berkeley.edu/~amc/idn/ +Adam M. Costello +http://www.nicemice.net/amc/ + +Disclaimer and license + + Regarding this entire document or any portion of it (including + the pseudocode and C code), the author makes no guarantees and + is not responsible for any damage resulting from its use. The + author grants irrevocable permission to anyone to use, modify, + and distribute it in any way that does not diminish the rights + of anyone else to use, modify, and distribute it, provided that + redistributed derivative works do not contain misleading author or + version information. Derivative works need not be licensed under + similar terms. +*/ +/* + * ICU modifications: + * - ICU data types and coding conventions + * - ICU string buffer handling with implicit source lengths + * and destination preflighting + * - UTF-16 handling + */ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_IDNA + +#include "ustr_imp.h" +#include "cstring.h" +#include "cmemory.h" +#include "punycode.h" +#include "unicode/ustring.h" + + +/* Punycode ----------------------------------------------------------------- */ + +/* Punycode parameters for Bootstring */ +#define BASE 36 +#define TMIN 1 +#define TMAX 26 +#define SKEW 38 +#define DAMP 700 +#define INITIAL_BIAS 72 +#define INITIAL_N 0x80 + +/* "Basic" Unicode/ASCII code points */ +#define _HYPHEN 0X2d +#define DELIMITER _HYPHEN + +#define _ZERO_ 0X30 +#define _NINE 0x39 + +#define _SMALL_A 0X61 +#define _SMALL_Z 0X7a + +#define _CAPITAL_A 0X41 +#define _CAPITAL_Z 0X5a + +#define IS_BASIC(c) ((c)<0x80) +#define IS_BASIC_UPPERCASE(c) (_CAPITAL_A<=(c) && (c)<=_CAPITAL_Z) + +/** + * digitToBasic() returns the basic code point whose value + * (when used for representing integers) is d, which must be in the + * range 0 to BASE-1. The lowercase form is used unless the uppercase flag is + * nonzero, in which case the uppercase form is used. + */ +static U_INLINE char +digitToBasic(int32_t digit, UBool uppercase) { + /* 0..25 map to ASCII a..z or A..Z */ + /* 26..35 map to ASCII 0..9 */ + if(digit<26) { + if(uppercase) { + return (char)(_CAPITAL_A+digit); + } else { + return (char)(_SMALL_A+digit); + } + } else { + return (char)((_ZERO_-26)+digit); + } +} + +/** + * basicToDigit[] contains the numeric value of a basic code + * point (for use in representing integers) in the range 0 to + * BASE-1, or -1 if b is does not represent a value. + */ +static const int8_t +basicToDigit[256]={ + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, -1, -1, -1, -1, -1, -1, + + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, + + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, + + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 +}; + +static U_INLINE char +asciiCaseMap(char b, UBool uppercase) { + if(uppercase) { + if(_SMALL_A<=b && b<=_SMALL_Z) { + b-=(_SMALL_A-_CAPITAL_A); + } + } else { + if(_CAPITAL_A<=b && b<=_CAPITAL_Z) { + b+=(_SMALL_A-_CAPITAL_A); + } + } + return b; +} + +/* Punycode-specific Bootstring code ---------------------------------------- */ + +/* + * The following code omits the {parts} of the pseudo-algorithm in the spec + * that are not used with the Punycode parameter set. + */ + +/* Bias adaptation function. */ +static int32_t +adaptBias(int32_t delta, int32_t length, UBool firstTime) { + int32_t count; + + if(firstTime) { + delta/=DAMP; + } else { + delta/=2; + } + + delta+=delta/length; + for(count=0; delta>((BASE-TMIN)*TMAX)/2; count+=BASE) { + delta/=(BASE-TMIN); + } + + return count+(((BASE-TMIN+1)*delta)/(delta+SKEW)); +} + +#define MAX_CP_COUNT 200 + +U_CFUNC int32_t +u_strToPunycode(const UChar *src, int32_t srcLength, + UChar *dest, int32_t destCapacity, + const UBool *caseFlags, + UErrorCode *pErrorCode) { + + int32_t cpBuffer[MAX_CP_COUNT]; + int32_t n, delta, handledCPCount, basicLength, destLength, bias, j, m, q, k, t, srcCPCount; + UChar c, c2; + + /* argument checking */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + if(src==NULL || srcLength<-1 || (dest==NULL && destCapacity!=0)) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* + * Handle the basic code points and + * convert extended ones to UTF-32 in cpBuffer (caseFlag in sign bit): + */ + srcCPCount=destLength=0; + if(srcLength==-1) { + /* NUL-terminated input */ + for(j=0; /* no condition */; ++j) { + if((c=src[j])==0) { + break; + } + if(srcCPCount==MAX_CP_COUNT) { + /* too many input code points */ + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + if(IS_BASIC(c)) { + cpBuffer[srcCPCount++]=0; + if(destLength0) { + if(destLength state to , but guard against overflow: + */ + if(m-n>(0x7fffffff-MAX_CP_COUNT-delta)/(handledCPCount+1)) { + *pErrorCode=U_INTERNAL_PROGRAM_ERROR; + return 0; + } + delta+=(m-n)*(handledCPCount+1); + n=m; + + /* Encode a sequence of same code points n */ + for(j=0; jTMAX) { + t=TMAX; + } + */ + + t=k-bias; + if(t=(bias+TMAX)) { + t=TMAX; + } + + if(q0;) { + if(src[--j]==DELIMITER) { + break; + } + } + destLength=basicLength=destCPCount=j; + + while(j>0) { + b=src[--j]; + if(!IS_BASIC(b)) { + *pErrorCode=U_INVALID_CHAR_FOUND; + return 0; + } + + if(j0 ? basicLength+1 : 0; in=srcLength) { + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + return 0; + } + + digit=basicToDigit[(uint8_t)src[in++]]; + if(digit<0) { + *pErrorCode=U_INVALID_CHAR_FOUND; + return 0; + } + if(digit>(0x7fffffff-i)/w) { + /* integer overflow */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + return 0; + } + + i+=digit*w; + /** RAM: comment out the old code for conformance with draft-ietf-idn-punycode-03.txt + t=k-bias; + if(tTMAX) { + t=TMAX; + } + */ + t=k-bias; + if(t=(bias+TMAX)) { + t=TMAX; + } + if(digit0x7fffffff/(BASE-t)) { + /* integer overflow */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + return 0; + } + w*=BASE-t; + } + + /* + * Modification from sample code: + * Increments destCPCount here, + * where needed instead of in for() loop tail. + */ + ++destCPCount; + bias=adaptBias(i-oldi, destCPCount, (UBool)(oldi==0)); + + /* + * i was supposed to wrap around from (incremented) destCPCount to 0, + * incrementing n each time, so we'll fix that now: + */ + if(i/destCPCount>(0x7fffffff-n)) { + /* integer overflow */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + return 0; + } + + n+=i/destCPCount; + i%=destCPCount; + /* not needed for Punycode: */ + /* if (decode_digit(n) <= BASE) return punycode_invalid_input; */ + + if(n>0x10ffff || UTF_IS_SURROGATE(n)) { + /* Unicode code point overflow */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + return 0; + } + + /* Insert n at position i of the output: */ + cpLength=UTF_CHAR_LENGTH(n); + if((destLength+cpLength)1) { + firstSupplementaryIndex=codeUnitIndex; + } else { + ++firstSupplementaryIndex; + } + } else { + codeUnitIndex=firstSupplementaryIndex; + UTF_FWD_N(dest, codeUnitIndex, destLength, i-codeUnitIndex); + } + + /* use the UChar index codeUnitIndex instead of the code point index i */ + if(codeUnitIndex=64). + * U_INDEX_OUTOFBOUNDS_ERROR is set if the limit is exceeded. + * @param srcLength Number of UChars in src, or -1 if NUL-terminated. + * @param dest Output Punycode array. + * @param destCapacity Size of dest. + * @param caseFlags Vector of boolean values, one per input UChar, + * indicating that the corresponding character is to be + * marked for the decoder optionally + * uppercasing (TRUE) or lowercasing (FALSE) + * the character. + * ASCII characters are output directly in the case as marked. + * Flags corresponding to trail surrogates are ignored. + * If caseFlags==NULL then input characters are not + * case-mapped. + * @param pErrorCode ICU in/out error code parameter. + * U_INVALID_CHAR_FOUND if src contains + * unmatched single surrogates. + * U_INDEX_OUTOFBOUNDS_ERROR if src contains + * too many code points. + * @return Number of ASCII characters in puny. + * + * @see u_strFromPunycode + */ +U_CFUNC int32_t +u_strToPunycode(const UChar *src, int32_t srcLength, + UChar *dest, int32_t destCapacity, + const UBool *caseFlags, + UErrorCode *pErrorCode); + +/** + * u_strFromPunycode() converts Punycode to Unicode. + * The Unicode string will be at most as long (in UChars) + * than the Punycode string (in chars). + * + * @param src Input Punycode string. + * @param srcLength Length of puny, or -1 if NUL-terminated + * @param dest Output Unicode string buffer. + * @param destCapacity Size of dest in number of UChars, + * and of caseFlags in numbers of UBools. + * @param caseFlags Output array for case flags as + * defined by the Punycode string. + * The caller should uppercase (TRUE) or lowercase (FASLE) + * the corresponding character in dest. + * For supplementary characters, only the lead surrogate + * is marked, and FALSE is stored for the trail surrogate. + * This is redundant and not necessary for ASCII characters + * because they are already in the case indicated. + * Can be NULL if the case flags are not needed. + * @param pErrorCode ICU in/out error code parameter. + * U_INVALID_CHAR_FOUND if a non-ASCII character + * precedes the last delimiter ('-'), + * or if an invalid character (not a-zA-Z0-9) is found + * after the last delimiter. + * U_ILLEGAL_CHAR_FOUND if the delta sequence is ill-formed. + * @return Number of UChars written to dest. + * + * @see u_strToPunycode + */ +U_CFUNC int32_t +u_strFromPunycode(const UChar *src, int32_t srcLength, + UChar *dest, int32_t destCapacity, + UBool *caseFlags, + UErrorCode *pErrorCode); + +#endif /* #if !UCONFIG_NO_IDNA */ + +#endif + +/* + * Hey, Emacs, please set the following: + * + * Local Variables: + * indent-tabs-mode: nil + * End: + * + */ diff --git a/Build/source/libs/icu/icu-xetex/common/putil.c b/Build/source/libs/icu/icu-xetex/common/putil.c new file mode 100644 index 00000000000..825aa575e55 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/putil.c @@ -0,0 +1,1675 @@ +/* +****************************************************************************** +* +* Copyright (C) 1997-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* +* FILE NAME : putil.c (previously putil.cpp and ptypes.cpp) +* +* Date Name Description +* 04/14/97 aliu Creation. +* 04/24/97 aliu Added getDefaultDataDirectory() and +* getDefaultLocaleID(). +* 04/28/97 aliu Rewritten to assume Unix and apply general methods +* for assumed case. Non-UNIX platforms must be +* special-cased. Rewrote numeric methods dealing +* with NaN and Infinity to be platform independent +* over all IEEE 754 platforms. +* 05/13/97 aliu Restored sign of timezone +* (semantics are hours West of GMT) +* 06/16/98 erm Added IEEE_754 stuff, cleaned up isInfinite, isNan, +* nextDouble.. +* 07/22/98 stephen Added remainder, max, min, trunc +* 08/13/98 stephen Added isNegativeInfinity, isPositiveInfinity +* 08/24/98 stephen Added longBitsFromDouble +* 09/08/98 stephen Minor changes for Mac Port +* 03/02/99 stephen Removed openFile(). Added AS400 support. +* Fixed EBCDIC tables +* 04/15/99 stephen Converted to C. +* 06/28/99 stephen Removed mutex locking in u_isBigEndian(). +* 08/04/99 jeffrey R. Added OS/2 changes +* 11/15/99 helena Integrated S/390 IEEE support. +* 04/26/01 Barry N. OS/400 support for uprv_getDefaultLocaleID +* 08/15/01 Steven H. OS/400 support for uprv_getDefaultCodepage +****************************************************************************** +*/ + +/* Define _XOPEN_SOURCE for Solaris and friends. */ +/* NetBSD needs it to be >= 4 */ +#if !defined(_XOPEN_SOURCE) +#if __STDC_VERSION__ >= 199901L +/* It is invalid to compile an XPG3, XPG4, XPG4v2 or XPG5 application using c99 on Solaris */ +#define _XOPEN_SOURCE 600 +#else +#define _XOPEN_SOURCE 4 +#endif +#endif + +/* Make sure things like readlink and such functions work. +Poorly upgraded Solaris machines can't have this defined. +Cleanly installed Solaris can use this #define. +*/ +#if !defined(_XOPEN_SOURCE_EXTENDED) && (!defined(__STDC_VERSION__) || __STDC_VERSION__ >= 199901L) +#define _XOPEN_SOURCE_EXTENDED 1 +#endif + +/* include ICU headers */ +#include "unicode/utypes.h" +#include "unicode/putil.h" +#include "unicode/ustring.h" +#include "putilimp.h" +#include "uassert.h" +#include "umutex.h" +#include "cmemory.h" +#include "cstring.h" +#include "locmap.h" +#include "ucln_cmn.h" + +/* Include standard headers. */ +#include +#include +#include +#include +#include +#include +#include + +/* include system headers */ +#ifdef U_WINDOWS +# define WIN32_LEAN_AND_MEAN +# define VC_EXTRALEAN +# define NOUSER +# define NOSERVICE +# define NOIME +# define NOMCX +# include +# include "wintz.h" +#elif defined(U_CYGWIN) && defined(__STRICT_ANSI__) +/* tzset isn't defined in strict ANSI on Cygwin. */ +# undef __STRICT_ANSI__ +#elif defined(OS400) +# include +# include /* error code structure */ +# include +# include /* EPT_CALL macro - this include must be after all other "QSYSINCs" */ +# include /* For uprv_maximumPtr */ +#elif defined(XP_MAC) +# include +# include +# include +# include +# include +# include +# define ICU_NO_USER_DATA_OVERRIDE 1 +#elif defined(OS390) +#include "unicode/ucnv.h" /* Needed for UCNV_SWAP_LFNL_OPTION_STRING */ +#elif defined(U_DARWIN) || defined(U_LINUX) || defined(U_BSD) +#include +#include +#elif defined(U_QNX) +#include +#endif + +#ifndef U_WINDOWS +#include +#endif + +/* + * Only include langinfo.h if we have a way to get the codeset. If we later + * depend on more feature, we can test on U_HAVE_NL_LANGINFO. + * + */ + +#if U_HAVE_NL_LANGINFO_CODESET +#include +#endif + +/* Define the extension for data files, again... */ +#define DATA_TYPE "dat" + +/* Leave this copyright notice here! */ +static const char copyright[] = U_COPYRIGHT_STRING; + +/* floating point implementations ------------------------------------------- */ + +/* We return QNAN rather than SNAN*/ +#define SIGN 0x80000000U + +/* Make it easy to define certain types of constants */ +typedef union { + int64_t i64; /* This must be defined first in order to allow the initialization to work. This is a C89 feature. */ + double d64; +} BitPatternConversion; +static const BitPatternConversion gNan = { (int64_t) INT64_C(0x7FF8000000000000) }; +static const BitPatternConversion gInf = { (int64_t) INT64_C(0x7FF0000000000000) }; + +/*--------------------------------------------------------------------------- + Platform utilities + Our general strategy is to assume we're on a POSIX platform. Platforms which + are non-POSIX must declare themselves so. The default POSIX implementation + will sometimes work for non-POSIX platforms as well (e.g., the NaN-related + functions). + ---------------------------------------------------------------------------*/ + +#if defined(U_WINDOWS) || defined(XP_MAC) || defined(OS400) +# undef U_POSIX_LOCALE +#else +# define U_POSIX_LOCALE 1 +#endif + +/* + WARNING! u_topNBytesOfDouble and u_bottomNBytesOfDouble + can't be properly optimized by the gcc compiler sometimes (i.e. gcc 3.2). +*/ +#if !IEEE_754 +static char* +u_topNBytesOfDouble(double* d, int n) +{ +#if U_IS_BIG_ENDIAN + return (char*)d; +#else + return (char*)(d + 1) - n; +#endif +} +#endif + +static char* +u_bottomNBytesOfDouble(double* d, int n) +{ +#if U_IS_BIG_ENDIAN + return (char*)(d + 1) - n; +#else + return (char*)d; +#endif +} + +#if defined(U_WINDOWS) +typedef union { + int64_t int64; + FILETIME fileTime; +} FileTimeConversion; /* This is like a ULARGE_INTEGER */ + +/* Number of 100 nanoseconds from 1/1/1601 to 1/1/1970 */ +#define EPOCH_BIAS INT64_C(116444736000000000) +#define HECTONANOSECOND_PER_MILLISECOND 10000 + +#endif + +/*--------------------------------------------------------------------------- + Universal Implementations + These are designed to work on all platforms. Try these, and if they + don't work on your platform, then special case your platform with new + implementations. +---------------------------------------------------------------------------*/ + +/* Return UTC (GMT) time measured in milliseconds since 0:00 on 1/1/70.*/ +U_CAPI UDate U_EXPORT2 +uprv_getUTCtime() +{ +#ifdef XP_MAC + time_t t, t1, t2; + struct tm tmrec; + + uprv_memset( &tmrec, 0, sizeof(tmrec) ); + tmrec.tm_year = 70; + tmrec.tm_mon = 0; + tmrec.tm_mday = 1; + t1 = mktime(&tmrec); /* seconds of 1/1/1970*/ + + time(&t); + uprv_memcpy( &tmrec, gmtime(&t), sizeof(tmrec) ); + t2 = mktime(&tmrec); /* seconds of current GMT*/ + return (UDate)(t2 - t1) * U_MILLIS_PER_SECOND; /* GMT (or UTC) in seconds since 1970*/ +#elif defined(U_WINDOWS) + + FileTimeConversion winTime; + GetSystemTimeAsFileTime(&winTime.fileTime); + return (UDate)((winTime.int64 - EPOCH_BIAS) / HECTONANOSECOND_PER_MILLISECOND); +#else +/* + struct timeval posixTime; + gettimeofday(&posixTime, NULL); + return (UDate)(((int64_t)posixTime.tv_sec * U_MILLIS_PER_SECOND) + (posixTime.tv_usec/1000)); +*/ + time_t epochtime; + time(&epochtime); + return (UDate)epochtime * U_MILLIS_PER_SECOND; +#endif +} + +/*----------------------------------------------------------------------------- + IEEE 754 + These methods detect and return NaN and infinity values for doubles + conforming to IEEE 754. Platforms which support this standard include X86, + Mac 680x0, Mac PowerPC, AIX RS/6000, and most others. + If this doesn't work on your platform, you have non-IEEE floating-point, and + will need to code your own versions. A naive implementation is to return 0.0 + for getNaN and getInfinity, and false for isNaN and isInfinite. + ---------------------------------------------------------------------------*/ + +U_CAPI UBool U_EXPORT2 +uprv_isNaN(double number) +{ +#if IEEE_754 + BitPatternConversion convertedNumber; + convertedNumber.d64 = number; + /* Infinity is 0x7FF0000000000000U. Anything greater than that is a NaN */ + return (UBool)((convertedNumber.i64 & U_INT64_MAX) > gInf.i64); + +#elif defined(OS390) + uint32_t highBits = *(uint32_t*)u_topNBytesOfDouble(&number, + sizeof(uint32_t)); + uint32_t lowBits = *(uint32_t*)u_bottomNBytesOfDouble(&number, + sizeof(uint32_t)); + + return ((highBits & 0x7F080000L) == 0x7F080000L) && + (lowBits == 0x00000000L); + +#else + /* If your platform doesn't support IEEE 754 but *does* have an NaN value,*/ + /* you'll need to replace this default implementation with what's correct*/ + /* for your platform.*/ + return number != number; +#endif +} + +U_CAPI UBool U_EXPORT2 +uprv_isInfinite(double number) +{ +#if IEEE_754 + BitPatternConversion convertedNumber; + convertedNumber.d64 = number; + /* Infinity is exactly 0x7FF0000000000000U. */ + return (UBool)((convertedNumber.i64 & U_INT64_MAX) == gInf.i64); +#elif defined(OS390) + uint32_t highBits = *(uint32_t*)u_topNBytesOfDouble(&number, + sizeof(uint32_t)); + uint32_t lowBits = *(uint32_t*)u_bottomNBytesOfDouble(&number, + sizeof(uint32_t)); + + return ((highBits & ~SIGN) == 0x70FF0000L) && (lowBits == 0x00000000L); + +#else + /* If your platform doesn't support IEEE 754 but *does* have an infinity*/ + /* value, you'll need to replace this default implementation with what's*/ + /* correct for your platform.*/ + return number == (2.0 * number); +#endif +} + +U_CAPI UBool U_EXPORT2 +uprv_isPositiveInfinity(double number) +{ +#if IEEE_754 || defined(OS390) + return (UBool)(number > 0 && uprv_isInfinite(number)); +#else + return uprv_isInfinite(number); +#endif +} + +U_CAPI UBool U_EXPORT2 +uprv_isNegativeInfinity(double number) +{ +#if IEEE_754 || defined(OS390) + return (UBool)(number < 0 && uprv_isInfinite(number)); + +#else + uint32_t highBits = *(uint32_t*)u_topNBytesOfDouble(&number, + sizeof(uint32_t)); + return((highBits & SIGN) && uprv_isInfinite(number)); + +#endif +} + +U_CAPI double U_EXPORT2 +uprv_getNaN() +{ +#if IEEE_754 || defined(OS390) + return gNan.d64; +#else + /* If your platform doesn't support IEEE 754 but *does* have an NaN value,*/ + /* you'll need to replace this default implementation with what's correct*/ + /* for your platform.*/ + return 0.0; +#endif +} + +U_CAPI double U_EXPORT2 +uprv_getInfinity() +{ +#if IEEE_754 || defined(OS390) + return gInf.d64; +#else + /* If your platform doesn't support IEEE 754 but *does* have an infinity*/ + /* value, you'll need to replace this default implementation with what's*/ + /* correct for your platform.*/ + return 0.0; +#endif +} + +U_CAPI double U_EXPORT2 +uprv_floor(double x) +{ + return floor(x); +} + +U_CAPI double U_EXPORT2 +uprv_ceil(double x) +{ + return ceil(x); +} + +U_CAPI double U_EXPORT2 +uprv_round(double x) +{ + return uprv_floor(x + 0.5); +} + +U_CAPI double U_EXPORT2 +uprv_fabs(double x) +{ + return fabs(x); +} + +U_CAPI double U_EXPORT2 +uprv_modf(double x, double* y) +{ + return modf(x, y); +} + +U_CAPI double U_EXPORT2 +uprv_fmod(double x, double y) +{ + return fmod(x, y); +} + +U_CAPI double U_EXPORT2 +uprv_pow(double x, double y) +{ + /* This is declared as "double pow(double x, double y)" */ + return pow(x, y); +} + +U_CAPI double U_EXPORT2 +uprv_pow10(int32_t x) +{ + return pow(10.0, (double)x); +} + +U_CAPI double U_EXPORT2 +uprv_fmax(double x, double y) +{ +#if IEEE_754 + int32_t lowBits; + + /* first handle NaN*/ + if(uprv_isNaN(x) || uprv_isNaN(y)) + return uprv_getNaN(); + + /* check for -0 and 0*/ + lowBits = *(uint32_t*) u_bottomNBytesOfDouble(&x, sizeof(uint32_t)); + if(x == 0.0 && y == 0.0 && (lowBits & SIGN)) + return y; + +#endif + + /* this should work for all flt point w/o NaN and Infpecial cases */ + return (x > y ? x : y); +} + +U_CAPI double U_EXPORT2 +uprv_fmin(double x, double y) +{ +#if IEEE_754 + int32_t lowBits; + + /* first handle NaN*/ + if(uprv_isNaN(x) || uprv_isNaN(y)) + return uprv_getNaN(); + + /* check for -0 and 0*/ + lowBits = *(uint32_t*) u_bottomNBytesOfDouble(&y, sizeof(uint32_t)); + if(x == 0.0 && y == 0.0 && (lowBits & SIGN)) + return y; + +#endif + + /* this should work for all flt point w/o NaN and Inf special cases */ + return (x > y ? y : x); +} + +/** + * Truncates the given double. + * trunc(3.3) = 3.0, trunc (-3.3) = -3.0 + * This is different than calling floor() or ceil(): + * floor(3.3) = 3, floor(-3.3) = -4 + * ceil(3.3) = 4, ceil(-3.3) = -3 + */ +U_CAPI double U_EXPORT2 +uprv_trunc(double d) +{ +#if IEEE_754 + int32_t lowBits; + + /* handle error cases*/ + if(uprv_isNaN(d)) + return uprv_getNaN(); + if(uprv_isInfinite(d)) + return uprv_getInfinity(); + + lowBits = *(uint32_t*) u_bottomNBytesOfDouble(&d, sizeof(uint32_t)); + if( (d == 0.0 && (lowBits & SIGN)) || d < 0) + return ceil(d); + else + return floor(d); + +#else + return d >= 0 ? floor(d) : ceil(d); + +#endif +} + +/** + * Return the largest positive number that can be represented by an integer + * type of arbitrary bit length. + */ +U_CAPI double U_EXPORT2 +uprv_maxMantissa(void) +{ + return pow(2.0, DBL_MANT_DIG + 1.0) - 1.0; +} + +U_CAPI double U_EXPORT2 +uprv_log(double d) +{ + return log(d); +} + +U_CAPI void * U_EXPORT2 +uprv_maximumPtr(void * base) +{ +#if defined(OS400) + /* + * With the provided function we should never be out of range of a given segment + * (a traditional/typical segment that is). Our segments have 5 bytes for the + * id and 3 bytes for the offset. The key is that the casting takes care of + * only retrieving the offset portion minus x1000. Hence, the smallest offset + * seen in a program is x001000 and when casted to an int would be 0. + * That's why we can only add 0xffefff. Otherwise, we would exceed the segment. + * + * Currently, 16MB is the current addressing limitation on i5/OS if the activation is + * non-TERASPACE. If it is TERASPACE it is 2GB - 4k(header information). + * This function determines the activation based on the pointer that is passed in and + * calculates the appropriate maximum available size for + * each pointer type (TERASPACE and non-TERASPACE) + * + * Unlike other operating systems, the pointer model isn't determined at + * compile time on i5/OS. + */ + if ((base != NULL) && (_TESTPTR(base, _C_TERASPACE_CHECK))) { + /* if it is a TERASPACE pointer the max is 2GB - 4k */ + return ((void *)(((char *)base)-((uint32_t)(base))+((uint32_t)0x7fffefff))); + } + /* otherwise 16MB since NULL ptr is not checkable or the ptr is not TERASPACE */ + return ((void *)(((char *)base)-((uint32_t)(base))+((uint32_t)0xffefff))); + +#else + return U_MAX_PTR(base); +#endif +} + +/*--------------------------------------------------------------------------- + Platform-specific Implementations + Try these, and if they don't work on your platform, then special case your + platform with new implementations. + ---------------------------------------------------------------------------*/ + +/* Generic time zone layer -------------------------------------------------- */ + +/* Time zone utilities */ +U_CAPI void U_EXPORT2 +uprv_tzset() +{ +#ifdef U_TZSET + U_TZSET(); +#else + /* no initialization*/ +#endif +} + +U_CAPI int32_t U_EXPORT2 +uprv_timezone() +{ +#ifdef U_TIMEZONE + return U_TIMEZONE; +#else + time_t t, t1, t2; + struct tm tmrec; + UBool dst_checked; + int32_t tdiff = 0; + + time(&t); + uprv_memcpy( &tmrec, localtime(&t), sizeof(tmrec) ); + dst_checked = (tmrec.tm_isdst != 0); /* daylight savings time is checked*/ + t1 = mktime(&tmrec); /* local time in seconds*/ + uprv_memcpy( &tmrec, gmtime(&t), sizeof(tmrec) ); + t2 = mktime(&tmrec); /* GMT (or UTC) in seconds*/ + tdiff = t2 - t1; + /* imitate NT behaviour, which returns same timezone offset to GMT for + winter and summer*/ + if (dst_checked) + tdiff += 3600; + return tdiff; +#endif +} + +/* Note that U_TZNAME does *not* have to be tzname, but if it is, + some platforms need to have it declared here. */ + +#if defined(U_TZNAME) && (defined(U_IRIX) || defined(U_DARWIN) || defined(U_CYGWIN)) +/* RS6000 and others reject char **tzname. */ +extern U_IMPORT char *U_TZNAME[]; +#endif + +#if !UCONFIG_NO_FILE_IO && (defined(U_DARWIN) || defined(U_LINUX) || defined(U_BSD)) +/* These platforms are likely to use Olson timezone IDs. */ +#define CHECK_LOCALTIME_LINK 1 +#if defined(U_DARWIN) +#include +#define TZZONEINFO (TZDIR "/") +#else +#define TZDEFAULT "/etc/localtime" +#define TZZONEINFO "/usr/share/zoneinfo/" +#endif +static char gTimeZoneBuffer[PATH_MAX]; +static char *gTimeZoneBufferPtr = NULL; +#endif + +#ifndef U_WINDOWS +#define isNonDigit(ch) (ch < '0' || '9' < ch) +static UBool isValidOlsonID(const char *id) { + int32_t idx = 0; + + /* Determine if this is something like Iceland (Olson ID) + or AST4ADT (non-Olson ID) */ + while (id[idx] && isNonDigit(id[idx]) && id[idx] != ',') { + idx++; + } + + /* If we went through the whole string, then it might be okay. + The timezone is sometimes set to "CST-7CDT", "CST6CDT5,J129,J131/19:30", + "GRNLNDST3GRNLNDDT" or similar, so we cannot use it. + The rest of the time it could be an Olson ID. George */ + return (UBool)(id[idx] == 0 + || uprv_strcmp(id, "PST8PDT") == 0 + || uprv_strcmp(id, "MST7MDT") == 0 + || uprv_strcmp(id, "CST6CDT") == 0 + || uprv_strcmp(id, "EST5EDT") == 0); +} +#endif + +#if defined(U_TZNAME) && !defined(U_WINDOWS) + +#define CONVERT_HOURS_TO_SECONDS(offset) (int32_t)(offset*3600) +typedef struct OffsetZoneMapping { + int32_t offsetSeconds; + int32_t daylightType; /* 1=daylight in June, 2=daylight in December*/ + const char *stdID; + const char *dstID; + const char *olsonID; +} OffsetZoneMapping; + +/* +This list tries to disambiguate a set of abbreviated timezone IDs and offsets +and maps it to an Olson ID. +Before adding anything to this list, take a look at +icu/source/tools/tzcode/tz.alias +Sometimes no daylight savings (0) is important to define due to aliases. +This list can be tested with icu/source/test/compat/tzone.pl +More values could be added to daylightType to increase precision. +*/ +static const struct OffsetZoneMapping OFFSET_ZONE_MAPPINGS[] = { + {-45900, 2, "CHAST", "CHADT", "Pacific/Chatham"}, + {-43200, 1, "PETT", "PETST", "Asia/Kamchatka"}, + {-43200, 2, "NZST", "NZDT", "Pacific/Auckland"}, + {-43200, 1, "ANAT", "ANAST", "Asia/Anadyr"}, + {-39600, 1, "MAGT", "MAGST", "Asia/Magadan"}, + {-37800, 2, "LHST", "LHST", "Australia/Lord_Howe"}, + {-36000, 2, "EST", "EST", "Australia/Sydney"}, + {-36000, 1, "SAKT", "SAKST", "Asia/Sakhalin"}, + {-36000, 1, "VLAT", "VLAST", "Asia/Vladivostok"}, + {-34200, 2, "CST", "CST", "Australia/South"}, + {-32400, 1, "YAKT", "YAKST", "Asia/Yakutsk"}, + {-32400, 1, "CHOT", "CHOST", "Asia/Choibalsan"}, + {-31500, 2, "CWST", "CWST", "Australia/Eucla"}, + {-28800, 1, "IRKT", "IRKST", "Asia/Irkutsk"}, + {-28800, 1, "ULAT", "ULAST", "Asia/Ulaanbaatar"}, + {-28800, 2, "WST", "WST", "Australia/West"}, + {-25200, 1, "HOVT", "HOVST", "Asia/Hovd"}, + {-25200, 1, "KRAT", "KRAST", "Asia/Krasnoyarsk"}, + {-21600, 1, "NOVT", "NOVST", "Asia/Novosibirsk"}, + {-21600, 1, "OMST", "OMSST", "Asia/Omsk"}, + {-18000, 1, "YEKT", "YEKST", "Asia/Yekaterinburg"}, + {-14400, 1, "SAMT", "SAMST", "Europe/Samara"}, + {-14400, 1, "AMT", "AMST", "Asia/Yerevan"}, + {-14400, 1, "AZT", "AZST", "Asia/Baku"}, + {-10800, 1, "AST", "ADT", "Asia/Baghdad"}, + {-10800, 1, "MSK", "MSD", "Europe/Moscow"}, + {-10800, 1, "VOLT", "VOLST", "Europe/Volgograd"}, + {-7200, 0, "EET", "CEST", "Africa/Tripoli"}, + {-7200, 1, "EET", "EEST", "Europe/Athens"}, /* Conflicts with Africa/Cairo */ + {-7200, 1, "IST", "IDT", "Asia/Jerusalem"}, + {-3600, 0, "CET", "WEST", "Africa/Algiers"}, + {-3600, 2, "WAT", "WAST", "Africa/Windhoek"}, + {0, 1, "GMT", "IST", "Europe/Dublin"}, + {0, 1, "GMT", "BST", "Europe/London"}, + {0, 0, "WET", "WEST", "Africa/Casablanca"}, + {0, 0, "WET", "WET", "Africa/El_Aaiun"}, + {3600, 1, "AZOT", "AZOST", "Atlantic/Azores"}, + {3600, 1, "EGT", "EGST", "America/Scoresbysund"}, + {10800, 1, "PMST", "PMDT", "America/Miquelon"}, + {10800, 2, "UYT", "UYST", "America/Montevideo"}, + {10800, 1, "WGT", "WGST", "America/Godthab"}, + {10800, 2, "BRT", "BRST", "Brazil/East"}, + {12600, 1, "NST", "NDT", "America/St_Johns"}, + {14400, 1, "AST", "ADT", "Canada/Atlantic"}, + {14400, 2, "AMT", "AMST", "America/Cuiaba"}, + {14400, 2, "CLT", "CLST", "Chile/Continental"}, + {14400, 2, "FKT", "FKST", "Atlantic/Stanley"}, + {14400, 2, "PYT", "PYST", "America/Asuncion"}, + {18000, 1, "CST", "CDT", "America/Havana"}, + {18000, 1, "EST", "EDT", "US/Eastern"}, /* Conflicts with America/Grand_Turk */ + {21600, 2, "EAST", "EASST", "Chile/EasterIsland"}, + {21600, 0, "CST", "MDT", "Canada/Saskatchewan"}, + {21600, 0, "CST", "CDT", "America/Guatemala"}, + {21600, 1, "CST", "CDT", "US/Central"}, /* Conflicts with Mexico/General */ + {25200, 1, "MST", "MDT", "US/Mountain"}, /* Conflicts with Mexico/BajaSur */ + {28800, 0, "PST", "PST", "Pacific/Pitcairn"}, + {28800, 1, "PST", "PDT", "US/Pacific"}, /* Conflicts with Mexico/BajaNorte */ + {32400, 1, "AKST", "AKDT", "US/Alaska"}, + {36000, 1, "HAST", "HADT", "US/Aleutian"} +}; + +/*#define DEBUG_TZNAME*/ + +static const char* remapShortTimeZone(const char *stdID, const char *dstID, int32_t daylightType, int32_t offset) +{ + int32_t idx; +#ifdef DEBUG_TZNAME + fprintf(stderr, "TZ=%s std=%s dst=%s daylight=%d offset=%d\n", getenv("TZ"), stdID, dstID, daylightType, offset); +#endif + for (idx = 0; idx < (int32_t)sizeof(OFFSET_ZONE_MAPPINGS)/sizeof(OFFSET_ZONE_MAPPINGS[0]); idx++) + { + if (offset == OFFSET_ZONE_MAPPINGS[idx].offsetSeconds + && daylightType == OFFSET_ZONE_MAPPINGS[idx].daylightType + && strcmp(OFFSET_ZONE_MAPPINGS[idx].stdID, stdID) == 0 + && strcmp(OFFSET_ZONE_MAPPINGS[idx].dstID, dstID) == 0) + { + return OFFSET_ZONE_MAPPINGS[idx].olsonID; + } + } + return NULL; +} +#endif + +U_CAPI const char* U_EXPORT2 +uprv_tzname(int n) +{ + const char *tzid = NULL; +#ifdef U_WINDOWS + tzid = uprv_detectWindowsTimeZone(); + + if (tzid != NULL) { + return tzid; + } +#else + +/*#if defined(U_DARWIN) + int ret; + + tzid = getenv("TZFILE"); + if (tzid != NULL) { + return tzid; + } +#endif*/ + +/* This code can be temporarily disabled to test tzname resolution later on. */ +#ifndef DEBUG_TZNAME + tzid = getenv("TZ"); + if (tzid != NULL && isValidOlsonID(tzid)) + { + /* This might be a good Olson ID. */ + if (uprv_strncmp(tzid, "posix/", 6) == 0 + || uprv_strncmp(tzid, "right/", 6) == 0) + { + /* Remove the posix/ or right/ prefix. */ + tzid += 6; + } + return tzid; + } + /* else U_TZNAME will give a better result. */ +#endif + +#if defined(CHECK_LOCALTIME_LINK) + /* Caller must handle threading issues */ + if (gTimeZoneBufferPtr == NULL) { + /* + This is a trick to look at the name of the link to get the Olson ID + because the tzfile contents is underspecified. + This isn't guaranteed to work because it may not be a symlink. + */ + int32_t ret = (int32_t)readlink(TZDEFAULT, gTimeZoneBuffer, sizeof(gTimeZoneBuffer)); + if (0 < ret) { + int32_t tzZoneInfoLen = uprv_strlen(TZZONEINFO); + gTimeZoneBuffer[ret] = 0; + if (uprv_strncmp(gTimeZoneBuffer, TZZONEINFO, tzZoneInfoLen) == 0 + && isValidOlsonID(gTimeZoneBuffer + tzZoneInfoLen)) + { + return (gTimeZoneBufferPtr = gTimeZoneBuffer + tzZoneInfoLen); + } + } + } + else { + return gTimeZoneBufferPtr; + } +#endif +#endif + +#ifdef U_TZNAME +#if !defined(U_WINDOWS) + /* + U_TZNAME is usually a non-unique abbreviation, which isn't normally usable. + So we remap the abbreviation to an olson ID. + + Since Windows exposes a little more timezone information, + we normally don't use this code on Windows because + uprv_detectWindowsTimeZone should have already given the correct answer. + */ + { + struct tm juneSol, decemberSol; + int daylightType; + static const time_t juneSolstice=1182478260; /*2007-06-21 18:11 UT*/ + static const time_t decemberSolstice=1198332540; /*2007-12-22 06:09 UT*/ + + /* This probing will tell us when daylight savings occurs. */ + localtime_r(&juneSolstice, &juneSol); + localtime_r(&decemberSolstice, &decemberSol); + daylightType = ((decemberSol.tm_isdst > 0) << 1) | (juneSol.tm_isdst > 0); + tzid = remapShortTimeZone(U_TZNAME[0], U_TZNAME[1], daylightType, uprv_timezone()); + if (tzid != NULL) { + return tzid; + } + } +#endif + return U_TZNAME[n]; +#else + return ""; +#endif +} + +/* Get and set the ICU data directory --------------------------------------- */ + +static char *gDataDirectory = NULL; +#if U_POSIX_LOCALE + static char *gCorrectedPOSIXLocale = NULL; /* Heap allocated */ +#endif + +static UBool U_CALLCONV putil_cleanup(void) +{ + if (gDataDirectory && *gDataDirectory) { + uprv_free(gDataDirectory); + } + gDataDirectory = NULL; +#if U_POSIX_LOCALE + if (gCorrectedPOSIXLocale) { + uprv_free(gCorrectedPOSIXLocale); + gCorrectedPOSIXLocale = NULL; + } +#endif + return TRUE; +} + +/* + * Set the data directory. + * Make a copy of the passed string, and set the global data dir to point to it. + * TODO: see bug #2849, regarding thread safety. + */ +U_CAPI void U_EXPORT2 +u_setDataDirectory(const char *directory) { + char *newDataDir; + int32_t length; + + if(directory==NULL || *directory==0) { + /* A small optimization to prevent the malloc and copy when the + shared library is used, and this is a way to make sure that NULL + is never returned. + */ + newDataDir = (char *)""; + } + else { + length=(int32_t)uprv_strlen(directory); + newDataDir = (char *)uprv_malloc(length + 2); + uprv_strcpy(newDataDir, directory); + +#if (U_FILE_SEP_CHAR != U_FILE_ALT_SEP_CHAR) + { + char *p; + while(p = uprv_strchr(newDataDir, U_FILE_ALT_SEP_CHAR)) { + *p = U_FILE_SEP_CHAR; + } + } +#endif + } + + umtx_lock(NULL); + if (gDataDirectory && *gDataDirectory) { + uprv_free(gDataDirectory); + } + gDataDirectory = newDataDir; + ucln_common_registerCleanup(UCLN_COMMON_PUTIL, putil_cleanup); + umtx_unlock(NULL); +} + +U_CAPI UBool U_EXPORT2 +uprv_pathIsAbsolute(const char *path) +{ + if(!path || !*path) { + return FALSE; + } + + if(*path == U_FILE_SEP_CHAR) { + return TRUE; + } + +#if (U_FILE_SEP_CHAR != U_FILE_ALT_SEP_CHAR) + if(*path == U_FILE_ALT_SEP_CHAR) { + return TRUE; + } +#endif + +#if defined(U_WINDOWS) + if( (((path[0] >= 'A') && (path[0] <= 'Z')) || + ((path[0] >= 'a') && (path[0] <= 'z'))) && + path[1] == ':' ) { + return TRUE; + } +#endif + + return FALSE; +} + +U_CAPI const char * U_EXPORT2 +u_getDataDirectory(void) { + const char *path = NULL; + + /* if we have the directory, then return it immediately */ + UMTX_CHECK(NULL, gDataDirectory, path); + + if(path) { + return path; + } + + /* + When ICU_NO_USER_DATA_OVERRIDE is defined, users aren't allowed to + override ICU's data with the ICU_DATA environment variable. This prevents + problems where multiple custom copies of ICU's specific version of data + are installed on a system. Either the application must define the data + directory with u_setDataDirectory, define ICU_DATA_DIR when compiling + ICU, set the data with udata_setCommonData or trust that all of the + required data is contained in ICU's data library that contains + the entry point defined by U_ICUDATA_ENTRY_POINT. + + There may also be some platforms where environment variables + are not allowed. + */ +# if !defined(ICU_NO_USER_DATA_OVERRIDE) && !UCONFIG_NO_FILE_IO + /* First try to get the environment variable */ + path=getenv("ICU_DATA"); +# endif + + /* ICU_DATA_DIR may be set as a compile option */ +# ifdef ICU_DATA_DIR + if(path==NULL || *path==0) { + path=ICU_DATA_DIR; + } +# endif + + if(path==NULL) { + /* It looks really bad, set it to something. */ + path = ""; + } + + u_setDataDirectory(path); + return gDataDirectory; +} + + + + + +/* Macintosh-specific locale information ------------------------------------ */ +#ifdef XP_MAC + +typedef struct { + int32_t script; + int32_t region; + int32_t lang; + int32_t date_region; + const char* posixID; +} mac_lc_rec; + +/* Todo: This will be updated with a newer version from www.unicode.org web + page when it's available.*/ +#define MAC_LC_MAGIC_NUMBER -5 +#define MAC_LC_INIT_NUMBER -9 + +static const mac_lc_rec mac_lc_recs[] = { + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 0, "en_US", + /* United States*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 1, "fr_FR", + /* France*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 2, "en_GB", + /* Great Britain*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 3, "de_DE", + /* Germany*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 4, "it_IT", + /* Italy*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 5, "nl_NL", + /* Metherlands*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 6, "fr_BE", + /* French for Belgium or Lxembourg*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 7, "sv_SE", + /* Sweden*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 9, "da_DK", + /* Denmark*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 10, "pt_PT", + /* Portugal*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 11, "fr_CA", + /* French Canada*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 13, "is_IS", + /* Israel*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 14, "ja_JP", + /* Japan*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 15, "en_AU", + /* Australia*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 16, "ar_AE", + /* the Arabic world (?)*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 17, "fi_FI", + /* Finland*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 18, "fr_CH", + /* French for Switzerland*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 19, "de_CH", + /* German for Switzerland*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 20, "el_GR", + /* Greece*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 21, "is_IS", + /* Iceland ===*/ + /*MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 22, "",*/ + /* Malta ===*/ + /*MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 23, "",*/ + /* Cyprus ===*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 24, "tr_TR", + /* Turkey ===*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 25, "sh_YU", + /* Croatian system for Yugoslavia*/ + /*MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 33, "",*/ + /* Hindi system for India*/ + /*MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 34, "",*/ + /* Pakistan*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 41, "lt_LT", + /* Lithuania*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 42, "pl_PL", + /* Poland*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 43, "hu_HU", + /* Hungary*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 44, "et_EE", + /* Estonia*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 45, "lv_LV", + /* Latvia*/ + /*MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 46, "",*/ + /* Lapland [Ask Rich for the data. HS]*/ + /*MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 47, "",*/ + /* Faeroe Islands*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 48, "fa_IR", + /* Iran*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 49, "ru_RU", + /* Russia*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 50, "en_IE", + /* Ireland*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 51, "ko_KR", + /* Korea*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 52, "zh_CN", + /* People's Republic of China*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 53, "zh_TW", + /* Taiwan*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, 54, "th_TH", + /* Thailand*/ + + /* fallback is en_US*/ + MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, MAC_LC_MAGIC_NUMBER, + MAC_LC_MAGIC_NUMBER, "en_US" +}; + +#endif + +#if U_POSIX_LOCALE +/* Return just the POSIX id, whatever happens to be in it */ +static const char *uprv_getPOSIXID(void) +{ + static const char* posixID = NULL; + if (posixID == 0) { + /* + * On Solaris two different calls to setlocale can result in + * different values. Only get this value once. + * + * We must check this first because an application can set this. + * + * LC_ALL can't be used because it's platform dependent. The LANG + * environment variable seems to affect LC_CTYPE variable by default. + * Here is what setlocale(LC_ALL, NULL) can return. + * HPUX can return 'C C C C C C C' + * Solaris can return /en_US/C/C/C/C/C on the second try. + * Linux can return LC_CTYPE=C;LC_NUMERIC=C;... + * + * The default codepage detection also needs to use LC_CTYPE. + * + * Do not call setlocale(LC_*, "")! Using an empty string instead + * of NULL, will modify the libc behavior. + */ + posixID = setlocale(LC_CTYPE, NULL); + if ((posixID == 0) + || (uprv_strcmp("C", posixID) == 0) + || (uprv_strcmp("POSIX", posixID) == 0)) + { + /* Maybe we got some garbage. Try something more reasonable */ + posixID = getenv("LC_ALL"); + if (posixID == 0) { + posixID = getenv("LC_CTYPE"); + if (posixID == 0) { + posixID = getenv("LANG"); + } + } + } + + if ((posixID==0) + || (uprv_strcmp("C", posixID) == 0) + || (uprv_strcmp("POSIX", posixID) == 0)) + { + /* Nothing worked. Give it a nice POSIX default value. */ + posixID = "en_US_POSIX"; + } + } + + return posixID; +} +#endif + +/* NOTE: The caller should handle thread safety */ +U_CAPI const char* U_EXPORT2 +uprv_getDefaultLocaleID() +{ +#if U_POSIX_LOCALE +/* + Note that: (a '!' means the ID is improper somehow) + LC_ALL ----> default_loc codepage +-------------------------------------------------------- + ab.CD ab CD + ab@CD ab__CD - + ab@CD.EF ab__CD EF + + ab_CD.EF@GH ab_CD_GH EF + +Some 'improper' ways to do the same as above: + ! ab_CD@GH.EF ab_CD_GH EF + ! ab_CD.EF@GH.IJ ab_CD_GH EF + ! ab_CD@ZZ.EF@GH.IJ ab_CD_GH EF + + _CD@GH _CD_GH - + _CD.EF@GH _CD_GH EF + +The variant cannot have dots in it. +The 'rightmost' variant (@xxx) wins. +The leftmost codepage (.xxx) wins. +*/ + char *correctedPOSIXLocale = 0; + const char* posixID = uprv_getPOSIXID(); + const char *p; + const char *q; + int32_t len; + + /* Format: (no spaces) + ll [ _CC ] [ . MM ] [ @ VV] + + l = lang, C = ctry, M = charmap, V = variant + */ + + if (gCorrectedPOSIXLocale != NULL) { + return gCorrectedPOSIXLocale; + } + + if ((p = uprv_strchr(posixID, '.')) != NULL) { + /* assume new locale can't be larger than old one? */ + correctedPOSIXLocale = uprv_malloc(uprv_strlen(posixID)+1); + uprv_strncpy(correctedPOSIXLocale, posixID, p-posixID); + correctedPOSIXLocale[p-posixID] = 0; + + /* do not copy after the @ */ + if ((p = uprv_strchr(correctedPOSIXLocale, '@')) != NULL) { + correctedPOSIXLocale[p-correctedPOSIXLocale] = 0; + } + } + + /* Note that we scan the *uncorrected* ID. */ + if ((p = uprv_strrchr(posixID, '@')) != NULL) { + if (correctedPOSIXLocale == NULL) { + correctedPOSIXLocale = uprv_malloc(uprv_strlen(posixID)+1); + uprv_strncpy(correctedPOSIXLocale, posixID, p-posixID); + correctedPOSIXLocale[p-posixID] = 0; + } + p++; + + /* Take care of any special cases here.. */ + if (!uprv_strcmp(p, "nynorsk")) { + p = "NY"; + /* Don't worry about no__NY. In practice, it won't appear. */ + } + + if (uprv_strchr(correctedPOSIXLocale,'_') == NULL) { + uprv_strcat(correctedPOSIXLocale, "__"); /* aa@b -> aa__b */ + } + else { + uprv_strcat(correctedPOSIXLocale, "_"); /* aa_CC@b -> aa_CC_b */ + } + + if ((q = uprv_strchr(p, '.')) != NULL) { + /* How big will the resulting string be? */ + len = (int32_t)(uprv_strlen(correctedPOSIXLocale) + (q-p)); + uprv_strncat(correctedPOSIXLocale, p, q-p); + correctedPOSIXLocale[len] = 0; + } + else { + /* Anything following the @ sign */ + uprv_strcat(correctedPOSIXLocale, p); + } + + /* Should there be a map from 'no@nynorsk' -> no_NO_NY here? + * How about 'russian' -> 'ru'? + * Many of the other locales using ISO codes will be handled by the + * canonicalization functions in uloc_getDefault. + */ + } + + /* Was a correction made? */ + if (correctedPOSIXLocale != NULL) { + posixID = correctedPOSIXLocale; + } + else { + /* copy it, just in case the original pointer goes away. See j2395 */ + correctedPOSIXLocale = (char *)uprv_malloc(uprv_strlen(posixID) + 1); + posixID = uprv_strcpy(correctedPOSIXLocale, posixID); + } + + if (gCorrectedPOSIXLocale == NULL) { + gCorrectedPOSIXLocale = correctedPOSIXLocale; + ucln_common_registerCleanup(UCLN_COMMON_PUTIL, putil_cleanup); + correctedPOSIXLocale = NULL; + } + + if (correctedPOSIXLocale != NULL) { /* Was already set - clean up. */ + uprv_free(correctedPOSIXLocale); + } + + return posixID; + +#elif defined(U_WINDOWS) + UErrorCode status = U_ZERO_ERROR; + LCID id = GetThreadLocale(); + const char* locID = uprv_convertToPosix(id, &status); + + if (U_FAILURE(status)) { + locID = "en_US"; + } + return locID; + +#elif defined(XP_MAC) + int32_t script = MAC_LC_INIT_NUMBER; + /* = IntlScript(); or GetScriptManagerVariable(smSysScript);*/ + int32_t region = MAC_LC_INIT_NUMBER; + /* = GetScriptManagerVariable(smRegionCode);*/ + int32_t lang = MAC_LC_INIT_NUMBER; + /* = GetScriptManagerVariable(smScriptLang);*/ + int32_t date_region = MAC_LC_INIT_NUMBER; + const char* posixID = 0; + int32_t count = sizeof(mac_lc_recs) / sizeof(mac_lc_rec); + int32_t i; + Intl1Hndl ih; + + ih = (Intl1Hndl) GetIntlResource(1); + if (ih) + date_region = ((uint16_t)(*ih)->intl1Vers) >> 8; + + for (i = 0; i < count; i++) { + if ( ((mac_lc_recs[i].script == MAC_LC_MAGIC_NUMBER) + || (mac_lc_recs[i].script == script)) + && ((mac_lc_recs[i].region == MAC_LC_MAGIC_NUMBER) + || (mac_lc_recs[i].region == region)) + && ((mac_lc_recs[i].lang == MAC_LC_MAGIC_NUMBER) + || (mac_lc_recs[i].lang == lang)) + && ((mac_lc_recs[i].date_region == MAC_LC_MAGIC_NUMBER) + || (mac_lc_recs[i].date_region == date_region)) + ) + { + posixID = mac_lc_recs[i].posixID; + break; + } + } + + return posixID; + +#elif defined(OS400) + /* locales are process scoped and are by definition thread safe */ + static char correctedLocale[64]; + const char *localeID = getenv("LC_ALL"); + char *p; + + if (localeID == NULL) + localeID = getenv("LANG"); + if (localeID == NULL) + localeID = setlocale(LC_ALL, NULL); + /* Make sure we have something... */ + if (localeID == NULL) + return "en_US_POSIX"; + + /* Extract the locale name from the path. */ + if((p = uprv_strrchr(localeID, '/')) != NULL) + { + /* Increment p to start of locale name. */ + p++; + localeID = p; + } + + /* Copy to work location. */ + uprv_strcpy(correctedLocale, localeID); + + /* Strip off the '.locale' extension. */ + if((p = uprv_strchr(correctedLocale, '.')) != NULL) { + *p = 0; + } + + /* Upper case the locale name. */ + T_CString_toUpperCase(correctedLocale); + + /* See if we are using the POSIX locale. Any of the + * following are equivalent and use the same QLGPGCMA + * (POSIX) locale. + * QLGPGCMA2 means UCS2 + * QLGPGCMA_4 means UTF-32 + * QLGPGCMA_8 means UTF-8 + */ + if ((uprv_strcmp("C", correctedLocale) == 0) || + (uprv_strcmp("POSIX", correctedLocale) == 0) || + (uprv_strncmp("QLGPGCMA", correctedLocale, 8) == 0)) + { + uprv_strcpy(correctedLocale, "en_US_POSIX"); + } + else + { + int16_t LocaleLen; + + /* Lower case the lang portion. */ + for(p = correctedLocale; *p != 0 && *p != '_'; p++) + { + *p = uprv_tolower(*p); + } + + /* Adjust for Euro. After '_E' add 'URO'. */ + LocaleLen = uprv_strlen(correctedLocale); + if (correctedLocale[LocaleLen - 2] == '_' && + correctedLocale[LocaleLen - 1] == 'E') + { + uprv_strcat(correctedLocale, "URO"); + } + + /* If using Lotus-based locale then convert to + * equivalent non Lotus. + */ + else if (correctedLocale[LocaleLen - 2] == '_' && + correctedLocale[LocaleLen - 1] == 'L') + { + correctedLocale[LocaleLen - 2] = 0; + } + + /* There are separate simplified and traditional + * locales called zh_HK_S and zh_HK_T. + */ + else if (uprv_strncmp(correctedLocale, "zh_HK", 5) == 0) + { + uprv_strcpy(correctedLocale, "zh_HK"); + } + + /* A special zh_CN_GBK locale... + */ + else if (uprv_strcmp(correctedLocale, "zh_CN_GBK") == 0) + { + uprv_strcpy(correctedLocale, "zh_CN"); + } + + } + + return correctedLocale; +#endif + +} + +#if U_POSIX_LOCALE +/* +Due to various platform differences, one platform may specify a charset, +when they really mean a different charset. Remap the names so that they are +compatible with ICU. Only conflicting/ambiguous aliases should be resolved +here. Before adding anything to this function, please consider adding unique +names to the ICU alias table in the data directory. +*/ +static const char* +remapPlatformDependentCodepage(const char *locale, const char *name) { + if (locale != NULL && *locale == 0) { + /* Make sure that an empty locale is handled the same way. */ + locale = NULL; + } + if (name == NULL) { + return NULL; + } +#if defined(U_AIX) + if (uprv_strcmp(name, "IBM-943") == 0) { + /* Use the ASCII compatible ibm-943 */ + name = "Shift-JIS"; + } + else if (uprv_strcmp(name, "IBM-1252") == 0) { + /* Use the windows-1252 that contains the Euro */ + name = "IBM-5348"; + } +#elif defined(U_SOLARIS) + if (locale != NULL && uprv_strcmp(name, "EUC") == 0) { + /* Solaris underspecifies the "EUC" name. */ + if (uprv_strcmp(locale, "zh_CN") == 0) { + name = "EUC-CN"; + } + else if (uprv_strcmp(locale, "zh_TW") == 0) { + name = "EUC-TW"; + } + else if (uprv_strcmp(locale, "ko_KR") == 0) { + name = "EUC-KR"; + } + } + else if (uprv_strcmp(name, "eucJP") == 0) { + /* + ibm-954 is the best match. + ibm-33722 is the default for eucJP (similar to Windows). + */ + name = "eucjis"; + } +#elif defined(U_DARWIN) + if (locale == NULL && *name == 0) { + /* + No locale was specified, and an empty name was passed in. + This usually indicates that nl_langinfo didn't return valid information. + Mac OS X uses UTF-8 by default (especially the locale data and console). + */ + name = "UTF-8"; + } +#elif defined(U_HPUX) + if (uprv_strcmp(name, "eucJP") == 0) { + /* + ibm-1350 is the best match, but unavailable. + ibm-954 is mostly a superset of ibm-1350. + ibm-33722 is the default for eucJP (similar to Windows). + */ + name = "eucjis"; + } +#elif defined(U_LINUX) + if (locale != NULL && uprv_strcmp(name, "euc") == 0) { + /* Linux underspecifies the "EUC" name. */ + if (uprv_strcmp(locale, "korean") == 0) { + name = "EUC-KR"; + } + else if (uprv_strcmp(locale, "japanese") == 0) { + /* See comment below about eucJP */ + name = "eucjis"; + } + } + else if (uprv_strcmp(name, "eucjp") == 0) { + /* + ibm-1350 is the best match, but unavailable. + ibm-954 is mostly a superset of ibm-1350. + ibm-33722 is the default for eucJP (similar to Windows). + */ + name = "eucjis"; + } +#endif + /* return NULL when "" is passed in */ + if (*name == 0) { + name = NULL; + } + return name; +} + +static const char* +getCodepageFromPOSIXID(const char *localeName, char * buffer, int32_t buffCapacity) +{ + char localeBuf[100]; + const char *name = NULL; + char *variant = NULL; + + if (localeName != NULL && (name = (uprv_strchr(localeName, '.'))) != NULL) { + size_t localeCapacity = uprv_min(sizeof(localeBuf), (name-localeName)+1); + uprv_strncpy(localeBuf, localeName, localeCapacity); + localeBuf[localeCapacity-1] = 0; /* ensure NULL termination */ + name = uprv_strncpy(buffer, name+1, buffCapacity); + buffer[buffCapacity-1] = 0; /* ensure NULL termination */ + if ((variant = (uprv_strchr(name, '@'))) != NULL) { + *variant = 0; + } + name = remapPlatformDependentCodepage(localeBuf, name); + } + return name; +} +#endif + +static const char* +int_getDefaultCodepage() +{ +#if defined(OS400) + uint32_t ccsid = 37; /* Default to ibm-37 */ + static char codepage[64]; + Qwc_JOBI0400_t jobinfo; + Qus_EC_t error = { sizeof(Qus_EC_t) }; /* SPI error code */ + + EPT_CALL(QUSRJOBI)(&jobinfo, sizeof(jobinfo), "JOBI0400", + "* ", " ", &error); + + if (error.Bytes_Available == 0) { + if (jobinfo.Coded_Char_Set_ID != 0xFFFF) { + ccsid = (uint32_t)jobinfo.Coded_Char_Set_ID; + } + else if (jobinfo.Default_Coded_Char_Set_Id != 0xFFFF) { + ccsid = (uint32_t)jobinfo.Default_Coded_Char_Set_Id; + } + /* else use the default */ + } + sprintf(codepage,"ibm-%d", ccsid); + return codepage; + +#elif defined(OS390) + static char codepage[64]; + sprintf(codepage,"%63s" UCNV_SWAP_LFNL_OPTION_STRING, nl_langinfo(CODESET)); + codepage[63] = 0; /* NULL terminate */ + return codepage; + +#elif defined(XP_MAC) + return "macintosh"; /* TODO: Macintosh Roman. There must be a better way. fixme! */ + +#elif defined(U_WINDOWS) + static char codepage[64]; + sprintf(codepage, "windows-%d", GetACP()); + return codepage; + +#elif U_POSIX_LOCALE + static char codesetName[100]; + const char *localeName = NULL; + const char *name = NULL; + + uprv_memset(codesetName, 0, sizeof(codesetName)); + + /* Use setlocale in a nice way, and then check some environment variables. + Maybe the application used setlocale already. + */ + localeName = uprv_getPOSIXID(); + name = getCodepageFromPOSIXID(localeName, codesetName, sizeof(codesetName)); + if (name) { + /* if we can find the codeset name from setlocale, return that. */ + return name; + } + /* else "C" was probably returned. That's underspecified. */ + +#if U_HAVE_NL_LANGINFO_CODESET + if (*codesetName) { + uprv_memset(codesetName, 0, sizeof(codesetName)); + } + /* When available, check nl_langinfo because it usually gives more + useful names. It depends on LC_CTYPE and not LANG or LC_ALL. + nl_langinfo may use the same buffer as setlocale. */ + { + const char *codeset = nl_langinfo(U_NL_LANGINFO_CODESET); + codeset = remapPlatformDependentCodepage(NULL, codeset); + if (codeset != NULL) { + uprv_strncpy(codesetName, codeset, sizeof(codesetName)); + codesetName[sizeof(codesetName)-1] = 0; + return codesetName; + } + } +#endif + + if (*codesetName == 0) + { + /* Everything failed. Return US ASCII (ISO 646). */ + (void)uprv_strcpy(codesetName, "US-ASCII"); + } + return codesetName; +#else + return "US-ASCII"; +#endif +} + + +U_CAPI const char* U_EXPORT2 +uprv_getDefaultCodepage() +{ + static char const *name = NULL; + umtx_lock(NULL); + if (name == NULL) { + name = int_getDefaultCodepage(); + } + umtx_unlock(NULL); + return name; +} + + +/* end of platform-specific implementation -------------- */ + +/* version handling --------------------------------------------------------- */ + +U_CAPI void U_EXPORT2 +u_versionFromString(UVersionInfo versionArray, const char *versionString) { + char *end; + uint16_t part=0; + + if(versionArray==NULL) { + return; + } + + if(versionString!=NULL) { + for(;;) { + versionArray[part]=(uint8_t)uprv_strtoul(versionString, &end, 10); + if(end==versionString || ++part==U_MAX_VERSION_LENGTH || *end!=U_VERSION_DELIMITER) { + break; + } + versionString=end+1; + } + } + + while(part0 && versionArray[count-1]==0; --count) { + } + + if(count <= 1) { + count = 2; + } + + /* write the first part */ + /* write the decimal field value */ + field=versionArray[0]; + if(field>=100) { + *versionString++=(char)('0'+field/100); + field%=100; + } + if(field>=10) { + *versionString++=(char)('0'+field/10); + field%=10; + } + *versionString++=(char)('0'+field); + + /* write the following parts */ + for(part=1; part=100) { + *versionString++=(char)('0'+field/100); + field%=100; + } + if(field>=10) { + *versionString++=(char)('0'+field/10); + field%=10; + } + *versionString++=(char)('0'+field); + } + + /* NUL-terminate */ + *versionString=0; +} + +U_CAPI void U_EXPORT2 +u_getVersion(UVersionInfo versionArray) { + u_versionFromString(versionArray, U_ICU_VERSION); +} + +/* + * Hey, Emacs, please set the following: + * + * Local Variables: + * indent-tabs-mode: nil + * End: + * + */ diff --git a/Build/source/libs/icu/icu-xetex/common/putilimp.h b/Build/source/libs/icu/icu-xetex/common/putilimp.h new file mode 100644 index 00000000000..c516821e84b --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/putilimp.h @@ -0,0 +1,277 @@ +/* +****************************************************************************** +* +* Copyright (C) 1997-2006, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* +* FILE NAME : putilimp.h +* +* Date Name Description +* 10/17/04 grhoten Move internal functions from putil.h to this file. +****************************************************************************** +*/ + +#ifndef PUTILIMP_H +#define PUTILIMP_H + +#include "unicode/utypes.h" +#include "unicode/putil.h" + +/*==========================================================================*/ +/* Platform utilities */ +/*==========================================================================*/ + +/** + * Platform utilities isolates the platform dependencies of the + * libarary. For each platform which this code is ported to, these + * functions may have to be re-implemented. + */ + +/** + * Floating point utility to determine if a double is Not a Number (NaN). + * @internal + */ +U_INTERNAL UBool U_EXPORT2 uprv_isNaN(double d); +/** + * Floating point utility to determine if a double has an infinite value. + * @internal + */ +U_INTERNAL UBool U_EXPORT2 uprv_isInfinite(double d); +/** + * Floating point utility to determine if a double has a positive infinite value. + * @internal + */ +U_INTERNAL UBool U_EXPORT2 uprv_isPositiveInfinity(double d); +/** + * Floating point utility to determine if a double has a negative infinite value. + * @internal + */ +U_INTERNAL UBool U_EXPORT2 uprv_isNegativeInfinity(double d); +/** + * Floating point utility that returns a Not a Number (NaN) value. + * @internal + */ +U_INTERNAL double U_EXPORT2 uprv_getNaN(void); +/** + * Floating point utility that returns an infinite value. + * @internal + */ +U_INTERNAL double U_EXPORT2 uprv_getInfinity(void); + +/** + * Floating point utility to truncate a double. + * @internal + */ +U_INTERNAL double U_EXPORT2 uprv_trunc(double d); +/** + * Floating point utility to calculate the floor of a double. + * @internal + */ +U_INTERNAL double U_EXPORT2 uprv_floor(double d); +/** + * Floating point utility to calculate the ceiling of a double. + * @internal + */ +U_INTERNAL double U_EXPORT2 uprv_ceil(double d); +/** + * Floating point utility to calculate the absolute value of a double. + * @internal + */ +U_INTERNAL double U_EXPORT2 uprv_fabs(double d); +/** + * Floating point utility to calculate the fractional and integer parts of a double. + * @internal + */ +U_INTERNAL double U_EXPORT2 uprv_modf(double d, double* pinteger); +/** + * Floating point utility to calculate the remainder of a double divided by another double. + * @internal + */ +U_INTERNAL double U_EXPORT2 uprv_fmod(double d, double y); +/** + * Floating point utility to calculate d to the power of exponent (d^exponent). + * @internal + */ +U_INTERNAL double U_EXPORT2 uprv_pow(double d, double exponent); +/** + * Floating point utility to calculate 10 to the power of exponent (10^exponent). + * @internal + */ +U_INTERNAL double U_EXPORT2 uprv_pow10(int32_t exponent); +/** + * Floating point utility to calculate the maximum value of two doubles. + * @internal + */ +U_INTERNAL double U_EXPORT2 uprv_fmax(double d, double y); +/** + * Floating point utility to calculate the minimum value of two doubles. + * @internal + */ +U_INTERNAL double U_EXPORT2 uprv_fmin(double d, double y); +/** + * Private utility to calculate the maximum value of two integers. + * @internal + */ +U_INTERNAL int32_t U_EXPORT2 uprv_max(int32_t d, int32_t y); +/** + * Private utility to calculate the minimum value of two integers. + * @internal + */ +U_INTERNAL int32_t U_EXPORT2 uprv_min(int32_t d, int32_t y); + +#if U_IS_BIG_ENDIAN +# define uprv_isNegative(number) (*((signed char *)&(number))<0) +#else +# define uprv_isNegative(number) (*((signed char *)&(number)+sizeof(number)-1)<0) +#endif + +/** + * Return the largest positive number that can be represented by an integer + * type of arbitrary bit length. + * @internal + */ +U_INTERNAL double U_EXPORT2 uprv_maxMantissa(void); + +/** + * Floating point utility to calculate the logarithm of a double. + * @internal + */ +U_INTERNAL double U_EXPORT2 uprv_log(double d); + +/** + * Does common notion of rounding e.g. uprv_floor(x + 0.5); + * @param x the double number + * @return the rounded double + * @internal + */ +U_INTERNAL double U_EXPORT2 uprv_round(double x); + +#if 0 +/** + * Returns the number of digits after the decimal point in a double number x. + * + * @param x the double number + * @return the number of digits after the decimal point in a double number x. + * @internal + */ +/*U_INTERNAL int32_t U_EXPORT2 uprv_digitsAfterDecimal(double x);*/ +#endif + +/** + * Time zone utilities + * + * Wrappers for C runtime library functions relating to timezones. + * The t_tzset() function (similar to tzset) uses the current setting + * of the environment variable TZ to assign values to three global + * variables: daylight, timezone, and tzname. These variables have the + * following meanings, and are declared in <time.h>. + * + * daylight Nonzero if daylight-saving-time zone (DST) is specified + * in TZ; otherwise, 0. Default value is 1. + * timezone Difference in seconds between coordinated universal + * time and local time. E.g., -28,800 for PST (GMT-8hrs) + * tzname(0) Three-letter time-zone name derived from TZ environment + * variable. E.g., "PST". + * tzname(1) Three-letter DST zone name derived from TZ environment + * variable. E.g., "PDT". If DST zone is omitted from TZ, + * tzname(1) is an empty string. + * + * Notes: For example, to set the TZ environment variable to correspond + * to the current time zone in Germany, you can use one of the + * following statements: + * + * set TZ=GST1GDT + * set TZ=GST+1GDT + * + * If the TZ value is not set, t_tzset() attempts to use the time zone + * information specified by the operating system. Under Windows NT + * and Windows 95, this information is specified in the Control Panel's + * Date/Time application. + * @internal + */ +U_INTERNAL void U_EXPORT2 uprv_tzset(void); + +/** + * Difference in seconds between coordinated universal + * time and local time. E.g., -28,800 for PST (GMT-8hrs) + * @return the difference in seconds between coordinated universal time and local time. + * @internal + */ +U_INTERNAL int32_t U_EXPORT2 uprv_timezone(void); + +/** + * tzname(0) Three-letter time-zone name derived from TZ environment + * variable. E.g., "PST". + * tzname(1) Three-letter DST zone name derived from TZ environment + * variable. E.g., "PDT". If DST zone is omitted from TZ, + * tzname(1) is an empty string. + * @internal + */ +U_INTERNAL const char* U_EXPORT2 uprv_tzname(int n); + +/** + * Get UTC (GMT) time measured in milliseconds since 0:00 on 1/1/1970. + * @return the UTC time measured in milliseconds + * @internal + */ +U_INTERNAL UDate U_EXPORT2 uprv_getUTCtime(void); + +/** + * Determine whether a pathname is absolute or not, as defined by the platform. + * @param path Pathname to test + * @return TRUE if the path is absolute + * @internal (ICU 3.0) + */ +U_INTERNAL UBool U_EXPORT2 uprv_pathIsAbsolute(const char *path); + +/** + * Use U_MAX_PTR instead of this function. + * @param void pointer to test + * @return the largest possible pointer greater than the base + * @internal (ICU 3.8) + */ +U_INTERNAL void * U_EXPORT2 uprv_maximumPtr(void *base); + +/** + * Maximum value of a (void*) - use to indicate the limit of an 'infinite' buffer. + * In fact, buffer sizes must not exceed 2GB so that the difference between + * the buffer limit and the buffer start can be expressed in an int32_t. + * + * The definition of U_MAX_PTR must fulfill the following conditions: + * - return the largest possible pointer greater than base + * - return a valid pointer according to the machine architecture (AS/400, 64-bit, etc.) + * - avoid wrapping around at high addresses + * - make sure that the returned pointer is not farther from base than 0x7fffffff + * + * @param base The beginning of a buffer to find the maximum offset from + * @internal + */ +#ifndef U_MAX_PTR +# if defined(OS390) && !defined(_LP64) + /* We have 31-bit pointers. */ +# define U_MAX_PTR(base) ((void *)0x7fffffff) +# elif defined(OS400) +# define U_MAX_PTR(base) uprv_maximumPtr((void *)base) +# elif defined(__GNUC__) && __GNUC__ >= 4 +/* + * Due to a compiler optimization bug, gcc 4 causes test failures when doing + * this math arithmetic on pointers on some platforms. It seems like the + * pointers are considered signed instead of unsigned. The uintptr_t type + * isn't available on all platforms (i.e MSVC 6) and pointers aren't always + * a scalar value (i.e. i5/OS see uprv_maximumPtr function). + */ +# define U_MAX_PTR(base) \ + ((void *)(((uintptr_t)(base)+0x7fffffffu) > (uintptr_t)(base) \ + ? ((uintptr_t)(base)+0x7fffffffu) \ + : (uintptr_t)-1)) +# else +# define U_MAX_PTR(base) \ + ((char *)(((char *)(base)+0x7fffffffu) > (char *)(base) \ + ? ((char *)(base)+0x7fffffffu) \ + : (char *)-1)) +# endif +#endif + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/rbbi.cpp b/Build/source/libs/icu/icu-xetex/common/rbbi.cpp new file mode 100644 index 00000000000..41d242fff9c --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/rbbi.cpp @@ -0,0 +1,1832 @@ +/* +*************************************************************************** +* Copyright (C) 1999-2007 International Business Machines Corporation * +* and others. All rights reserved. * +*************************************************************************** +*/ +// +// file: rbbi.c Contains the implementation of the rule based break iterator +// runtime engine and the API implementation for +// class RuleBasedBreakIterator +// + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_BREAK_ITERATION + +#include "unicode/rbbi.h" +#include "unicode/schriter.h" +#include "unicode/uchriter.h" +#include "unicode/udata.h" +#include "unicode/uclean.h" +#include "rbbidata.h" +#include "rbbirb.h" +#include "cmemory.h" +#include "cstring.h" +#include "umutex.h" +#include "ucln_cmn.h" +#include "brkeng.h" + +#include "uassert.h" +#include "uvector.h" + +// if U_LOCAL_SERVICE_HOOK is defined, then localsvc.cpp is expected to be included. +#if U_LOCAL_SERVICE_HOOK +#include "localsvc.h" +#endif + +#ifdef RBBI_DEBUG +static UBool fTrace = FALSE; +#endif + +U_NAMESPACE_BEGIN + + +static const int16_t START_STATE = 1; // The state number of the starting state +static const int16_t STOP_STATE = 0; // The state-transition value indicating "stop" + + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(RuleBasedBreakIterator) + + +//======================================================================= +// constructors +//======================================================================= + +/** + * Constructs a RuleBasedBreakIterator that uses the already-created + * tables object that is passed in as a parameter. + */ +RuleBasedBreakIterator::RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status) +{ + init(); + fData = new RBBIDataWrapper(data, status); // status checked in constructor + if (U_FAILURE(status)) {return;} + if(fData == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } +} + +//------------------------------------------------------------------------------- +// +// Constructor from a UDataMemory handle to precompiled break rules +// stored in an ICU data file. +// +//------------------------------------------------------------------------------- +RuleBasedBreakIterator::RuleBasedBreakIterator(UDataMemory* udm, UErrorCode &status) +{ + init(); + fData = new RBBIDataWrapper(udm, status); // status checked in constructor + if (U_FAILURE(status)) {return;} + if(fData == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } +} + + + +//------------------------------------------------------------------------------- +// +// Constructor from a set of rules supplied as a string. +// +//------------------------------------------------------------------------------- +RuleBasedBreakIterator::RuleBasedBreakIterator( const UnicodeString &rules, + UParseError &parseError, + UErrorCode &status) +{ + init(); + if (U_FAILURE(status)) {return;} + RuleBasedBreakIterator *bi = (RuleBasedBreakIterator *) + RBBIRuleBuilder::createRuleBasedBreakIterator(rules, parseError, status); + // Note: This is a bit awkward. The RBBI ruleBuilder has a factory method that + // creates and returns a complete RBBI. From here, in a constructor, we + // can't just return the object created by the builder factory, hence + // the assignment of the factory created object to "this". + if (U_SUCCESS(status)) { + *this = *bi; + delete bi; + } +} + + +//------------------------------------------------------------------------------- +// +// Default Constructor. Create an empty shell that can be set up later. +// Used when creating a RuleBasedBreakIterator from a set +// of rules. +//------------------------------------------------------------------------------- +RuleBasedBreakIterator::RuleBasedBreakIterator() { + init(); +} + + +//------------------------------------------------------------------------------- +// +// Copy constructor. Will produce a break iterator with the same behavior, +// and which iterates over the same text, as the one passed in. +// +//------------------------------------------------------------------------------- +RuleBasedBreakIterator::RuleBasedBreakIterator(const RuleBasedBreakIterator& other) +: BreakIterator(other) +{ + this->init(); + *this = other; +} + + +/** + * Destructor + */ +RuleBasedBreakIterator::~RuleBasedBreakIterator() { + if (fCharIter!=fSCharIter && fCharIter!=fDCharIter) { + // fCharIter was adopted from the outside. + delete fCharIter; + } + fCharIter = NULL; + delete fSCharIter; + fCharIter = NULL; + delete fDCharIter; + fDCharIter = NULL; + + utext_close(fText); + + if (fData != NULL) { + fData->removeReference(); + fData = NULL; + } + if (fCachedBreakPositions) { + uprv_free(fCachedBreakPositions); + fCachedBreakPositions = NULL; + } + if (fLanguageBreakEngines) { + delete fLanguageBreakEngines; + fLanguageBreakEngines = NULL; + } + if (fUnhandledBreakEngine) { + delete fUnhandledBreakEngine; + fUnhandledBreakEngine = NULL; + } +} + +/** + * Assignment operator. Sets this iterator to have the same behavior, + * and iterate over the same text, as the one passed in. + */ +RuleBasedBreakIterator& +RuleBasedBreakIterator::operator=(const RuleBasedBreakIterator& that) { + if (this == &that) { + return *this; + } + reset(); // Delete break cache information + fBreakType = that.fBreakType; + if (fLanguageBreakEngines != NULL) { + delete fLanguageBreakEngines; + fLanguageBreakEngines = NULL; // Just rebuild for now + } + // TODO: clone fLanguageBreakEngines from "that" + UErrorCode status = U_ZERO_ERROR; + fText = utext_clone(fText, that.fText, FALSE, TRUE, &status); + + if (fCharIter!=fSCharIter && fCharIter!=fDCharIter) { + delete fCharIter; + } + fCharIter = NULL; + + if (that.fCharIter != NULL ) { + // This is a little bit tricky - it will intially appear that + // this->fCharIter is adopted, even if that->fCharIter was + // not adopted. That's ok. + fCharIter = that.fCharIter->clone(); + } + + if (fData != NULL) { + fData->removeReference(); + fData = NULL; + } + if (that.fData != NULL) { + fData = that.fData->addReference(); + } + + return *this; +} + + + +//----------------------------------------------------------------------------- +// +// init() Shared initialization routine. Used by all the constructors. +// Initializes all fields, leaving the object in a consistent state. +// +//----------------------------------------------------------------------------- +void RuleBasedBreakIterator::init() { + UErrorCode status = U_ZERO_ERROR; + fBufferClone = FALSE; + fText = utext_openUChars(NULL, NULL, 0, &status); + fCharIter = NULL; + fSCharIter = NULL; + fDCharIter = NULL; + fData = NULL; + fLastRuleStatusIndex = 0; + fLastStatusIndexValid = TRUE; + fDictionaryCharCount = 0; + fBreakType = -1; + + fCachedBreakPositions = NULL; + fLanguageBreakEngines = NULL; + fUnhandledBreakEngine = NULL; + fNumCachedBreakPositions = 0; + fPositionInCache = 0; + +#ifdef RBBI_DEBUG + static UBool debugInitDone = FALSE; + if (debugInitDone == FALSE) { + char *debugEnv = getenv("U_RBBIDEBUG"); + if (debugEnv && uprv_strstr(debugEnv, "trace")) { + fTrace = TRUE; + } + debugInitDone = TRUE; + } +#endif +} + + + +//----------------------------------------------------------------------------- +// +// clone - Returns a newly-constructed RuleBasedBreakIterator with the same +// behavior, and iterating over the same text, as this one. +// Virtual function: does the right thing with subclasses. +// +//----------------------------------------------------------------------------- +BreakIterator* +RuleBasedBreakIterator::clone(void) const { + return new RuleBasedBreakIterator(*this); +} + +/** + * Equality operator. Returns TRUE if both BreakIterators are of the + * same class, have the same behavior, and iterate over the same text. + */ +UBool +RuleBasedBreakIterator::operator==(const BreakIterator& that) const { + if (that.getDynamicClassID() != getDynamicClassID()) { + return FALSE; + } + + const RuleBasedBreakIterator& that2 = (const RuleBasedBreakIterator&) that; + + if (!utext_equals(fText, that2.fText)) { + // The two break iterators are operating on different text, + // or have a different interation position. + return FALSE; + }; + + // TODO: need a check for when in a dictionary region at different offsets. + + if (that2.fData == fData || + (fData != NULL && that2.fData != NULL && *that2.fData == *fData)) { + // The two break iterators are using the same rules. + return TRUE; + } + return FALSE; +} + +/** + * Compute a hash code for this BreakIterator + * @return A hash code + */ +int32_t +RuleBasedBreakIterator::hashCode(void) const { + int32_t hash = 0; + if (fData != NULL) { + hash = fData->hashCode(); + } + return hash; +} + + +void RuleBasedBreakIterator::setText(UText *ut, UErrorCode &status) { + if (U_FAILURE(status)) { + return; + } + reset(); + fText = utext_clone(fText, ut, FALSE, TRUE, &status); + + // Set up a dummy CharacterIterator to be returned if anyone + // calls getText(). With input from UText, there is no reasonable + // way to return a characterIterator over the actual input text. + // Return one over an empty string instead - this is the closest + // we can come to signaling a failure. + // (GetText() is obsolete, this failure is sort of OK) + if (fDCharIter == NULL) { + static UChar c = 0; + fDCharIter = new UCharCharacterIterator(&c, 0); + } + + if (fCharIter!=fSCharIter && fCharIter!=fDCharIter) { + // existing fCharIter was adopted from the outside. Delete it now. + delete fCharIter; + } + fCharIter = fDCharIter; + + this->first(); +} + + +UText *RuleBasedBreakIterator::getUText(UText *fillIn, UErrorCode &status) const { + UText *result = utext_clone(fillIn, fText, FALSE, TRUE, &status); + return result; +} + + + +/** + * Returns the description used to create this iterator + */ +const UnicodeString& +RuleBasedBreakIterator::getRules() const { + if (fData != NULL) { + return fData->getRuleSourceString(); + } else { + static const UnicodeString *s; + if (s == NULL) { + // TODO: something more elegant here. + // perhaps API should return the string by value. + // Note: thread unsafe init & leak are semi-ok, better than + // what was before. Sould be cleaned up, though. + s = new UnicodeString; + } + return *s; + } +} + +//======================================================================= +// BreakIterator overrides +//======================================================================= + +/** + * Return a CharacterIterator over the text being analyzed. + */ +CharacterIterator& +RuleBasedBreakIterator::getText() const { + return *fCharIter; +} + +/** + * Set the iterator to analyze a new piece of text. This function resets + * the current iteration position to the beginning of the text. + * @param newText An iterator over the text to analyze. + */ +void +RuleBasedBreakIterator::adoptText(CharacterIterator* newText) { + // If we are holding a CharacterIterator adopted from a + // previous call to this function, delete it now. + if (fCharIter!=fSCharIter && fCharIter!=fDCharIter) { + delete fCharIter; + } + + fCharIter = newText; + UErrorCode status = U_ZERO_ERROR; + reset(); + if (newText==NULL || newText->startIndex() != 0) { + // startIndex !=0 wants to be an error, but there's no way to report it. + // Make the iterator text be an empty string. + fText = utext_openUChars(fText, NULL, 0, &status); + } else { + fText = utext_openCharacterIterator(fText, newText, &status); + } + this->first(); +} + +/** + * Set the iterator to analyze a new piece of text. This function resets + * the current iteration position to the beginning of the text. + * @param newText An iterator over the text to analyze. + */ +void +RuleBasedBreakIterator::setText(const UnicodeString& newText) { + UErrorCode status = U_ZERO_ERROR; + reset(); + fText = utext_openConstUnicodeString(fText, &newText, &status); + + // Set up a character iterator on the string. + // Needed in case someone calls getText(). + // Can not, unfortunately, do this lazily on the (probably never) + // call to getText(), because getText is const. + if (fSCharIter == NULL) { + fSCharIter = new StringCharacterIterator(newText); + } else { + fSCharIter->setText(newText); + } + + if (fCharIter!=fSCharIter && fCharIter!=fDCharIter) { + // old fCharIter was adopted from the outside. Delete it. + delete fCharIter; + } + fCharIter = fSCharIter; + + this->first(); +} + + + +/** + * Sets the current iteration position to the beginning of the text. + * @return The offset of the beginning of the text. + */ +int32_t RuleBasedBreakIterator::first(void) { + reset(); + fLastRuleStatusIndex = 0; + fLastStatusIndexValid = TRUE; + //if (fText == NULL) + // return BreakIterator::DONE; + + utext_setNativeIndex(fText, 0); + return 0; +} + +/** + * Sets the current iteration position to the end of the text. + * @return The text's past-the-end offset. + */ +int32_t RuleBasedBreakIterator::last(void) { + reset(); + if (fText == NULL) { + fLastRuleStatusIndex = 0; + fLastStatusIndexValid = TRUE; + return BreakIterator::DONE; + } + + fLastStatusIndexValid = FALSE; + int32_t pos = (int32_t)utext_nativeLength(fText); + utext_setNativeIndex(fText, pos); + return pos; +} + +/** + * Advances the iterator either forward or backward the specified number of steps. + * Negative values move backward, and positive values move forward. This is + * equivalent to repeatedly calling next() or previous(). + * @param n The number of steps to move. The sign indicates the direction + * (negative is backwards, and positive is forwards). + * @return The character offset of the boundary position n boundaries away from + * the current one. + */ +int32_t RuleBasedBreakIterator::next(int32_t n) { + int32_t result = current(); + while (n > 0) { + result = next(); + --n; + } + while (n < 0) { + result = previous(); + ++n; + } + return result; +} + +/** + * Advances the iterator to the next boundary position. + * @return The position of the first boundary after this one. + */ +int32_t RuleBasedBreakIterator::next(void) { + // if we have cached break positions and we're still in the range + // covered by them, just move one step forward in the cache + if (fCachedBreakPositions != NULL) { + if (fPositionInCache < fNumCachedBreakPositions - 1) { + ++fPositionInCache; + int32_t pos = fCachedBreakPositions[fPositionInCache]; + utext_setNativeIndex(fText, pos); + return pos; + } + else { + reset(); + } + } + + int32_t startPos = current(); + int32_t result = handleNext(fData->fForwardTable); + if (fDictionaryCharCount > 0) { + result = checkDictionary(startPos, result, FALSE); + } + return result; +} + +/** + * Advances the iterator backwards, to the last boundary preceding this one. + * @return The position of the last boundary position preceding this one. + */ +int32_t RuleBasedBreakIterator::previous(void) { + int32_t result; + int32_t startPos; + + // if we have cached break positions and we're still in the range + // covered by them, just move one step backward in the cache + if (fCachedBreakPositions != NULL) { + if (fPositionInCache > 0) { + --fPositionInCache; + // If we're at the beginning of the cache, need to reevaluate the + // rule status + if (fPositionInCache <= 0) { + fLastStatusIndexValid = FALSE; + } + int32_t pos = fCachedBreakPositions[fPositionInCache]; + utext_setNativeIndex(fText, pos); + return pos; + } + else { + reset(); + } + } + + // if we're already sitting at the beginning of the text, return DONE + if (fText == NULL || (startPos = current()) == 0) { + fLastRuleStatusIndex = 0; + fLastStatusIndexValid = TRUE; + return BreakIterator::DONE; + } + + if (fData->fSafeRevTable != NULL || fData->fSafeFwdTable != NULL) { + result = handlePrevious(fData->fReverseTable); + if (fDictionaryCharCount > 0) { + result = checkDictionary(result, startPos, TRUE); + } + return result; + } + + // old rule syntax + // set things up. handlePrevious() will back us up to some valid + // break position before the current position (we back our internal + // iterator up one step to prevent handlePrevious() from returning + // the current position), but not necessarily the last one before + + // where we started + + int32_t start = current(); + + UTEXT_PREVIOUS32(fText); + int32_t lastResult = handlePrevious(fData->fReverseTable); + if (lastResult == UBRK_DONE) { + lastResult = 0; + utext_setNativeIndex(fText, 0); + } + result = lastResult; + int32_t lastTag = 0; + UBool breakTagValid = FALSE; + + // iterate forward from the known break position until we pass our + // starting point. The last break position before the starting + // point is our return value + + for (;;) { + result = next(); + if (result == BreakIterator::DONE || result >= start) { + break; + } + lastResult = result; + lastTag = fLastRuleStatusIndex; + breakTagValid = TRUE; + } + + // fLastBreakTag wants to have the value for section of text preceding + // the result position that we are to return (in lastResult.) If + // the backwards rules overshot and the above loop had to do two or more + // next()s to move up to the desired return position, we will have a valid + // tag value. But, if handlePrevious() took us to exactly the correct result positon, + // we wont have a tag value for that position, which is only set by handleNext(). + + // set the current iteration position to be the last break position + // before where we started, and then return that value + utext_setNativeIndex(fText, lastResult); + fLastRuleStatusIndex = lastTag; // for use by getRuleStatus() + fLastStatusIndexValid = breakTagValid; + + // No need to check the dictionary; it will have been handled by + // next() + + return lastResult; +} + +/** + * Sets the iterator to refer to the first boundary position following + * the specified position. + * @offset The position from which to begin searching for a break position. + * @return The position of the first break after the current position. + */ +int32_t RuleBasedBreakIterator::following(int32_t offset) { + // if we have cached break positions and offset is in the range + // covered by them, use them + // TODO: could use binary search + // TODO: what if offset is outside range, but break is not? + if (fCachedBreakPositions != NULL) { + if (offset >= fCachedBreakPositions[0] + && offset < fCachedBreakPositions[fNumCachedBreakPositions - 1]) { + fPositionInCache = 0; + // We are guaranteed not to leave the array due to range test above + while (offset >= fCachedBreakPositions[fPositionInCache]) { + ++fPositionInCache; + } + int32_t pos = fCachedBreakPositions[fPositionInCache]; + utext_setNativeIndex(fText, pos); + return pos; + } + else { + reset(); + } + } + + // if the offset passed in is already past the end of the text, + // just return DONE; if it's before the beginning, return the + // text's starting offset + fLastRuleStatusIndex = 0; + fLastStatusIndexValid = TRUE; + if (fText == NULL || offset >= utext_nativeLength(fText)) { + last(); + return next(); + } + else if (offset < 0) { + return first(); + } + + // otherwise, set our internal iteration position (temporarily) + // to the position passed in. If this is the _beginning_ position, + // then we can just use next() to get our return value + + int32_t result = 0; + + if (fData->fSafeRevTable != NULL) { + // new rule syntax + utext_setNativeIndex(fText, offset); + // move forward one codepoint to prepare for moving back to a + // safe point. + // this handles offset being between a supplementary character + UTEXT_NEXT32(fText); + // handlePrevious will move most of the time to < 1 boundary away + handlePrevious(fData->fSafeRevTable); + int32_t result = next(); + while (result <= offset) { + result = next(); + } + return result; + } + if (fData->fSafeFwdTable != NULL) { + // backup plan if forward safe table is not available + utext_setNativeIndex(fText, offset); + UTEXT_PREVIOUS32(fText); + // handle next will give result >= offset + handleNext(fData->fSafeFwdTable); + // previous will give result 0 or 1 boundary away from offset, + // most of the time + // we have to + int32_t oldresult = previous(); + while (oldresult > offset) { + int32_t result = previous(); + if (result <= offset) { + return oldresult; + } + oldresult = result; + } + int32_t result = next(); + if (result <= offset) { + return next(); + } + return result; + } + // otherwise, we have to sync up first. Use handlePrevious() to back + // up to a known break position before the specified position (if + // we can determine that the specified position is a break position, + // we don't back up at all). This may or may not be the last break + // position at or before our starting position. Advance forward + // from here until we've passed the starting position. The position + // we stop on will be the first break position after the specified one. + // old rule syntax + + utext_setNativeIndex(fText, offset); + if (offset==0 || + offset==1 && utext_getNativeIndex(fText)==0) { + return next(); + } + result = previous(); + + while (result != BreakIterator::DONE && result <= offset) { + result = next(); + } + + return result; +} + +/** + * Sets the iterator to refer to the last boundary position before the + * specified position. + * @offset The position to begin searching for a break from. + * @return The position of the last boundary before the starting position. + */ +int32_t RuleBasedBreakIterator::preceding(int32_t offset) { + // if we have cached break positions and offset is in the range + // covered by them, use them + if (fCachedBreakPositions != NULL) { + // TODO: binary search? + // TODO: What if offset is outside range, but break is not? + if (offset > fCachedBreakPositions[0] + && offset <= fCachedBreakPositions[fNumCachedBreakPositions - 1]) { + fPositionInCache = 0; + while (fPositionInCache < fNumCachedBreakPositions + && offset > fCachedBreakPositions[fPositionInCache]) + ++fPositionInCache; + --fPositionInCache; + // If we're at the beginning of the cache, need to reevaluate the + // rule status + if (fPositionInCache <= 0) { + fLastStatusIndexValid = FALSE; + } + utext_setNativeIndex(fText, fCachedBreakPositions[fPositionInCache]); + return fCachedBreakPositions[fPositionInCache]; + } + else { + reset(); + } + } + + // if the offset passed in is already past the end of the text, + // just return DONE; if it's before the beginning, return the + // text's starting offset + if (fText == NULL || offset > utext_nativeLength(fText)) { + // return BreakIterator::DONE; + return last(); + } + else if (offset < 0) { + return first(); + } + + // if we start by updating the current iteration position to the + // position specified by the caller, we can just use previous() + // to carry out this operation + + if (fData->fSafeFwdTable != NULL) { + // new rule syntax + utext_setNativeIndex(fText, offset); + int32_t newOffset = (int32_t)UTEXT_GETNATIVEINDEX(fText); + if (newOffset != offset) { + // Will come here if specified offset was not a code point boundary AND + // the underlying implmentation is using UText, which snaps any non-code-point-boundary + // indices to the containing code point. + // For breakitereator::preceding only, these non-code-point indices need to be moved + // up to refer to the following codepoint. + UTEXT_NEXT32(fText); + offset = (int32_t)UTEXT_GETNATIVEINDEX(fText); + } + + // TODO: (synwee) would it be better to just check for being in the middle of a surrogate pair, + // rather than adjusting the position unconditionally? + // (Change would interact with safe rules.) + // TODO: change RBBI behavior for off-boundary indices to match that of UText? + // affects only preceding(), seems cleaner, but is slightly different. + UTEXT_PREVIOUS32(fText); + handleNext(fData->fSafeFwdTable); + int32_t result = (int32_t)UTEXT_GETNATIVEINDEX(fText); + while (result >= offset) { + result = previous(); + } + return result; + } + if (fData->fSafeRevTable != NULL) { + // backup plan if forward safe table is not available + // TODO: check whether this path can be discarded + // It's probably OK to say that rules must supply both safe tables + // if they use safe tables at all. We have certainly never described + // to anyone how to work with just one safe table. + utext_setNativeIndex(fText, offset); + UTEXT_NEXT32(fText); + + // handle previous will give result <= offset + handlePrevious(fData->fSafeRevTable); + + // next will give result 0 or 1 boundary away from offset, + // most of the time + // we have to + int32_t oldresult = next(); + while (oldresult < offset) { + int32_t result = next(); + if (result >= offset) { + return oldresult; + } + oldresult = result; + } + int32_t result = previous(); + if (result >= offset) { + return previous(); + } + return result; + } + + // old rule syntax + utext_setNativeIndex(fText, offset); + return previous(); +} + +/** + * Returns true if the specfied position is a boundary position. As a side + * effect, leaves the iterator pointing to the first boundary position at + * or after "offset". + * @param offset the offset to check. + * @return True if "offset" is a boundary position. + */ +UBool RuleBasedBreakIterator::isBoundary(int32_t offset) { + // the beginning index of the iterator is always a boundary position by definition + if (offset == 0) { + first(); // For side effects on current position, tag values. + return TRUE; + } + + if (offset == (int32_t)utext_nativeLength(fText)) { + last(); // For side effects on current position, tag values. + return TRUE; + } + + // out-of-range indexes are never boundary positions + if (offset < 0) { + first(); // For side effects on current position, tag values. + return FALSE; + } + + if (offset > utext_nativeLength(fText)) { + last(); // For side effects on current position, tag values. + return FALSE; + } + + // otherwise, we can use following() on the position before the specified + // one and return true if the position we get back is the one the user + // specified + utext_previous32From(fText, offset); + int32_t backOne = (int32_t)UTEXT_GETNATIVEINDEX(fText); + UBool result = following(backOne) == offset; + return result; +} + +/** + * Returns the current iteration position. + * @return The current iteration position. + */ +int32_t RuleBasedBreakIterator::current(void) const { + int32_t pos = (int32_t)UTEXT_GETNATIVEINDEX(fText); + return pos; +} + +//======================================================================= +// implementation +//======================================================================= + +// +// RBBIRunMode - the state machine runs an extra iteration at the beginning and end +// of user text. A variable with this enum type keeps track of where we +// are. The state machine only fetches user input while in the RUN mode. +// +enum RBBIRunMode { + RBBI_START, // state machine processing is before first char of input + RBBI_RUN, // state machine processing is in the user text + RBBI_END // state machine processing is after end of user text. +}; + + +//----------------------------------------------------------------------------------- +// +// handleNext(stateTable) +// This method is the actual implementation of the rbbi next() method. +// This method initializes the state machine to state 1 +// and advances through the text character by character until we reach the end +// of the text or the state machine transitions to state 0. We update our return +// value every time the state machine passes through an accepting state. +// +//----------------------------------------------------------------------------------- +int32_t RuleBasedBreakIterator::handleNext(const RBBIStateTable *statetable) { + int32_t state; + int16_t category = 0; + RBBIRunMode mode; + + RBBIStateTableRow *row; + UChar32 c; + int32_t lookaheadStatus = 0; + int32_t lookaheadTagIdx = 0; + int32_t result = 0; + int32_t initialPosition = 0; + int32_t lookaheadResult = 0; + UBool lookAheadHardBreak = (statetable->fFlags & RBBI_LOOKAHEAD_HARD_BREAK) != 0; + const char *tableData = statetable->fTableData; + uint32_t tableRowLen = statetable->fRowLen; + + #ifdef RBBI_DEBUG + if (fTrace) { + RBBIDebugPuts("Handle Next pos char state category"); + } + #endif + + // No matter what, handleNext alway correctly sets the break tag value. + fLastStatusIndexValid = TRUE; + fLastRuleStatusIndex = 0; + + // if we're already at the end of the text, return DONE. + initialPosition = (int32_t)UTEXT_GETNATIVEINDEX(fText); + result = initialPosition; + c = UTEXT_NEXT32(fText); + if (fData == NULL || c==U_SENTINEL) { + return BreakIterator::DONE; + } + + // Set the initial state for the state machine + state = START_STATE; + row = (RBBIStateTableRow *) + //(statetable->fTableData + (statetable->fRowLen * state)); + (tableData + tableRowLen * state); + + + mode = RBBI_RUN; + if (statetable->fFlags & RBBI_BOF_REQUIRED) { + category = 2; + mode = RBBI_START; + } + + + // loop until we reach the end of the text or transition to state 0 + // + for (;;) { + if (c == U_SENTINEL) { + // Reached end of input string. + if (mode == RBBI_END) { + // We have already run the loop one last time with the + // character set to the psueudo {eof} value. Now it is time + // to unconditionally bail out. + if (lookaheadResult > result) { + // We ran off the end of the string with a pending look-ahead match. + // Treat this as if the look-ahead condition had been met, and return + // the match at the / position from the look-ahead rule. + result = lookaheadResult; + fLastRuleStatusIndex = lookaheadTagIdx; + lookaheadStatus = 0; + } + break; + } + // Run the loop one last time with the fake end-of-input character category. + mode = RBBI_END; + category = 1; + } + + // + // Get the char category. An incoming category of 1 or 2 means that + // we are preset for doing the beginning or end of input, and + // that we shouldn't get a category from an actual text input character. + // + if (mode == RBBI_RUN) { + // look up the current character's character category, which tells us + // which column in the state table to look at. + // Note: the 16 in UTRIE_GET16 refers to the size of the data being returned, + // not the size of the character going in, which is a UChar32. + // + UTRIE_GET16(&fData->fTrie, c, category); + + // Check the dictionary bit in the character's category. + // Counter is only used by dictionary based iterators (subclasses). + // Chars that need to be handled by a dictionary have a flag bit set + // in their category values. + // + if ((category & 0x4000) != 0) { + fDictionaryCharCount++; + // And off the dictionary flag bit. + category &= ~0x4000; + } + } + + #ifdef RBBI_DEBUG + if (fTrace) { + RBBIDebugPrintf(" %4d ", utext_getNativeIndex(fText)); + if (0x20<=c && c<0x7f) { + RBBIDebugPrintf("\"%c\" ", c); + } else { + RBBIDebugPrintf("%5x ", c); + } + RBBIDebugPrintf("%3d %3d\n", state, category); + } + #endif + + // State Transition - move machine to its next state + // + state = row->fNextState[category]; + row = (RBBIStateTableRow *) + // (statetable->fTableData + (statetable->fRowLen * state)); + (tableData + tableRowLen * state); + + + if (row->fAccepting == -1) { + // Match found, common case. + if (mode != RBBI_START) { + result = (int32_t)UTEXT_GETNATIVEINDEX(fText); + } + fLastRuleStatusIndex = row->fTagIdx; // Remember the break status (tag) values. + } + + if (row->fLookAhead != 0) { + if (lookaheadStatus != 0 + && row->fAccepting == lookaheadStatus) { + // Lookahead match is completed. + result = lookaheadResult; + fLastRuleStatusIndex = lookaheadTagIdx; + lookaheadStatus = 0; + // TODO: make a standalone hard break in a rule work. + if (lookAheadHardBreak) { + UTEXT_SETNATIVEINDEX(fText, result); + return result; + } + // Look-ahead completed, but other rules may match further. Continue on + // TODO: junk this feature? I don't think it's used anywhwere. + goto continueOn; + } + + int32_t r = (int32_t)UTEXT_GETNATIVEINDEX(fText); + lookaheadResult = r; + lookaheadStatus = row->fLookAhead; + lookaheadTagIdx = row->fTagIdx; + goto continueOn; + } + + + if (row->fAccepting != 0) { + // Because this is an accepting state, any in-progress look-ahead match + // is no longer relavant. Clear out the pending lookahead status. + lookaheadStatus = 0; // clear out any pending look-ahead match. + } + +continueOn: + if (state == STOP_STATE) { + // This is the normal exit from the lookup state machine. + // We have advanced through the string until it is certain that no + // longer match is possible, no matter what characters follow. + break; + } + + // Advance to the next character. + // If this is a beginning-of-input loop iteration, don't advance + // the input position. The next iteration will be processing the + // first real input character. + if (mode == RBBI_RUN) { + c = UTEXT_NEXT32(fText); + } else { + if (mode == RBBI_START) { + mode = RBBI_RUN; + } + } + + + } + + // The state machine is done. Check whether it found a match... + + // If the iterator failed to advance in the match engine, force it ahead by one. + // (This really indicates a defect in the break rules. They should always match + // at least one character.) + if (result == initialPosition) { + UTEXT_SETNATIVEINDEX(fText, initialPosition); + UTEXT_NEXT32(fText); + result = (int32_t)UTEXT_GETNATIVEINDEX(fText); + } + + // Leave the iterator at our result position. + UTEXT_SETNATIVEINDEX(fText, result); + #ifdef RBBI_DEBUG + if (fTrace) { + RBBIDebugPrintf("result = %d\n\n", result); + } + #endif + return result; +} + + + +//----------------------------------------------------------------------------------- +// +// handlePrevious() +// +// Iterate backwards, according to the logic of the reverse rules. +// This version handles the exact style backwards rules. +// +// The logic of this function is very similar to handleNext(), above. +// +//----------------------------------------------------------------------------------- +int32_t RuleBasedBreakIterator::handlePrevious(const RBBIStateTable *statetable) { + int32_t state; + int16_t category = 0; + RBBIRunMode mode; + RBBIStateTableRow *row; + UChar32 c; + int32_t lookaheadStatus = 0; + int32_t result = 0; + int32_t initialPosition = 0; + int32_t lookaheadResult = 0; + UBool lookAheadHardBreak = (statetable->fFlags & RBBI_LOOKAHEAD_HARD_BREAK) != 0; + + #ifdef RBBI_DEBUG + if (fTrace) { + RBBIDebugPuts("Handle Previous pos char state category"); + } + #endif + + // handlePrevious() never gets the rule status. + // Flag the status as invalid; if the user ever asks for status, we will need + // to back up, then re-find the break position using handleNext(), which does + // get the status value. + fLastStatusIndexValid = FALSE; + fLastRuleStatusIndex = 0; + + // if we're already at the start of the text, return DONE. + if (fText == NULL || fData == NULL || UTEXT_GETNATIVEINDEX(fText)==0) { + return BreakIterator::DONE; + } + + // Set up the starting char. + initialPosition = (int32_t)UTEXT_GETNATIVEINDEX(fText); + result = initialPosition; + c = UTEXT_PREVIOUS32(fText); + + // Set the initial state for the state machine + state = START_STATE; + row = (RBBIStateTableRow *) + (statetable->fTableData + (statetable->fRowLen * state)); + category = 3; + mode = RBBI_RUN; + if (statetable->fFlags & RBBI_BOF_REQUIRED) { + category = 2; + mode = RBBI_START; + } + + + // loop until we reach the start of the text or transition to state 0 + // + for (;;) { + if (c == U_SENTINEL) { + // Reached end of input string. + if (mode == RBBI_END || + *(int32_t *)fData->fHeader->fFormatVersion == 1 ) { + // We have already run the loop one last time with the + // character set to the psueudo {eof} value. Now it is time + // to unconditionally bail out. + // (Or we have an old format binary rule file that does not support {eof}.) + if (lookaheadResult < result) { + // We ran off the end of the string with a pending look-ahead match. + // Treat this as if the look-ahead condition had been met, and return + // the match at the / position from the look-ahead rule. + result = lookaheadResult; + lookaheadStatus = 0; + } else if (result == initialPosition) { + // Ran off start, no match found. + // move one index one (towards the start, since we are doing a previous()) + UTEXT_SETNATIVEINDEX(fText, initialPosition); + UTEXT_PREVIOUS32(fText); // TODO: shouldn't be necessary. We're already at beginning. Check. + } + break; + } + // Run the loop one last time with the fake end-of-input character category. + mode = RBBI_END; + category = 1; + } + + // + // Get the char category. An incoming category of 1 or 2 means that + // we are preset for doing the beginning or end of input, and + // that we shouldn't get a category from an actual text input character. + // + if (mode == RBBI_RUN) { + // look up the current character's character category, which tells us + // which column in the state table to look at. + // Note: the 16 in UTRIE_GET16 refers to the size of the data being returned, + // not the size of the character going in, which is a UChar32. + // + UTRIE_GET16(&fData->fTrie, c, category); + + // Check the dictionary bit in the character's category. + // Counter is only used by dictionary based iterators (subclasses). + // Chars that need to be handled by a dictionary have a flag bit set + // in their category values. + // + if ((category & 0x4000) != 0) { + fDictionaryCharCount++; + // And off the dictionary flag bit. + category &= ~0x4000; + } + } + + #ifdef RBBI_DEBUG + if (fTrace) { + RBBIDebugPrintf(" %4d ", (int32_t)utext_getNativeIndex(fText)); + if (0x20<=c && c<0x7f) { + RBBIDebugPrintf("\"%c\" ", c); + } else { + RBBIDebugPrintf("%5x ", c); + } + RBBIDebugPrintf("%3d %3d\n", state, category); + } + #endif + + // State Transition - move machine to its next state + // + state = row->fNextState[category]; + row = (RBBIStateTableRow *) + (statetable->fTableData + (statetable->fRowLen * state)); + + if (row->fAccepting == -1) { + // Match found, common case. + result = (int32_t)UTEXT_GETNATIVEINDEX(fText); + } + + if (row->fLookAhead != 0) { + if (lookaheadStatus != 0 + && row->fAccepting == lookaheadStatus) { + // Lookahead match is completed. + result = lookaheadResult; + lookaheadStatus = 0; + // TODO: make a standalone hard break in a rule work. + if (lookAheadHardBreak) { + UTEXT_SETNATIVEINDEX(fText, result); + return result; + } + // Look-ahead completed, but other rules may match further. Continue on + // TODO: junk this feature? I don't think it's used anywhwere. + goto continueOn; + } + + int32_t r = (int32_t)UTEXT_GETNATIVEINDEX(fText); + lookaheadResult = r; + lookaheadStatus = row->fLookAhead; + goto continueOn; + } + + + if (row->fAccepting != 0) { + // Because this is an accepting state, any in-progress look-ahead match + // is no longer relavant. Clear out the pending lookahead status. + lookaheadStatus = 0; + } + +continueOn: + if (state == STOP_STATE) { + // This is the normal exit from the lookup state machine. + // We have advanced through the string until it is certain that no + // longer match is possible, no matter what characters follow. + break; + } + + // Move (backwards) to the next character to process. + // If this is a beginning-of-input loop iteration, don't advance + // the input position. The next iteration will be processing the + // first real input character. + if (mode == RBBI_RUN) { + c = UTEXT_PREVIOUS32(fText); + } else { + if (mode == RBBI_START) { + mode = RBBI_RUN; + } + } + } + + // The state machine is done. Check whether it found a match... + + // If the iterator failed to advance in the match engine, force it ahead by one. + // (This really indicates a defect in the break rules. They should always match + // at least one character.) + if (result == initialPosition) { + UTEXT_SETNATIVEINDEX(fText, initialPosition); + UTEXT_PREVIOUS32(fText); + result = (int32_t)UTEXT_GETNATIVEINDEX(fText); + } + + // Leave the iterator at our result position. + UTEXT_SETNATIVEINDEX(fText, result); + #ifdef RBBI_DEBUG + if (fTrace) { + RBBIDebugPrintf("result = %d\n\n", result); + } + #endif + return result; +} + + +void +RuleBasedBreakIterator::reset() +{ + if (fCachedBreakPositions) { + uprv_free(fCachedBreakPositions); + } + fCachedBreakPositions = NULL; + fNumCachedBreakPositions = 0; + fDictionaryCharCount = 0; + fPositionInCache = 0; +} + + + +//------------------------------------------------------------------------------- +// +// getRuleStatus() Return the break rule tag associated with the current +// iterator position. If the iterator arrived at its current +// position by iterating forwards, the value will have been +// cached by the handleNext() function. +// +// If no cached status value is available, the status is +// found by doing a previous() followed by a next(), which +// leaves the iterator where it started, and computes the +// status while doing the next(). +// +//------------------------------------------------------------------------------- +void RuleBasedBreakIterator::makeRuleStatusValid() { + if (fLastStatusIndexValid == FALSE) { + // No cached status is available. + if (fText == NULL || current() == 0) { + // At start of text, or there is no text. Status is always zero. + fLastRuleStatusIndex = 0; + fLastStatusIndexValid = TRUE; + } else { + // Not at start of text. Find status the tedious way. + int32_t pa = current(); + previous(); + if (fNumCachedBreakPositions > 0) { + reset(); // Blow off the dictionary cache + } + int32_t pb = next(); + if (pa != pb) { + // note: the if (pa != pb) test is here only to eliminate warnings for + // unused local variables on gcc. Logically, it isn't needed. + U_ASSERT(pa == pb); + } + } + } + U_ASSERT(fLastRuleStatusIndex >= 0 && fLastRuleStatusIndex < fData->fStatusMaxIdx); +} + + +int32_t RuleBasedBreakIterator::getRuleStatus() const { + RuleBasedBreakIterator *nonConstThis = (RuleBasedBreakIterator *)this; + nonConstThis->makeRuleStatusValid(); + + // fLastRuleStatusIndex indexes to the start of the appropriate status record + // (the number of status values.) + // This function returns the last (largest) of the array of status values. + int32_t idx = fLastRuleStatusIndex + fData->fRuleStatusTable[fLastRuleStatusIndex]; + int32_t tagVal = fData->fRuleStatusTable[idx]; + + return tagVal; +} + + + + +int32_t RuleBasedBreakIterator::getRuleStatusVec( + int32_t *fillInVec, int32_t capacity, UErrorCode &status) +{ + if (U_FAILURE(status)) { + return 0; + } + + RuleBasedBreakIterator *nonConstThis = (RuleBasedBreakIterator *)this; + nonConstThis->makeRuleStatusValid(); + int32_t numVals = fData->fRuleStatusTable[fLastRuleStatusIndex]; + int32_t numValsToCopy = numVals; + if (numVals > capacity) { + status = U_BUFFER_OVERFLOW_ERROR; + numValsToCopy = capacity; + } + int i; + for (i=0; ifRuleStatusTable[fLastRuleStatusIndex + i + 1]; + } + return numVals; +} + + + +//------------------------------------------------------------------------------- +// +// getBinaryRules Access to the compiled form of the rules, +// for use by build system tools that save the data +// for standard iterator types. +// +//------------------------------------------------------------------------------- +const uint8_t *RuleBasedBreakIterator::getBinaryRules(uint32_t &length) { + const uint8_t *retPtr = NULL; + length = 0; + + if (fData != NULL) { + retPtr = (const uint8_t *)fData->fHeader; + length = fData->fHeader->fLength; + } + return retPtr; +} + + + + +//------------------------------------------------------------------------------- +// +// BufferClone TODO: In my (Andy) opinion, this function should be deprecated. +// Saving one heap allocation isn't worth the trouble. +// Cloning shouldn't be done in tight loops, and +// making the clone copy involves other heap operations anyway. +// And the application code for correctly dealing with buffer +// size problems and the eventual object destruction is ugly. +// +//------------------------------------------------------------------------------- +BreakIterator * RuleBasedBreakIterator::createBufferClone(void *stackBuffer, + int32_t &bufferSize, + UErrorCode &status) +{ + if (U_FAILURE(status)){ + return NULL; + } + + // + // If user buffer size is zero this is a preflight operation to + // obtain the needed buffer size, allowing for worst case misalignment. + // + if (bufferSize == 0) { + bufferSize = sizeof(RuleBasedBreakIterator) + U_ALIGNMENT_OFFSET_UP(0); + return NULL; + } + + + // + // Check the alignment and size of the user supplied buffer. + // Allocate heap memory if the user supplied memory is insufficient. + // + char *buf = (char *)stackBuffer; + uint32_t s = bufferSize; + + if (stackBuffer == NULL) { + s = 0; // Ignore size, force allocation if user didn't give us a buffer. + } + if (U_ALIGNMENT_OFFSET(stackBuffer) != 0) { + uint32_t offsetUp = (uint32_t)U_ALIGNMENT_OFFSET_UP(buf); + s -= offsetUp; + buf += offsetUp; + } + if (s < sizeof(RuleBasedBreakIterator)) { + // Not enough room in the caller-supplied buffer. + // Do a plain-vanilla heap based clone and return that, along with + // a warning that the clone was allocated. + RuleBasedBreakIterator *clonedBI = new RuleBasedBreakIterator(*this); + if (clonedBI == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + } else { + status = U_SAFECLONE_ALLOCATED_WARNING; + } + return clonedBI; + } + + // + // Clone the source BI into the caller-supplied buffer. + // TODO: using an overloaded operator new to directly initialize the + // copy in the user's buffer would be better, but it doesn't seem + // to get along with namespaces. Investigate why. + // + // The memcpy is only safe with an empty (default constructed) + // break iterator. Use on others can screw up reference counts + // to data. memcpy-ing objects is not really a good idea... + // + RuleBasedBreakIterator localIter; // Empty break iterator, source for memcpy + RuleBasedBreakIterator *clone = (RuleBasedBreakIterator *)buf; + uprv_memcpy(clone, &localIter, sizeof(RuleBasedBreakIterator)); // init C++ gorp, BreakIterator base class part + clone->init(); // Init RuleBasedBreakIterator part, (user default constructor) + *clone = *this; // clone = the real BI we want. + clone->fBufferClone = TRUE; // Flag to prevent deleting storage on close (From C code) + + return clone; +} + + +//------------------------------------------------------------------------------- +// +// isDictionaryChar Return true if the category lookup for this char +// indicates that it is in the set of dictionary lookup +// chars. +// +// This function is intended for use by dictionary based +// break iterators. +// +//------------------------------------------------------------------------------- +/*UBool RuleBasedBreakIterator::isDictionaryChar(UChar32 c) { + if (fData == NULL) { + return FALSE; + } + uint16_t category; + UTRIE_GET16(&fData->fTrie, c, category); + return (category & 0x4000) != 0; +}*/ + + +//------------------------------------------------------------------------------- +// +// checkDictionary This function handles all processing of characters in +// the "dictionary" set. It will determine the appropriate +// course of action, and possibly set up a cache in the +// process. +// +//------------------------------------------------------------------------------- +int32_t RuleBasedBreakIterator::checkDictionary(int32_t startPos, + int32_t endPos, + UBool reverse) { + // Reset the old break cache first. + uint32_t dictionaryCount = fDictionaryCharCount; + reset(); + + if (dictionaryCount <= 1 || (endPos - startPos) <= 1) { + return (reverse ? startPos : endPos); + } + + // Starting from the starting point, scan towards the proposed result, + // looking for the first dictionary character (which may be the one + // we're on, if we're starting in the middle of a range). + utext_setNativeIndex(fText, reverse ? endPos : startPos); + if (reverse) { + UTEXT_PREVIOUS32(fText); + } + + int32_t rangeStart = startPos; + int32_t rangeEnd = endPos; + + uint16_t category; + int32_t current; + UErrorCode status = U_ZERO_ERROR; + UStack breaks(status); + int32_t foundBreakCount = 0; + UChar32 c = utext_current32(fText); + + UTRIE_GET16(&fData->fTrie, c, category); + + // Is the character we're starting on a dictionary character? If so, we + // need to back up to include the entire run; otherwise the results of + // the break algorithm will differ depending on where we start. Since + // the result is cached and there is typically a non-dictionary break + // within a small number of words, there should be little performance impact. + if (category & 0x4000) { + if (reverse) { + do { + utext_next32(fText); // TODO: recast to work directly with postincrement. + c = utext_current32(fText); + UTRIE_GET16(&fData->fTrie, c, category); + } while (c != U_SENTINEL && (category & 0x4000)); + // Back up to the last dictionary character + rangeEnd = (int32_t)UTEXT_GETNATIVEINDEX(fText); + if (c == U_SENTINEL) { + // c = fText->last32(); + // TODO: why was this if needed? + c = UTEXT_PREVIOUS32(fText); + } + else { + c = UTEXT_PREVIOUS32(fText); + } + } + else { + do { + c = UTEXT_PREVIOUS32(fText); + UTRIE_GET16(&fData->fTrie, c, category); + } + while (c != U_SENTINEL && (category & 0x4000)); + // Back up to the last dictionary character + if (c == U_SENTINEL) { + // c = fText->first32(); + c = utext_current32(fText); + } + else { + utext_next32(fText); + c = utext_current32(fText); + } + rangeStart = (int32_t)UTEXT_GETNATIVEINDEX(fText);; + } + UTRIE_GET16(&fData->fTrie, c, category); + } + + // Loop through the text, looking for ranges of dictionary characters. + // For each span, find the appropriate break engine, and ask it to find + // any breaks within the span. + // Note: we always do this in the forward direction, so that the break + // cache is built in the right order. + if (reverse) { + utext_setNativeIndex(fText, rangeStart); + c = utext_current32(fText); + UTRIE_GET16(&fData->fTrie, c, category); + } + while(U_SUCCESS(status)) { + while((current = (int32_t)UTEXT_GETNATIVEINDEX(fText)) < rangeEnd && (category & 0x4000) == 0) { + utext_next32(fText); // TODO: tweak for post-increment operation + c = utext_current32(fText); + UTRIE_GET16(&fData->fTrie, c, category); + } + if (current >= rangeEnd) { + break; + } + + // We now have a dictionary character. Get the appropriate language object + // to deal with it. + const LanguageBreakEngine *lbe = getLanguageBreakEngine(c); + + // Ask the language object if there are any breaks. It will leave the text + // pointer on the other side of its range, ready to search for the next one. + if (lbe != NULL) { + foundBreakCount += lbe->findBreaks(fText, rangeStart, rangeEnd, FALSE, fBreakType, breaks); + } + + // Reload the loop variables for the next go-round + c = utext_current32(fText); + UTRIE_GET16(&fData->fTrie, c, category); + } + + // If we found breaks, build a new break cache. The first and last entries must + // be the original starting and ending position. + if (foundBreakCount > 0) { + int32_t totalBreaks = foundBreakCount; + if (startPos < breaks.elementAti(0)) { + totalBreaks += 1; + } + if (endPos > breaks.peeki()) { + totalBreaks += 1; + } + fCachedBreakPositions = (int32_t *)uprv_malloc(totalBreaks * sizeof(int32_t)); + if (fCachedBreakPositions != NULL) { + int32_t out = 0; + fNumCachedBreakPositions = totalBreaks; + if (startPos < breaks.elementAti(0)) { + fCachedBreakPositions[out++] = startPos; + } + for (int32_t i = 0; i < foundBreakCount; ++i) { + fCachedBreakPositions[out++] = breaks.elementAti(i); + } + if (endPos > fCachedBreakPositions[out-1]) { + fCachedBreakPositions[out] = endPos; + } + // If there are breaks, then by definition, we are replacing the original + // proposed break by one of the breaks we found. Use following() and + // preceding() to do the work. They should never recurse in this case. + if (reverse) { + return preceding(endPos - 1); + } + else { + return following(startPos); + } + } + // If the allocation failed, just fall through to the "no breaks found" case. + } + + // If we get here, there were no language-based breaks. Set the text pointer + // to the original proposed break. + utext_setNativeIndex(fText, reverse ? startPos : endPos); + return (reverse ? startPos : endPos); +} + +U_NAMESPACE_END + +// defined in ucln_cmn.h + +static U_NAMESPACE_QUALIFIER UStack *gLanguageBreakFactories = NULL; + +/** + * Release all static memory held by breakiterator. + */ +U_CDECL_BEGIN +static UBool U_CALLCONV breakiterator_cleanup_dict(void) { + if (gLanguageBreakFactories) { + delete gLanguageBreakFactories; + gLanguageBreakFactories = NULL; + } + return TRUE; +} +U_CDECL_END + +U_CDECL_BEGIN +static void U_CALLCONV _deleteFactory(void *obj) { + delete (U_NAMESPACE_QUALIFIER LanguageBreakFactory *) obj; +} +U_CDECL_END +U_NAMESPACE_BEGIN + +static const LanguageBreakEngine* +getLanguageBreakEngineFromFactory(UChar32 c, int32_t breakType) +{ + UBool needsInit; + UErrorCode status = U_ZERO_ERROR; + UMTX_CHECK(NULL, (UBool)(gLanguageBreakFactories == NULL), needsInit); + + if (needsInit) { + UStack *factories = new UStack(_deleteFactory, NULL, status); + if (U_SUCCESS(status)) { + ICULanguageBreakFactory *builtIn = new ICULanguageBreakFactory(status); + factories->push(builtIn, status); +#ifdef U_LOCAL_SERVICE_HOOK + LanguageBreakFactory *extra = (LanguageBreakFactory *)uprv_svc_hook("languageBreakFactory", &status); + if (extra != NULL) { + factories->push(extra, status); + } +#endif + } + umtx_lock(NULL); + if (gLanguageBreakFactories == NULL) { + gLanguageBreakFactories = factories; + factories = NULL; + ucln_common_registerCleanup(UCLN_COMMON_BREAKITERATOR_DICT, breakiterator_cleanup_dict); + } + umtx_unlock(NULL); + delete factories; + } + + if (gLanguageBreakFactories == NULL) { + return NULL; + } + + int32_t i = gLanguageBreakFactories->size(); + const LanguageBreakEngine *lbe = NULL; + while (--i >= 0) { + LanguageBreakFactory *factory = (LanguageBreakFactory *)(gLanguageBreakFactories->elementAt(i)); + lbe = factory->getEngineFor(c, breakType); + if (lbe != NULL) { + break; + } + } + return lbe; +} + + +//------------------------------------------------------------------------------- +// +// getLanguageBreakEngine Find an appropriate LanguageBreakEngine for the +// the characer c. +// +//------------------------------------------------------------------------------- +const LanguageBreakEngine * +RuleBasedBreakIterator::getLanguageBreakEngine(UChar32 c) { + const LanguageBreakEngine *lbe = NULL; + UErrorCode status = U_ZERO_ERROR; + + if (fLanguageBreakEngines == NULL) { + fLanguageBreakEngines = new UStack(status); + if (U_FAILURE(status)) { + delete fLanguageBreakEngines; + fLanguageBreakEngines = 0; + return NULL; + } + } + + int32_t i = fLanguageBreakEngines->size(); + while (--i >= 0) { + lbe = (const LanguageBreakEngine *)(fLanguageBreakEngines->elementAt(i)); + if (lbe->handles(c, fBreakType)) { + return lbe; + } + } + + // No existing dictionary took the character. See if a factory wants to + // give us a new LanguageBreakEngine for this character. + lbe = getLanguageBreakEngineFromFactory(c, fBreakType); + + // If we got one, use it and push it on our stack. + if (lbe != NULL) { + fLanguageBreakEngines->push((void *)lbe, status); + // Even if we can't remember it, we can keep looking it up, so + // return it even if the push fails. + return lbe; + } + + // No engine is forthcoming for this character. Add it to the + // reject set. Create the reject break engine if needed. + if (fUnhandledBreakEngine == NULL) { + fUnhandledBreakEngine = new UnhandledEngine(status); + if (U_SUCCESS(status) && fUnhandledBreakEngine == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + } + // Put it last so that scripts for which we have an engine get tried + // first. + fLanguageBreakEngines->insertElementAt(fUnhandledBreakEngine, 0, status); + // If we can't insert it, or creation failed, get rid of it + if (U_FAILURE(status)) { + delete fUnhandledBreakEngine; + fUnhandledBreakEngine = 0; + return NULL; + } + } + + // Tell the reject engine about the character; at its discretion, it may + // add more than just the one character. + fUnhandledBreakEngine->handleCharacter(c, fBreakType); + + return fUnhandledBreakEngine; +} + + + +/*int32_t RuleBasedBreakIterator::getBreakType() const { + return fBreakType; +}*/ + +void RuleBasedBreakIterator::setBreakType(int32_t type) { + fBreakType = type; + reset(); +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_BREAK_ITERATION */ diff --git a/Build/source/libs/icu/icu-xetex/common/rbbicst.pl b/Build/source/libs/icu/icu-xetex/common/rbbicst.pl new file mode 100755 index 00000000000..98b06cbc27d --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/rbbicst.pl @@ -0,0 +1,453 @@ +#************************************************************************** +# Copyright (C) 2002-2005 International Business Machines Corporation * +# and others. All rights reserved. * +#************************************************************************** +# +# rbbicst Compile the RBBI rule paser state table data into initialized C data. +# Usage: +# cd icu/source/common +# perl rbbicst.pl < rbbirpt.txt > rbbirpt.h +# perl rbbicst.pl -j < rbbirpt.txt > RBBIRuleParseTable.java +# +# The output file, rbbrpt.h, is included by some of the .cpp rbbi +# implementation files. This perl script is NOT run as part +# of a normal ICU build. It is run by hand when needed, and the +# rbbirpt.h generated file is put back into cvs. +# +# See rbbirpt.txt for a description of the input format for this script. +# + +if ($ARGV[0] eq "-j") { + $javaOutput = 1; + shift @ARGV; +} + + +$num_states = 1; # Always the state number for the line being compiled. +$line_num = 0; # The line number in the input file. + +$states{"pop"} = 255; # Add the "pop" to the list of defined state names. + # This prevents any state from being labelled with "pop", + # and resolves references to "pop" in the next state field. + +line_loop: while (<>) { + chomp(); + $line = $_; + @fields = split(); + $line_num++; + + # Remove # comments, which are any fields beginning with a #, plus all + # that follow on the line. + for ($i=0; $i<@fields; $i++) { + if ($fields[$i] =~ /^#/) { + @fields = @fields[0 .. $i-1]; + last; + } + } + # ignore blank lines, and those with no fields left after stripping comments.. + if (@fields == 0) { + next; + } + + # + # State Label: handling. + # Does the first token end with a ":"? If so, it's the name of a state. + # Put in a hash, together with the current state number, + # so that we can later look up the number from the name. + # + if (@fields[0] =~ /.*:$/) { + $state_name = @fields[0]; + $state_name =~ s/://; # strip off the colon from the state name. + + if ($states{$state_name} != 0) { + print " rbbicst: at line $line-num duplicate definition of state $state_name\n"; + } + $states{$state_name} = $num_states; + $stateNames[$num_states] = $state_name; + + # if the label was the only thing on this line, go on to the next line, + # otherwise assume that a state definition is on the same line and fall through. + if (@fields == 1) { + next line_loop; + } + shift @fields; # shift off label field in preparation + # for handling the rest of the line. + } + + # + # State Transition line. + # syntax is this, + # character [n] target-state [^push-state] [function-name] + # where + # [something] is an optional something + # character is either a single quoted character e.g. '[' + # or a name of a character class, e.g. white_space + # + + $state_line_num[$num_states] = $line_num; # remember line number with each state + # so we can make better error messages later. + # + # First field, character class or literal character for this transition. + # + if ($fields[0] =~ /^'.'$/) { + # We've got a quoted literal character. + $state_literal_chars[$num_states] = $fields[0]; + $state_literal_chars[$num_states] =~ s/'//g; + } else { + # We've got the name of a character class. + $state_char_class[$num_states] = $fields[0]; + if ($fields[0] =~ /[\W]/) { + print " rbbicsts: at line $line_num, bad character literal or character class name.\n"; + print " scanning $fields[0]\n"; + exit(-1); + } + } + shift @fields; + + # + # do the 'n' flag + # + $state_flag[$num_states] = $javaOutput? "false" : "FALSE"; + if ($fields[0] eq "n") { + $state_flag[$num_states] = $javaOutput? "true": "TRUE"; + shift @fields; + } + + # + # do the destination state. + # + $state_dest_state[$num_states] = $fields[0]; + if ($fields[0] eq "") { + print " rbbicsts: at line $line_num, destination state missing.\n"; + exit(-1); + } + shift @fields; + + # + # do the push state, if present. + # + if ($fields[0] =~ /^\^/) { + $fields[0] =~ s/^\^//; + $state_push_state[$num_states] = $fields[0]; + if ($fields[0] eq "" ) { + print " rbbicsts: at line $line_num, expected state after ^ (no spaces).\n"; + exit(-1); + } + shift @fields; + } + + # + # Lastly, do the optional action name. + # + if ($fields[0] ne "") { + $state_func_name[$num_states] = $fields[0]; + shift @fields; + } + + # + # There should be no fields left on the line at this point. + # + if (@fields > 0) { + print " rbbicsts: at line $line_num, unexpected extra stuff on input line.\n"; + print " scanning $fields[0]\n"; + } + $num_states++; +} + +# +# We've read in the whole file, now go back and output the +# C source code for the state transition table. +# +# We read all states first, before writing anything, so that the state numbers +# for the destination states are all available to be written. +# + +# +# Make hashes for the names of the character classes and +# for the names of the actions that appeared. +# +for ($state=1; $state < $num_states; $state++) { + if ($state_char_class[$state] ne "") { + if ($charClasses{$state_char_class[$state]} == 0) { + $charClasses{$state_char_class[$state]} = 1; + } + } + if ($state_func_name[$state] eq "") { + $state_func_name[$state] = "doNOP"; + } + if ($actions{$state_action_name[$state]} == 0) { + $actions{$state_func_name[$state]} = 1; + } +} + +# +# Check that all of the destination states have been defined +# +# +$states{"exit"} = 0; # Predefined state name, terminates state machine. +for ($state=1; $state<$num_states; $state++) { + if ($states{$state_dest_state[$state]} == 0 && $state_dest_state[$state] ne "exit") { + print "Error at line $state_line_num[$state]: target state \"$state_dest_state[$state]\" is not defined.\n"; + $errors++; + } + if ($state_push_state[$state] ne "" && $states{$state_push_state[$state]} == 0) { + print "Error at line $state_line_num[$state]: target state \"$state_push_state[$state]\" is not defined.\n"; + $errors++; + } +} + +die if ($errors>0); + +# +# Assign numbers to each of the character classes classes used. +# Sets are numbered from 128 - 250 +# The values 0-127 in the state table are used for matching +# individual ASCII characters (the only thing that can appear in the rules.) +# The "set" names appearing in the code below (default, etc.) need special +# handling because they do not correspond to a normal set of characters, +# but trigger special handling by code in the state machine. +# +$i = 128; +foreach $setName (sort keys %charClasses) { + if ($setName eq "default") { + $charClasses{$setName} = 255;} + elsif ($setName eq "escaped") { + $charClasses{$setName} = 254;} + elsif ($setName eq "escapedP") { + $charClasses{$setName} = 253;} + elsif ($setName eq "eof") { + $charClasses{$setName} = 252;} + else { + # Normal (single) character class. Number them. + $charClasses{$setName} = $i; + $i++; + } +} + + +my ($sec, $min, $hour, , $day, $mon, $year, $wday, $yday, $isdst) = localtime; +$year += 1900; + +if ($javaOutput) { + print "/*\n"; + print " *******************************************************************************\n"; + print " * Copyright (C) 2003-$year,\n"; + print " * International Business Machines Corporation and others. All Rights Reserved.\n"; + print " *******************************************************************************\n"; + print " */\n"; + print " \n"; + print "package com.ibm.icu.text;\n"; + print " \n"; + print "/**\n"; + print " * Generated Java File. Do not edit by hand.\n"; + print " * This file contains the state table for the ICU Rule Based Break Iterator\n"; + print " * rule parser.\n"; + print " * It is generated by the Perl script \"rbbicst.pl\" from\n"; + print " * the rule parser state definitions file \"rbbirpt.txt\".\n"; + print " * \@internal \n"; + print " *\n"; + print " */\n"; + + print "class RBBIRuleParseTable\n"; + print "{\n"; + + # + # Emit the constants for the actions to be performed. + # + $n = 1; + foreach $act (sort keys %actions) { + print " static final short $act = $n;\n"; + $n++; + } + print " \n"; + + # + # Emit constants for char class names + # + foreach $setName (sort keys %charClasses) { + print " static final short kRuleSet_$setName = $charClasses{$setName};\n"; + } + print "\n\n"; + + + print " static class RBBIRuleTableElement { \n"; + print " short fAction; \n"; + print " short fCharClass; \n"; + print " short fNextState; \n"; + print " short fPushState; \n"; + print " boolean fNextChar; \n"; + print " String fStateName; \n"; + print " RBBIRuleTableElement(short a, int cc, int ns, int ps, boolean nc, String sn) { \n"; + print " fAction = a; \n"; + print " fCharClass = (short)cc; \n"; + print " fNextState = (short)ns; \n"; + print " fPushState = (short)ps; \n"; + print " fNextChar = nc; \n"; + print " fStateName = sn; \n"; + print " } \n"; + print " }; \n"; + print " \n"; + + + print " static RBBIRuleTableElement[] gRuleParseStateTable = { \n "; + print " new RBBIRuleTableElement(doNOP, 0, 0,0, true, null ) // 0 \n"; #output the unused state 0. + for ($state=1; $state < $num_states; $state++) { + print " , new RBBIRuleTableElement($state_func_name[$state],"; + if ($state_literal_chars[$state] ne "") { + $c = $state_literal_chars[$state]; + print("'$c', "); + }else { + print " $charClasses{$state_char_class[$state]},"; + } + print " $states{$state_dest_state[$state]},"; + + # The push-state field is optional. If omitted, fill field with a zero, which flags + # the state machine that there is no push state. + if ($state_push_state[$state] eq "") { + print "0, "; + } else { + print " $states{$state_push_state[$state]},"; + } + print " $state_flag[$state], "; + + # if this is the first row of the table for this state, put out the state name. + if ($stateNames[$state] ne "") { + print " \"$stateNames[$state]\") "; + } else { + print " null ) "; + } + + # Put out a comment showing the number (index) of this state row, + print " // $state "; + print "\n"; + } + print " };\n"; + + print "}; \n"; + +} +else +{ + # + # C++ Output ... + # + + + print "//---------------------------------------------------------------------------------\n"; + print "//\n"; + print "// Generated Header File. Do not edit by hand.\n"; + print "// This file contains the state table for the ICU Rule Based Break Iterator\n"; + print "// rule parser.\n"; + print "// It is generated by the Perl script \"rbbicst.pl\" from\n"; + print "// the rule parser state definitions file \"rbbirpt.txt\".\n"; + print "//\n"; + print "// Copyright (C) 2002-$year International Business Machines Corporation \n"; + print "// and others. All rights reserved. \n"; + print "//\n"; + print "//---------------------------------------------------------------------------------\n"; + print "#ifndef RBBIRPT_H\n"; + print "#define RBBIRPT_H\n"; + print "\n"; + print "U_NAMESPACE_BEGIN\n"; + + # + # Emit the constants for indicies of Unicode Sets + # Define one constant for each of the character classes encountered. + # At the same time, store the index corresponding to the set name back into hash. + # + print "//\n"; + print "// Character classes for RBBI rule scanning.\n"; + print "//\n"; + foreach $setName (sort keys %charClasses) { + if ($charClasses{$setName} < 250) { + # Normal character class. + print " static const uint8_t kRuleSet_$setName = $charClasses{$setName};\n"; + } + } + print "\n\n"; + + # + # Emit the enum for the actions to be performed. + # + print "enum RBBI_RuleParseAction {\n"; + foreach $act (sort keys %actions) { + print " $act,\n"; + } + print " rbbiLastAction};\n\n"; + + # + # Emit the struct definition for transtion table elements. + # + print "//-------------------------------------------------------------------------------\n"; + print "//\n"; + print "// RBBIRuleTableEl represents the structure of a row in the transition table\n"; + print "// for the rule parser state machine.\n"; + print "//-------------------------------------------------------------------------------\n"; + print "struct RBBIRuleTableEl {\n"; + print " RBBI_RuleParseAction fAction;\n"; + print " uint8_t fCharClass; // 0-127: an individual ASCII character\n"; + print " // 128-255: character class index\n"; + print " uint8_t fNextState; // 0-250: normal next-stat numbers\n"; + print " // 255: pop next-state from stack.\n"; + print " uint8_t fPushState;\n"; + print " UBool fNextChar;\n"; + print "};\n\n"; + + # + # emit the state transition table + # + print "static const struct RBBIRuleTableEl gRuleParseStateTable[] = {\n"; + print " {doNOP, 0, 0, 0, TRUE}\n"; # State 0 is a dummy. Real states start with index = 1. + for ($state=1; $state < $num_states; $state++) { + print " , {$state_func_name[$state],"; + if ($state_literal_chars[$state] ne "") { + $c = $state_literal_chars[$state]; + printf(" %d /* $c */,", ord($c)); # use numeric value, so EBCDIC machines are ok. + }else { + print " $charClasses{$state_char_class[$state]},"; + } + print " $states{$state_dest_state[$state]},"; + + # The push-state field is optional. If omitted, fill field with a zero, which flags + # the state machine that there is no push state. + if ($state_push_state[$state] eq "") { + print "0, "; + } else { + print " $states{$state_push_state[$state]},"; + } + print " $state_flag[$state]} "; + + # Put out a C++ comment showing the number (index) of this state row, + # and, if this is the first row of the table for this state, the state name. + print " // $state "; + if ($stateNames[$state] ne "") { + print " $stateNames[$state]"; + } + print "\n"; + }; + print " };\n"; + + + # + # emit a mapping array from state numbers to state names. + # + # This array is used for producing debugging output from the rule parser. + # + print "#ifdef RBBI_DEBUG\n"; + print "static const char * const RBBIRuleStateNames[] = {"; + for ($state=0; $state<$num_states; $state++) { + if ($stateNames[$state] ne "") { + print " \"$stateNames[$state]\",\n"; + } else { + print " 0,\n"; + } + } + print " 0};\n"; + print "#endif\n\n"; + + print "U_NAMESPACE_END\n"; + print "#endif\n"; +} + + + diff --git a/Build/source/libs/icu/icu-xetex/common/rbbidata.cpp b/Build/source/libs/icu/icu-xetex/common/rbbidata.cpp new file mode 100644 index 00000000000..7897b44e7b9 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/rbbidata.cpp @@ -0,0 +1,444 @@ +/* +*************************************************************************** +* Copyright (C) 1999-2007 International Business Machines Corporation * +* and others. All rights reserved. * +*************************************************************************** +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_BREAK_ITERATION + +#include "unicode/utypes.h" +#include "rbbidata.h" +#include "rbbirb.h" +#include "utrie.h" +#include "udatamem.h" +#include "cmemory.h" +#include "cstring.h" +#include "umutex.h" + +#include "uassert.h" + + +//----------------------------------------------------------------------------------- +// +// Trie access folding function. Copied as-is from properties code in uchar.c +// +//----------------------------------------------------------------------------------- +U_CDECL_BEGIN +static int32_t U_CALLCONV +getFoldingOffset(uint32_t data) { + /* if bit 15 is set, then the folding offset is in bits 14..0 of the 16-bit trie result */ + if(data&0x8000) { + return (int32_t)(data&0x7fff); + } else { + return 0; + } +} +U_CDECL_END + +U_NAMESPACE_BEGIN + +//----------------------------------------------------------------------------- +// +// Constructors. +// +//----------------------------------------------------------------------------- +RBBIDataWrapper::RBBIDataWrapper(const RBBIDataHeader *data, UErrorCode &status) { + init(data, status); +} + +RBBIDataWrapper::RBBIDataWrapper(UDataMemory* udm, UErrorCode &status) { + const RBBIDataHeader *d = (const RBBIDataHeader *) + // ((char *)&(udm->pHeader->info) + udm->pHeader->info.size); + // taking into consideration the padding added in by udata_write + ((char *)(udm->pHeader) + udm->pHeader->dataHeader.headerSize); + init(d, status); + fUDataMem = udm; +} + +//----------------------------------------------------------------------------- +// +// init(). Does most of the work of construction, shared between the +// constructors. +// +//----------------------------------------------------------------------------- +void RBBIDataWrapper::init(const RBBIDataHeader *data, UErrorCode &status) { + if (U_FAILURE(status)) { + return; + } + fHeader = data; + if (fHeader->fMagic != 0xb1a0 || + !(fHeader->fFormatVersion[0] == 3 || // ICU 3.4 + *(int32_t *)fHeader->fFormatVersion == 1)) // ICU 3.2 and earlier. + { + status = U_INVALID_FORMAT_ERROR; + return; + } + + fUDataMem = NULL; + fReverseTable = NULL; + fSafeFwdTable = NULL; + fSafeRevTable = NULL; + if (data->fFTableLen != 0) { + fForwardTable = (RBBIStateTable *)((char *)data + fHeader->fFTable); + } + if (data->fRTableLen != 0) { + fReverseTable = (RBBIStateTable *)((char *)data + fHeader->fRTable); + } + if (data->fSFTableLen != 0) { + fSafeFwdTable = (RBBIStateTable *)((char *)data + fHeader->fSFTable); + } + if (data->fSRTableLen != 0) { + fSafeRevTable = (RBBIStateTable *)((char *)data + fHeader->fSRTable); + } + + + utrie_unserialize(&fTrie, + (uint8_t *)data + fHeader->fTrie, + fHeader->fTrieLen, + &status); + if (U_FAILURE(status)) { + return; + } + fTrie.getFoldingOffset=getFoldingOffset; + + + fRuleSource = (UChar *)((char *)data + fHeader->fRuleSource); + fRuleString.setTo(TRUE, fRuleSource, -1); + U_ASSERT(data->fRuleSourceLen > 0); + + fRuleStatusTable = (int32_t *)((char *)data + fHeader->fStatusTable); + fStatusMaxIdx = data->fStatusTableLen / sizeof(int32_t); + + fRefCount = 1; + +#ifdef RBBI_DEBUG + char *debugEnv = getenv("U_RBBIDEBUG"); + if (debugEnv && uprv_strstr(debugEnv, "data")) {this->printData();} +#endif +} + + +//----------------------------------------------------------------------------- +// +// Destructor. Don't call this - use removeReference() instead. +// +//----------------------------------------------------------------------------- +RBBIDataWrapper::~RBBIDataWrapper() { + U_ASSERT(fRefCount == 0); + if (fUDataMem) { + udata_close(fUDataMem); + } else { + uprv_free((void *)fHeader); + } +} + + + +//----------------------------------------------------------------------------- +// +// Operator == Consider two RBBIDataWrappers to be equal if they +// refer to the same underlying data. Although +// the data wrappers are normally shared between +// iterator instances, it's possible to independently +// open the same data twice, and get two instances, which +// should still be ==. +// +//----------------------------------------------------------------------------- +UBool RBBIDataWrapper::operator ==(const RBBIDataWrapper &other) const { + if (fHeader == other.fHeader) { + return TRUE; + } + if (fHeader->fLength != other.fHeader->fLength) { + return FALSE; + } + if (uprv_memcmp(fHeader, other.fHeader, fHeader->fLength) == 0) { + return TRUE; + } + return FALSE; +} + +int32_t RBBIDataWrapper::hashCode() { + return fHeader->fFTableLen; +} + + + +//----------------------------------------------------------------------------- +// +// Reference Counting. A single RBBIDataWrapper object is shared among +// however many RulesBasedBreakIterator instances are +// referencing the same data. +// +//----------------------------------------------------------------------------- +void RBBIDataWrapper::removeReference() { + if (umtx_atomic_dec(&fRefCount) == 0) { + delete this; + } +} + + +RBBIDataWrapper *RBBIDataWrapper::addReference() { + umtx_atomic_inc(&fRefCount); + return this; +} + + + +//----------------------------------------------------------------------------- +// +// getRuleSourceString +// +//----------------------------------------------------------------------------- +const UnicodeString &RBBIDataWrapper::getRuleSourceString() const { + return fRuleString; +} + + +//----------------------------------------------------------------------------- +// +// print - debugging function to dump the runtime data tables. +// +//----------------------------------------------------------------------------- +#ifdef RBBI_DEBUG +void RBBIDataWrapper::printTable(const char *heading, const RBBIStateTable *table) { + uint32_t c; + uint32_t s; + + RBBIDebugPrintf(" %s\n", heading); + + RBBIDebugPrintf("State | Acc LA TagIx"); + for (c=0; cfCatCount; c++) {RBBIDebugPrintf("%3d ", c);} + RBBIDebugPrintf("\n------|---------------"); for (c=0;cfCatCount; c++) { + RBBIDebugPrintf("----"); + } + RBBIDebugPrintf("\n"); + + if (table == NULL) { + RBBIDebugPrintf(" N U L L T A B L E\n\n"); + return; + } + for (s=0; sfNumStates; s++) { + RBBIStateTableRow *row = (RBBIStateTableRow *) + (table->fTableData + (table->fRowLen * s)); + RBBIDebugPrintf("%4d | %3d %3d %3d ", s, row->fAccepting, row->fLookAhead, row->fTagIdx); + for (c=0; cfCatCount; c++) { + RBBIDebugPrintf("%3d ", row->fNextState[c]); + } + RBBIDebugPrintf("\n"); + } + RBBIDebugPrintf("\n"); +} +#endif + + +#ifdef RBBI_DEBUG +void RBBIDataWrapper::printData() { + RBBIDebugPrintf("RBBI Data at %p\n", (void *)fHeader); + RBBIDebugPrintf(" Version = {%d %d %d %d}\n", fHeader->fFormatVersion[0], fHeader->fFormatVersion[1], + fHeader->fFormatVersion[2], fHeader->fFormatVersion[3]); + RBBIDebugPrintf(" total length of data = %d\n", fHeader->fLength); + RBBIDebugPrintf(" number of character categories = %d\n\n", fHeader->fCatCount); + + printTable("Forward State Transition Table", fForwardTable); + printTable("Reverse State Transition Table", fReverseTable); + printTable("Safe Forward State Transition Table", fSafeFwdTable); + printTable("Safe Reverse State Transition Table", fSafeRevTable); + + RBBIDebugPrintf("\nOrignal Rules source:\n"); + for (int32_t c=0; fRuleSource[c] != 0; c++) { + RBBIDebugPrintf("%c", fRuleSource[c]); + } + RBBIDebugPrintf("\n\n"); +} +#endif + + +U_NAMESPACE_END +U_NAMESPACE_USE + +//----------------------------------------------------------------------------- +// +// ubrk_swap - byte swap and char encoding swap of RBBI data +// +//----------------------------------------------------------------------------- + +U_CAPI int32_t U_EXPORT2 +ubrk_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData, + UErrorCode *status) { + + if (status == NULL || U_FAILURE(*status)) { + return 0; + } + if(ds==NULL || inData==NULL || length<-1 || (length>0 && outData==NULL)) { + *status=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + // + // Check that the data header is for for break data. + // (Header contents are defined in genbrk.cpp) + // + const UDataInfo *pInfo = (const UDataInfo *)((const char *)inData+4); + if(!( pInfo->dataFormat[0]==0x42 && /* dataFormat="Brk " */ + pInfo->dataFormat[1]==0x72 && + pInfo->dataFormat[2]==0x6b && + pInfo->dataFormat[3]==0x20 && + pInfo->formatVersion[0]==3 )) { + udata_printError(ds, "ubrk_swap(): data format %02x.%02x.%02x.%02x (format version %02x) is not recognized\n", + pInfo->dataFormat[0], pInfo->dataFormat[1], + pInfo->dataFormat[2], pInfo->dataFormat[3], + pInfo->formatVersion[0]); + *status=U_UNSUPPORTED_ERROR; + return 0; + } + + // + // Swap the data header. (This is the generic ICU Data Header, not the RBBI Specific + // RBBIDataHeader). This swap also conveniently gets us + // the size of the ICU d.h., which lets us locate the start + // of the RBBI specific data. + // + int32_t headerSize=udata_swapDataHeader(ds, inData, length, outData, status); + + + // + // Get the RRBI Data Header, and check that it appears to be OK. + // + // Note: ICU 3.2 and earlier, RBBIDataHeader::fDataFormat was actually + // an int32_t with a value of 1. Starting with ICU 3.4, + // RBBI's fDataFormat matches the dataFormat field from the + // UDataInfo header, four int8_t bytes. The value is {3,1,0,0} + // + const uint8_t *inBytes =(const uint8_t *)inData+headerSize; + RBBIDataHeader *rbbiDH = (RBBIDataHeader *)inBytes; + UBool formatVersionOne = ds->readUInt32(*(int32_t *)rbbiDH->fFormatVersion) == 1; + if (ds->readUInt32(rbbiDH->fMagic) != 0xb1a0 || + !(formatVersionOne || rbbiDH->fFormatVersion[0] == 3) || + ds->readUInt32(rbbiDH->fLength) < sizeof(RBBIDataHeader)) + { + udata_printError(ds, "ubrk_swap(): RBBI Data header is invalid.\n"); + *status=U_UNSUPPORTED_ERROR; + return 0; + } + + // + // Prefight operation? Just return the size + // + int32_t breakDataLength = ds->readUInt32(rbbiDH->fLength); + int32_t totalSize = headerSize + breakDataLength; + if (length < 0) { + return totalSize; + } + + // + // Check that length passed in is consistent with length from RBBI data header. + // + if (length < totalSize) { + udata_printError(ds, "ubrk_swap(): too few bytes (%d after ICU Data header) for break data.\n", + breakDataLength); + *status=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + + + // + // Swap the Data. Do the data itself first, then the RBBI Data Header, because + // we need to reference the header to locate the data, and an + // inplace swap of the header leaves it unusable. + // + uint8_t *outBytes = (uint8_t *)outData + headerSize; + RBBIDataHeader *outputDH = (RBBIDataHeader *)outBytes; + + int32_t tableStartOffset; + int32_t tableLength; + + // + // If not swapping in place, zero out the output buffer before starting. + // Individual tables and other data items within are aligned to 8 byte boundaries + // when originally created. Any unused space between items needs to be zero. + // + if (inBytes != outBytes) { + uprv_memset(outBytes, 0, breakDataLength); + } + + // + // Each state table begins with several 32 bit fields. Calculate the size + // in bytes of these. + // + int32_t topSize = offsetof(RBBIStateTable, fTableData); + + // Forward state table. + tableStartOffset = ds->readUInt32(rbbiDH->fFTable); + tableLength = ds->readUInt32(rbbiDH->fFTableLen); + + if (tableLength > 0) { + ds->swapArray32(ds, inBytes+tableStartOffset, topSize, + outBytes+tableStartOffset, status); + ds->swapArray16(ds, inBytes+tableStartOffset+topSize, tableLength-topSize, + outBytes+tableStartOffset+topSize, status); + } + + // Reverse state table. Same layout as forward table, above. + tableStartOffset = ds->readUInt32(rbbiDH->fRTable); + tableLength = ds->readUInt32(rbbiDH->fRTableLen); + + if (tableLength > 0) { + ds->swapArray32(ds, inBytes+tableStartOffset, topSize, + outBytes+tableStartOffset, status); + ds->swapArray16(ds, inBytes+tableStartOffset+topSize, tableLength-topSize, + outBytes+tableStartOffset+topSize, status); + } + + // Safe Forward state table. Same layout as forward table, above. + tableStartOffset = ds->readUInt32(rbbiDH->fSFTable); + tableLength = ds->readUInt32(rbbiDH->fSFTableLen); + + if (tableLength > 0) { + ds->swapArray32(ds, inBytes+tableStartOffset, topSize, + outBytes+tableStartOffset, status); + ds->swapArray16(ds, inBytes+tableStartOffset+topSize, tableLength-topSize, + outBytes+tableStartOffset+topSize, status); + } + + // Safe Reverse state table. Same layout as forward table, above. + tableStartOffset = ds->readUInt32(rbbiDH->fSRTable); + tableLength = ds->readUInt32(rbbiDH->fSRTableLen); + + if (tableLength > 0) { + ds->swapArray32(ds, inBytes+tableStartOffset, topSize, + outBytes+tableStartOffset, status); + ds->swapArray16(ds, inBytes+tableStartOffset+topSize, tableLength-topSize, + outBytes+tableStartOffset+topSize, status); + } + + // Trie table for character categories + utrie_swap(ds, inBytes+ds->readUInt32(rbbiDH->fTrie), ds->readUInt32(rbbiDH->fTrieLen), + outBytes+ds->readUInt32(rbbiDH->fTrie), status); + + // Source Rules Text. It's UChar data + ds->swapArray16(ds, inBytes+ds->readUInt32(rbbiDH->fRuleSource), ds->readUInt32(rbbiDH->fRuleSourceLen), + outBytes+ds->readUInt32(rbbiDH->fRuleSource), status); + + // Table of rule status values. It's all int_32 values + ds->swapArray32(ds, inBytes+ds->readUInt32(rbbiDH->fStatusTable), ds->readUInt32(rbbiDH->fStatusTableLen), + outBytes+ds->readUInt32(rbbiDH->fStatusTable), status); + + // And, last, the header. + // For the old version one format, the entire header consists of int32_t values. + // For the newer formats, the fDataFormat field is an array of four bytes. + // Swap the whole thing as int32_t, then, for the newer format, re-swap the one field. + // + ds->swapArray32(ds, inBytes, sizeof(RBBIDataHeader), outBytes, status); + if (formatVersionOne == FALSE) { + ds->swapArray32(ds, outputDH->fFormatVersion, 4, outputDH->fFormatVersion, status); + } + + + return totalSize; +} + + +#endif /* #if !UCONFIG_NO_BREAK_ITERATION */ diff --git a/Build/source/libs/icu/icu-xetex/common/rbbidata.h b/Build/source/libs/icu/icu-xetex/common/rbbidata.h new file mode 100644 index 00000000000..7ba1db05d65 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/rbbidata.h @@ -0,0 +1,193 @@ +/* +******************************************************************************* +* +* Copyright (C) 1999-2005, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: rbbidata.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* RBBI data formats Includes +* +* Structs that describes the format of the Binary RBBI data, +* as it is stored in ICU's data file. +* +* RBBIDataWrapper - Instances of this class sit between the +* raw data structs and the RulesBasedBreakIterator objects +* that are created by applications. The wrapper class +* provides reference counting for the underlying data, +* and direct pointers to data that would not otherwise +* be accessible without ugly pointer arithmetic. The +* wrapper does not attempt to provide any higher level +* abstractions for the data itself. +* +* There will be only one instance of RBBIDataWrapper for any +* set of RBBI run time data being shared by instances +* (clones) of RulesBasedBreakIterator. +*/ + +#ifndef __RBBIDATA_H__ +#define __RBBIDATA_H__ + +#include "unicode/utypes.h" +#include "unicode/udata.h" +#include "udataswp.h" + +/** + * Swap RBBI data. See udataswp.h. + * @internal + */ +U_CAPI int32_t U_EXPORT2 +ubrk_swap(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode); + +#ifdef XP_CPLUSPLUS + +#include "unicode/uobject.h" +#include "unicode/unistr.h" +#include "utrie.h" + +U_NAMESPACE_BEGIN + +/* + * The following structs map exactly onto the raw data from ICU common data file. + */ +struct RBBIDataHeader { + uint32_t fMagic; /* == 0xbla0 */ + uint8_t fFormatVersion[4]; /* Data Format. Same as the value in struct UDataInfo */ + /* if there is one associated with this data. */ + /* (version originates in rbbi, is copied to UDataInfo) */ + /* For ICU 3.2 and earlier, this field was */ + /* uint32_t fVersion */ + /* with a value of 1. */ + uint32_t fLength; /* Total length in bytes of this RBBI Data, */ + /* including all sections, not just the header. */ + uint32_t fCatCount; /* Number of character categories. */ + + /* */ + /* Offsets and sizes of each of the subsections within the RBBI data. */ + /* All offsets are bytes from the start of the RBBIDataHeader. */ + /* All sizes are in bytes. */ + /* */ + uint32_t fFTable; /* forward state transition table. */ + uint32_t fFTableLen; + uint32_t fRTable; /* Offset to the reverse state transition table. */ + uint32_t fRTableLen; + uint32_t fSFTable; /* safe point forward transition table */ + uint32_t fSFTableLen; + uint32_t fSRTable; /* safe point reverse transition table */ + uint32_t fSRTableLen; + uint32_t fTrie; /* Offset to Trie data for character categories */ + uint32_t fTrieLen; + uint32_t fRuleSource; /* Offset to the source for for the break */ + uint32_t fRuleSourceLen; /* rules. Stored UChar *. */ + uint32_t fStatusTable; /* Offset to the table of rule status values */ + uint32_t fStatusTableLen; + + uint32_t fReserved[6]; /* Reserved for expansion */ + +}; + + + +struct RBBIStateTableRow { + int16_t fAccepting; /* Non-zero if this row is for an accepting state. */ + /* Value 0: not an accepting state. */ + /* -1: Unconditional Accepting state. */ + /* positive: Look-ahead match has completed. */ + /* Actual boundary position happened earlier */ + /* Value here == fLookAhead in earlier */ + /* state, at actual boundary pos. */ + int16_t fLookAhead; /* Non-zero if this row is for a state that */ + /* corresponds to a '/' in the rule source. */ + /* Value is the same as the fAccepting */ + /* value for the rule (which will appear */ + /* in a different state. */ + int16_t fTagIdx; /* Non-zero if this row covers a {tagged} position */ + /* from a rule. Value is the index in the */ + /* StatusTable of the set of matching */ + /* tags (rule status values) */ + int16_t fReserved; + uint16_t fNextState[2]; /* Next State, indexed by char category. */ + /* Array Size is fNumCols from the */ + /* state table header. */ + /* CAUTION: see RBBITableBuilder::getTableSize() */ + /* before changing anything here. */ +}; + + +struct RBBIStateTable { + uint32_t fNumStates; /* Number of states. */ + uint32_t fRowLen; /* Length of a state table row, in bytes. */ + uint32_t fFlags; /* Option Flags for this state table */ + uint32_t fReserved; /* reserved */ + char fTableData[4]; /* First RBBIStateTableRow begins here. */ + /* (making it char[] simplifies ugly address */ + /* arithmetic for indexing variable length rows.) */ +}; + +typedef enum { + RBBI_LOOKAHEAD_HARD_BREAK = 1, + RBBI_BOF_REQUIRED = 2 +} RBBIStateTableFlags; + + +/* */ +/* The reference counting wrapper class */ +/* */ +class RBBIDataWrapper : public UMemory { +public: + RBBIDataWrapper(const RBBIDataHeader *data, UErrorCode &status); + RBBIDataWrapper(UDataMemory* udm, UErrorCode &status); + ~RBBIDataWrapper(); + + void init(const RBBIDataHeader *data, UErrorCode &status); + RBBIDataWrapper *addReference(); + void removeReference(); + UBool operator ==(const RBBIDataWrapper &other) const; + int32_t hashCode(); + const UnicodeString &getRuleSourceString() const; +#ifdef RBBI_DEBUG + void printData(); + void printTable(const char *heading, const RBBIStateTable *table); +#else + #define printData() + #define printTable(heading, table) +#endif + + /* */ + /* Pointers to items within the data */ + /* */ + const RBBIDataHeader *fHeader; + const RBBIStateTable *fForwardTable; + const RBBIStateTable *fReverseTable; + const RBBIStateTable *fSafeFwdTable; + const RBBIStateTable *fSafeRevTable; + const UChar *fRuleSource; + const int32_t *fRuleStatusTable; + + /* number of int32_t values in the rule status table. Used to sanity check indexing */ + int32_t fStatusMaxIdx; + + UTrie fTrie; + +private: + int32_t fRefCount; + UDataMemory *fUDataMem; + UnicodeString fRuleString; + + RBBIDataWrapper(const RBBIDataWrapper &other); /* forbid copying of this class */ + RBBIDataWrapper &operator=(const RBBIDataWrapper &other); /* forbid copying of this class */ +}; + + + +U_NAMESPACE_END + +#endif /* C++ */ + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/rbbinode.cpp b/Build/source/libs/icu/icu-xetex/common/rbbinode.cpp new file mode 100644 index 00000000000..af467b6d064 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/rbbinode.cpp @@ -0,0 +1,355 @@ +/* +*************************************************************************** +* Copyright (C) 2002-2006 International Business Machines Corporation * +* and others. All rights reserved. * +*************************************************************************** +*/ + +// +// File: rbbinode.cpp +// +// Implementation of class RBBINode, which represents a node in the +// tree generated when parsing the Rules Based Break Iterator rules. +// +// This "Class" is actually closer to a struct. +// Code using it is expected to directly access fields much of the time. +// + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_BREAK_ITERATION + +#include "unicode/unistr.h" +#include "unicode/uniset.h" +#include "unicode/uchar.h" +#include "unicode/parsepos.h" +#include "uvector.h" + +#include "rbbirb.h" +#include "rbbinode.h" + +#include "uassert.h" + + +U_NAMESPACE_BEGIN + +#ifdef RBBI_DEBUG +static int gLastSerial = 0; +#endif + + +//------------------------------------------------------------------------- +// +// Constructor. Just set the fields to reasonable default values. +// +//------------------------------------------------------------------------- +RBBINode::RBBINode(NodeType t) : UMemory() { +#ifdef RBBI_DEBUG + fSerialNum = ++gLastSerial; +#endif + fType = t; + fParent = NULL; + fLeftChild = NULL; + fRightChild = NULL; + fInputSet = NULL; + fFirstPos = 0; + fLastPos = 0; + fNullable = FALSE; + fLookAheadEnd = FALSE; + fVal = 0; + fPrecedence = precZero; + + UErrorCode status = U_ZERO_ERROR; + fFirstPosSet = new UVector(status); // TODO - get a real status from somewhere + fLastPosSet = new UVector(status); + fFollowPos = new UVector(status); + if (t==opCat) {fPrecedence = precOpCat;} + else if (t==opOr) {fPrecedence = precOpOr;} + else if (t==opStart) {fPrecedence = precStart;} + else if (t==opLParen) {fPrecedence = precLParen;} + +} + + +RBBINode::RBBINode(const RBBINode &other) : UMemory(other) { +#ifdef RBBI_DEBUG + fSerialNum = ++gLastSerial; +#endif + fType = other.fType; + fParent = NULL; + fLeftChild = NULL; + fRightChild = NULL; + fInputSet = other.fInputSet; + fPrecedence = other.fPrecedence; + fText = other.fText; + fFirstPos = other.fFirstPos; + fLastPos = other.fLastPos; + fNullable = other.fNullable; + fVal = other.fVal; + UErrorCode status = U_ZERO_ERROR; + fFirstPosSet = new UVector(status); // TODO - get a real status from somewhere + fLastPosSet = new UVector(status); + fFollowPos = new UVector(status); +} + + +//------------------------------------------------------------------------- +// +// Destructor. Deletes both this node AND any child nodes, +// except in the case of variable reference nodes. For +// these, the l. child points back to the definition, which +// is common for all references to the variable, meaning +// it can't be deleted here. +// +//------------------------------------------------------------------------- +RBBINode::~RBBINode() { + // printf("deleting node %8x serial %4d\n", this, this->fSerialNum); + delete fInputSet; + fInputSet = NULL; + + switch (this->fType) { + case varRef: + case setRef: + // for these node types, multiple instances point to the same "children" + // Storage ownership of children handled elsewhere. Don't delete here. + break; + + default: + delete fLeftChild; + fLeftChild = NULL; + delete fRightChild; + fRightChild = NULL; + } + + + delete fFirstPosSet; + delete fLastPosSet; + delete fFollowPos; + +} + + +//------------------------------------------------------------------------- +// +// cloneTree Make a copy of the subtree rooted at this node. +// Discard any variable references encountered along the way, +// and replace with copies of the variable's definitions. +// Used to replicate the expression underneath variable +// references in preparation for generating the DFA tables. +// +//------------------------------------------------------------------------- +RBBINode *RBBINode::cloneTree() { + RBBINode *n; + + if (fType == RBBINode::varRef) { + // If the current node is a variable reference, skip over it + // and clone the definition of the variable instead. + n = fLeftChild->cloneTree(); + } else if (fType == RBBINode::uset) { + n = this; + } else { + n = new RBBINode(*this); + if (fLeftChild != NULL) { + n->fLeftChild = fLeftChild->cloneTree(); + n->fLeftChild->fParent = n; + } + if (fRightChild != NULL) { + n->fRightChild = fRightChild->cloneTree(); + n->fRightChild->fParent = n; + } + } + return n; +} + + + +//------------------------------------------------------------------------- +// +// flattenVariables Walk a parse tree, replacing any variable +// references with a copy of the variable's definition. +// Aside from variables, the tree is not changed. +// +// Return the root of the tree. If the root was not a variable +// reference, it remains unchanged - the root we started with +// is the root we return. If, however, the root was a variable +// reference, the root of the newly cloned replacement tree will +// be returned, and the original tree deleted. +// +// This function works by recursively walking the tree +// without doing anything until a variable reference is +// found, then calling cloneTree() at that point. Any +// nested references are handled by cloneTree(), not here. +// +//------------------------------------------------------------------------- +RBBINode *RBBINode::flattenVariables() { + if (fType == varRef) { + RBBINode *retNode = fLeftChild->cloneTree(); + delete this; + return retNode; + } + + if (fLeftChild != NULL) { + fLeftChild = fLeftChild->flattenVariables(); + fLeftChild->fParent = this; + } + if (fRightChild != NULL) { + fRightChild = fRightChild->flattenVariables(); + fRightChild->fParent = this; + } + return this; +} + + +//------------------------------------------------------------------------- +// +// flattenSets Walk the parse tree, replacing any nodes of type setRef +// with a copy of the expression tree for the set. A set's +// equivalent expression tree is precomputed and saved as +// the left child of the uset node. +// +//------------------------------------------------------------------------- +void RBBINode::flattenSets() { + U_ASSERT(fType != setRef); + + if (fLeftChild != NULL) { + if (fLeftChild->fType==setRef) { + RBBINode *setRefNode = fLeftChild; + RBBINode *usetNode = setRefNode->fLeftChild; + RBBINode *replTree = usetNode->fLeftChild; + fLeftChild = replTree->cloneTree(); + fLeftChild->fParent = this; + delete setRefNode; + } else { + fLeftChild->flattenSets(); + } + } + + if (fRightChild != NULL) { + if (fRightChild->fType==setRef) { + RBBINode *setRefNode = fRightChild; + RBBINode *usetNode = setRefNode->fLeftChild; + RBBINode *replTree = usetNode->fLeftChild; + fRightChild = replTree->cloneTree(); + fRightChild->fParent = this; + delete setRefNode; + } else { + fRightChild->flattenSets(); + } + } +} + + + +//------------------------------------------------------------------------- +// +// findNodes() Locate all the nodes of the specified type, starting +// at the specified root. +// +//------------------------------------------------------------------------- +void RBBINode::findNodes(UVector *dest, RBBINode::NodeType kind, UErrorCode &status) { + /* test for buffer overflows */ + if (U_FAILURE(status)) { + return; + } + if (fType == kind) { + dest->addElement(this, status); + } + if (fLeftChild != NULL) { + fLeftChild->findNodes(dest, kind, status); + } + if (fRightChild != NULL) { + fRightChild->findNodes(dest, kind, status); + } +} + + +//------------------------------------------------------------------------- +// +// print. Print out a single node, for debugging. +// +//------------------------------------------------------------------------- +#ifdef RBBI_DEBUG +void RBBINode::printNode() { + static const char * const nodeTypeNames[] = { + "setRef", + "uset", + "varRef", + "leafChar", + "lookAhead", + "tag", + "endMark", + "opStart", + "opCat", + "opOr", + "opStar", + "opPlus", + "opQuestion", + "opBreak", + "opReverse", + "opLParen" + }; + + if (this==NULL) { + RBBIDebugPrintf("%10p", (void *)this); + } else { + RBBIDebugPrintf("%10p %12s %10p %10p %10p %4d %6d %d ", + (void *)this, nodeTypeNames[fType], (void *)fParent, (void *)fLeftChild, (void *)fRightChild, + fSerialNum, fFirstPos, fVal); + if (fType == varRef) { + RBBI_DEBUG_printUnicodeString(fText); + } + } + RBBIDebugPrintf("\n"); +} +#endif + + +#ifdef RBBI_DEBUG +U_CFUNC void RBBI_DEBUG_printUnicodeString(const UnicodeString &s, int minWidth) +{ + int i; + for (i=0; iprintNode(); + if (this != NULL) { + // Only dump the definition under a variable reference if asked to. + // Unconditinally dump children of all other node types. + if (fType != varRef) { + if (fLeftChild != NULL) { + fLeftChild->printTree(FALSE); + } + + if (fRightChild != NULL) { + fRightChild->printTree(FALSE); + } + } + } +} +#endif + + + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_BREAK_ITERATION */ diff --git a/Build/source/libs/icu/icu-xetex/common/rbbinode.h b/Build/source/libs/icu/icu-xetex/common/rbbinode.h new file mode 100644 index 00000000000..0cbf6a71abe --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/rbbinode.h @@ -0,0 +1,118 @@ +/******************************************************************** + * COPYRIGHT: + * Copyright (c) 2001-2006, International Business Machines Corporation and + * others. All Rights Reserved. + ********************************************************************/ + +#ifndef RBBINODE_H +#define RBBINODE_H + +#include "unicode/utypes.h" +#include "unicode/uobject.h" + +// +// class RBBINode +// +// Represents a node in the parse tree generated when reading +// a rule file. +// + +U_NAMESPACE_BEGIN + +class UnicodeSet; +class UVector; + +class RBBINode : public UMemory { + public: + enum NodeType { + setRef, + uset, + varRef, + leafChar, + lookAhead, + tag, + endMark, + opStart, + opCat, + opOr, + opStar, + opPlus, + opQuestion, + opBreak, + opReverse, + opLParen + }; + + enum OpPrecedence { + precZero, + precStart, + precLParen, + precOpOr, + precOpCat + }; + + NodeType fType; + RBBINode *fParent; + RBBINode *fLeftChild; + RBBINode *fRightChild; + UnicodeSet *fInputSet; // For uset nodes only. + OpPrecedence fPrecedence; // For binary ops only. + + UnicodeString fText; // Text corresponding to this node. + // May be lazily evaluated when (if) needed + // for some node types. + int fFirstPos; // Position in the rule source string of the + // first text associated with the node. + // If there's a left child, this will be the same + // as that child's left pos. + int fLastPos; // Last position in the rule source string + // of any text associated with this node. + // If there's a right child, this will be the same + // as that child's last postion. + + UBool fNullable; // See Aho. + int32_t fVal; // For leafChar nodes, the value. + // Values are the character category, + // corresponds to columns in the final + // state transition table. + + UBool fLookAheadEnd; // For endMark nodes, set TRUE if + // marking the end of a look-ahead rule. + + UVector *fFirstPosSet; + UVector *fLastPosSet; // TODO: rename fFirstPos & fLastPos to avoid confusion. + UVector *fFollowPos; + + + RBBINode(NodeType t); + RBBINode(const RBBINode &other); + ~RBBINode(); + + RBBINode *cloneTree(); + RBBINode *flattenVariables(); + void flattenSets(); + void findNodes(UVector *dest, RBBINode::NodeType kind, UErrorCode &status); + +#ifdef RBBI_DEBUG + void printNode(); + void printTree(UBool withHeading); +#endif + + private: + RBBINode &operator = (const RBBINode &other); // No defs. + UBool operator == (const RBBINode &other); // Private, so these functions won't accidently be used. + +#ifdef RBBI_DEBUG + int fSerialNum; // Debugging aids. +#endif +}; + +#ifdef RBBI_DEBUG +U_CFUNC void +RBBI_DEBUG_printUnicodeString(const UnicodeString &s, int minWidth=0); +#endif + +U_NAMESPACE_END + +#endif + diff --git a/Build/source/libs/icu/icu-xetex/common/rbbirb.cpp b/Build/source/libs/icu/icu-xetex/common/rbbirb.cpp new file mode 100644 index 00000000000..a3d32c396ff --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/rbbirb.cpp @@ -0,0 +1,312 @@ +// +// file: rbbirb.cpp +// +// Copyright (C) 2002-2005, International Business Machines Corporation and others. +// All Rights Reserved. +// +// This file contains the RBBIRuleBuilder class implementation. This is the main class for +// building (compiling) break rules into the tables required by the runtime +// RBBI engine. +// + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_BREAK_ITERATION + +#include "unicode/brkiter.h" +#include "unicode/rbbi.h" +#include "unicode/ubrk.h" +#include "unicode/unistr.h" +#include "unicode/uniset.h" +#include "unicode/uchar.h" +#include "unicode/uchriter.h" +#include "unicode/parsepos.h" +#include "unicode/parseerr.h" +#include "cmemory.h" +#include "cstring.h" + +#include "rbbirb.h" +#include "rbbinode.h" + +#include "rbbiscan.h" +#include "rbbisetb.h" +#include "rbbitblb.h" +#include "rbbidata.h" + + +U_NAMESPACE_BEGIN + + +//---------------------------------------------------------------------------------------- +// +// Constructor. +// +//---------------------------------------------------------------------------------------- +RBBIRuleBuilder::RBBIRuleBuilder(const UnicodeString &rules, + UParseError &parseErr, + UErrorCode &status) + : fRules(rules) +{ + fStatus = &status; // status is checked below + fParseError = &parseErr; + fDebugEnv = NULL; +#ifdef RBBI_DEBUG + fDebugEnv = getenv("U_RBBIDEBUG"); +#endif + + + fForwardTree = NULL; + fReverseTree = NULL; + fSafeFwdTree = NULL; + fSafeRevTree = NULL; + fDefaultTree = &fForwardTree; + fForwardTables = NULL; + fReverseTables = NULL; + fSafeFwdTables = NULL; + fSafeRevTables = NULL; + fRuleStatusVals = NULL; + fChainRules = FALSE; + fLBCMNoChain = FALSE; + fLookAheadHardBreak = FALSE; + fUSetNodes = NULL; + fRuleStatusVals = NULL; + fScanner = NULL; + fSetBuilder = NULL; + + if (U_FAILURE(status)) { + return; + } + + fUSetNodes = new UVector(status); // bcos status gets overwritten here + fRuleStatusVals = new UVector(status); + fScanner = new RBBIRuleScanner(this); + fSetBuilder = new RBBISetBuilder(this); + if (U_FAILURE(status)) { + return; + } + if(fSetBuilder == 0 || fScanner == 0 || fUSetNodes == 0 || fRuleStatusVals == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + } +} + + + +//---------------------------------------------------------------------------------------- +// +// Destructor +// +//---------------------------------------------------------------------------------------- +RBBIRuleBuilder::~RBBIRuleBuilder() { + + int i; + for (i=0; ; i++) { + RBBINode *n = (RBBINode *)fUSetNodes->elementAt(i); + if (n==NULL) { + break; + } + delete n; + } + + delete fUSetNodes; + delete fSetBuilder; + delete fForwardTables; + delete fReverseTables; + delete fSafeFwdTables; + delete fSafeRevTables; + + delete fForwardTree; + delete fReverseTree; + delete fSafeFwdTree; + delete fSafeRevTree; + delete fScanner; + delete fRuleStatusVals; +} + + + + + +//---------------------------------------------------------------------------------------- +// +// flattenData() - Collect up the compiled RBBI rule data and put it into +// the format for saving in ICU data files, +// which is also the format needed by the RBBI runtime engine. +// +//---------------------------------------------------------------------------------------- +static int32_t align8(int32_t i) {return (i+7) & 0xfffffff8;} + +RBBIDataHeader *RBBIRuleBuilder::flattenData() { + int32_t i; + + if (U_FAILURE(*fStatus)) { + return NULL; + } + + // Remove comments and whitespace from the rules to make it smaller. + UnicodeString strippedRules((const UnicodeString&)RBBIRuleScanner::stripRules(fRules)); + + // Calculate the size of each section in the data. + // Sizes here are padded up to a multiple of 8 for better memory alignment. + // Sections sizes actually stored in the header are for the actual data + // without the padding. + // + int32_t headerSize = align8(sizeof(RBBIDataHeader)); + int32_t forwardTableSize = align8(fForwardTables->getTableSize()); + int32_t reverseTableSize = align8(fReverseTables->getTableSize()); + int32_t safeFwdTableSize = align8(fSafeFwdTables->getTableSize()); + int32_t safeRevTableSize = align8(fSafeRevTables->getTableSize()); + int32_t trieSize = align8(fSetBuilder->getTrieSize()); + int32_t statusTableSize = align8(fRuleStatusVals->size() * sizeof(int32_t)); + int32_t rulesSize = align8((strippedRules.length()+1) * sizeof(UChar)); + + int32_t totalSize = headerSize + forwardTableSize + reverseTableSize + + safeFwdTableSize + safeRevTableSize + + statusTableSize + trieSize + rulesSize; + + RBBIDataHeader *data = (RBBIDataHeader *)uprv_malloc(totalSize); + if (data == NULL) { + *fStatus = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + uprv_memset(data, 0, totalSize); + + + data->fMagic = 0xb1a0; + data->fFormatVersion[0] = 3; + data->fFormatVersion[1] = 1; + data->fFormatVersion[2] = 0; + data->fFormatVersion[3] = 0; + data->fLength = totalSize; + data->fCatCount = fSetBuilder->getNumCharCategories(); + + data->fFTable = headerSize; + data->fFTableLen = forwardTableSize; + data->fRTable = data->fFTable + forwardTableSize; + data->fRTableLen = reverseTableSize; + data->fSFTable = data->fRTable + reverseTableSize; + data->fSFTableLen = safeFwdTableSize; + data->fSRTable = data->fSFTable + safeFwdTableSize; + data->fSRTableLen = safeRevTableSize; + + data->fTrie = data->fSRTable + safeRevTableSize; + data->fTrieLen = fSetBuilder->getTrieSize(); + data->fStatusTable = data->fTrie + trieSize; + data->fStatusTableLen= statusTableSize; + data->fRuleSource = data->fStatusTable + statusTableSize; + data->fRuleSourceLen = strippedRules.length() * sizeof(UChar); + + uprv_memset(data->fReserved, 0, sizeof(data->fReserved)); + + fForwardTables->exportTable((uint8_t *)data + data->fFTable); + fReverseTables->exportTable((uint8_t *)data + data->fRTable); + fSafeFwdTables->exportTable((uint8_t *)data + data->fSFTable); + fSafeRevTables->exportTable((uint8_t *)data + data->fSRTable); + fSetBuilder->serializeTrie ((uint8_t *)data + data->fTrie); + + int32_t *ruleStatusTable = (int32_t *)((uint8_t *)data + data->fStatusTable); + for (i=0; isize(); i++) { + ruleStatusTable[i] = fRuleStatusVals->elementAti(i); + } + + strippedRules.extract((UChar *)((uint8_t *)data+data->fRuleSource), rulesSize/2+1, *fStatus); + + return data; +} + + + + + + +//---------------------------------------------------------------------------------------- +// +// createRuleBasedBreakIterator construct from source rules that are passed in +// in a UnicodeString +// +//---------------------------------------------------------------------------------------- +BreakIterator * +RBBIRuleBuilder::createRuleBasedBreakIterator( const UnicodeString &rules, + UParseError &parseError, + UErrorCode &status) +{ + // status checked below + + // + // Read the input rules, generate a parse tree, symbol table, + // and list of all Unicode Sets referenced by the rules. + // + RBBIRuleBuilder builder(rules, parseError, status); + builder.fScanner->parse(); + if (U_FAILURE(status)) { // status checked here bcos build below doesn't + return NULL; + } + + // + // UnicodeSet processing. + // Munge the Unicode Sets to create a set of character categories. + // Generate the mapping tables (TRIE) from input 32-bit characters to + // the character categories. + // + builder.fSetBuilder->build(); + + + // + // Generate the DFA state transition table. + // + builder.fForwardTables = new RBBITableBuilder(&builder, &builder.fForwardTree); + builder.fReverseTables = new RBBITableBuilder(&builder, &builder.fReverseTree); + builder.fSafeFwdTables = new RBBITableBuilder(&builder, &builder.fSafeFwdTree); + builder.fSafeRevTables = new RBBITableBuilder(&builder, &builder.fSafeRevTree); + if (U_SUCCESS(status) + && (builder.fForwardTables == NULL || builder.fReverseTables == NULL || + builder.fSafeFwdTables == NULL || builder.fSafeRevTables == NULL)) + { + status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + + builder.fForwardTables->build(); + builder.fReverseTables->build(); + builder.fSafeFwdTables->build(); + builder.fSafeRevTables->build(); + if (U_FAILURE(status)) { + return NULL; + } + +#ifdef RBBI_DEBUG + if (builder.fDebugEnv && uprv_strstr(builder.fDebugEnv, "states")) { + builder.fForwardTables->printRuleStatusTable(); + } +#endif + + // + // Package up the compiled data into a memory image + // in the run-time format. + // + RBBIDataHeader *data = builder.flattenData(); // returns NULL if error + + + // + // Clean up the compiler related stuff + // + + + // + // Create a break iterator from the compiled rules. + // (Identical to creation from stored pre-compiled rules) + // + // status is checked after init in construction. + RuleBasedBreakIterator *This = new RuleBasedBreakIterator(data, status); + if (U_FAILURE(status)) { + delete This; + This = NULL; + } + else if(This == NULL) { // test for NULL + status = U_MEMORY_ALLOCATION_ERROR; + } + return This; +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_BREAK_ITERATION */ diff --git a/Build/source/libs/icu/icu-xetex/common/rbbirb.h b/Build/source/libs/icu/icu-xetex/common/rbbirb.h new file mode 100644 index 00000000000..d7a5383606c --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/rbbirb.h @@ -0,0 +1,211 @@ +// +// rbbirb.h +// +// Copyright (C) 2002-2005, International Business Machines Corporation and others. +// All Rights Reserved. +// +// This file contains declarations for several classes from the +// Rule Based Break Iterator rule builder. +// + + +#ifndef RBBIRB_H +#define RBBIRB_H + +#include "unicode/utypes.h" +#include "unicode/uobject.h" +#include "unicode/rbbi.h" +#include "unicode/uniset.h" +#include "unicode/parseerr.h" +#include "uhash.h" +#include "uvector.h" +#include "unicode/symtable.h"// For UnicodeSet parsing, is the interface that + // looks up references to $variables within a set. + + + +U_NAMESPACE_BEGIN + +class RBBIRuleScanner; +struct RBBIRuleTableEl; +class RBBISetBuilder; +class RBBINode; +class RBBITableBuilder; + + + +//-------------------------------------------------------------------------------- +// +// RBBISymbolTable. Implements SymbolTable interface that is used by the +// UnicodeSet parser to resolve references to $variables. +// +//-------------------------------------------------------------------------------- +class RBBISymbolTableEntry : public UMemory { // The symbol table hash table contains one +public: // of these structs for each entry. + RBBISymbolTableEntry(); + UnicodeString key; + RBBINode *val; + ~RBBISymbolTableEntry(); + +private: + RBBISymbolTableEntry(const RBBISymbolTableEntry &other); // forbid copying of this class + RBBISymbolTableEntry &operator=(const RBBISymbolTableEntry &other); // forbid copying of this class +}; + + +class RBBISymbolTable : public UMemory, public SymbolTable { +private: + const UnicodeString &fRules; + UHashtable *fHashTable; + RBBIRuleScanner *fRuleScanner; + + // These next two fields are part of the mechanism for passing references to + // already-constructed UnicodeSets back to the UnicodeSet constructor + // when the pattern includes $variable references. + const UnicodeString ffffString; // = "/uffff" + UnicodeSet *fCachedSetLookup; + +public: + // API inherited from class SymbolTable + virtual const UnicodeString* lookup(const UnicodeString& s) const; + virtual const UnicodeFunctor* lookupMatcher(UChar32 ch) const; + virtual UnicodeString parseReference(const UnicodeString& text, + ParsePosition& pos, int32_t limit) const; + + // Additional Functions + RBBISymbolTable(RBBIRuleScanner *, const UnicodeString &fRules, UErrorCode &status); + virtual ~RBBISymbolTable(); + + virtual RBBINode *lookupNode(const UnicodeString &key) const; + virtual void addEntry (const UnicodeString &key, RBBINode *val, UErrorCode &err); + +#ifdef RBBI_DEBUG + virtual void rbbiSymtablePrint() const; +#else + // A do-nothing inline function for non-debug builds. Member funcs can't be empty + // or the call sites won't compile. + int32_t fFakeField; + #define rbbiSymtablePrint() fFakeField=0; +#endif + +private: + RBBISymbolTable(const RBBISymbolTable &other); // forbid copying of this class + RBBISymbolTable &operator=(const RBBISymbolTable &other); // forbid copying of this class +}; + + +//-------------------------------------------------------------------------------- +// +// class RBBIRuleBuilder The top-level class handling RBBI rule compiling. +// +//-------------------------------------------------------------------------------- +class RBBIRuleBuilder : public UMemory { +public: + + // Create a rule based break iterator from a set of rules. + // This function is the main entry point into the rule builder. The + // public ICU API for creating RBBIs uses this function to do the actual work. + // + static BreakIterator * createRuleBasedBreakIterator( const UnicodeString &rules, + UParseError &parseError, + UErrorCode &status); + +public: + // The "public" functions and data members that appear below are accessed + // (and shared) by the various parts that make up the rule builder. They + // are NOT intended to be accessed by anything outside of the + // rule builder implementation. + RBBIRuleBuilder(const UnicodeString &rules, + UParseError &parseErr, + UErrorCode &status + ); + + virtual ~RBBIRuleBuilder(); + char *fDebugEnv; // controls debug trace output + UErrorCode *fStatus; // Error reporting. Keeping status + UParseError *fParseError; // here avoids passing it everywhere. + const UnicodeString &fRules; // The rule string that we are compiling + + RBBIRuleScanner *fScanner; // The scanner. + RBBINode *fForwardTree; // The parse trees, generated by the scanner, + RBBINode *fReverseTree; // then manipulated by subsequent steps. + RBBINode *fSafeFwdTree; + RBBINode *fSafeRevTree; + + RBBINode **fDefaultTree; // For rules not qualified with a ! + // the tree to which they belong to. + + UBool fChainRules; // True for chained Unicode TR style rules. + // False for traditional regexp rules. + + UBool fLBCMNoChain; // True: suppress chaining of rules on + // chars with LineBreak property == CM. + + UBool fLookAheadHardBreak; // True: Look ahead matches cause an + // immediate break, no continuing for the + // longest match. + + RBBISetBuilder *fSetBuilder; // Set and Character Category builder. + UVector *fUSetNodes; // Vector of all uset nodes. + + RBBITableBuilder *fForwardTables; // State transition tables + RBBITableBuilder *fReverseTables; + RBBITableBuilder *fSafeFwdTables; + RBBITableBuilder *fSafeRevTables; + + UVector *fRuleStatusVals; // The values that can be returned + // from getRuleStatus(). + + RBBIDataHeader *flattenData(); // Create the flattened (runtime format) + // data tables.. +private: + RBBIRuleBuilder(const RBBIRuleBuilder &other); // forbid copying of this class + RBBIRuleBuilder &operator=(const RBBIRuleBuilder &other); // forbid copying of this class +}; + + + + +//---------------------------------------------------------------------------- +// +// RBBISetTableEl is an entry in the hash table of UnicodeSets that have +// been encountered. The val Node will be of nodetype uset +// and contain pointers to the actual UnicodeSets. +// The Key is the source string for initializing the set. +// +// The hash table is used to avoid creating duplicate +// unnamed (not $var references) UnicodeSets. +// +// Memory Management: +// The Hash Table owns these RBBISetTableEl structs and +// the key strings. It does NOT own the val nodes. +// +//---------------------------------------------------------------------------- +struct RBBISetTableEl { + UnicodeString *key; + RBBINode *val; +}; + + +//---------------------------------------------------------------------------- +// +// RBBIDebugPrintf Printf equivalent, for debugging output. +// Conditional compilation of the implementation lets us +// get rid of the stdio dependency in environments where it +// is unavailable. +// +//---------------------------------------------------------------------------- +#ifdef RBBI_DEBUG +#include +#define RBBIDebugPrintf printf +#define RBBIDebugPuts puts +#else +#undef RBBIDebugPrintf +#define RBBIDebugPuts(arg) +#endif + +U_NAMESPACE_END +#endif + + + diff --git a/Build/source/libs/icu/icu-xetex/common/rbbirpt.h b/Build/source/libs/icu/icu-xetex/common/rbbirpt.h new file mode 100644 index 00000000000..deea57b7add --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/rbbirpt.h @@ -0,0 +1,275 @@ +//--------------------------------------------------------------------------------- +// +// Generated Header File. Do not edit by hand. +// This file contains the state table for the ICU Rule Based Break Iterator +// rule parser. +// It is generated by the Perl script "rbbicst.pl" from +// the rule parser state definitions file "rbbirpt.txt". +// +// Copyright (C) 2002-2005 International Business Machines Corporation +// and others. All rights reserved. +// +//--------------------------------------------------------------------------------- +#ifndef RBBIRPT_H +#define RBBIRPT_H + +U_NAMESPACE_BEGIN +// +// Character classes for RBBI rule scanning. +// + static const uint8_t kRuleSet_digit_char = 128; + static const uint8_t kRuleSet_name_char = 129; + static const uint8_t kRuleSet_name_start_char = 130; + static const uint8_t kRuleSet_rule_char = 131; + static const uint8_t kRuleSet_white_space = 132; + + +enum RBBI_RuleParseAction { + doCheckVarDef, + doDotAny, + doEndAssign, + doEndOfRule, + doEndVariableName, + doExit, + doExprCatOperator, + doExprFinished, + doExprOrOperator, + doExprRParen, + doExprStart, + doLParen, + doNOP, + doOptionEnd, + doOptionStart, + doReverseDir, + doRuleChar, + doRuleError, + doRuleErrorAssignExpr, + doScanUnicodeSet, + doSlash, + doStartAssign, + doStartTagValue, + doStartVariableName, + doTagDigit, + doTagExpectedError, + doTagValue, + doUnaryOpPlus, + doUnaryOpQuestion, + doUnaryOpStar, + doVariableNameExpectedErr, + rbbiLastAction}; + +//------------------------------------------------------------------------------- +// +// RBBIRuleTableEl represents the structure of a row in the transition table +// for the rule parser state machine. +//------------------------------------------------------------------------------- +struct RBBIRuleTableEl { + RBBI_RuleParseAction fAction; + uint8_t fCharClass; // 0-127: an individual ASCII character + // 128-255: character class index + uint8_t fNextState; // 0-250: normal next-stat numbers + // 255: pop next-state from stack. + uint8_t fPushState; + UBool fNextChar; +}; + +static const struct RBBIRuleTableEl gRuleParseStateTable[] = { + {doNOP, 0, 0, 0, TRUE} + , {doExprStart, 254, 21, 8, FALSE} // 1 start + , {doNOP, 132, 1,0, TRUE} // 2 + , {doExprStart, 36 /* $ */, 80, 90, FALSE} // 3 + , {doNOP, 33 /* ! */, 11,0, TRUE} // 4 + , {doNOP, 59 /* ; */, 1,0, TRUE} // 5 + , {doNOP, 252, 0,0, FALSE} // 6 + , {doExprStart, 255, 21, 8, FALSE} // 7 + , {doEndOfRule, 59 /* ; */, 1,0, TRUE} // 8 break-rule-end + , {doNOP, 132, 8,0, TRUE} // 9 + , {doRuleError, 255, 95,0, FALSE} // 10 + , {doNOP, 33 /* ! */, 13,0, TRUE} // 11 rev-option + , {doReverseDir, 255, 20, 8, FALSE} // 12 + , {doOptionStart, 130, 15,0, TRUE} // 13 option-scan1 + , {doRuleError, 255, 95,0, FALSE} // 14 + , {doNOP, 129, 15,0, TRUE} // 15 option-scan2 + , {doOptionEnd, 255, 17,0, FALSE} // 16 + , {doNOP, 59 /* ; */, 1,0, TRUE} // 17 option-scan3 + , {doNOP, 132, 17,0, TRUE} // 18 + , {doRuleError, 255, 95,0, FALSE} // 19 + , {doExprStart, 255, 21, 8, FALSE} // 20 reverse-rule + , {doRuleChar, 254, 30,0, TRUE} // 21 term + , {doNOP, 132, 21,0, TRUE} // 22 + , {doRuleChar, 131, 30,0, TRUE} // 23 + , {doNOP, 91 /* [ */, 86, 30, FALSE} // 24 + , {doLParen, 40 /* ( */, 21, 30, TRUE} // 25 + , {doNOP, 36 /* $ */, 80, 29, FALSE} // 26 + , {doDotAny, 46 /* . */, 30,0, TRUE} // 27 + , {doRuleError, 255, 95,0, FALSE} // 28 + , {doCheckVarDef, 255, 30,0, FALSE} // 29 term-var-ref + , {doNOP, 132, 30,0, TRUE} // 30 expr-mod + , {doUnaryOpStar, 42 /* * */, 35,0, TRUE} // 31 + , {doUnaryOpPlus, 43 /* + */, 35,0, TRUE} // 32 + , {doUnaryOpQuestion, 63 /* ? */, 35,0, TRUE} // 33 + , {doNOP, 255, 35,0, FALSE} // 34 + , {doExprCatOperator, 254, 21,0, FALSE} // 35 expr-cont + , {doNOP, 132, 35,0, TRUE} // 36 + , {doExprCatOperator, 131, 21,0, FALSE} // 37 + , {doExprCatOperator, 91 /* [ */, 21,0, FALSE} // 38 + , {doExprCatOperator, 40 /* ( */, 21,0, FALSE} // 39 + , {doExprCatOperator, 36 /* $ */, 21,0, FALSE} // 40 + , {doExprCatOperator, 46 /* . */, 21,0, FALSE} // 41 + , {doExprCatOperator, 47 /* / */, 47,0, FALSE} // 42 + , {doExprCatOperator, 123 /* { */, 59,0, TRUE} // 43 + , {doExprOrOperator, 124 /* | */, 21,0, TRUE} // 44 + , {doExprRParen, 41 /* ) */, 255,0, TRUE} // 45 + , {doExprFinished, 255, 255,0, FALSE} // 46 + , {doSlash, 47 /* / */, 49,0, TRUE} // 47 look-ahead + , {doNOP, 255, 95,0, FALSE} // 48 + , {doExprCatOperator, 254, 21,0, FALSE} // 49 expr-cont-no-slash + , {doNOP, 132, 35,0, TRUE} // 50 + , {doExprCatOperator, 131, 21,0, FALSE} // 51 + , {doExprCatOperator, 91 /* [ */, 21,0, FALSE} // 52 + , {doExprCatOperator, 40 /* ( */, 21,0, FALSE} // 53 + , {doExprCatOperator, 36 /* $ */, 21,0, FALSE} // 54 + , {doExprCatOperator, 46 /* . */, 21,0, FALSE} // 55 + , {doExprOrOperator, 124 /* | */, 21,0, TRUE} // 56 + , {doExprRParen, 41 /* ) */, 255,0, TRUE} // 57 + , {doExprFinished, 255, 255,0, FALSE} // 58 + , {doNOP, 132, 59,0, TRUE} // 59 tag-open + , {doStartTagValue, 128, 62,0, FALSE} // 60 + , {doTagExpectedError, 255, 95,0, FALSE} // 61 + , {doNOP, 132, 66,0, TRUE} // 62 tag-value + , {doNOP, 125 /* } */, 66,0, FALSE} // 63 + , {doTagDigit, 128, 62,0, TRUE} // 64 + , {doTagExpectedError, 255, 95,0, FALSE} // 65 + , {doNOP, 132, 66,0, TRUE} // 66 tag-close + , {doTagValue, 125 /* } */, 69,0, TRUE} // 67 + , {doTagExpectedError, 255, 95,0, FALSE} // 68 + , {doExprCatOperator, 254, 21,0, FALSE} // 69 expr-cont-no-tag + , {doNOP, 132, 69,0, TRUE} // 70 + , {doExprCatOperator, 131, 21,0, FALSE} // 71 + , {doExprCatOperator, 91 /* [ */, 21,0, FALSE} // 72 + , {doExprCatOperator, 40 /* ( */, 21,0, FALSE} // 73 + , {doExprCatOperator, 36 /* $ */, 21,0, FALSE} // 74 + , {doExprCatOperator, 46 /* . */, 21,0, FALSE} // 75 + , {doExprCatOperator, 47 /* / */, 47,0, FALSE} // 76 + , {doExprOrOperator, 124 /* | */, 21,0, TRUE} // 77 + , {doExprRParen, 41 /* ) */, 255,0, TRUE} // 78 + , {doExprFinished, 255, 255,0, FALSE} // 79 + , {doStartVariableName, 36 /* $ */, 82,0, TRUE} // 80 scan-var-name + , {doNOP, 255, 95,0, FALSE} // 81 + , {doNOP, 130, 84,0, TRUE} // 82 scan-var-start + , {doVariableNameExpectedErr, 255, 95,0, FALSE} // 83 + , {doNOP, 129, 84,0, TRUE} // 84 scan-var-body + , {doEndVariableName, 255, 255,0, FALSE} // 85 + , {doScanUnicodeSet, 91 /* [ */, 255,0, TRUE} // 86 scan-unicode-set + , {doScanUnicodeSet, 112 /* p */, 255,0, TRUE} // 87 + , {doScanUnicodeSet, 80 /* P */, 255,0, TRUE} // 88 + , {doNOP, 255, 95,0, FALSE} // 89 + , {doNOP, 132, 90,0, TRUE} // 90 assign-or-rule + , {doStartAssign, 61 /* = */, 21, 93, TRUE} // 91 + , {doNOP, 255, 29, 8, FALSE} // 92 + , {doEndAssign, 59 /* ; */, 1,0, TRUE} // 93 assign-end + , {doRuleErrorAssignExpr, 255, 95,0, FALSE} // 94 + , {doExit, 255, 95,0, TRUE} // 95 errorDeath + }; +#ifdef RBBI_DEBUG +static const char * const RBBIRuleStateNames[] = { 0, + "start", + 0, + 0, + 0, + 0, + 0, + 0, + "break-rule-end", + 0, + 0, + "rev-option", + 0, + "option-scan1", + 0, + "option-scan2", + 0, + "option-scan3", + 0, + 0, + "reverse-rule", + "term", + 0, + 0, + 0, + 0, + 0, + 0, + 0, + "term-var-ref", + "expr-mod", + 0, + 0, + 0, + 0, + "expr-cont", + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + "look-ahead", + 0, + "expr-cont-no-slash", + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + "tag-open", + 0, + 0, + "tag-value", + 0, + 0, + 0, + "tag-close", + 0, + 0, + "expr-cont-no-tag", + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + "scan-var-name", + 0, + "scan-var-start", + 0, + "scan-var-body", + 0, + "scan-unicode-set", + 0, + 0, + 0, + "assign-or-rule", + 0, + 0, + "assign-end", + 0, + "errorDeath", + 0}; +#endif + +U_NAMESPACE_END +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/rbbirpt.txt b/Build/source/libs/icu/icu-xetex/common/rbbirpt.txt new file mode 100644 index 00000000000..8e932a6037a --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/rbbirpt.txt @@ -0,0 +1,315 @@ + +#***************************************************************************** +# +# Copyright (C) 2002-2003, International Business Machines Corporation and others. +# All Rights Reserved. +# +#***************************************************************************** +# +# file: rbbirpt.txt +# ICU Break Iterator Rule Parser State Table +# +# This state table is used when reading and parsing a set of RBBI rules +# The rule parser uses a state machine; the data in this file define the +# state transitions that occur for each input character. +# +# *** This file defines the RBBI rule grammar. This is it. +# *** The determination of what is accepted is here. +# +# This file is processed by a perl script "rbbicst.pl" to produce initialized C arrays +# that are then built with the rule parser. +# + +# +# Here is the syntax of the state definitions in this file: +# +# +#StateName: +# input-char n next-state ^push-state action +# input-char n next-state ^push-state action +# | | | | | +# | | | | |--- action to be performed by state machine +# | | | | See function RBBIRuleScanner::doParseActions() +# | | | | +# | | | |--- Push this named state onto the state stack. +# | | | Later, when next state is specified as "pop", +# | | | the pushed state will become the current state. +# | | | +# | | |--- Transition to this state if the current input character matches the input +# | | character or char class in the left hand column. "pop" causes the next +# | | state to be popped from the state stack. +# | | +# | |--- When making the state transition specified on this line, advance to the next +# | character from the input only if 'n' appears here. +# | +# |--- Character or named character classes to test for. If the current character being scanned +# matches, peform the actions and go to the state specified on this line. +# The input character is tested sequentally, in the order written. The characters and +# character classes tested for do not need to be mutually exclusive. The first match wins. +# + + + + +# +# start state, scan position is at the beginning of the rules file, or in between two rules. +# +start: + escaped term ^break-rule-end doExprStart + white_space n start + '$' scan-var-name ^assign-or-rule doExprStart + '!' n rev-option + ';' n start # ignore empty rules. + eof exit + default term ^break-rule-end doExprStart + +# +# break-rule-end: Returned from doing a break-rule expression. +# +break-rule-end: + ';' n start doEndOfRule + white_space n break-rule-end + default errorDeath doRuleError + + +# +# ! We've just scanned a '!', indicating either a !!key word flag or a +# !Reverse rule. +# +rev-option: + '!' n option-scan1 + default reverse-rule ^break-rule-end doReverseDir + +option-scan1: + name_start_char n option-scan2 doOptionStart + default errorDeath doRuleError + +option-scan2: + name_char n option-scan2 + default option-scan3 doOptionEnd + +option-scan3: + ';' n start + white_space n option-scan3 + default errorDeath doRuleError + + +reverse-rule: + default term ^break-rule-end doExprStart + + +# +# term. Eat through a single rule character, or a composite thing, which +# could be a parenthesized expression, a variable name, or a Unicode Set. +# +term: + escaped n expr-mod doRuleChar + white_space n term + rule_char n expr-mod doRuleChar + '[' scan-unicode-set ^expr-mod + '(' n term ^expr-mod doLParen + '$' scan-var-name ^term-var-ref + '.' n expr-mod doDotAny + default errorDeath doRuleError + + + +# +# term-var-ref We've just finished scanning a reference to a $variable. +# Check that the variable was defined. +# The variable name scanning is in common with assignment statements, +# so the check can't be done there. +term-var-ref: + default expr-mod doCheckVarDef + + +# +# expr-mod We've just finished scanning a term, now look for the optional +# trailing '*', '?', '+' +# +expr-mod: + white_space n expr-mod + '*' n expr-cont doUnaryOpStar + '+' n expr-cont doUnaryOpPlus + '?' n expr-cont doUnaryOpQuestion + default expr-cont + + +# +# expr-cont Expression, continuation. At a point where additional terms are +# allowed, but not required. +# +expr-cont: + escaped term doExprCatOperator + white_space n expr-cont + rule_char term doExprCatOperator + '[' term doExprCatOperator + '(' term doExprCatOperator + '$' term doExprCatOperator + '.' term doExprCatOperator + '/' look-ahead doExprCatOperator + '{' n tag-open doExprCatOperator + '|' n term doExprOrOperator + ')' n pop doExprRParen + default pop doExprFinished + + +# +# look-ahead Scanning a '/', which identifies a break point, assuming that the +# remainder of the expression matches. +# +# Generate a parse tree as if this was a special kind of input symbol +# appearing in an otherwise normal concatenation expression. +# +look-ahead: + '/' n expr-cont-no-slash doSlash + default errorDeath + + +# +# expr-cont-no-slash Expression, continuation. At a point where additional terms are +# allowed, but not required. Just like +# expr-cont, above, except that no '/' +# look-ahead symbol is permitted. +# +expr-cont-no-slash: + escaped term doExprCatOperator + white_space n expr-cont + rule_char term doExprCatOperator + '[' term doExprCatOperator + '(' term doExprCatOperator + '$' term doExprCatOperator + '.' term doExprCatOperator + '|' n term doExprOrOperator + ')' n pop doExprRParen + default pop doExprFinished + + +# +# tags scanning a '{', the opening delimiter for a tag that identifies +# the kind of match. Scan the whole {dddd} tag, where d=digit +# +tag-open: + white_space n tag-open + digit_char tag-value doStartTagValue + default errorDeath doTagExpectedError + +tag-value: + white_space n tag-close + '}' tag-close + digit_char n tag-value doTagDigit + default errorDeath doTagExpectedError + +tag-close: + white_space n tag-close + '}' n expr-cont-no-tag doTagValue + default errorDeath doTagExpectedError + + + +# +# expr-cont-no-tag Expression, continuation. At a point where additional terms are +# allowed, but not required. Just like +# expr-cont, above, except that no "{ddd}" +# tagging is permitted. +# +expr-cont-no-tag: + escaped term doExprCatOperator + white_space n expr-cont-no-tag + rule_char term doExprCatOperator + '[' term doExprCatOperator + '(' term doExprCatOperator + '$' term doExprCatOperator + '.' term doExprCatOperator + '/' look-ahead doExprCatOperator + '|' n term doExprOrOperator + ')' n pop doExprRParen + default pop doExprFinished + + + + +# +# Variable Name Scanning. +# +# The state that branched to here must have pushed a return state +# to go to after completion of the variable name scanning. +# +# The current input character must be the $ that introduces the name. +# The $ is consummed here rather than in the state that first detected it +# so that the doStartVariableName action only needs to happen in one +# place (here), and the other states don't need to worry about it. +# +scan-var-name: + '$' n scan-var-start doStartVariableName + default errorDeath + + +scan-var-start: + name_start_char n scan-var-body + default errorDeath doVariableNameExpectedErr + +scan-var-body: + name_char n scan-var-body + default pop doEndVariableName + + + +# +# scan-unicode-set Unicode Sets are parsed by the the UnicodeSet class. +# Within the RBBI parser, after finding the first character +# of a Unicode Set, we just hand the rule input at that +# point of to the Unicode Set constructor, then pick +# up parsing after the close of the set. +# +# The action for this state invokes the UnicodeSet parser. +# +scan-unicode-set: + '[' n pop doScanUnicodeSet + 'p' n pop doScanUnicodeSet + 'P' n pop doScanUnicodeSet + default errorDeath + + + + + + + +# +# assign-or-rule. A $variable was encountered at the start of something, could be +# either an assignment statement or a rule, depending on whether an '=' +# follows the variable name. We get to this state when the variable name +# scanning does a return. +# +assign-or-rule: + white_space n assign-or-rule + '=' n term ^assign-end doStartAssign # variable was target of assignment + default term-var-ref ^break-rule-end # variable was a term in a rule + + + +# +# assign-end This state is entered when the end of the expression on the +# right hand side of an assignment is found. We get here via +# a pop; this state is pushed when the '=' in an assignment is found. +# +# The only thing allowed at this point is a ';'. The RHS of an +# assignment must look like a rule expression, and we come here +# when what is being scanned no longer looks like an expression. +# +assign-end: + ';' n start doEndAssign + default errorDeath doRuleErrorAssignExpr + + + +# +# errorDeath. This state is specified as the next state whenever a syntax error +# in the source rules is detected. Barring bugs, the state machine will never +# actually get here, but will stop because of the action associated with the error. +# But, just in case, this state asks the state machine to exit. +errorDeath: + default n errorDeath doExit + + diff --git a/Build/source/libs/icu/icu-xetex/common/rbbiscan.cpp b/Build/source/libs/icu/icu-xetex/common/rbbiscan.cpp new file mode 100644 index 00000000000..2d5c5ec1f02 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/rbbiscan.cpp @@ -0,0 +1,1193 @@ + +// +// file: rbbiscan.cpp +// +// Copyright (C) 2002-2007, International Business Machines Corporation and others. +// All Rights Reserved. +// +// This file contains the Rule Based Break Iterator Rule Builder functions for +// scanning the rules and assembling a parse tree. This is the first phase +// of compiling the rules. +// +// The overall of the rules is managed by class RBBIRuleBuilder, which will +// create and use an instance of this class as part of the process. +// + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_BREAK_ITERATION + +#include "unicode/unistr.h" +#include "unicode/uniset.h" +#include "unicode/uchar.h" +#include "unicode/uchriter.h" +#include "unicode/parsepos.h" +#include "unicode/parseerr.h" +#include "util.h" +#include "cmemory.h" +#include "cstring.h" + +#include "rbbirpt.h" // Contains state table for the rbbi rules parser. + // generated by a Perl script. +#include "rbbirb.h" +#include "rbbinode.h" +#include "rbbiscan.h" +#include "rbbitblb.h" + +#include "uassert.h" + + +//---------------------------------------------------------------------------------------- +// +// Unicode Set init strings for each of the character classes needed for parsing a rule file. +// (Initialized with hex values for portability to EBCDIC based machines. +// Really ugly, but there's no good way to avoid it.) +// +// The sets are referred to by name in the rbbirpt.txt, which is the +// source form of the state transition table for the RBBI rule parser. +// +//---------------------------------------------------------------------------------------- +static const UChar gRuleSet_rule_char_pattern[] = { + // [ ^ [ \ p { Z } \ u 0 0 2 0 + 0x5b, 0x5e, 0x5b, 0x5c, 0x70, 0x7b, 0x5a, 0x7d, 0x5c, 0x75, 0x30, 0x30, 0x32, 0x30, + // - \ u 0 0 7 f ] - [ \ p + 0x2d, 0x5c, 0x75, 0x30, 0x30, 0x37, 0x66, 0x5d, 0x2d, 0x5b, 0x5c, 0x70, + // { L } ] - [ \ p { N } ] ] + 0x7b, 0x4c, 0x7d, 0x5d, 0x2d, 0x5b, 0x5c, 0x70, 0x7b, 0x4e, 0x7d, 0x5d, 0x5d, 0}; + +static const UChar gRuleSet_name_char_pattern[] = { +// [ _ \ p { L } \ p { N } ] + 0x5b, 0x5f, 0x5c, 0x70, 0x7b, 0x4c, 0x7d, 0x5c, 0x70, 0x7b, 0x4e, 0x7d, 0x5d, 0}; + +static const UChar gRuleSet_digit_char_pattern[] = { +// [ 0 - 9 ] + 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0}; + +static const UChar gRuleSet_name_start_char_pattern[] = { +// [ _ \ p { L } ] + 0x5b, 0x5f, 0x5c, 0x70, 0x7b, 0x4c, 0x7d, 0x5d, 0 }; + +static const UChar kAny[] = {0x61, 0x6e, 0x79, 0x00}; // "any" + + +U_CDECL_BEGIN +static void U_CALLCONV RBBISetTable_deleter(void *p) { + U_NAMESPACE_QUALIFIER RBBISetTableEl *px = (U_NAMESPACE_QUALIFIER RBBISetTableEl *)p; + delete px->key; + // Note: px->val is owned by the linked list "fSetsListHead" in scanner. + // Don't delete the value nodes here. + uprv_free(px); +} +U_CDECL_END + +U_NAMESPACE_BEGIN + +//---------------------------------------------------------------------------------------- +// +// Constructor. +// +//---------------------------------------------------------------------------------------- +RBBIRuleScanner::RBBIRuleScanner(RBBIRuleBuilder *rb) +{ + fRB = rb; + fStackPtr = 0; + fStack[fStackPtr] = 0; + fNodeStackPtr = 0; + fRuleNum = 0; + fNodeStack[0] = NULL; + + fRuleSets[kRuleSet_rule_char-128] = NULL; + fRuleSets[kRuleSet_white_space-128] = NULL; + fRuleSets[kRuleSet_name_char-128] = NULL; + fRuleSets[kRuleSet_name_start_char-128] = NULL; + fRuleSets[kRuleSet_digit_char-128] = NULL; + fSymbolTable = NULL; + fSetTable = NULL; + + fScanIndex = 0; + fNextIndex = 0; + + fReverseRule = FALSE; + fLookAheadRule = FALSE; + + fLineNum = 1; + fCharNum = 0; + fQuoteMode = FALSE; + + // Do not check status until after all critical fields are sufficiently initialized + // that the destructor can run cleanly. + if (U_FAILURE(*rb->fStatus)) { + return; + } + + // + // Set up the constant Unicode Sets. + // Note: These could be made static, lazily initialized, and shared among + // all instances of RBBIRuleScanners. BUT this is quite a bit simpler, + // and the time to build these few sets should be small compared to a + // full break iterator build. + fRuleSets[kRuleSet_rule_char-128] = new UnicodeSet(gRuleSet_rule_char_pattern, *rb->fStatus); + fRuleSets[kRuleSet_white_space-128] = uprv_openRuleWhiteSpaceSet(rb->fStatus); + fRuleSets[kRuleSet_name_char-128] = new UnicodeSet(gRuleSet_name_char_pattern, *rb->fStatus); + fRuleSets[kRuleSet_name_start_char-128] = new UnicodeSet(gRuleSet_name_start_char_pattern, *rb->fStatus); + fRuleSets[kRuleSet_digit_char-128] = new UnicodeSet(gRuleSet_digit_char_pattern, *rb->fStatus); + if (*rb->fStatus == U_ILLEGAL_ARGUMENT_ERROR) { + // This case happens if ICU's data is missing. UnicodeSet tries to look up property + // names from the init string, can't find them, and claims an illegal arguement. + // Change the error so that the actual problem will be clearer to users. + *rb->fStatus = U_BRK_INIT_ERROR; + } + if (U_FAILURE(*rb->fStatus)) { + return; + } + + fSymbolTable = new RBBISymbolTable(this, rb->fRules, *rb->fStatus); + fSetTable = uhash_open(uhash_hashUnicodeString, uhash_compareUnicodeString, NULL, rb->fStatus); + uhash_setValueDeleter(fSetTable, RBBISetTable_deleter); +} + + + +//---------------------------------------------------------------------------------------- +// +// Destructor +// +//---------------------------------------------------------------------------------------- +RBBIRuleScanner::~RBBIRuleScanner() { + delete fRuleSets[kRuleSet_rule_char-128]; + delete fRuleSets[kRuleSet_white_space-128]; + delete fRuleSets[kRuleSet_name_char-128]; + delete fRuleSets[kRuleSet_name_start_char-128]; + delete fRuleSets[kRuleSet_digit_char-128]; + + delete fSymbolTable; + if (fSetTable != NULL) { + uhash_close(fSetTable); + fSetTable = NULL; + + } + + + // Node Stack. + // Normally has one entry, which is the entire parse tree for the rules. + // If errors occured, there may be additional subtrees left on the stack. + while (fNodeStackPtr > 0) { + delete fNodeStack[fNodeStackPtr]; + fNodeStackPtr--; + } + +} + +//---------------------------------------------------------------------------------------- +// +// doParseAction Do some action during rule parsing. +// Called by the parse state machine. +// Actions build the parse tree and Unicode Sets, +// and maintain the parse stack for nested expressions. +// +// TODO: unify EParseAction and RBBI_RuleParseAction enum types. +// They represent exactly the same thing. They're separate +// only to work around enum forward declaration restrictions +// in some compilers, while at the same time avoiding multiple +// definitions problems. I'm sure that there's a better way. +// +//---------------------------------------------------------------------------------------- +UBool RBBIRuleScanner::doParseActions(int32_t action) +{ + RBBINode *n = NULL; + + UBool returnVal = TRUE; + + switch (action) { + + case doExprStart: + pushNewNode(RBBINode::opStart); + fRuleNum++; + break; + + + case doExprOrOperator: + { + fixOpStack(RBBINode::precOpCat); + RBBINode *operandNode = fNodeStack[fNodeStackPtr--]; + RBBINode *orNode = pushNewNode(RBBINode::opOr); + orNode->fLeftChild = operandNode; + operandNode->fParent = orNode; + } + break; + + case doExprCatOperator: + // concatenation operator. + // For the implicit concatenation of adjacent terms in an expression that are + // not separated by any other operator. Action is invoked between the + // actions for the two terms. + { + fixOpStack(RBBINode::precOpCat); + RBBINode *operandNode = fNodeStack[fNodeStackPtr--]; + RBBINode *catNode = pushNewNode(RBBINode::opCat); + catNode->fLeftChild = operandNode; + operandNode->fParent = catNode; + } + break; + + case doLParen: + // Open Paren. + // The openParen node is a dummy operation type with a low precedence, + // which has the affect of ensuring that any real binary op that + // follows within the parens binds more tightly to the operands than + // stuff outside of the parens. + pushNewNode(RBBINode::opLParen); + break; + + case doExprRParen: + fixOpStack(RBBINode::precLParen); + break; + + case doNOP: + break; + + case doStartAssign: + // We've just scanned "$variable = " + // The top of the node stack has the $variable ref node. + + // Save the start position of the RHS text in the StartExpression node + // that precedes the $variableReference node on the stack. + // This will eventually be used when saving the full $variable replacement + // text as a string. + n = fNodeStack[fNodeStackPtr-1]; + n->fFirstPos = fNextIndex; // move past the '=' + + // Push a new start-of-expression node; needed to keep parse of the + // RHS expression happy. + pushNewNode(RBBINode::opStart); + break; + + + + + case doEndAssign: + { + // We have reached the end of an assignement statement. + // Current scan char is the ';' that terminates the assignment. + + // Terminate expression, leaves expression parse tree rooted in TOS node. + fixOpStack(RBBINode::precStart); + + RBBINode *startExprNode = fNodeStack[fNodeStackPtr-2]; + RBBINode *varRefNode = fNodeStack[fNodeStackPtr-1]; + RBBINode *RHSExprNode = fNodeStack[fNodeStackPtr]; + + // Save original text of right side of assignment, excluding the terminating ';' + // in the root of the node for the right-hand-side expression. + RHSExprNode->fFirstPos = startExprNode->fFirstPos; + RHSExprNode->fLastPos = fScanIndex; + fRB->fRules.extractBetween(RHSExprNode->fFirstPos, RHSExprNode->fLastPos, RHSExprNode->fText); + + // Expression parse tree becomes l. child of the $variable reference node. + varRefNode->fLeftChild = RHSExprNode; + RHSExprNode->fParent = varRefNode; + + // Make a symbol table entry for the $variableRef node. + fSymbolTable->addEntry(varRefNode->fText, varRefNode, *fRB->fStatus); + if (U_FAILURE(*fRB->fStatus)) { + // This is a round-about way to get the parse position set + // so that duplicate symbols error messages include a line number. + UErrorCode t = *fRB->fStatus; + *fRB->fStatus = U_ZERO_ERROR; + error(t); + } + + // Clean up the stack. + delete startExprNode; + fNodeStackPtr-=3; + break; + } + + case doEndOfRule: + { + fixOpStack(RBBINode::precStart); // Terminate expression, leaves expression + if (U_FAILURE(*fRB->fStatus)) { // parse tree rooted in TOS node. + break; + } +#ifdef RBBI_DEBUG + if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "rtree")) {printNodeStack("end of rule");} +#endif + U_ASSERT(fNodeStackPtr == 1); + + // If this rule includes a look-ahead '/', add a endMark node to the + // expression tree. + if (fLookAheadRule) { + RBBINode *thisRule = fNodeStack[fNodeStackPtr]; + RBBINode *endNode = pushNewNode(RBBINode::endMark); + RBBINode *catNode = pushNewNode(RBBINode::opCat); + fNodeStackPtr -= 2; + catNode->fLeftChild = thisRule; + catNode->fRightChild = endNode; + fNodeStack[fNodeStackPtr] = catNode; + endNode->fVal = fRuleNum; + endNode->fLookAheadEnd = TRUE; + } + + // All rule expressions are ORed together. + // The ';' that terminates an expression really just functions as a '|' with + // a low operator prededence. + // + // Each of the four sets of rules are collected separately. + // (forward, reverse, safe_forward, safe_reverse) + // OR this rule into the appropriate group of them. + // + RBBINode **destRules = (fReverseRule? &fRB->fReverseTree : fRB->fDefaultTree); + + if (*destRules != NULL) { + // This is not the first rule encounted. + // OR previous stuff (from *destRules) + // with the current rule expression (on the Node Stack) + // with the resulting OR expression going to *destRules + // + RBBINode *thisRule = fNodeStack[fNodeStackPtr]; + RBBINode *prevRules = *destRules; + RBBINode *orNode = pushNewNode(RBBINode::opOr); + orNode->fLeftChild = prevRules; + prevRules->fParent = orNode; + orNode->fRightChild = thisRule; + thisRule->fParent = orNode; + *destRules = orNode; + } + else + { + // This is the first rule encountered (for this direction). + // Just move its parse tree from the stack to *destRules. + *destRules = fNodeStack[fNodeStackPtr]; + } + fReverseRule = FALSE; // in preparation for the next rule. + fLookAheadRule = FALSE; + fNodeStackPtr = 0; + } + break; + + + case doRuleError: + error(U_BRK_RULE_SYNTAX); + returnVal = FALSE; + break; + + + case doVariableNameExpectedErr: + error(U_BRK_RULE_SYNTAX); + break; + + + // + // Unary operands + ? * + // These all appear after the operand to which they apply. + // When we hit one, the operand (may be a whole sub expression) + // will be on the top of the stack. + // Unary Operator becomes TOS, with the old TOS as its one child. + case doUnaryOpPlus: + { + RBBINode *operandNode = fNodeStack[fNodeStackPtr--]; + RBBINode *plusNode = pushNewNode(RBBINode::opPlus); + plusNode->fLeftChild = operandNode; + operandNode->fParent = plusNode; + } + break; + + case doUnaryOpQuestion: + { + RBBINode *operandNode = fNodeStack[fNodeStackPtr--]; + RBBINode *qNode = pushNewNode(RBBINode::opQuestion); + qNode->fLeftChild = operandNode; + operandNode->fParent = qNode; + } + break; + + case doUnaryOpStar: + { + RBBINode *operandNode = fNodeStack[fNodeStackPtr--]; + RBBINode *starNode = pushNewNode(RBBINode::opStar); + starNode->fLeftChild = operandNode; + operandNode->fParent = starNode; + } + break; + + case doRuleChar: + // A "Rule Character" is any single character that is a literal part + // of the regular expression. Like a, b and c in the expression "(abc*) | [:L:]" + // These are pretty uncommon in break rules; the terms are more commonly + // sets. To keep things uniform, treat these characters like as + // sets that just happen to contain only one character. + { + n = pushNewNode(RBBINode::setRef); + findSetFor(fC.fChar, n); + n->fFirstPos = fScanIndex; + n->fLastPos = fNextIndex; + fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText); + break; + } + + case doDotAny: + // scanned a ".", meaning match any single character. + { + n = pushNewNode(RBBINode::setRef); + findSetFor(kAny, n); + n->fFirstPos = fScanIndex; + n->fLastPos = fNextIndex; + fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText); + break; + } + + case doSlash: + // Scanned a '/', which identifies a look-ahead break position in a rule. + n = pushNewNode(RBBINode::lookAhead); + n->fVal = fRuleNum; + n->fFirstPos = fScanIndex; + n->fLastPos = fNextIndex; + fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText); + fLookAheadRule = TRUE; + break; + + + case doStartTagValue: + // Scanned a '{', the opening delimiter for a tag value within a rule. + n = pushNewNode(RBBINode::tag); + n->fVal = 0; + n->fFirstPos = fScanIndex; + n->fLastPos = fNextIndex; + break; + + case doTagDigit: + // Just scanned a decimal digit that's part of a tag value + { + n = fNodeStack[fNodeStackPtr]; + uint32_t v = u_charDigitValue(fC.fChar); + U_ASSERT(v < 10); + n->fVal = n->fVal*10 + v; + break; + } + + case doTagValue: + n = fNodeStack[fNodeStackPtr]; + n->fLastPos = fNextIndex; + fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText); + break; + + case doTagExpectedError: + error(U_BRK_MALFORMED_RULE_TAG); + returnVal = FALSE; + break; + + case doOptionStart: + // Scanning a !!option. At the start of string. + fOptionStart = fScanIndex; + break; + + case doOptionEnd: + { + UnicodeString opt(fRB->fRules, fOptionStart, fScanIndex-fOptionStart); + if (opt == UNICODE_STRING("chain", 5)) { + fRB->fChainRules = TRUE; + } else if (opt == UNICODE_STRING("LBCMNoChain", 11)) { + fRB->fLBCMNoChain = TRUE; + } else if (opt == UNICODE_STRING("forward", 7)) { + fRB->fDefaultTree = &fRB->fForwardTree; + } else if (opt == UNICODE_STRING("reverse", 7)) { + fRB->fDefaultTree = &fRB->fReverseTree; + } else if (opt == UNICODE_STRING("safe_forward", 12)) { + fRB->fDefaultTree = &fRB->fSafeFwdTree; + } else if (opt == UNICODE_STRING("safe_reverse", 12)) { + fRB->fDefaultTree = &fRB->fSafeRevTree; + } else if (opt == UNICODE_STRING("lookAheadHardBreak", 18)) { + fRB->fLookAheadHardBreak = TRUE; + } else { + error(U_BRK_UNRECOGNIZED_OPTION); + } + } + break; + + case doReverseDir: + fReverseRule = TRUE; + break; + + case doStartVariableName: + n = pushNewNode(RBBINode::varRef); + if (U_FAILURE(*fRB->fStatus)) { + break; + } + n->fFirstPos = fScanIndex; + break; + + case doEndVariableName: + n = fNodeStack[fNodeStackPtr]; + if (n==NULL || n->fType != RBBINode::varRef) { + error(U_BRK_INTERNAL_ERROR); + break; + } + n->fLastPos = fScanIndex; + fRB->fRules.extractBetween(n->fFirstPos+1, n->fLastPos, n->fText); + // Look the newly scanned name up in the symbol table + // If there's an entry, set the l. child of the var ref to the replacement expression. + // (We also pass through here when scanning assignments, but no harm is done, other + // than a slight wasted effort that seems hard to avoid. Lookup will be null) + n->fLeftChild = fSymbolTable->lookupNode(n->fText); + break; + + case doCheckVarDef: + n = fNodeStack[fNodeStackPtr]; + if (n->fLeftChild == NULL) { + error(U_BRK_UNDEFINED_VARIABLE); + returnVal = FALSE; + } + break; + + case doExprFinished: + break; + + case doRuleErrorAssignExpr: + error(U_BRK_ASSIGN_ERROR); + returnVal = FALSE; + break; + + case doExit: + returnVal = FALSE; + break; + + case doScanUnicodeSet: + scanSet(); + break; + + default: + error(U_BRK_INTERNAL_ERROR); + returnVal = FALSE; + break; + } + return returnVal; +} + + + + +//---------------------------------------------------------------------------------------- +// +// Error Report a rule parse error. +// Only report it if no previous error has been recorded. +// +//---------------------------------------------------------------------------------------- +void RBBIRuleScanner::error(UErrorCode e) { + if (U_SUCCESS(*fRB->fStatus)) { + *fRB->fStatus = e; + fRB->fParseError->line = fLineNum; + fRB->fParseError->offset = fCharNum; + fRB->fParseError->preContext[0] = 0; + fRB->fParseError->preContext[0] = 0; + } +} + + + + +//---------------------------------------------------------------------------------------- +// +// fixOpStack The parse stack holds partially assembled chunks of the parse tree. +// An entry on the stack may be as small as a single setRef node, +// or as large as the parse tree +// for an entire expression (this will be the one item left on the stack +// when the parsing of an RBBI rule completes. +// +// This function is called when a binary operator is encountered. +// It looks back up the stack for operators that are not yet associated +// with a right operand, and if the precedence of the stacked operator >= +// the precedence of the current operator, binds the operand left, +// to the previously encountered operator. +// +//---------------------------------------------------------------------------------------- +void RBBIRuleScanner::fixOpStack(RBBINode::OpPrecedence p) { + RBBINode *n; + // printNodeStack("entering fixOpStack()"); + for (;;) { + n = fNodeStack[fNodeStackPtr-1]; // an operator node + if (n->fPrecedence == 0) { + RBBIDebugPuts("RBBIRuleScanner::fixOpStack, bad operator node"); + error(U_BRK_INTERNAL_ERROR); + return; + } + + if (n->fPrecedence < p || n->fPrecedence <= RBBINode::precLParen) { + // The most recent operand goes with the current operator, + // not with the previously stacked one. + break; + } + // Stack operator is a binary op ( '|' or concatenation) + // TOS operand becomes right child of this operator. + // Resulting subexpression becomes the TOS operand. + n->fRightChild = fNodeStack[fNodeStackPtr]; + fNodeStack[fNodeStackPtr]->fParent = n; + fNodeStackPtr--; + // printNodeStack("looping in fixOpStack() "); + } + + if (p <= RBBINode::precLParen) { + // Scan is at a right paren or end of expression. + // The scanned item must match the stack, or else there was an error. + // Discard the left paren (or start expr) node from the stack, + // leaving the completed (sub)expression as TOS. + if (n->fPrecedence != p) { + // Right paren encountered matched start of expression node, or + // end of expression matched with a left paren node. + error(U_BRK_MISMATCHED_PAREN); + } + fNodeStack[fNodeStackPtr-1] = fNodeStack[fNodeStackPtr]; + fNodeStackPtr--; + // Delete the now-discarded LParen or Start node. + delete n; + } + // printNodeStack("leaving fixOpStack()"); +} + + + + +//---------------------------------------------------------------------------------------- +// +// findSetFor given a UnicodeString, +// - find the corresponding Unicode Set (uset node) +// (create one if necessary) +// - Set fLeftChild of the caller's node (should be a setRef node) +// to the uset node +// Maintain a hash table of uset nodes, so the same one is always used +// for the same string. +// If a "to adopt" set is provided and we haven't seen this key before, +// add the provided set to the hash table. +// If the string is one (32 bit) char in length, the set contains +// just one element which is the char in question. +// If the string is "any", return a set containing all chars. +// +//---------------------------------------------------------------------------------------- +void RBBIRuleScanner::findSetFor(const UnicodeString &s, RBBINode *node, UnicodeSet *setToAdopt) { + + RBBISetTableEl *el; + + // First check whether we've already cached a set for this string. + // If so, just use the cached set in the new node. + // delete any set provided by the caller, since we own it. + el = (RBBISetTableEl *)uhash_get(fSetTable, &s); + if (el != NULL) { + delete setToAdopt; + node->fLeftChild = el->val; + U_ASSERT(node->fLeftChild->fType == RBBINode::uset); + return; + } + + // Haven't seen this set before. + // If the caller didn't provide us with a prebuilt set, + // create a new UnicodeSet now. + if (setToAdopt == NULL) { + if (s.compare(kAny, -1) == 0) { + setToAdopt = new UnicodeSet(0x000000, 0x10ffff); + } else { + UChar32 c; + c = s.char32At(0); + setToAdopt = new UnicodeSet(c, c); + } + } + + // + // Make a new uset node to refer to this UnicodeSet + // This new uset node becomes the child of the caller's setReference node. + // + RBBINode *usetNode = new RBBINode(RBBINode::uset); + usetNode->fInputSet = setToAdopt; + usetNode->fParent = node; + node->fLeftChild = usetNode; + usetNode->fText = s; + + + // + // Add the new uset node to the list of all uset nodes. + // + fRB->fUSetNodes->addElement(usetNode, *fRB->fStatus); + + + // + // Add the new set to the set hash table. + // + el = (RBBISetTableEl *)uprv_malloc(sizeof(RBBISetTableEl)); + UnicodeString *tkey = new UnicodeString(s); + if (tkey == NULL || el == NULL || setToAdopt == NULL) { + error(U_MEMORY_ALLOCATION_ERROR); + return; + } + el->key = tkey; + el->val = usetNode; + uhash_put(fSetTable, el->key, el, fRB->fStatus); + + return; +} + + + +// +// Assorted Unicode character constants. +// Numeric because there is no portable way to enter them as literals. +// (Think EBCDIC). +// +static const UChar chCR = 0x0d; // New lines, for terminating comments. +static const UChar chLF = 0x0a; +static const UChar chNEL = 0x85; // NEL newline variant +static const UChar chLS = 0x2028; // Unicode Line Separator +static const UChar chApos = 0x27; // single quote, for quoted chars. +static const UChar chPound = 0x23; // '#', introduces a comment. +static const UChar chBackSlash = 0x5c; // '\' introduces a char escape +static const UChar chLParen = 0x28; +static const UChar chRParen = 0x29; + + +//---------------------------------------------------------------------------------------- +// +// stripRules Return a rules string without unnecessary +// characters. +// +//---------------------------------------------------------------------------------------- +UnicodeString RBBIRuleScanner::stripRules(const UnicodeString &rules) { + UnicodeString strippedRules; + int rulesLength = rules.length(); + for (int idx = 0; idx < rulesLength; ) { + UChar ch = rules[idx++]; + if (ch == chPound) { + while (idx < rulesLength + && ch != chCR && ch != chLF && ch != chNEL) + { + ch = rules[idx++]; + } + } + if (!u_isISOControl(ch)) { + strippedRules.append(ch); + } + } + // strippedRules = strippedRules.unescape(); + return strippedRules; +} + + +//---------------------------------------------------------------------------------------- +// +// nextCharLL Low Level Next Char from rule input source. +// Get a char from the input character iterator, +// keep track of input position for error reporting. +// +//---------------------------------------------------------------------------------------- +UChar32 RBBIRuleScanner::nextCharLL() { + UChar32 ch; + + if (fNextIndex >= fRB->fRules.length()) { + return (UChar32)-1; + } + ch = fRB->fRules.char32At(fNextIndex); + fNextIndex = fRB->fRules.moveIndex32(fNextIndex, 1); + + if (ch == chCR || + ch == chNEL || + ch == chLS || + ch == chLF && fLastChar != chCR) { + // Character is starting a new line. Bump up the line number, and + // reset the column to 0. + fLineNum++; + fCharNum=0; + if (fQuoteMode) { + error(U_BRK_NEW_LINE_IN_QUOTED_STRING); + fQuoteMode = FALSE; + } + } + else { + // Character is not starting a new line. Except in the case of a + // LF following a CR, increment the column position. + if (ch != chLF) { + fCharNum++; + } + } + fLastChar = ch; + return ch; +} + + +//--------------------------------------------------------------------------------- +// +// nextChar for rules scanning. At this level, we handle stripping +// out comments and processing backslash character escapes. +// The rest of the rules grammar is handled at the next level up. +// +//--------------------------------------------------------------------------------- +void RBBIRuleScanner::nextChar(RBBIRuleChar &c) { + + // Unicode Character constants needed for the processing done by nextChar(), + // in hex because literals wont work on EBCDIC machines. + + fScanIndex = fNextIndex; + c.fChar = nextCharLL(); + c.fEscaped = FALSE; + + // + // check for '' sequence. + // These are recognized in all contexts, whether in quoted text or not. + // + if (c.fChar == chApos) { + if (fRB->fRules.char32At(fNextIndex) == chApos) { + c.fChar = nextCharLL(); // get nextChar officially so character counts + c.fEscaped = TRUE; // stay correct. + } + else + { + // Single quote, by itself. + // Toggle quoting mode. + // Return either '(' or ')', because quotes cause a grouping of the quoted text. + fQuoteMode = !fQuoteMode; + if (fQuoteMode == TRUE) { + c.fChar = chLParen; + } else { + c.fChar = chRParen; + } + c.fEscaped = FALSE; // The paren that we return is not escaped. + return; + } + } + + if (fQuoteMode) { + c.fEscaped = TRUE; + } + else + { + // We are not in a 'quoted region' of the source. + // + if (c.fChar == chPound) { + // Start of a comment. Consume the rest of it. + // The new-line char that terminates the comment is always returned. + // It will be treated as white-space, and serves to break up anything + // that might otherwise incorrectly clump together with a comment in + // the middle (a variable name, for example.) + for (;;) { + c.fChar = nextCharLL(); + if (c.fChar == (UChar32)-1 || // EOF + c.fChar == chCR || + c.fChar == chLF || + c.fChar == chNEL || + c.fChar == chLS) {break;} + } + } + if (c.fChar == (UChar32)-1) { + return; + } + + // + // check for backslash escaped characters. + // Use UnicodeString::unescapeAt() to handle them. + // + if (c.fChar == chBackSlash) { + c.fEscaped = TRUE; + int32_t startX = fNextIndex; + c.fChar = fRB->fRules.unescapeAt(fNextIndex); + if (fNextIndex == startX) { + error(U_BRK_HEX_DIGITS_EXPECTED); + } + fCharNum += fNextIndex-startX; + } + } + // putc(c.fChar, stdout); +} + +//--------------------------------------------------------------------------------- +// +// Parse RBBI rules. The state machine for rules parsing is here. +// The state tables are hand-written in the file rbbirpt.txt, +// and converted to the form used here by a perl +// script rbbicst.pl +// +//--------------------------------------------------------------------------------- +void RBBIRuleScanner::parse() { + uint16_t state; + const RBBIRuleTableEl *tableEl; + + if (U_FAILURE(*fRB->fStatus)) { + return; + } + + state = 1; + nextChar(fC); + // + // Main loop for the rule parsing state machine. + // Runs once per state transition. + // Each time through optionally performs, depending on the state table, + // - an advance to the the next input char + // - an action to be performed. + // - pushing or popping a state to/from the local state return stack. + // + for (;;) { + // Bail out if anything has gone wrong. + // RBBI rule file parsing stops on the first error encountered. + if (U_FAILURE(*fRB->fStatus)) { + break; + } + + // Quit if state == 0. This is the normal way to exit the state machine. + // + if (state == 0) { + break; + } + + // Find the state table element that matches the input char from the rule, or the + // class of the input character. Start with the first table row for this + // state, then linearly scan forward until we find a row that matches the + // character. The last row for each state always matches all characters, so + // the search will stop there, if not before. + // + tableEl = &gRuleParseStateTable[state]; + #ifdef RBBI_DEBUG + if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { + RBBIDebugPrintf("char, line, col = (\'%c\', %d, %d) state=%s ", + fC.fChar, fLineNum, fCharNum, RBBIRuleStateNames[state]); + } + #endif + + for (;;) { + #ifdef RBBI_DEBUG + if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPrintf(".");} + #endif + if (tableEl->fCharClass < 127 && fC.fEscaped == FALSE && tableEl->fCharClass == fC.fChar) { + // Table row specified an individual character, not a set, and + // the input character is not escaped, and + // the input character matched it. + break; + } + if (tableEl->fCharClass == 255) { + // Table row specified default, match anything character class. + break; + } + if (tableEl->fCharClass == 254 && fC.fEscaped) { + // Table row specified "escaped" and the char was escaped. + break; + } + if (tableEl->fCharClass == 253 && fC.fEscaped && + (fC.fChar == 0x50 || fC.fChar == 0x70 )) { + // Table row specified "escaped P" and the char is either 'p' or 'P'. + break; + } + if (tableEl->fCharClass == 252 && fC.fChar == (UChar32)-1) { + // Table row specified eof and we hit eof on the input. + break; + } + + if (tableEl->fCharClass >= 128 && tableEl->fCharClass < 240 && // Table specs a char class && + fC.fEscaped == FALSE && // char is not escaped && + fC.fChar != (UChar32)-1) { // char is not EOF + UnicodeSet *uniset = fRuleSets[tableEl->fCharClass-128]; + if (uniset->contains(fC.fChar)) { + // Table row specified a character class, or set of characters, + // and the current char matches it. + break; + } + } + + // No match on this row, advance to the next row for this state, + tableEl++; + } + if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPuts("");} + + // + // We've found the row of the state table that matches the current input + // character from the rules string. + // Perform any action specified by this row in the state table. + if (doParseActions((int32_t)tableEl->fAction) == FALSE) { + // Break out of the state machine loop if the + // the action signalled some kind of error, or + // the action was to exit, occurs on normal end-of-rules-input. + break; + } + + if (tableEl->fPushState != 0) { + fStackPtr++; + if (fStackPtr >= kStackSize) { + error(U_BRK_INTERNAL_ERROR); + RBBIDebugPuts("RBBIRuleScanner::parse() - state stack overflow."); + fStackPtr--; + } + fStack[fStackPtr] = tableEl->fPushState; + } + + if (tableEl->fNextChar) { + nextChar(fC); + } + + // Get the next state from the table entry, or from the + // state stack if the next state was specified as "pop". + if (tableEl->fNextState != 255) { + state = tableEl->fNextState; + } else { + state = fStack[fStackPtr]; + fStackPtr--; + if (fStackPtr < 0) { + error(U_BRK_INTERNAL_ERROR); + RBBIDebugPuts("RBBIRuleScanner::parse() - state stack underflow."); + fStackPtr++; + } + } + + } + + // + // If there were NO user specified reverse rules, set up the equivalent of ".*;" + // + if (fRB->fReverseTree == NULL) { + fRB->fReverseTree = pushNewNode(RBBINode::opStar); + RBBINode *operand = pushNewNode(RBBINode::setRef); + findSetFor(kAny, operand); + fRB->fReverseTree->fLeftChild = operand; + operand->fParent = fRB->fReverseTree; + fNodeStackPtr -= 2; + } + + + // + // Parsing of the input RBBI rules is complete. + // We now have a parse tree for the rule expressions + // and a list of all UnicodeSets that are referenced. + // +#ifdef RBBI_DEBUG + if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "symbols")) {fSymbolTable->rbbiSymtablePrint();} + if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "ptree")) + { + RBBIDebugPrintf("Completed Forward Rules Parse Tree...\n"); + fRB->fForwardTree->printTree(TRUE); + RBBIDebugPrintf("\nCompleted Reverse Rules Parse Tree...\n"); + fRB->fReverseTree->printTree(TRUE); + RBBIDebugPrintf("\nCompleted Safe Point Forward Rules Parse Tree...\n"); + fRB->fSafeFwdTree->printTree(TRUE); + RBBIDebugPrintf("\nCompleted Safe Point Reverse Rules Parse Tree...\n"); + fRB->fSafeRevTree->printTree(TRUE); + } +#endif +} + + +//--------------------------------------------------------------------------------- +// +// printNodeStack for debugging... +// +//--------------------------------------------------------------------------------- +#ifdef RBBI_DEBUG +void RBBIRuleScanner::printNodeStack(const char *title) { + int i; + RBBIDebugPrintf("%s. Dumping node stack...\n", title); + for (i=fNodeStackPtr; i>0; i--) {fNodeStack[i]->printTree(TRUE);} +} +#endif + + + + +//--------------------------------------------------------------------------------- +// +// pushNewNode create a new RBBINode of the specified type and push it +// onto the stack of nodes. +// +//--------------------------------------------------------------------------------- +RBBINode *RBBIRuleScanner::pushNewNode(RBBINode::NodeType t) { + fNodeStackPtr++; + if (fNodeStackPtr >= kStackSize) { + error(U_BRK_INTERNAL_ERROR); + RBBIDebugPuts("RBBIRuleScanner::pushNewNode - stack overflow."); + *fRB->fStatus = U_BRK_INTERNAL_ERROR; + return NULL; + } + fNodeStack[fNodeStackPtr] = new RBBINode(t); + if (fNodeStack[fNodeStackPtr] == NULL) { + *fRB->fStatus = U_MEMORY_ALLOCATION_ERROR; + } + return fNodeStack[fNodeStackPtr]; +} + + + +//--------------------------------------------------------------------------------- +// +// scanSet Construct a UnicodeSet from the text at the current scan +// position. Advance the scan position to the first character +// after the set. +// +// A new RBBI setref node referring to the set is pushed onto the node +// stack. +// +// The scan position is normally under the control of the state machine +// that controls rule parsing. UnicodeSets, however, are parsed by +// the UnicodeSet constructor, not by the RBBI rule parser. +// +//--------------------------------------------------------------------------------- +void RBBIRuleScanner::scanSet() { + UnicodeSet *uset; + ParsePosition pos; + int startPos; + int i; + + if (U_FAILURE(*fRB->fStatus)) { + return; + } + + pos.setIndex(fScanIndex); + startPos = fScanIndex; + UErrorCode localStatus = U_ZERO_ERROR; + uset = new UnicodeSet(fRB->fRules, pos, USET_IGNORE_SPACE, + fSymbolTable, + localStatus); + if (U_FAILURE(localStatus)) { + // TODO: Get more accurate position of the error from UnicodeSet's return info. + // UnicodeSet appears to not be reporting correctly at this time. + #ifdef RBBI_DEBUG + RBBIDebugPrintf("UnicodeSet parse postion.ErrorIndex = %d\n", pos.getIndex()); + #endif + error(localStatus); + delete uset; + return; + } + + // Verify that the set contains at least one code point. + // + if (uset->isEmpty()) { + // This set is empty. + // Make it an error, because it almost certainly is not what the user wanted. + // Also, avoids having to think about corner cases in the tree manipulation code + // that occurs later on. + error(U_BRK_RULE_EMPTY_SET); + delete uset; + return; + } + + + // Advance the RBBI parse postion over the UnicodeSet pattern. + // Don't just set fScanIndex because the line/char positions maintained + // for error reporting would be thrown off. + i = pos.getIndex(); + for (;;) { + if (fNextIndex >= i) { + break; + } + nextCharLL(); + } + + if (U_SUCCESS(*fRB->fStatus)) { + RBBINode *n; + + n = pushNewNode(RBBINode::setRef); + n->fFirstPos = startPos; + n->fLastPos = fNextIndex; + fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText); + // findSetFor() serves several purposes here: + // - Adopts storage for the UnicodeSet, will be responsible for deleting. + // - Mantains collection of all sets in use, needed later for establishing + // character categories for run time engine. + // - Eliminates mulitiple instances of the same set. + // - Creates a new uset node if necessary (if this isn't a duplicate.) + findSetFor(n->fText, n, uset); + } + +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_BREAK_ITERATION */ diff --git a/Build/source/libs/icu/icu-xetex/common/rbbiscan.h b/Build/source/libs/icu/icu-xetex/common/rbbiscan.h new file mode 100644 index 00000000000..cd0bf94a4fb --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/rbbiscan.h @@ -0,0 +1,160 @@ +// +// rbbiscan.h +// +// Copyright (C) 2002-2007, International Business Machines Corporation and others. +// All Rights Reserved. +// +// This file contains declarations for class RBBIRuleScanner +// + + +#ifndef RBBISCAN_H +#define RBBISCAN_H + +#include "unicode/utypes.h" +#include "unicode/uobject.h" +#include "unicode/rbbi.h" +#include "unicode/uniset.h" +#include "unicode/parseerr.h" +#include "uhash.h" +#include "uvector.h" +#include "unicode/symtable.h"// For UnicodeSet parsing, is the interface that + // looks up references to $variables within a set. +#include "rbbinode.h" +//#include "rbbitblb.h" + + + +U_NAMESPACE_BEGIN + +class RBBIRuleBuilder; +class RBBISymbolTable; + + +//-------------------------------------------------------------------------------- +// +// class RBBIRuleScanner does the lowest level, character-at-a-time +// scanning of break iterator rules. +// +// The output of the scanner is parse trees for +// the rule expressions and a list of all Unicode Sets +// encountered. +// +//-------------------------------------------------------------------------------- +static const int kStackSize = 100; // The size of the state stack for + // rules parsing. Corresponds roughly + // to the depth of parentheses nesting + // that is allowed in the rules. + +class RBBIRuleScanner : public UMemory { +public: + + struct RBBIRuleChar { + UChar32 fChar; + UBool fEscaped; + }; + + RBBIRuleScanner(RBBIRuleBuilder *rb); + + + virtual ~RBBIRuleScanner(); + + void nextChar(RBBIRuleChar &c); // Get the next char from the input stream. + // Return false if at end. + + UBool push(const RBBIRuleChar &c); // Push (unget) one character. + // Only a single character may be pushed. + + void parse(); // Parse the rules, generating two parse + // trees, one each for the forward and + // reverse rules, + // and a list of UnicodeSets encountered. + + /** + * Return a rules string without unnecessary + * characters. + */ + static UnicodeString stripRules(const UnicodeString &rules); +private: + + UBool doParseActions(int32_t a); + void error(UErrorCode e); // error reporting convenience function. + void fixOpStack(RBBINode::OpPrecedence p); + // a character. + void findSetFor(const UnicodeString &s, RBBINode *node, UnicodeSet *setToAdopt = NULL); + + UChar32 nextCharLL(); +#ifdef RBBI_DEBUG + void printNodeStack(const char *title); +#endif + RBBINode *pushNewNode(RBBINode::NodeType t); + void scanSet(); + + + RBBIRuleBuilder *fRB; // The rule builder that we are part of. + + int32_t fScanIndex; // Index of current character being processed + // in the rule input string. + int32_t fNextIndex; // Index of the next character, which + // is the first character not yet scanned. + UBool fQuoteMode; // Scan is in a 'quoted region' + int32_t fLineNum; // Line number in input file. + int32_t fCharNum; // Char position within the line. + UChar32 fLastChar; // Previous char, needed to count CR-LF + // as a single line, not two. + + RBBIRuleChar fC; // Current char for parse state machine + // processing. + UnicodeString fVarName; // $variableName, valid when we've just + // scanned one. + + RBBIRuleTableEl **fStateTable; // State Transition Table for RBBI Rule + // parsing. index by p[state][char-class] + + uint16_t fStack[kStackSize]; // State stack, holds state pushes + int32_t fStackPtr; // and pops as specified in the state + // transition rules. + + RBBINode *fNodeStack[kStackSize]; // Node stack, holds nodes created + // during the parse of a rule + int32_t fNodeStackPtr; + + + UBool fReverseRule; // True if the rule currently being scanned + // is a reverse direction rule (if it + // starts with a '!') + + UBool fLookAheadRule; // True if the rule includes a '/' + // somewhere within it. + + RBBISymbolTable *fSymbolTable; // symbol table, holds definitions of + // $variable symbols. + + UHashtable *fSetTable; // UnicocodeSet hash table, holds indexes to + // the sets created while parsing rules. + // The key is the string used for creating + // the set. + + UnicodeSet *fRuleSets[10]; // Unicode Sets that are needed during + // the scanning of RBBI rules. The + // indicies for these are assigned by the + // perl script that builds the state tables. + // See rbbirpt.h. + + int32_t fRuleNum; // Counts each rule as it is scanned. + + int32_t fOptionStart; // Input index of start of a !!option + // keyword, while being scanned. + + UnicodeSet *gRuleSet_rule_char; + UnicodeSet *gRuleSet_white_space; + UnicodeSet *gRuleSet_name_char; + UnicodeSet *gRuleSet_name_start_char; + + RBBIRuleScanner(const RBBIRuleScanner &other); // forbid copying of this class + RBBIRuleScanner &operator=(const RBBIRuleScanner &other); // forbid copying of this class +}; + +U_NAMESPACE_END + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/rbbisetb.cpp b/Build/source/libs/icu/icu-xetex/common/rbbisetb.cpp new file mode 100644 index 00000000000..0bbe8e920ee --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/rbbisetb.cpp @@ -0,0 +1,684 @@ +// +// rbbisetb.cpp +// +/* +*************************************************************************** +* Copyright (C) 2002-2005 International Business Machines Corporation * +* and others. All rights reserved. * +*************************************************************************** +*/ +// +// RBBISetBuilder Handles processing of Unicode Sets from RBBI rules +// (part of the rule building process.) +// +// Starting with the rules parse tree from the scanner, +// +// - Enumerate the set of UnicodeSets that are referenced +// by the RBBI rules. +// - compute a set of non-overlapping character ranges +// with all characters within a range belonging to the same +// set of input uniocde sets. +// - Derive a set of non-overlapping UnicodeSet (like things) +// that will correspond to columns in the state table for +// the RBBI execution engine. All characters within one +// of these sets belong to the same set of the original +// UnicodeSets from the user's rules. +// - construct the trie table that maps input characters +// to the index of the matching non-overlapping set of set from +// the previous step. +// + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_BREAK_ITERATION + +#include "unicode/uniset.h" +#include "utrie.h" +#include "uvector.h" +#include "uassert.h" +#include "cmemory.h" +#include "cstring.h" + +#include "rbbisetb.h" +#include "rbbinode.h" + + +//------------------------------------------------------------------------ +// +// getFoldedRBBIValue Call-back function used during building of Trie table. +// Folding value: just store the offset (16 bits) +// if there is any non-0 entry. +// (It'd really be nice if the Trie builder would provide a +// simple default, so this function could go away from here.) +// +//------------------------------------------------------------------------ +/* folding value: just store the offset (16 bits) if there is any non-0 entry */ +U_CDECL_BEGIN +static uint32_t U_CALLCONV +getFoldedRBBIValue(UNewTrie *trie, UChar32 start, int32_t offset) { + uint32_t value; + UChar32 limit; + UBool inBlockZero; + + limit=start+0x400; + while(startfStatus; + fRangeList = 0; + fTrie = 0; + fTrieSize = 0; + fGroupCount = 0; + fSawBOF = FALSE; +} + + +//------------------------------------------------------------------------ +// +// Destructor +// +//------------------------------------------------------------------------ +RBBISetBuilder::~RBBISetBuilder() +{ + RangeDescriptor *nextRangeDesc; + + // Walk through & delete the linked list of RangeDescriptors + for (nextRangeDesc = fRangeList; nextRangeDesc!=NULL;) { + RangeDescriptor *r = nextRangeDesc; + nextRangeDesc = r->fNext; + delete r; + } + + utrie_close(fTrie); +} + + + + +//------------------------------------------------------------------------ +// +// build Build the list of non-overlapping character ranges +// from the Unicode Sets. +// +//------------------------------------------------------------------------ +void RBBISetBuilder::build() { + RBBINode *usetNode; + RangeDescriptor *rlRange; + + if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "usets")) {printSets();} + + // + // Initialize the process by creating a single range encompassing all characters + // that is in no sets. + // + fRangeList = new RangeDescriptor(*fStatus); // will check for status here + fRangeList->fStartChar = 0; + fRangeList->fEndChar = 0x10ffff; + + if (U_FAILURE(*fStatus)) { + return; + } + + // + // Find the set of non-overlapping ranges of characters + // + int ni; + for (ni=0; ; ni++) { // Loop over each of the UnicodeSets encountered in the input rules + usetNode = (RBBINode *)this->fRB->fUSetNodes->elementAt(ni); + if (usetNode==NULL) { + break; + } + + UnicodeSet *inputSet = usetNode->fInputSet; + int32_t inputSetRangeCount = inputSet->getRangeCount(); + int inputSetRangeIndex = 0; + rlRange = fRangeList; + + for (;;) { + if (inputSetRangeIndex >= inputSetRangeCount) { + break; + } + UChar32 inputSetRangeBegin = inputSet->getRangeStart(inputSetRangeIndex); + UChar32 inputSetRangeEnd = inputSet->getRangeEnd(inputSetRangeIndex); + + // skip over ranges from the range list that are completely + // below the current range from the input unicode set. + while (rlRange->fEndChar < inputSetRangeBegin) { + rlRange = rlRange->fNext; + } + + // If the start of the range from the range list is before with + // the start of the range from the unicode set, split the range list range + // in two, with one part being before (wholly outside of) the unicode set + // and the other containing the rest. + // Then continue the loop; the post-split current range will then be skipped + // over + if (rlRange->fStartChar < inputSetRangeBegin) { + rlRange->split(inputSetRangeBegin, *fStatus); + if (U_FAILURE(*fStatus)) { + return; + } + continue; + } + + // Same thing at the end of the ranges... + // If the end of the range from the range list doesn't coincide with + // the end of the range from the unicode set, split the range list + // range in two. The first part of the split range will be + // wholly inside the Unicode set. + if (rlRange->fEndChar > inputSetRangeEnd) { + rlRange->split(inputSetRangeEnd+1, *fStatus); + if (U_FAILURE(*fStatus)) { + return; + } + } + + // The current rlRange is now entirely within the UnicodeSet range. + // Add this unicode set to the list of sets for this rlRange + if (rlRange->fIncludesSets->indexOf(usetNode) == -1) { + rlRange->fIncludesSets->addElement(usetNode, *fStatus); + if (U_FAILURE(*fStatus)) { + return; + } + } + + // Advance over ranges that we are finished with. + if (inputSetRangeEnd == rlRange->fEndChar) { + inputSetRangeIndex++; + } + rlRange = rlRange->fNext; + } + } + + if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "range")) { printRanges();} + + // + // Group the above ranges, with each group consisting of one or more + // ranges that are in exactly the same set of original UnicodeSets. + // The groups are numbered, and these group numbers are the set of + // input symbols recognized by the run-time state machine. + // + // Numbering: # 0 (state table column 0) is unused. + // # 1 is reserved - table column 1 is for end-of-input + // # 2 is reserved - table column 2 is for beginning-in-input + // # 3 is the first range list. + // + RangeDescriptor *rlSearchRange; + for (rlRange = fRangeList; rlRange!=0; rlRange=rlRange->fNext) { + for (rlSearchRange=fRangeList; rlSearchRange != rlRange; rlSearchRange=rlSearchRange->fNext) { + if (rlRange->fIncludesSets->equals(*rlSearchRange->fIncludesSets)) { + rlRange->fNum = rlSearchRange->fNum; + break; + } + } + if (rlRange->fNum == 0) { + fGroupCount ++; + rlRange->fNum = fGroupCount+2; + rlRange->setDictionaryFlag(); + addValToSets(rlRange->fIncludesSets, fGroupCount+2); + } + } + + // Handle input sets that contain the special string {eof}. + // Column 1 of the state table is reserved for EOF on input. + // Column 2 is reserved for before-the-start-input. + // (This column can be optimized away later if there are no rule + // references to {bof}.) + // Add this column value (1 or 2) to the equivalent expression + // subtree for each UnicodeSet that contains the string {eof} + // Because {bof} and {eof} are not a characters in the normal sense, + // they doesn't affect the computation of ranges or TRIE. + static const UChar eofUString[] = {0x65, 0x6f, 0x66, 0}; + static const UChar bofUString[] = {0x62, 0x6f, 0x66, 0}; + + UnicodeString eofString(eofUString); + UnicodeString bofString(bofUString); + for (ni=0; ; ni++) { // Loop over each of the UnicodeSets encountered in the input rules + usetNode = (RBBINode *)this->fRB->fUSetNodes->elementAt(ni); + if (usetNode==NULL) { + break; + } + UnicodeSet *inputSet = usetNode->fInputSet; + if (inputSet->contains(eofString)) { + addValToSet(usetNode, 1); + } + if (inputSet->contains(bofString)) { + addValToSet(usetNode, 2); + fSawBOF = TRUE; + } + } + + + if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "rgroup")) {printRangeGroups();} + if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "esets")) {printSets();} + + // + // Build the Trie table for mapping UChar32 values to the corresponding + // range group number + // + fTrie = utrie_open(NULL, // Pre-existing trie to be filled in + NULL, // Data array (utrie will allocate one) + 100000, // Max Data Length + 0, // Initial value for all code points + 0, // Lead surrogate unit value + TRUE); // Keep Latin 1 in separately + + + for (rlRange = fRangeList; rlRange!=0; rlRange=rlRange->fNext) { + utrie_setRange32(fTrie, rlRange->fStartChar, rlRange->fEndChar+1, rlRange->fNum, TRUE); + } +} + + + +//----------------------------------------------------------------------------------- +// +// getTrieSize() Return the size that will be required to serialize the Trie. +// +//----------------------------------------------------------------------------------- +int32_t RBBISetBuilder::getTrieSize() /*const*/ { + fTrieSize = utrie_serialize(fTrie, + NULL, // Buffer + 0, // Capacity + getFoldedRBBIValue, + TRUE, // Reduce to 16 bits + fStatus); + // RBBIDebugPrintf("Trie table size is %d\n", trieSize); + return fTrieSize; +} + + +//----------------------------------------------------------------------------------- +// +// serializeTrie() Put the serialized trie at the specified address. +// Trust the caller to have given us enough memory. +// getTrieSize() MUST be called first. +// +//----------------------------------------------------------------------------------- +void RBBISetBuilder::serializeTrie(uint8_t *where) { + utrie_serialize(fTrie, + where, // Buffer + fTrieSize, // Capacity + getFoldedRBBIValue, + TRUE, // Reduce to 16 bits + fStatus); +} + +//------------------------------------------------------------------------ +// +// addValToSets Add a runtime-mapped input value to each uset from a +// list of uset nodes. (val corresponds to a state table column.) +// For each of the original Unicode sets - which correspond +// directly to uset nodes - a logically equivalent expression +// is constructed in terms of the remapped runtime input +// symbol set. This function adds one runtime input symbol to +// a list of sets. +// +// The "logically equivalent expression" is the tree for an +// or-ing together of all of the symbols that go into the set. +// +//------------------------------------------------------------------------ +void RBBISetBuilder::addValToSets(UVector *sets, uint32_t val) { + int32_t ix; + + for (ix=0; ixsize(); ix++) { + RBBINode *usetNode = (RBBINode *)sets->elementAt(ix); + addValToSet(usetNode, val); + } +} + +void RBBISetBuilder::addValToSet(RBBINode *usetNode, uint32_t val) { + RBBINode *leafNode = new RBBINode(RBBINode::leafChar); + leafNode->fVal = (unsigned short)val; + if (usetNode->fLeftChild == NULL) { + usetNode->fLeftChild = leafNode; + leafNode->fParent = usetNode; + } else { + // There are already input symbols present for this set. + // Set up an OR node, with the previous stuff as the left child + // and the new value as the right child. + RBBINode *orNode = new RBBINode(RBBINode::opOr); + orNode->fLeftChild = usetNode->fLeftChild; + orNode->fRightChild = leafNode; + orNode->fLeftChild->fParent = orNode; + orNode->fRightChild->fParent = orNode; + usetNode->fLeftChild = orNode; + orNode->fParent = usetNode; + } +} + + +//------------------------------------------------------------------------ +// +// getNumCharCategories +// +//------------------------------------------------------------------------ +int32_t RBBISetBuilder::getNumCharCategories() const { + return fGroupCount + 3; +} + + +//------------------------------------------------------------------------ +// +// sawBOF +// +//------------------------------------------------------------------------ +UBool RBBISetBuilder::sawBOF() const { + return fSawBOF; +} + + +//------------------------------------------------------------------------ +// +// getFirstChar Given a runtime RBBI character category, find +// the first UChar32 that is in the set of chars +// in the category. +//------------------------------------------------------------------------ +UChar32 RBBISetBuilder::getFirstChar(int32_t category) const { + RangeDescriptor *rlRange; + UChar32 retVal = (UChar32)-1; + for (rlRange = fRangeList; rlRange!=0; rlRange=rlRange->fNext) { + if (rlRange->fNum == category) { + retVal = rlRange->fStartChar; + break; + } + } + return retVal; +} + + + +//------------------------------------------------------------------------ +// +// printRanges A debugging function. +// dump out all of the range definitions. +// +//------------------------------------------------------------------------ +#ifdef RBBI_DEBUG +void RBBISetBuilder::printRanges() { + RangeDescriptor *rlRange; + int i; + + RBBIDebugPrintf("\n\n Nonoverlapping Ranges ...\n"); + for (rlRange = fRangeList; rlRange!=0; rlRange=rlRange->fNext) { + RBBIDebugPrintf("%2i %4x-%4x ", rlRange->fNum, rlRange->fStartChar, rlRange->fEndChar); + + for (i=0; ifIncludesSets->size(); i++) { + RBBINode *usetNode = (RBBINode *)rlRange->fIncludesSets->elementAt(i); + UnicodeString setName = UNICODE_STRING("anon", 4); + RBBINode *setRef = usetNode->fParent; + if (setRef != NULL) { + RBBINode *varRef = setRef->fParent; + if (varRef != NULL && varRef->fType == RBBINode::varRef) { + setName = varRef->fText; + } + } + RBBI_DEBUG_printUnicodeString(setName); RBBIDebugPrintf(" "); + } + RBBIDebugPrintf("\n"); + } +} +#endif + + +//------------------------------------------------------------------------ +// +// printRangeGroups A debugging function. +// dump out all of the range groups. +// +//------------------------------------------------------------------------ +#ifdef RBBI_DEBUG +void RBBISetBuilder::printRangeGroups() { + RangeDescriptor *rlRange; + RangeDescriptor *tRange; + int i; + int lastPrintedGroupNum = 0; + + RBBIDebugPrintf("\nRanges grouped by Unicode Set Membership...\n"); + for (rlRange = fRangeList; rlRange!=0; rlRange=rlRange->fNext) { + int groupNum = rlRange->fNum & 0xbfff; + if (groupNum > lastPrintedGroupNum) { + lastPrintedGroupNum = groupNum; + RBBIDebugPrintf("%2i ", groupNum); + + if (rlRange->fNum & 0x4000) { RBBIDebugPrintf(" ");} + + for (i=0; ifIncludesSets->size(); i++) { + RBBINode *usetNode = (RBBINode *)rlRange->fIncludesSets->elementAt(i); + UnicodeString setName = UNICODE_STRING("anon", 4); + RBBINode *setRef = usetNode->fParent; + if (setRef != NULL) { + RBBINode *varRef = setRef->fParent; + if (varRef != NULL && varRef->fType == RBBINode::varRef) { + setName = varRef->fText; + } + } + RBBI_DEBUG_printUnicodeString(setName); RBBIDebugPrintf(" "); + } + + i = 0; + for (tRange = rlRange; tRange != 0; tRange = tRange->fNext) { + if (tRange->fNum == rlRange->fNum) { + if (i++ % 5 == 0) { + RBBIDebugPrintf("\n "); + } + RBBIDebugPrintf(" %05x-%05x", tRange->fStartChar, tRange->fEndChar); + } + } + RBBIDebugPrintf("\n"); + } + } + RBBIDebugPrintf("\n"); +} +#endif + + +//------------------------------------------------------------------------ +// +// printSets A debugging function. +// dump out all of the set definitions. +// +//------------------------------------------------------------------------ +#ifdef RBBI_DEBUG +void RBBISetBuilder::printSets() { + int i; + + RBBIDebugPrintf("\n\nUnicode Sets List\n------------------\n"); + for (i=0; ; i++) { + RBBINode *usetNode; + RBBINode *setRef; + RBBINode *varRef; + UnicodeString setName; + + usetNode = (RBBINode *)fRB->fUSetNodes->elementAt(i); + if (usetNode == NULL) { + break; + } + + RBBIDebugPrintf("%3d ", i); + setName = UNICODE_STRING("anonymous", 9); + setRef = usetNode->fParent; + if (setRef != NULL) { + varRef = setRef->fParent; + if (varRef != NULL && varRef->fType == RBBINode::varRef) { + setName = varRef->fText; + } + } + RBBI_DEBUG_printUnicodeString(setName); + RBBIDebugPrintf(" "); + RBBI_DEBUG_printUnicodeString(usetNode->fText); + RBBIDebugPrintf("\n"); + if (usetNode->fLeftChild != NULL) { + usetNode->fLeftChild->printTree(TRUE); + } + } + RBBIDebugPrintf("\n"); +} +#endif + + + +//------------------------------------------------------------------------------------- +// +// RangeDescriptor copy constructor +// +//------------------------------------------------------------------------------------- + +RangeDescriptor::RangeDescriptor(const RangeDescriptor &other, UErrorCode &status) { + int i; + + this->fStartChar = other.fStartChar; + this->fEndChar = other.fEndChar; + this->fNum = other.fNum; + this->fNext = NULL; + UErrorCode oldstatus = status; + this->fIncludesSets = new UVector(status); + if (U_FAILURE(oldstatus)) { + status = oldstatus; + } + if (U_FAILURE(status)) { + return; + } + /* test for NULL */ + if (this->fIncludesSets == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + + for (i=0; isize(); i++) { + this->fIncludesSets->addElement(other.fIncludesSets->elementAt(i), status); + } +} + + +//------------------------------------------------------------------------------------- +// +// RangeDesriptor default constructor +// +//------------------------------------------------------------------------------------- +RangeDescriptor::RangeDescriptor(UErrorCode &status) { + this->fStartChar = 0; + this->fEndChar = 0; + this->fNum = 0; + this->fNext = NULL; + UErrorCode oldstatus = status; + this->fIncludesSets = new UVector(status); + if (U_FAILURE(oldstatus)) { + status = oldstatus; + } + if (U_FAILURE(status)) { + return; + } + /* test for NULL */ + if(this->fIncludesSets == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + +} + + +//------------------------------------------------------------------------------------- +// +// RangeDesriptor Destructor +// +//------------------------------------------------------------------------------------- +RangeDescriptor::~RangeDescriptor() { + delete fIncludesSets; + fIncludesSets = NULL; +} + +//------------------------------------------------------------------------------------- +// +// RangeDesriptor::split() +// +//------------------------------------------------------------------------------------- +void RangeDescriptor::split(UChar32 where, UErrorCode &status) { + U_ASSERT(where>fStartChar && where<=fEndChar); + RangeDescriptor *nr = new RangeDescriptor(*this, status); + if (U_FAILURE(status)) { + return; + } + /* test for NULL */ + if(nr == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + + // RangeDescriptor copy constructor copies all fields. + // Only need to update those that are different after the split. + nr->fStartChar = where; + this->fEndChar = where-1; + nr->fNext = this->fNext; + this->fNext = nr; +} + + +//------------------------------------------------------------------------------------- +// +// RangeDescriptor::setDictionaryFlag +// +// Character Category Numbers that include characters from +// the original Unicode Set named "dictionary" have bit 14 +// set to 1. The RBBI runtime engine uses this to trigger +// use of the word dictionary. +// +// This function looks through the Unicode Sets that it +// (the range) includes, and sets the bit in fNum when +// "dictionary" is among them. +// +// TODO: a faster way would be to find the set node for +// "dictionary" just once, rather than looking it +// up by name every time. +// +//------------------------------------------------------------------------------------- +void RangeDescriptor::setDictionaryFlag() { + int i; + + for (i=0; ifIncludesSets->size(); i++) { + RBBINode *usetNode = (RBBINode *)fIncludesSets->elementAt(i); + UnicodeString setName; + RBBINode *setRef = usetNode->fParent; + if (setRef != NULL) { + RBBINode *varRef = setRef->fParent; + if (varRef != NULL && varRef->fType == RBBINode::varRef) { + setName = varRef->fText; + } + } + if (setName.compare(UNICODE_STRING("dictionary", 10)) == 0) { // TODO: no string literals. + this->fNum |= 0x4000; + break; + } + } +} + + + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_BREAK_ITERATION */ diff --git a/Build/source/libs/icu/icu-xetex/common/rbbisetb.h b/Build/source/libs/icu/icu-xetex/common/rbbisetb.h new file mode 100644 index 00000000000..c8bc1df7786 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/rbbisetb.h @@ -0,0 +1,130 @@ +// +// rbbisetb.h +/* +********************************************************************** +* Copyright (c) 2001-2005, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +*/ + +#ifndef RBBISETB_H +#define RBBISETB_H + +#include "unicode/utypes.h" +#include "unicode/uobject.h" +#include "rbbirb.h" +#include "uvector.h" + +struct UNewTrie; + +U_NAMESPACE_BEGIN + +// +// RBBISetBuilder Derives the character categories used by the runtime RBBI engine +// from the Unicode Sets appearing in the source RBBI rules, and +// creates the TRIE table used to map from Unicode to the +// character categories. +// + + +// +// RangeDescriptor +// +// Each of the non-overlapping character ranges gets one of these descriptors. +// All of them are strung together in a linked list, which is kept in order +// (by character) +// +class RangeDescriptor : public UMemory { +public: + UChar32 fStartChar; // Start of range, unicode 32 bit value. + UChar32 fEndChar; // End of range, unicode 32 bit value. + int32_t fNum; // runtime-mapped input value for this range. + UVector *fIncludesSets; // vector of the the original + // Unicode sets that include this range. + // (Contains ptrs to uset nodes) + RangeDescriptor *fNext; // Next RangeDescriptor in the linked list. + + RangeDescriptor(UErrorCode &status); + RangeDescriptor(const RangeDescriptor &other, UErrorCode &status); + ~RangeDescriptor(); + void split(UChar32 where, UErrorCode &status); // Spit this range in two at "where", with + // where appearing in the second (higher) part. + void setDictionaryFlag(); // Check whether this range appears as part of + // the Unicode set named "dictionary" + +private: + RangeDescriptor(const RangeDescriptor &other); // forbid copying of this class + RangeDescriptor &operator=(const RangeDescriptor &other); // forbid copying of this class +}; + + +// +// RBBISetBuilder Handles processing of Unicode Sets from RBBI rules. +// +// Starting with the rules parse tree from the scanner, +// +// - Enumerate the set of UnicodeSets that are referenced +// by the RBBI rules. +// - compute a derived set of non-overlapping UnicodeSets +// that will correspond to columns in the state table for +// the RBBI execution engine. +// - construct the trie table that maps input characters +// to set numbers in the non-overlapping set of sets. +// + + +class RBBISetBuilder : public UMemory { +public: + RBBISetBuilder(RBBIRuleBuilder *rb); + ~RBBISetBuilder(); + + void build(); + void addValToSets(UVector *sets, uint32_t val); + void addValToSet (RBBINode *usetNode, uint32_t val); + int32_t getNumCharCategories() const; // CharCategories are the same as input symbol set to the + // runtime state machine, which are the same as + // columns in the DFA state table + int32_t getTrieSize() /*const*/; // Size in bytes of the serialized Trie. + void serializeTrie(uint8_t *where); // write out the serialized Trie. + UChar32 getFirstChar(int32_t val) const; + UBool sawBOF() const; // Indicate whether any references to the {bof} pseudo + // character were encountered. +#ifdef RBBI_DEBUG + void printSets(); + void printRanges(); + void printRangeGroups(); +#else + #define printSets() + #define printRanges() + #define printRangeGroups() +#endif + +private: + void numberSets(); + + RBBIRuleBuilder *fRB; // The RBBI Rule Compiler that owns us. + UErrorCode *fStatus; + + RangeDescriptor *fRangeList; // Head of the linked list of RangeDescriptors + + UNewTrie *fTrie; // The mapping TRIE that is the end result of processing + uint32_t fTrieSize; // the Unicode Sets. + + // Groups correspond to character categories - + // groups of ranges that are in the same original UnicodeSets. + // fGroupCount is the index of the last used group. + // fGroupCount+1 is also the number of columns in the RBBI state table being compiled. + // State table column 0 is not used. Column 1 is for end-of-input. + // column 2 is for group 0. Funny counting. + int32_t fGroupCount; + + UBool fSawBOF; + + RBBISetBuilder(const RBBISetBuilder &other); // forbid copying of this class + RBBISetBuilder &operator=(const RBBISetBuilder &other); // forbid copying of this class +}; + + + +U_NAMESPACE_END +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/rbbistbl.cpp b/Build/source/libs/icu/icu-xetex/common/rbbistbl.cpp new file mode 100644 index 00000000000..cbcd3815509 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/rbbistbl.cpp @@ -0,0 +1,269 @@ +// +// file: rbbistbl.cpp Implementation of the ICU RBBISymbolTable class +// +/* +*************************************************************************** +* Copyright (C) 2002-2006 International Business Machines Corporation * +* and others. All rights reserved. * +*************************************************************************** +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_BREAK_ITERATION + +#include "unicode/unistr.h" +#include "unicode/uniset.h" +#include "unicode/uchar.h" +#include "unicode/parsepos.h" + +#include "umutex.h" + +#include "rbbirb.h" +#include "rbbinode.h" + + +// +// RBBISymbolTableEntry_deleter Used by the UHashTable to delete the contents +// when the hash table is deleted. +// +U_CDECL_BEGIN +static void U_CALLCONV RBBISymbolTableEntry_deleter(void *p) { + U_NAMESPACE_QUALIFIER RBBISymbolTableEntry *px = (U_NAMESPACE_QUALIFIER RBBISymbolTableEntry *)p; + delete px; +} +U_CDECL_END + + + +U_NAMESPACE_BEGIN + +RBBISymbolTable::RBBISymbolTable(RBBIRuleScanner *rs, const UnicodeString &rules, UErrorCode &status) + :fRules(rules), fRuleScanner(rs), ffffString(UChar(0xffff)) +{ + fHashTable = NULL; + fCachedSetLookup = NULL; + + fHashTable = uhash_open(uhash_hashUnicodeString, uhash_compareUnicodeString, NULL, &status); + // uhash_open checks status + if (U_FAILURE(status)) { + return; + } + uhash_setValueDeleter(fHashTable, RBBISymbolTableEntry_deleter); +} + + + +RBBISymbolTable::~RBBISymbolTable() +{ + uhash_close(fHashTable); +} + + +// +// RBBISymbolTable::lookup This function from the abstract symbol table inteface +// looks up a variable name and returns a UnicodeString +// containing the substitution text. +// +// The variable name does NOT include the leading $. +// +const UnicodeString *RBBISymbolTable::lookup(const UnicodeString& s) const +{ + RBBISymbolTableEntry *el; + RBBINode *varRefNode; + RBBINode *exprNode; + RBBINode *usetNode; + const UnicodeString *retString; + RBBISymbolTable *This = (RBBISymbolTable *)this; // cast off const + + el = (RBBISymbolTableEntry *)uhash_get(fHashTable, &s); + if (el == NULL) { + return NULL; + } + + varRefNode = el->val; + exprNode = varRefNode->fLeftChild; // Root node of expression for variable + if (exprNode->fType == RBBINode::setRef) { + // The $variable refers to a single UnicodeSet + // return the ffffString, which will subsequently be interpreted as a + // stand-in character for the set by RBBISymbolTable::lookupMatcher() + usetNode = exprNode->fLeftChild; + This->fCachedSetLookup = usetNode->fInputSet; + retString = &ffffString; + } + else + { + // The variable refers to something other than just a set. + // return the original source string for the expression + retString = &exprNode->fText; + This->fCachedSetLookup = NULL; + } + return retString; +} + + + +// +// RBBISymbolTable::lookupMatcher This function from the abstract symbol table +// interface maps a single stand-in character to a +// pointer to a Unicode Set. The Unicode Set code uses this +// mechanism to get all references to the same $variable +// name to refer to a single common Unicode Set instance. +// +// This implementation cheats a little, and does not maintain a map of stand-in chars +// to sets. Instead, it takes advantage of the fact that the UnicodeSet +// constructor will always call this function right after calling lookup(), +// and we just need to remember what set to return between these two calls. +const UnicodeFunctor *RBBISymbolTable::lookupMatcher(UChar32 ch) const +{ + UnicodeSet *retVal = NULL; + RBBISymbolTable *This = (RBBISymbolTable *)this; // cast off const + if (ch == 0xffff) { + retVal = fCachedSetLookup; + This->fCachedSetLookup = 0; + } + return retVal; +} + +// +// RBBISymbolTable::parseReference This function from the abstract symbol table interface +// looks for a $variable name in the source text. +// It does not look it up, only scans for it. +// It is used by the UnicodeSet parser. +// +// This implementation is lifted pretty much verbatim +// from the rules based transliterator implementation. +// I didn't see an obvious way of sharing it. +// +UnicodeString RBBISymbolTable::parseReference(const UnicodeString& text, + ParsePosition& pos, int32_t limit) const +{ + int32_t start = pos.getIndex(); + int32_t i = start; + UnicodeString result; + while (i < limit) { + UChar c = text.charAt(i); + if ((i==start && !u_isIDStart(c)) || !u_isIDPart(c)) { + break; + } + ++i; + } + if (i == start) { // No valid name chars + return result; // Indicate failure with empty string + } + pos.setIndex(i); + text.extractBetween(start, i, result); + return result; +} + + + +// +// RBBISymbolTable::lookupNode Given a key (a variable name), return the +// corresponding RBBI Node. If there is no entry +// in the table for this name, return NULL. +// +RBBINode *RBBISymbolTable::lookupNode(const UnicodeString &key) const{ + + RBBINode *retNode = NULL; + RBBISymbolTableEntry *el; + + el = (RBBISymbolTableEntry *)uhash_get(fHashTable, &key); + if (el != NULL) { + retNode = el->val; + } + return retNode; +} + + +// +// RBBISymbolTable::addEntry Add a new entry to the symbol table. +// Indicate an error if the name already exists - +// this will only occur in the case of duplicate +// variable assignments. +// +void RBBISymbolTable::addEntry (const UnicodeString &key, RBBINode *val, UErrorCode &err) { + RBBISymbolTableEntry *e; + /* test for buffer overflows */ + if (U_FAILURE(err)) { + return; + } + e = (RBBISymbolTableEntry *)uhash_get(fHashTable, &key); + if (e != NULL) { + err = U_BRK_VARIABLE_REDFINITION; + return; + } + + e = new RBBISymbolTableEntry; + if (e == NULL) { + err = U_MEMORY_ALLOCATION_ERROR; + return; + } + e->key = key; + e->val = val; + uhash_put( fHashTable, &e->key, e, &err); +} + + +RBBISymbolTableEntry::RBBISymbolTableEntry() : UMemory(), key(), val(NULL) {} + +RBBISymbolTableEntry::~RBBISymbolTableEntry() { + // The "val" of a symbol table entry is a variable reference node. + // The l. child of the val is the rhs expression from the assignment. + // Unlike other node types, children of variable reference nodes are not + // automatically recursively deleted. We do it manually here. + delete val->fLeftChild; + val->fLeftChild = NULL; + + delete val; + + // Note: the key UnicodeString is destructed by virtue of being in the object by value. +} + + +// +// RBBISymbolTable::print Debugging function, dump out the symbol table contents. +// +#ifdef RBBI_DEBUG +void RBBISymbolTable::rbbiSymtablePrint() const { + RBBIDebugPrintf("Variable Definitions\n" + "Name Node Val String Val\n" + "----------------------------------------------------------------------\n"); + + int32_t pos = -1; + const UHashElement *e = NULL; + for (;;) { + e = uhash_nextElement(fHashTable, &pos); + if (e == NULL ) { + break; + } + RBBISymbolTableEntry *s = (RBBISymbolTableEntry *)e->value.pointer; + + RBBI_DEBUG_printUnicodeString(s->key, 15); + RBBIDebugPrintf(" %8p ", (void *)s->val); + RBBI_DEBUG_printUnicodeString(s->val->fLeftChild->fText); + RBBIDebugPrintf("\n"); + } + + RBBIDebugPrintf("\nParsed Variable Definitions\n"); + pos = -1; + for (;;) { + e = uhash_nextElement(fHashTable, &pos); + if (e == NULL ) { + break; + } + RBBISymbolTableEntry *s = (RBBISymbolTableEntry *)e->value.pointer; + RBBI_DEBUG_printUnicodeString(s->key); + s->val->fLeftChild->printTree(TRUE); + RBBIDebugPrintf("\n"); + } +} +#endif + + + + + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_BREAK_ITERATION */ diff --git a/Build/source/libs/icu/icu-xetex/common/rbbitblb.cpp b/Build/source/libs/icu/icu-xetex/common/rbbitblb.cpp new file mode 100644 index 00000000000..78fd1c51d9c --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/rbbitblb.cpp @@ -0,0 +1,1232 @@ +/* +********************************************************************** +* Copyright (c) 2002-2007, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +*/ +// +// rbbitblb.cpp +// + + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_BREAK_ITERATION + +#include "unicode/unistr.h" +#include "rbbitblb.h" +#include "rbbirb.h" +#include "rbbisetb.h" +#include "rbbidata.h" +#include "cstring.h" +#include "uassert.h" +#include "cmemory.h" + +U_NAMESPACE_BEGIN + +RBBITableBuilder::RBBITableBuilder(RBBIRuleBuilder *rb, RBBINode **rootNode) : + fTree(*rootNode) { + fRB = rb; + fStatus = fRB->fStatus; + UErrorCode status = U_ZERO_ERROR; + fDStates = new UVector(status); + if (U_FAILURE(*fStatus)) { + return; + } + if (U_FAILURE(status)) { + *fStatus = status; + return; + } + if (fDStates == NULL) { + *fStatus = U_MEMORY_ALLOCATION_ERROR;; + } +} + + + +RBBITableBuilder::~RBBITableBuilder() { + int i; + for (i=0; isize(); i++) { + delete (RBBIStateDescriptor *)fDStates->elementAt(i); + } + delete fDStates; +} + + +//----------------------------------------------------------------------------- +// +// RBBITableBuilder::build - This is the main function for building the DFA state transtion +// table from the RBBI rules parse tree. +// +//----------------------------------------------------------------------------- +void RBBITableBuilder::build() { + + if (U_FAILURE(*fStatus)) { + return; + } + + // If there were no rules, just return. This situation can easily arise + // for the reverse rules. + if (fTree==NULL) { + return; + } + + // + // Walk through the tree, replacing any references to $variables with a copy of the + // parse tree for the substition expression. + // + fTree = fTree->flattenVariables(); +#ifdef RBBI_DEBUG + if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "ftree")) { + RBBIDebugPuts("Parse tree after flattening variable references."); + fTree->printTree(TRUE); + } +#endif + + // + // If the rules contained any references to {bof} + // add a {bof} to the + // tree. Means that all matches must start out with the + // {bof} fake character. + // + if (fRB->fSetBuilder->sawBOF()) { + RBBINode *bofTop = new RBBINode(RBBINode::opCat); + RBBINode *bofLeaf = new RBBINode(RBBINode::leafChar); + bofTop->fLeftChild = bofLeaf; + bofTop->fRightChild = fTree; + bofLeaf->fParent = bofTop; + bofLeaf->fVal = 2; // Reserved value for {bof}. + fTree = bofTop; + } + + // + // Add a unique right-end marker to the expression. + // Appears as a cat-node, left child being the original tree, + // right child being the end marker. + // + RBBINode *cn = new RBBINode(RBBINode::opCat); + cn->fLeftChild = fTree; + fTree->fParent = cn; + cn->fRightChild = new RBBINode(RBBINode::endMark); + cn->fRightChild->fParent = cn; + fTree = cn; + + // + // Replace all references to UnicodeSets with the tree for the equivalent + // expression. + // + fTree->flattenSets(); +#ifdef RBBI_DEBUG + if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "stree")) { + RBBIDebugPuts("Parse tree after flattening Unicode Set references."); + fTree->printTree(TRUE); + } +#endif + + + // + // calculate the functions nullable, firstpos, lastpos and followpos on + // nodes in the parse tree. + // See the alogrithm description in Aho. + // Understanding how this works by looking at the code alone will be + // nearly impossible. + // + calcNullable(fTree); + calcFirstPos(fTree); + calcLastPos(fTree); + calcFollowPos(fTree); + if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "pos")) { + RBBIDebugPuts("\n"); + printPosSets(fTree); + } + + // + // For "chained" rules, modify the followPos sets + // + if (fRB->fChainRules) { + calcChainedFollowPos(fTree); + } + + // + // BOF (start of input) test fixup. + // + if (fRB->fSetBuilder->sawBOF()) { + bofFixup(); + } + + // + // Build the DFA state transition tables. + // + buildStateTable(); + flagAcceptingStates(); + flagLookAheadStates(); + flagTaggedStates(); + + // + // Update the global table of rule status {tag} values + // The rule builder has a global vector of status values that are common + // for all tables. Merge the ones from this table into the global set. + // + mergeRuleStatusVals(); + + if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "states")) {printStates();}; +} + + + +//----------------------------------------------------------------------------- +// +// calcNullable. Impossible to explain succinctly. See Aho, section 3.9 +// +//----------------------------------------------------------------------------- +void RBBITableBuilder::calcNullable(RBBINode *n) { + if (n == NULL) { + return; + } + if (n->fType == RBBINode::setRef || + n->fType == RBBINode::endMark ) { + // These are non-empty leaf node types. + n->fNullable = FALSE; + return; + } + + if (n->fType == RBBINode::lookAhead || n->fType == RBBINode::tag) { + // Lookahead marker node. It's a leaf, so no recursion on children. + // It's nullable because it does not match any literal text from the input stream. + n->fNullable = TRUE; + return; + } + + + // The node is not a leaf. + // Calculate nullable on its children. + calcNullable(n->fLeftChild); + calcNullable(n->fRightChild); + + // Apply functions from table 3.40 in Aho + if (n->fType == RBBINode::opOr) { + n->fNullable = n->fLeftChild->fNullable || n->fRightChild->fNullable; + } + else if (n->fType == RBBINode::opCat) { + n->fNullable = n->fLeftChild->fNullable && n->fRightChild->fNullable; + } + else if (n->fType == RBBINode::opStar || n->fType == RBBINode::opQuestion) { + n->fNullable = TRUE; + } + else { + n->fNullable = FALSE; + } +} + + + + +//----------------------------------------------------------------------------- +// +// calcFirstPos. Impossible to explain succinctly. See Aho, section 3.9 +// +//----------------------------------------------------------------------------- +void RBBITableBuilder::calcFirstPos(RBBINode *n) { + if (n == NULL) { + return; + } + if (n->fType == RBBINode::leafChar || + n->fType == RBBINode::endMark || + n->fType == RBBINode::lookAhead || + n->fType == RBBINode::tag) { + // These are non-empty leaf node types. + // Note: In order to maintain the sort invariant on the set, + // this function should only be called on a node whose set is + // empty to start with. + n->fFirstPosSet->addElement(n, *fStatus); + return; + } + + // The node is not a leaf. + // Calculate firstPos on its children. + calcFirstPos(n->fLeftChild); + calcFirstPos(n->fRightChild); + + // Apply functions from table 3.40 in Aho + if (n->fType == RBBINode::opOr) { + setAdd(n->fFirstPosSet, n->fLeftChild->fFirstPosSet); + setAdd(n->fFirstPosSet, n->fRightChild->fFirstPosSet); + } + else if (n->fType == RBBINode::opCat) { + setAdd(n->fFirstPosSet, n->fLeftChild->fFirstPosSet); + if (n->fLeftChild->fNullable) { + setAdd(n->fFirstPosSet, n->fRightChild->fFirstPosSet); + } + } + else if (n->fType == RBBINode::opStar || + n->fType == RBBINode::opQuestion || + n->fType == RBBINode::opPlus) { + setAdd(n->fFirstPosSet, n->fLeftChild->fFirstPosSet); + } +} + + + +//----------------------------------------------------------------------------- +// +// calcLastPos. Impossible to explain succinctly. See Aho, section 3.9 +// +//----------------------------------------------------------------------------- +void RBBITableBuilder::calcLastPos(RBBINode *n) { + if (n == NULL) { + return; + } + if (n->fType == RBBINode::leafChar || + n->fType == RBBINode::endMark || + n->fType == RBBINode::lookAhead || + n->fType == RBBINode::tag) { + // These are non-empty leaf node types. + // Note: In order to maintain the sort invariant on the set, + // this function should only be called on a node whose set is + // empty to start with. + n->fLastPosSet->addElement(n, *fStatus); + return; + } + + // The node is not a leaf. + // Calculate lastPos on its children. + calcLastPos(n->fLeftChild); + calcLastPos(n->fRightChild); + + // Apply functions from table 3.40 in Aho + if (n->fType == RBBINode::opOr) { + setAdd(n->fLastPosSet, n->fLeftChild->fLastPosSet); + setAdd(n->fLastPosSet, n->fRightChild->fLastPosSet); + } + else if (n->fType == RBBINode::opCat) { + setAdd(n->fLastPosSet, n->fRightChild->fLastPosSet); + if (n->fRightChild->fNullable) { + setAdd(n->fLastPosSet, n->fLeftChild->fLastPosSet); + } + } + else if (n->fType == RBBINode::opStar || + n->fType == RBBINode::opQuestion || + n->fType == RBBINode::opPlus) { + setAdd(n->fLastPosSet, n->fLeftChild->fLastPosSet); + } +} + + + +//----------------------------------------------------------------------------- +// +// calcFollowPos. Impossible to explain succinctly. See Aho, section 3.9 +// +//----------------------------------------------------------------------------- +void RBBITableBuilder::calcFollowPos(RBBINode *n) { + if (n == NULL || + n->fType == RBBINode::leafChar || + n->fType == RBBINode::endMark) { + return; + } + + calcFollowPos(n->fLeftChild); + calcFollowPos(n->fRightChild); + + // Aho rule #1 + if (n->fType == RBBINode::opCat) { + RBBINode *i; // is 'i' in Aho's description + uint32_t ix; + + UVector *LastPosOfLeftChild = n->fLeftChild->fLastPosSet; + + for (ix=0; ix<(uint32_t)LastPosOfLeftChild->size(); ix++) { + i = (RBBINode *)LastPosOfLeftChild->elementAt(ix); + setAdd(i->fFollowPos, n->fRightChild->fFirstPosSet); + } + } + + // Aho rule #2 + if (n->fType == RBBINode::opStar || + n->fType == RBBINode::opPlus) { + RBBINode *i; // again, n and i are the names from Aho's description. + uint32_t ix; + + for (ix=0; ix<(uint32_t)n->fLastPosSet->size(); ix++) { + i = (RBBINode *)n->fLastPosSet->elementAt(ix); + setAdd(i->fFollowPos, n->fFirstPosSet); + } + } + + + +} + + +//----------------------------------------------------------------------------- +// +// calcChainedFollowPos. Modify the previously calculated followPos sets +// to implement rule chaining. NOT described by Aho +// +//----------------------------------------------------------------------------- +void RBBITableBuilder::calcChainedFollowPos(RBBINode *tree) { + + UVector endMarkerNodes(*fStatus); + UVector leafNodes(*fStatus); + int32_t i; + + if (U_FAILURE(*fStatus)) { + return; + } + + // get a list of all endmarker nodes. + tree->findNodes(&endMarkerNodes, RBBINode::endMark, *fStatus); + + // get a list all leaf nodes + tree->findNodes(&leafNodes, RBBINode::leafChar, *fStatus); + if (U_FAILURE(*fStatus)) { + return; + } + + // Get all nodes that can be the start a match, which is FirstPosition() + // of the portion of the tree corresponding to user-written rules. + // See the tree description in bofFixup(). + RBBINode *userRuleRoot = tree; + if (fRB->fSetBuilder->sawBOF()) { + userRuleRoot = tree->fLeftChild->fRightChild; + } + U_ASSERT(userRuleRoot != NULL); + UVector *matchStartNodes = userRuleRoot->fFirstPosSet; + + + // Iteratate over all leaf nodes, + // + int32_t endNodeIx; + int32_t startNodeIx; + + for (endNodeIx=0; endNodeIxfFollowPos->contains(endMarkerNodes.elementAt(i))) { + endNode = tNode; + break; + } + } + if (endNode == NULL) { + // node wasn't an end node. Try again with the next. + continue; + } + + // We've got a node that can end a match. + + // Line Break Specific hack: If this node's val correspond to the $CM char class, + // don't chain from it. + // TODO: Add rule syntax for this behavior, get specifics out of here and + // into the rule file. + if (fRB->fLBCMNoChain) { + UChar32 c = this->fRB->fSetBuilder->getFirstChar(endNode->fVal); + if (c != -1) { + // c == -1 occurs with sets containing only the {eof} marker string. + ULineBreak cLBProp = (ULineBreak)u_getIntPropertyValue(c, UCHAR_LINE_BREAK); + if (cLBProp == U_LB_COMBINING_MARK) { + continue; + } + } + } + + + // Now iterate over the nodes that can start a match, looking for ones + // with the same char class as our ending node. + RBBINode *startNode; + for (startNodeIx = 0; startNodeIxsize(); startNodeIx++) { + startNode = (RBBINode *)matchStartNodes->elementAt(startNodeIx); + if (startNode->fType != RBBINode::leafChar) { + continue; + } + + if (endNode->fVal == startNode->fVal) { + // The end val (character class) of one possible match is the + // same as the start of another. + + // Add all nodes from the followPos of the start node to the + // followPos set of the end node, which will have the effect of + // letting matches transition from a match state at endNode + // to the second char of a match starting with startNode. + setAdd(endNode->fFollowPos, startNode->fFollowPos); + } + } + } +} + + +//----------------------------------------------------------------------------- +// +// bofFixup. Fixup for state tables that include {bof} beginning of input testing. +// Do an swizzle similar to chaining, modifying the followPos set of +// the bofNode to include the followPos nodes from other {bot} nodes +// scattered through the tree. +// +// This function has much in common with calcChainedFollowPos(). +// +//----------------------------------------------------------------------------- +void RBBITableBuilder::bofFixup() { + + if (U_FAILURE(*fStatus)) { + return; + } + + // The parse tree looks like this ... + // fTree root ---> + // / \ . + // <#end node> + // / \ . + // rest + // of tree + // + // We will be adding things to the followPos set of the + // + RBBINode *bofNode = fTree->fLeftChild->fLeftChild; + U_ASSERT(bofNode->fType == RBBINode::leafChar); + U_ASSERT(bofNode->fVal == 2); + + // Get all nodes that can be the start a match of the user-written rules + // (excluding the fake bofNode) + // We want the nodes that can start a match in the + // part labeled "rest of tree" + // + UVector *matchStartNodes = fTree->fLeftChild->fRightChild->fFirstPosSet; + + RBBINode *startNode; + int startNodeIx; + for (startNodeIx = 0; startNodeIxsize(); startNodeIx++) { + startNode = (RBBINode *)matchStartNodes->elementAt(startNodeIx); + if (startNode->fType != RBBINode::leafChar) { + continue; + } + + if (startNode->fVal == bofNode->fVal) { + // We found a leaf node corresponding to a {bof} that was + // explicitly written into a rule. + // Add everything from the followPos set of this node to the + // followPos set of the fake bofNode at the start of the tree. + // + setAdd(bofNode->fFollowPos, startNode->fFollowPos); + } + } +} + +//----------------------------------------------------------------------------- +// +// buildStateTable() Determine the set of runtime DFA states and the +// transition tables for these states, by the algorithm +// of fig. 3.44 in Aho. +// +// Most of the comments are quotes of Aho's psuedo-code. +// +//----------------------------------------------------------------------------- +void RBBITableBuilder::buildStateTable() { + if (U_FAILURE(*fStatus)) { + return; + } + // + // Add a dummy state 0 - the stop state. Not from Aho. + int lastInputSymbol = fRB->fSetBuilder->getNumCharCategories() - 1; + RBBIStateDescriptor *failState = new RBBIStateDescriptor(lastInputSymbol, fStatus); + failState->fPositions = new UVector(*fStatus); + if (U_FAILURE(*fStatus)) { + return; + } + fDStates->addElement(failState, *fStatus); + if (U_FAILURE(*fStatus)) { + return; + } + + // initially, the only unmarked state in Dstates is firstpos(root), + // where toot is the root of the syntax tree for (r)#; + RBBIStateDescriptor *initialState = new RBBIStateDescriptor(lastInputSymbol, fStatus); + if (U_FAILURE(*fStatus)) { + return; + } + initialState->fPositions = new UVector(*fStatus); + if (U_FAILURE(*fStatus)) { + return; + } + setAdd(initialState->fPositions, fTree->fFirstPosSet); + fDStates->addElement(initialState, *fStatus); + if (U_FAILURE(*fStatus)) { + return; + } + + // while there is an unmarked state T in Dstates do begin + for (;;) { + RBBIStateDescriptor *T = NULL; + int32_t tx; + for (tx=1; txsize(); tx++) { + RBBIStateDescriptor *temp; + temp = (RBBIStateDescriptor *)fDStates->elementAt(tx); + if (temp->fMarked == FALSE) { + T = temp; + break; + } + } + if (T == NULL) { + break; + } + + // mark T; + T->fMarked = TRUE; + + // for each input symbol a do begin + int32_t a; + for (a = 1; a<=lastInputSymbol; a++) { + // let U be the set of positions that are in followpos(p) + // for some position p in T + // such that the symbol at position p is a; + UVector *U = NULL; + RBBINode *p; + int32_t px; + for (px=0; pxfPositions->size(); px++) { + p = (RBBINode *)T->fPositions->elementAt(px); + if ((p->fType == RBBINode::leafChar) && (p->fVal == a)) { + if (U == NULL) { + U = new UVector(*fStatus); + } + setAdd(U, p->fFollowPos); + } + } + + // if U is not empty and not in DStates then + int32_t ux = 0; + UBool UinDstates = FALSE; + if (U != NULL) { + U_ASSERT(U->size() > 0); + int ix; + for (ix=0; ixsize(); ix++) { + RBBIStateDescriptor *temp2; + temp2 = (RBBIStateDescriptor *)fDStates->elementAt(ix); + if (setEquals(U, temp2->fPositions)) { + delete U; + U = temp2->fPositions; + ux = ix; + UinDstates = TRUE; + break; + } + } + + // Add U as an unmarked state to Dstates + if (!UinDstates) + { + RBBIStateDescriptor *newState = new RBBIStateDescriptor(lastInputSymbol, fStatus); + if (U_FAILURE(*fStatus)) { + return; + } + newState->fPositions = U; + fDStates->addElement(newState, *fStatus); + if (U_FAILURE(*fStatus)) { + return; + } + ux = fDStates->size()-1; + } + + // Dtran[T, a] := U; + T->fDtran->setElementAt(ux, a); + } + } + } +} + + + +//----------------------------------------------------------------------------- +// +// flagAcceptingStates Identify accepting states. +// First get a list of all of the end marker nodes. +// Then, for each state s, +// if s contains one of the end marker nodes in its list of tree positions then +// s is an accepting state. +// +//----------------------------------------------------------------------------- +void RBBITableBuilder::flagAcceptingStates() { + if (U_FAILURE(*fStatus)) { + return; + } + UVector endMarkerNodes(*fStatus); + RBBINode *endMarker; + int32_t i; + int32_t n; + + if (U_FAILURE(*fStatus)) { + return; + } + + fTree->findNodes(&endMarkerNodes, RBBINode::endMark, *fStatus); + if (U_FAILURE(*fStatus)) { + return; + } + + for (i=0; isize(); n++) { + RBBIStateDescriptor *sd = (RBBIStateDescriptor *)fDStates->elementAt(n); + if (sd->fPositions->indexOf(endMarker) >= 0) { + // Any non-zero value for fAccepting means this is an accepting node. + // The value is what will be returned to the user as the break status. + // If no other value was specified, force it to -1. + + if (sd->fAccepting==0) { + // State hasn't been marked as accepting yet. Do it now. + sd->fAccepting = endMarker->fVal; + if (sd->fAccepting == 0) { + sd->fAccepting = -1; + } + } + if (sd->fAccepting==-1 && endMarker->fVal != 0) { + // Both lookahead and non-lookahead accepting for this state. + // Favor the look-ahead. Expedient for line break. + // TODO: need a more elegant resolution for conflicting rules. + sd->fAccepting = endMarker->fVal; + } + // implicit else: + // if sd->fAccepting already had a value other than 0 or -1, leave it be. + + // If the end marker node is from a look-ahead rule, set + // the fLookAhead field or this state also. + if (endMarker->fLookAheadEnd) { + // TODO: don't change value if already set? + // TODO: allow for more than one active look-ahead rule in engine. + // Make value here an index to a side array in engine? + sd->fLookAhead = sd->fAccepting; + } + } + } + } +} + + +//----------------------------------------------------------------------------- +// +// flagLookAheadStates Very similar to flagAcceptingStates, above. +// +//----------------------------------------------------------------------------- +void RBBITableBuilder::flagLookAheadStates() { + if (U_FAILURE(*fStatus)) { + return; + } + UVector lookAheadNodes(*fStatus); + RBBINode *lookAheadNode; + int32_t i; + int32_t n; + + fTree->findNodes(&lookAheadNodes, RBBINode::lookAhead, *fStatus); + if (U_FAILURE(*fStatus)) { + return; + } + for (i=0; isize(); n++) { + RBBIStateDescriptor *sd = (RBBIStateDescriptor *)fDStates->elementAt(n); + if (sd->fPositions->indexOf(lookAheadNode) >= 0) { + sd->fLookAhead = lookAheadNode->fVal; + } + } + } +} + + + + +//----------------------------------------------------------------------------- +// +// flagTaggedStates +// +//----------------------------------------------------------------------------- +void RBBITableBuilder::flagTaggedStates() { + if (U_FAILURE(*fStatus)) { + return; + } + UVector tagNodes(*fStatus); + RBBINode *tagNode; + int32_t i; + int32_t n; + + if (U_FAILURE(*fStatus)) { + return; + } + fTree->findNodes(&tagNodes, RBBINode::tag, *fStatus); + if (U_FAILURE(*fStatus)) { + return; + } + for (i=0; isize(); n++) { // For each state s (row in the state table) + RBBIStateDescriptor *sd = (RBBIStateDescriptor *)fDStates->elementAt(n); + if (sd->fPositions->indexOf(tagNode) >= 0) { // if s include the tag node t + sortedAdd(&sd->fTagVals, tagNode->fVal); + } + } + } +} + + + + +//----------------------------------------------------------------------------- +// +// mergeRuleStatusVals +// +// Update the global table of rule status {tag} values +// The rule builder has a global vector of status values that are common +// for all tables. Merge the ones from this table into the global set. +// +//----------------------------------------------------------------------------- +void RBBITableBuilder::mergeRuleStatusVals() { + // + // The basic outline of what happens here is this... + // + // for each state in this state table + // if the status tag list for this state is in the global statuses list + // record where and + // continue with the next state + // else + // add the tag list for this state to the global list. + // + int i; + int n; + + // Pre-set a single tag of {0} into the table. + // We will need this as a default, for rule sets with no explicit tagging. + if (fRB->fRuleStatusVals->size() == 0) { + fRB->fRuleStatusVals->addElement(1, *fStatus); // Num of statuses in group + fRB->fRuleStatusVals->addElement((int32_t)0, *fStatus); // and our single status of zero + } + + // For each state + for (n=0; nsize(); n++) { + RBBIStateDescriptor *sd = (RBBIStateDescriptor *)fDStates->elementAt(n); + UVector *thisStatesTagValues = sd->fTagVals; + if (thisStatesTagValues == NULL) { + // No tag values are explicitly associated with this state. + // Set the default tag value. + sd->fTagsIdx = 0; + continue; + } + + // There are tag(s) associated with this state. + // fTagsIdx will be the index into the global tag list for this state's tag values. + // Initial value of -1 flags that we haven't got it set yet. + sd->fTagsIdx = -1; + int32_t thisTagGroupStart = 0; // indexes into the global rule status vals list + int32_t nextTagGroupStart = 0; + + // Loop runs once per group of tags in the global list + while (nextTagGroupStart < fRB->fRuleStatusVals->size()) { + thisTagGroupStart = nextTagGroupStart; + nextTagGroupStart += fRB->fRuleStatusVals->elementAti(thisTagGroupStart) + 1; + if (thisStatesTagValues->size() != fRB->fRuleStatusVals->elementAti(thisTagGroupStart)) { + // The number of tags for this state is different from + // the number of tags in this group from the global list. + // Continue with the next group from the global list. + continue; + } + // The lengths match, go ahead and compare the actual tag values + // between this state and the group from the global list. + for (i=0; isize(); i++) { + if (thisStatesTagValues->elementAti(i) != + fRB->fRuleStatusVals->elementAti(thisTagGroupStart + 1 + i) ) { + // Mismatch. + break; + } + } + + if (i == thisStatesTagValues->size()) { + // We found a set of tag values in the global list that match + // those for this state. Use them. + sd->fTagsIdx = thisTagGroupStart; + break; + } + } + + if (sd->fTagsIdx == -1) { + // No suitable entry in the global tag list already. Add one + sd->fTagsIdx = fRB->fRuleStatusVals->size(); + fRB->fRuleStatusVals->addElement(thisStatesTagValues->size(), *fStatus); + for (i=0; isize(); i++) { + fRB->fRuleStatusVals->addElement(thisStatesTagValues->elementAti(i), *fStatus); + } + } + } +} + + + + + + + +//----------------------------------------------------------------------------- +// +// sortedAdd Add a value to a vector of sorted values (ints). +// Do not replicate entries; if the value is already there, do not +// add a second one. +// Lazily create the vector if it does not already exist. +// +//----------------------------------------------------------------------------- +void RBBITableBuilder::sortedAdd(UVector **vector, int32_t val) { + int32_t i; + + if (*vector == NULL) { + *vector = new UVector(*fStatus); + } + if (*vector == NULL || U_FAILURE(*fStatus)) { + return; + } + UVector *vec = *vector; + int32_t vSize = vec->size(); + for (i=0; ielementAti(i); + if (valAtI == val) { + // The value is already in the vector. Don't add it again. + return; + } + if (valAtI > val) { + break; + } + } + vec->insertElementAt(val, i, *fStatus); +} + + + +//----------------------------------------------------------------------------- +// +// setAdd Set operation on UVector +// dest = dest union source +// Elements may only appear once and must be sorted. +// +//----------------------------------------------------------------------------- +void RBBITableBuilder::setAdd(UVector *dest, UVector *source) { + int32_t destOriginalSize = dest->size(); + int32_t sourceSize = source->size(); + int32_t di = 0; + void *(destS[16]), *(sourceS[16]); // Handle small cases without malloc + void **destH = 0, **sourceH = 0; + void **destBuff, **sourceBuff; + void **destLim, **sourceLim; + + if (destOriginalSize > (int32_t)(sizeof(destS)/sizeof(destS[0]))) { + destH = (void **)uprv_malloc(sizeof(void *) * destOriginalSize); + destBuff = destH; + } + else { + destBuff = destS; + } + if (destBuff == 0) { + return; + } + destLim = destBuff + destOriginalSize; + + if (sourceSize > (int32_t)(sizeof(sourceS)/sizeof(sourceS[0]))) { + sourceH = (void **)uprv_malloc(sizeof(void *) * sourceSize); + sourceBuff = sourceH; + } + else { + sourceBuff = sourceS; + } + if (sourceBuff == 0) { + if (destH) { + uprv_free(destH); + } + return; + } + sourceLim = sourceBuff + sourceSize; + + // Avoid multiple "get element" calls by getting the contents into arrays + (void) dest->toArray(destBuff); + (void) source->toArray(sourceBuff); + + dest->setSize(sourceSize+destOriginalSize); + + while (sourceBuff < sourceLim && destBuff < destLim) { + if (*destBuff == *sourceBuff) { + dest->setElementAt(*sourceBuff++, di++); + destBuff++; + } + // This check is required for machines with segmented memory, like i5/OS. + // Direct pointer comparison is not recommended. + else if (uprv_memcmp(destBuff, sourceBuff, sizeof(void *)) < 0) { + dest->setElementAt(*destBuff++, di++); + } + else { /* *sourceBuff < *destBuff */ + dest->setElementAt(*sourceBuff++, di++); + } + } + + // At most one of these two cleanup loops will execute + while (destBuff < destLim) { + dest->setElementAt(*destBuff++, di++); + } + while (sourceBuff < sourceLim) { + dest->setElementAt(*sourceBuff++, di++); + } + + dest->setSize(di); + if (destH) { + uprv_free(destH); + } + if (sourceH) { + uprv_free(sourceH); + } +} + + + +//----------------------------------------------------------------------------- +// +// setEqual Set operation on UVector. +// Compare for equality. +// Elements must be sorted. +// +//----------------------------------------------------------------------------- +UBool RBBITableBuilder::setEquals(UVector *a, UVector *b) { + return a->equals(*b); +} + + +//----------------------------------------------------------------------------- +// +// printPosSets Debug function. Dump Nullable, firstpos, lastpos and followpos +// for each node in the tree. +// +//----------------------------------------------------------------------------- +#ifdef RBBI_DEBUG +void RBBITableBuilder::printPosSets(RBBINode *n) { + if (n==NULL) { + return; + } + n->printNode(); + RBBIDebugPrintf(" Nullable: %s\n", n->fNullable?"TRUE":"FALSE"); + + RBBIDebugPrintf(" firstpos: "); + printSet(n->fFirstPosSet); + + RBBIDebugPrintf(" lastpos: "); + printSet(n->fLastPosSet); + + RBBIDebugPrintf(" followpos: "); + printSet(n->fFollowPos); + + printPosSets(n->fLeftChild); + printPosSets(n->fRightChild); +} +#endif + + + +//----------------------------------------------------------------------------- +// +// getTableSize() Calculate the size of the runtime form of this +// state transition table. +// +//----------------------------------------------------------------------------- +int32_t RBBITableBuilder::getTableSize() const { + int32_t size = 0; + int32_t numRows; + int32_t numCols; + int32_t rowSize; + + if (fTree == NULL) { + return 0; + } + + size = sizeof(RBBIStateTable) - 4; // The header, with no rows to the table. + + numRows = fDStates->size(); + numCols = fRB->fSetBuilder->getNumCharCategories(); + + // Note The declaration of RBBIStateTableRow is for a table of two columns. + // Therefore we subtract two from numCols when determining + // how much storage to add to a row for the total columns. + rowSize = sizeof(RBBIStateTableRow) + sizeof(uint16_t)*(numCols-2); + size += numRows * rowSize; + return size; +} + + + +//----------------------------------------------------------------------------- +// +// exportTable() export the state transition table in the format required +// by the runtime engine. getTableSize() bytes of memory +// must be available at the output address "where". +// +//----------------------------------------------------------------------------- +void RBBITableBuilder::exportTable(void *where) { + RBBIStateTable *table = (RBBIStateTable *)where; + uint32_t state; + int col; + + if (U_FAILURE(*fStatus) || fTree == NULL) { + return; + } + + if (fRB->fSetBuilder->getNumCharCategories() > 0x7fff || + fDStates->size() > 0x7fff) { + *fStatus = U_BRK_INTERNAL_ERROR; + return; + } + + table->fRowLen = sizeof(RBBIStateTableRow) + + sizeof(uint16_t) * (fRB->fSetBuilder->getNumCharCategories() - 2); + table->fNumStates = fDStates->size(); + table->fFlags = 0; + if (fRB->fLookAheadHardBreak) { + table->fFlags |= RBBI_LOOKAHEAD_HARD_BREAK; + } + if (fRB->fSetBuilder->sawBOF()) { + table->fFlags |= RBBI_BOF_REQUIRED; + } + table->fReserved = 0; + + for (state=0; statefNumStates; state++) { + RBBIStateDescriptor *sd = (RBBIStateDescriptor *)fDStates->elementAt(state); + RBBIStateTableRow *row = (RBBIStateTableRow *)(table->fTableData + state*table->fRowLen); + U_ASSERT (-32768 < sd->fAccepting && sd->fAccepting <= 32767); + U_ASSERT (-32768 < sd->fLookAhead && sd->fLookAhead <= 32767); + row->fAccepting = (int16_t)sd->fAccepting; + row->fLookAhead = (int16_t)sd->fLookAhead; + row->fTagIdx = (int16_t)sd->fTagsIdx; + for (col=0; colfSetBuilder->getNumCharCategories(); col++) { + row->fNextState[col] = (uint16_t)sd->fDtran->elementAti(col); + } + } +} + + + +//----------------------------------------------------------------------------- +// +// printSet Debug function. Print the contents of a UVector +// +//----------------------------------------------------------------------------- +#ifdef RBBI_DEBUG +void RBBITableBuilder::printSet(UVector *s) { + int32_t i; + for (i=0; isize(); i++) { + void *v = s->elementAt(i); + RBBIDebugPrintf("%10p", v); + } + RBBIDebugPrintf("\n"); +} +#endif + + +//----------------------------------------------------------------------------- +// +// printStates Debug Function. Dump the fully constructed state transition table. +// +//----------------------------------------------------------------------------- +#ifdef RBBI_DEBUG +void RBBITableBuilder::printStates() { + int c; // input "character" + int n; // state number + + RBBIDebugPrintf("state | i n p u t s y m b o l s \n"); + RBBIDebugPrintf(" | Acc LA Tag"); + for (c=0; cfSetBuilder->getNumCharCategories(); c++) { + RBBIDebugPrintf(" %2d", c); + } + RBBIDebugPrintf("\n"); + RBBIDebugPrintf(" |---------------"); + for (c=0; cfSetBuilder->getNumCharCategories(); c++) { + RBBIDebugPrintf("---"); + } + RBBIDebugPrintf("\n"); + + for (n=0; nsize(); n++) { + RBBIStateDescriptor *sd = (RBBIStateDescriptor *)fDStates->elementAt(n); + RBBIDebugPrintf(" %3d | " , n); + RBBIDebugPrintf("%3d %3d %5d ", sd->fAccepting, sd->fLookAhead, sd->fTagsIdx); + for (c=0; cfSetBuilder->getNumCharCategories(); c++) { + RBBIDebugPrintf(" %2d", sd->fDtran->elementAti(c)); + } + RBBIDebugPrintf("\n"); + } + RBBIDebugPrintf("\n\n"); +} +#endif + + + +//----------------------------------------------------------------------------- +// +// printRuleStatusTable Debug Function. Dump the common rule status table +// +//----------------------------------------------------------------------------- +#ifdef RBBI_DEBUG +void RBBITableBuilder::printRuleStatusTable() { + int32_t thisRecord = 0; + int32_t nextRecord = 0; + int i; + UVector *tbl = fRB->fRuleStatusVals; + + RBBIDebugPrintf("index | tags \n"); + RBBIDebugPrintf("-------------------\n"); + + while (nextRecord < tbl->size()) { + thisRecord = nextRecord; + nextRecord = thisRecord + tbl->elementAti(thisRecord) + 1; + RBBIDebugPrintf("%4d ", thisRecord); + for (i=thisRecord+1; ielementAti(i)); + } + RBBIDebugPrintf("\n"); + } + RBBIDebugPrintf("\n\n"); +} +#endif + + +//----------------------------------------------------------------------------- +// +// RBBIStateDescriptor Methods. This is a very struct-like class +// Most access is directly to the fields. +// +//----------------------------------------------------------------------------- + +RBBIStateDescriptor::RBBIStateDescriptor(int lastInputSymbol, UErrorCode *fStatus) { + fMarked = FALSE; + fAccepting = 0; + fLookAhead = 0; + fTagsIdx = 0; + fTagVals = NULL; + fPositions = NULL; + fDtran = NULL; + + fDtran = new UVector(lastInputSymbol+1, *fStatus); + if (U_FAILURE(*fStatus)) { + return; + } + if (fDtran == NULL) { + *fStatus = U_MEMORY_ALLOCATION_ERROR; + return; + } + fDtran->setSize(lastInputSymbol+1); // fDtran needs to be pre-sized. + // It is indexed by input symbols, and will + // hold the next state number for each + // symbol. +} + + +RBBIStateDescriptor::~RBBIStateDescriptor() { + delete fPositions; + delete fDtran; + delete fTagVals; + fPositions = NULL; + fDtran = NULL; + fTagVals = NULL; +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_BREAK_ITERATION */ diff --git a/Build/source/libs/icu/icu-xetex/common/rbbitblb.h b/Build/source/libs/icu/icu-xetex/common/rbbitblb.h new file mode 100644 index 00000000000..3805b6752a3 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/rbbitblb.h @@ -0,0 +1,127 @@ +// +// rbbitblb.h +// + +/* +********************************************************************** +* Copyright (c) 2002-2005, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +*/ + +#ifndef RBBITBLB_H +#define RBBITBLB_H + +#include "unicode/utypes.h" +#include "unicode/uobject.h" +#include "unicode/rbbi.h" +#include "rbbinode.h" + + +U_NAMESPACE_BEGIN + +class RBBIRuleScanner; +class RBBIRuleBuilder; + +// +// class RBBITableBuilder is part of the RBBI rule compiler. +// It builds the state transition table used by the RBBI runtime +// from the expression syntax tree generated by the rule scanner. +// +// This class is part of the RBBI implementation only. +// There is no user-visible public API here. +// + +class RBBITableBuilder : public UMemory { +public: + RBBITableBuilder(RBBIRuleBuilder *rb, RBBINode **rootNode); + ~RBBITableBuilder(); + + void build(); + int32_t getTableSize() const; // Return the runtime size in bytes of + // the built state table + void exportTable(void *where); // fill in the runtime state table. + // Sufficient memory must exist at + // the specified location. + + +private: + void calcNullable(RBBINode *n); + void calcFirstPos(RBBINode *n); + void calcLastPos(RBBINode *n); + void calcFollowPos(RBBINode *n); + void calcChainedFollowPos(RBBINode *n); + void bofFixup(); + void buildStateTable(); + void flagAcceptingStates(); + void flagLookAheadStates(); + void flagTaggedStates(); + void mergeRuleStatusVals(); + + // Set functions for UVector. + // TODO: make a USet subclass of UVector + + void setAdd(UVector *dest, UVector *source); + UBool setEquals(UVector *a, UVector *b); + + void sortedAdd(UVector **dest, int32_t val); + +public: +#ifdef RBBI_DEBUG + void printSet(UVector *s); + void printPosSets(RBBINode *n /* = NULL*/); + void printStates(); + void printRuleStatusTable(); +#else + #define printSet(s) + #define printPosSets(n) + #define printStates() + #define printRuleStatusTable() +#endif + +private: + RBBIRuleBuilder *fRB; + RBBINode *&fTree; // The root node of the parse tree to build a + // table for. + UErrorCode *fStatus; + + UVector *fDStates; // D states (Aho's terminology) + // Index is state number + // Contents are RBBIStateDescriptor pointers. + + + RBBITableBuilder(const RBBITableBuilder &other); // forbid copying of this class + RBBITableBuilder &operator=(const RBBITableBuilder &other); // forbid copying of this class +}; + +// +// RBBIStateDescriptor - The DFA is constructed as a set of these descriptors, +// one for each state. +class RBBIStateDescriptor : public UMemory { +public: + UBool fMarked; + int32_t fAccepting; + int32_t fLookAhead; + UVector *fTagVals; + int32_t fTagsIdx; + UVector *fPositions; // Set of parse tree positions associated + // with this state. Unordered (it's a set). + // UVector contents are RBBINode * + + UVector *fDtran; // Transitions out of this state. + // indexed by input character + // contents is int index of dest state + // in RBBITableBuilder.fDStates + + RBBIStateDescriptor(int maxInputSymbol, UErrorCode *fStatus); + ~RBBIStateDescriptor(); + +private: + RBBIStateDescriptor(const RBBIStateDescriptor &other); // forbid copying of this class + RBBIStateDescriptor &operator=(const RBBIStateDescriptor &other); // forbid copying of this class +}; + + + +U_NAMESPACE_END +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/resbund.cpp b/Build/source/libs/icu/icu-xetex/common/resbund.cpp new file mode 100644 index 00000000000..c4b853293ea --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/resbund.cpp @@ -0,0 +1,396 @@ +/* +********************************************************************** +* Copyright (C) 1997-2006, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* +* File resbund.cpp +* +* Modification History: +* +* Date Name Description +* 02/05/97 aliu Fixed bug in chopLocale. Added scanForLocaleInFile +* based on code taken from scanForLocale. Added +* constructor which attempts to read resource bundle +* from a specific file, without searching other files. +* 02/11/97 aliu Added UErrorCode return values to constructors. Fixed +* infinite loops in scanForFile and scanForLocale. +* Modified getRawResourceData to not delete storage in +* localeData and resourceData which it doesn't own. +* Added Mac compatibility #ifdefs for tellp() and +* ios::nocreate. +* 03/04/97 aliu Modified to use ExpandingDataSink objects instead of +* the highly inefficient ostrstream objects. +* 03/13/97 aliu Rewrote to load in entire resource bundle and store +* it as a Hashtable of ResourceBundleData objects. +* Added state table to govern parsing of files. +* Modified to load locale index out of new file distinct +* from default.txt. +* 03/25/97 aliu Modified to support 2-d arrays, needed for timezone data. +* Added support for custom file suffixes. Again, needed +* to support timezone data. Improved error handling to +* detect duplicate tags and subtags. +* 04/07/97 aliu Fixed bug in getHashtableForLocale(). Fixed handling +* of failing UErrorCode values on entry to API methods. +* Fixed bugs in getArrayItem() for negative indices. +* 04/29/97 aliu Update to use new Hashtable deletion protocol. +* 05/06/97 aliu Flattened kTransitionTable for HP compiler. +* Fixed usage of CharString. +* 06/11/99 stephen Removed parsing of .txt files. +* Reworked to use new binary format. +* Cleaned up. +* 06/14/99 stephen Removed methods taking a filename suffix. +* 06/22/99 stephen Added missing T_FileStream_close in parse() +* 11/09/99 weiv Added getLocale(), rewritten constructForLocale() +* March 2000 weiv complete overhaul. +****************************************************************************** +*/ + +#include "unicode/utypes.h" +#include "unicode/resbund.h" +#include "umutex.h" + +#include "uresimp.h" + +U_NAMESPACE_BEGIN + +/*----------------------------------------------------------------------------- + * Implementation Notes + * + * Resource bundles are read in once, and thereafter cached. + * ResourceBundle statically keeps track of which files have been + * read, so we are guaranteed that each file is read at most once. + * Resource bundles can be loaded from different data directories and + * will be treated as distinct, even if they are for the same locale. + * + * Resource bundles are lightweight objects, which have pointers to + * one or more shared Hashtable objects containing all the data. + * Copying would be cheap, but there is no copy constructor, since + * there wasn't one in the original API. + * + * The ResourceBundle parsing mechanism is implemented as a transition + * network, for easy maintenance and modification. The network is + * implemented as a matrix (instead of in code) to make this even + * easier. The matrix contains Transition objects. Each Transition + * object describes a destination node and an action to take before + * moving to the destination node. The source node is encoded by the + * index of the object in the array that contains it. The pieces + * needed to understand the transition network are the enums for node + * IDs and actions, the parse() method, which walks through the + * network and implements the actions, and the network itself. The + * network guarantees certain conditions, for example, that a new + * resource will not be closed until one has been opened first; or + * that data will not be stored into a TaggedList until a TaggedList + * has been created. Nonetheless, the code in parse() does some + * consistency checks as it runs the network, and fails with an + * U_INTERNAL_PROGRAM_ERROR if one of these checks fails. If the input + * data has a bad format, an U_INVALID_FORMAT_ERROR is returned. If you + * see an U_INTERNAL_PROGRAM_ERROR the transition matrix has a bug in + * it. + * + * Old functionality of multiple locales in a single file is still + * supported. For this reason, LOCALE names override FILE names. If + * data for en_US is located in the en.txt file, once it is loaded, + * the code will not care where it came from (other than remembering + * which directory it came from). However, if there is an en_US + * resource in en_US.txt, that will take precedence. There is no + * limit to the number or type of resources that can be stored in a + * file, however, files are only searched in a specific way. If + * en_US_CA is requested, then first en_US_CA.txt is searched, then + * en_US.txt, then en.txt, then default.txt. So it only makes sense + * to put certain locales in certain files. In this example, it would + * be logical to put en_US_CA, en_US, and en into the en.txt file, + * since they would be found there if asked for. The extreme example + * is to place all locale resources into default.txt, which should + * also work. + * + * Inheritance is implemented. For example, xx_YY_zz inherits as + * follows: xx_YY_zz, xx_YY, xx, default. Inheritance is implemented + * as an array of hashtables. There will be from 1 to 4 hashtables in + * the array. + * + * Fallback files are implemented. The fallback pattern is Language + * Country Variant (LCV) -> LC -> L. Fallback is first done for the + * requested locale. Then it is done for the default locale, as + * returned by Locale::getDefault(). Then the special file + * default.txt is searched for the default locale. The overall FILE + * fallback path is LCV -> LC -> L -> dLCV -> dLC -> dL -> default. + * + * Note that although file name searching includes the default locale, + * once a ResourceBundle object is constructed, the inheritance path + * no longer includes the default locale. The path is LCV -> LC -> L + * -> default. + * + * File parsing is lazy. Nothing is parsed unless it is called for by + * someone. So when a ResourceBundle for xx_YY_zz is constructed, + * only that locale is parsed (along with anything else in the same + * file). Later, if the FooBar tag is asked for, and if it isn't + * found in xx_YY_zz, then xx_YY.txt will be parsed and checked, and + * so forth, until the chain is exhausted or the tag is found. + * + * Thread-safety is implemented around caches, both the cache that + * stores all the resouce data, and the cache that stores flags + * indicating whether or not a file has been visited. These caches + * delete their storage at static cleanup time, when the process + * quits. + * + * ResourceBundle supports TableCollation as a special case. This + * involves having special ResourceBundle objects which DO own their + * data, since we don't want large collation rule strings in the + * ResourceBundle cache (these are already cached in the + * TableCollation cache). TableCollation files (.ctx files) have the + * same format as normal resource data files, with a different + * interpretation, from the standpoint of ResourceBundle. .ctx files + * are loaded into otherwise ordinary ResourceBundle objects. They + * don't inherit (that's implemented by TableCollation) and they own + * their data (as mentioned above). However, they still support + * possible multiple locales in a single .ctx file. (This is in + * practice a bad idea, since you only want the one locale you're + * looking for, and only one tag will be present + * ("CollationElements"), so you don't need an inheritance chain of + * multiple locales.) Up to 4 locale resources will be loaded from a + * .ctx file; everything after the first 4 is ignored (parsed and + * deleted). (Normal .txt files have no limit.) Instead of being + * loaded into the cache, and then looked up as needed, the locale + * resources are read straight into the ResourceBundle object. + * + * The Index, which used to reside in default.txt, has been moved to a + * new file, index.txt. This file contains a slightly modified format + * with the addition of the "InstalledLocales" tag; it looks like: + * + * Index { + * InstalledLocales { + * ar + * .. + * zh_TW + * } + * } + */ +//----------------------------------------------------------------------------- + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(ResourceBundle) + +ResourceBundle::ResourceBundle(UErrorCode &err) + :UObject(), fLocale(NULL) +{ + fResource = ures_open(0, Locale::getDefault().getName(), &err); +} + +ResourceBundle::ResourceBundle(const ResourceBundle &other) + :UObject(other), fLocale(NULL) +{ + UErrorCode status = U_ZERO_ERROR; + + if (other.fResource) { + fResource = ures_copyResb(0, other.fResource, &status); + } else { + /* Copying a bad resource bundle */ + fResource = NULL; + } +} + +ResourceBundle::ResourceBundle(UResourceBundle *res, UErrorCode& err) + :UObject(), fLocale(NULL) +{ + if (res) { + fResource = ures_copyResb(0, res, &err); + } else { + /* Copying a bad resource bundle */ + fResource = NULL; + } +} + +ResourceBundle::ResourceBundle(const char* path, const Locale& locale, UErrorCode& err) + :UObject(), fLocale(NULL) +{ + fResource = ures_open(path, locale.getName(), &err); +} + + +ResourceBundle& ResourceBundle::operator=(const ResourceBundle& other) +{ + if(this == &other) { + return *this; + } + if(fResource != 0) { + ures_close(fResource); + fResource = NULL; + } + UErrorCode status = U_ZERO_ERROR; + if (other.fResource) { + fResource = ures_copyResb(0, other.fResource, &status); + } else { + /* Copying a bad resource bundle */ + fResource = NULL; + } + return *this; +} + +ResourceBundle::~ResourceBundle() +{ + if(fResource != 0) { + ures_close(fResource); + } + if(fLocale != NULL) { + delete(fLocale); + } +} + +ResourceBundle * +ResourceBundle::clone() const { + return new ResourceBundle(*this); +} + +UnicodeString ResourceBundle::getString(UErrorCode& status) const { + int32_t len = 0; + const UChar *r = ures_getString(fResource, &len, &status); + return UnicodeString(TRUE, r, len); +} + +const uint8_t *ResourceBundle::getBinary(int32_t& len, UErrorCode& status) const { + return ures_getBinary(fResource, &len, &status); +} + +const int32_t *ResourceBundle::getIntVector(int32_t& len, UErrorCode& status) const { + return ures_getIntVector(fResource, &len, &status); +} + +uint32_t ResourceBundle::getUInt(UErrorCode& status) const { + return ures_getUInt(fResource, &status); +} + +int32_t ResourceBundle::getInt(UErrorCode& status) const { + return ures_getInt(fResource, &status); +} + +const char *ResourceBundle::getName(void) const { + return ures_getName(fResource); +} + +const char *ResourceBundle::getKey(void) const { + return ures_getKey(fResource); +} + +UResType ResourceBundle::getType(void) const { + return ures_getType(fResource); +} + +int32_t ResourceBundle::getSize(void) const { + return ures_getSize(fResource); +} + +UBool ResourceBundle::hasNext(void) const { + return ures_hasNext(fResource); +} + +void ResourceBundle::resetIterator(void) { + ures_resetIterator(fResource); +} + +ResourceBundle ResourceBundle::getNext(UErrorCode& status) { + UResourceBundle r; + + ures_initStackObject(&r); + ures_getNextResource(fResource, &r, &status); + ResourceBundle res(&r, status); + if (U_SUCCESS(status)) { + ures_close(&r); + } + return res; +} + +UnicodeString ResourceBundle::getNextString(UErrorCode& status) { + int32_t len = 0; + const UChar* r = ures_getNextString(fResource, &len, 0, &status); + return UnicodeString(TRUE, r, len); +} + +UnicodeString ResourceBundle::getNextString(const char ** key, UErrorCode& status) { + int32_t len = 0; + const UChar* r = ures_getNextString(fResource, &len, key, &status); + return UnicodeString(TRUE, r, len); +} + +ResourceBundle ResourceBundle::get(int32_t indexR, UErrorCode& status) const { + UResourceBundle r; + + ures_initStackObject(&r); + ures_getByIndex(fResource, indexR, &r, &status); + ResourceBundle res(&r, status); + if (U_SUCCESS(status)) { + ures_close(&r); + } + return res; +} + +UnicodeString ResourceBundle::getStringEx(int32_t indexS, UErrorCode& status) const { + int32_t len = 0; + const UChar* r = ures_getStringByIndex(fResource, indexS, &len, &status); + return UnicodeString(TRUE, r, len); +} + +ResourceBundle ResourceBundle::get(const char* key, UErrorCode& status) const { + UResourceBundle r; + + ures_initStackObject(&r); + ures_getByKey(fResource, key, &r, &status); + ResourceBundle res(&r, status); + if (U_SUCCESS(status)) { + ures_close(&r); + } + return res; +} + +ResourceBundle ResourceBundle::getWithFallback(const char* key, UErrorCode& status){ + UResourceBundle r; + ures_initStackObject(&r); + ures_getByKeyWithFallback(fResource, key, &r, &status); + ResourceBundle res(&r, status); + if(U_SUCCESS(status)){ + ures_close(&r); + } + return res; +} +UnicodeString ResourceBundle::getStringEx(const char* key, UErrorCode& status) const { + int32_t len = 0; + const UChar* r = ures_getStringByKey(fResource, key, &len, &status); + return UnicodeString(TRUE, r, len); +} + +const char* +ResourceBundle::getVersionNumber() const +{ + return ures_getVersionNumber(fResource); +} + +void ResourceBundle::getVersion(UVersionInfo versionInfo) const { + ures_getVersion(fResource, versionInfo); +} + +const Locale &ResourceBundle::getLocale(void) const +{ + UBool needInit; + UMTX_CHECK(NULL, (fLocale == NULL), needInit); + if(needInit) { + UErrorCode status = U_ZERO_ERROR; + const char *localeName = ures_getLocale(fResource, &status); + Locale *tLocale = new Locale(localeName); + umtx_lock(NULL); + ResourceBundle *me = (ResourceBundle *)this; // semantically const + if (me->fLocale == NULL) { + me->fLocale = tLocale; + tLocale = NULL; + } + umtx_unlock(NULL); + delete tLocale; + } + return *fLocale; +} + +const Locale ResourceBundle::getLocale(ULocDataLocaleType type, UErrorCode &status) const +{ + return ures_getLocaleByType(fResource, type, &status); +} + +//eof +U_NAMESPACE_END diff --git a/Build/source/libs/icu/icu-xetex/common/resbund_cnv.cpp b/Build/source/libs/icu/icu-xetex/common/resbund_cnv.cpp new file mode 100644 index 00000000000..a18e57e3106 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/resbund_cnv.cpp @@ -0,0 +1,55 @@ +/* +******************************************************************************* +* +* Copyright (C) 1997-2006, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: resbund_cnv.cpp +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2004aug25 +* created by: Markus W. Scherer +* +* Character conversion functions moved here from resbund.cpp +*/ + +#include "unicode/utypes.h" +#include "unicode/resbund.h" +#include "uinvchar.h" + +U_NAMESPACE_BEGIN + +ResourceBundle::ResourceBundle( const UnicodeString& path, + const Locale& locale, + UErrorCode& error) + :UObject(), fLocale(NULL) +{ + constructForLocale(path, locale, error); +} + +ResourceBundle::ResourceBundle( const UnicodeString& path, + UErrorCode& error) + :UObject(), fLocale(NULL) +{ + constructForLocale(path, Locale::getDefault(), error); +} + +void +ResourceBundle::constructForLocale(const UnicodeString& path, + const Locale& locale, + UErrorCode& error) +{ + if (path.isEmpty()) { + fResource = ures_open(NULL, locale.getName(), &error); + } + else { + UnicodeString nullTerminatedPath(path); + nullTerminatedPath.append((UChar)0); + fResource = ures_openU(nullTerminatedPath.getBuffer(), locale.getName(), &error); + } +} + +U_NAMESPACE_END diff --git a/Build/source/libs/icu/icu-xetex/common/ruleiter.cpp b/Build/source/libs/icu/icu-xetex/common/ruleiter.cpp new file mode 100644 index 00000000000..b99a831513d --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ruleiter.cpp @@ -0,0 +1,160 @@ +/* +********************************************************************** +* Copyright (c) 2003-2007, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Author: Alan Liu +* Created: September 24 2003 +* Since: ICU 2.8 +********************************************************************** +*/ +#include "ruleiter.h" +#include "unicode/parsepos.h" +#include "unicode/unistr.h" +#include "unicode/symtable.h" +#include "util.h" + +/* \U87654321 or \ud800\udc00 */ +#define MAX_U_NOTATION_LEN 12 + +U_NAMESPACE_BEGIN + +RuleCharacterIterator::RuleCharacterIterator(const UnicodeString& theText, const SymbolTable* theSym, + ParsePosition& thePos) : + text(theText), + pos(thePos), + sym(theSym), + buf(0), + bufPos(0) +{} + +UBool RuleCharacterIterator::atEnd() const { + return buf == 0 && pos.getIndex() == text.length(); +} + +UChar32 RuleCharacterIterator::next(int32_t options, UBool& isEscaped, UErrorCode& ec) { + if (U_FAILURE(ec)) return DONE; + + UChar32 c = DONE; + isEscaped = FALSE; + + for (;;) { + c = _current(); + _advance(UTF_CHAR_LENGTH(c)); + + if (c == SymbolTable::SYMBOL_REF && buf == 0 && + (options & PARSE_VARIABLES) != 0 && sym != 0) { + UnicodeString name = sym->parseReference(text, pos, text.length()); + // If name is empty there was an isolated SYMBOL_REF; + // return it. Caller must be prepared for this. + if (name.length() == 0) { + break; + } + bufPos = 0; + buf = sym->lookup(name); + if (buf == 0) { + ec = U_UNDEFINED_VARIABLE; + return DONE; + } + // Handle empty variable value + if (buf->length() == 0) { + buf = 0; + } + continue; + } + + if ((options & SKIP_WHITESPACE) != 0 && + uprv_isRuleWhiteSpace(c)) { + continue; + } + + if (c == 0x5C /*'\\'*/ && (options & PARSE_ESCAPES) != 0) { + UnicodeString tempEscape; + int32_t offset = 0; + c = lookahead(tempEscape, MAX_U_NOTATION_LEN).unescapeAt(offset); + jumpahead(offset); + isEscaped = TRUE; + if (c < 0) { + ec = U_MALFORMED_UNICODE_ESCAPE; + return DONE; + } + } + + break; + } + + return c; +} + +void RuleCharacterIterator::getPos(RuleCharacterIterator::Pos& p) const { + p.buf = buf; + p.pos = pos.getIndex(); + p.bufPos = bufPos; +} + +void RuleCharacterIterator::setPos(const RuleCharacterIterator::Pos& p) { + buf = p.buf; + pos.setIndex(p.pos); + bufPos = p.bufPos; +} + +void RuleCharacterIterator::skipIgnored(int32_t options) { + if ((options & SKIP_WHITESPACE) != 0) { + for (;;) { + UChar32 a = _current(); + if (!uprv_isRuleWhiteSpace(a)) break; + _advance(UTF_CHAR_LENGTH(a)); + } + } +} + +UnicodeString& RuleCharacterIterator::lookahead(UnicodeString& result, int32_t maxLookAhead) const { + if (maxLookAhead < 0) { + maxLookAhead = 0x7FFFFFFF; + } + if (buf != 0) { + buf->extract(bufPos, maxLookAhead, result); + } else { + text.extract(pos.getIndex(), maxLookAhead, result); + } + return result; +} + +void RuleCharacterIterator::jumpahead(int32_t count) { + _advance(count); +} + +/* +UnicodeString& RuleCharacterIterator::toString(UnicodeString& result) const { + int32_t b = pos.getIndex(); + text.extract(0, b, result); + return result.append((UChar) 0x7C).append(text, b, 0x7FFFFFFF); // Insert '|' at index +} +*/ + +UChar32 RuleCharacterIterator::_current() const { + if (buf != 0) { + return buf->char32At(bufPos); + } else { + int i = pos.getIndex(); + return (i < text.length()) ? text.char32At(i) : (UChar32)DONE; + } +} + +void RuleCharacterIterator::_advance(int32_t count) { + if (buf != 0) { + bufPos += count; + if (bufPos == buf->length()) { + buf = 0; + } + } else { + pos.setIndex(pos.getIndex() + count); + if (pos.getIndex() > text.length()) { + pos.setIndex(text.length()); + } + } +} + +U_NAMESPACE_END + +//eof diff --git a/Build/source/libs/icu/icu-xetex/common/ruleiter.h b/Build/source/libs/icu/icu-xetex/common/ruleiter.h new file mode 100644 index 00000000000..cc4e8475fd2 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ruleiter.h @@ -0,0 +1,232 @@ +/* +********************************************************************** +* Copyright (c) 2003-2007, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Author: Alan Liu +* Created: September 24 2003 +* Since: ICU 2.8 +********************************************************************** +*/ +#ifndef _RULEITER_H_ +#define _RULEITER_H_ + +#include "unicode/uobject.h" + +U_NAMESPACE_BEGIN + +class UnicodeString; +class ParsePosition; +class SymbolTable; + +/** + * An iterator that returns 32-bit code points. This class is deliberately + * not related to any of the ICU character iterator classes + * in order to minimize complexity. + * @author Alan Liu + * @since ICU 2.8 + */ +class RuleCharacterIterator : public UMemory { + + // TODO: Ideas for later. (Do not implement if not needed, lest the + // code coverage numbers go down due to unused methods.) + // 1. Add a copy constructor, operator==() method. + // 2. Rather than return DONE, throw an exception if the end + // is reached -- this is an alternate usage model, probably not useful. + +private: + /** + * Text being iterated. + */ + const UnicodeString& text; + + /** + * Position of iterator. + */ + ParsePosition& pos; + + /** + * Symbol table used to parse and dereference variables. May be 0. + */ + const SymbolTable* sym; + + /** + * Current variable expansion, or 0 if none. + */ + const UnicodeString* buf; + + /** + * Position within buf. Meaningless if buf == 0. + */ + int32_t bufPos; + +public: + /** + * Value returned when there are no more characters to iterate. + */ + enum { DONE = -1 }; + + /** + * Bitmask option to enable parsing of variable names. If (options & + * PARSE_VARIABLES) != 0, then an embedded variable will be expanded to + * its value. Variables are parsed using the SymbolTable API. + */ + enum { PARSE_VARIABLES = 1 }; + + /** + * Bitmask option to enable parsing of escape sequences. If (options & + * PARSE_ESCAPES) != 0, then an embedded escape sequence will be expanded + * to its value. Escapes are parsed using Utility.unescapeAt(). + */ + enum { PARSE_ESCAPES = 2 }; + + /** + * Bitmask option to enable skipping of whitespace. If (options & + * SKIP_WHITESPACE) != 0, then whitespace characters will be silently + * skipped, as if they were not present in the input. Whitespace + * characters are defined by UCharacterProperty.isRuleWhiteSpace(). + */ + enum { SKIP_WHITESPACE = 4 }; + + /** + * Constructs an iterator over the given text, starting at the given + * position. + * @param text the text to be iterated + * @param sym the symbol table, or null if there is none. If sym is null, + * then variables will not be deferenced, even if the PARSE_VARIABLES + * option is set. + * @param pos upon input, the index of the next character to return. If a + * variable has been dereferenced, then pos will not increment as + * characters of the variable value are iterated. + */ + RuleCharacterIterator(const UnicodeString& text, const SymbolTable* sym, + ParsePosition& pos); + + /** + * Returns true if this iterator has no more characters to return. + */ + UBool atEnd() const; + + /** + * Returns the next character using the given options, or DONE if there + * are no more characters, and advance the position to the next + * character. + * @param options one or more of the following options, bitwise-OR-ed + * together: PARSE_VARIABLES, PARSE_ESCAPES, SKIP_WHITESPACE. + * @param isEscaped output parameter set to TRUE if the character + * was escaped + * @param ec input-output error code. An error will only be set by + * this routing if options includes PARSE_VARIABLES and an unknown + * variable name is seen, or if options includes PARSE_ESCAPES and + * an invalid escape sequence is seen. + * @return the current 32-bit code point, or DONE + */ + UChar32 next(int32_t options, UBool& isEscaped, UErrorCode& ec); + + /** + * Returns true if this iterator is currently within a variable expansion. + */ + inline UBool inVariable() const; + + /** + * An opaque object representing the position of a RuleCharacterIterator. + */ + struct Pos : public UMemory { + private: + const UnicodeString* buf; + int32_t pos; + int32_t bufPos; + friend class RuleCharacterIterator; + }; + + /** + * Sets an object which, when later passed to setPos(), will + * restore this iterator's position. Usage idiom: + * + * RuleCharacterIterator iterator = ...; + * RuleCharacterIterator::Pos pos; + * iterator.getPos(pos); + * for (;;) { + * iterator.getPos(pos); + * int c = iterator.next(...); + * ... + * } + * iterator.setPos(pos); + * + * @param p a position object to be set to this iterator's + * current position. + */ + void getPos(Pos& p) const; + + /** + * Restores this iterator to the position it had when getPos() + * set the given object. + * @param p a position object previously set by getPos() + */ + void setPos(const Pos& p); + + /** + * Skips ahead past any ignored characters, as indicated by the given + * options. This is useful in conjunction with the lookahead() method. + * + * Currently, this only has an effect for SKIP_WHITESPACE. + * @param options one or more of the following options, bitwise-OR-ed + * together: PARSE_VARIABLES, PARSE_ESCAPES, SKIP_WHITESPACE. + */ + void skipIgnored(int32_t options); + + /** + * Returns a string containing the remainder of the characters to be + * returned by this iterator, without any option processing. If the + * iterator is currently within a variable expansion, this will only + * extend to the end of the variable expansion. This method is provided + * so that iterators may interoperate with string-based APIs. The typical + * sequence of calls is to call skipIgnored(), then call lookahead(), then + * parse the string returned by lookahead(), then call jumpahead() to + * resynchronize the iterator. + * @param result a string to receive the characters to be returned + * by future calls to next() + * @param maxLookAhead The maximum to copy into the result. + * @return a reference to result + */ + UnicodeString& lookahead(UnicodeString& result, int32_t maxLookAhead = -1) const; + + /** + * Advances the position by the given number of 16-bit code units. + * This is useful in conjunction with the lookahead() method. + * @param count the number of 16-bit code units to jump over + */ + void jumpahead(int32_t count); + + /** + * Returns a string representation of this object, consisting of the + * characters being iterated, with a '|' marking the current position. + * Position within an expanded variable is not indicated. + * @param result output parameter to receive a string + * representation of this object + */ +// UnicodeString& toString(UnicodeString& result) const; + +private: + /** + * Returns the current 32-bit code point without parsing escapes, parsing + * variables, or skipping whitespace. + * @return the current 32-bit code point + */ + UChar32 _current() const; + + /** + * Advances the position by the given amount. + * @param count the number of 16-bit code units to advance past + */ + void _advance(int32_t count); +}; + +inline UBool RuleCharacterIterator::inVariable() const { + return buf != 0; +} + +U_NAMESPACE_END + +#endif // _RULEITER_H_ +//eof diff --git a/Build/source/libs/icu/icu-xetex/common/schriter.cpp b/Build/source/libs/icu/icu-xetex/common/schriter.cpp new file mode 100644 index 00000000000..29f15f9749e --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/schriter.cpp @@ -0,0 +1,115 @@ +/* +****************************************************************************** +* Copyright (C) 1998-2004, International Business Machines Corporation and * +* others. All Rights Reserved. * +****************************************************************************** +* +* File schriter.cpp +* +* Modification History: +* +* Date Name Description +* 05/05/99 stephen Cleaned up. +****************************************************************************** +*/ + +#include "unicode/chariter.h" +#include "unicode/schriter.h" + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(StringCharacterIterator) + +StringCharacterIterator::StringCharacterIterator() + : UCharCharacterIterator(), + text() +{ + // NEVER DEFAULT CONSTRUCT! +} + +StringCharacterIterator::StringCharacterIterator(const UnicodeString& textStr) + : UCharCharacterIterator(textStr.fArray, textStr.length()), + text(textStr) +{ + // we had set the input parameter's array, now we need to set our copy's array + UCharCharacterIterator::text = this->text.fArray; +} + +StringCharacterIterator::StringCharacterIterator(const UnicodeString& textStr, + int32_t textPos) + : UCharCharacterIterator(textStr.fArray, textStr.length(), textPos), + text(textStr) +{ + // we had set the input parameter's array, now we need to set our copy's array + UCharCharacterIterator::text = this->text.fArray; +} + +StringCharacterIterator::StringCharacterIterator(const UnicodeString& textStr, + int32_t textBegin, + int32_t textEnd, + int32_t textPos) + : UCharCharacterIterator(textStr.fArray, textStr.length(), textBegin, textEnd, textPos), + text(textStr) +{ + // we had set the input parameter's array, now we need to set our copy's array + UCharCharacterIterator::text = this->text.fArray; +} + +StringCharacterIterator::StringCharacterIterator(const StringCharacterIterator& that) + : UCharCharacterIterator(that), + text(that.text) +{ + // we had set the input parameter's array, now we need to set our copy's array + UCharCharacterIterator::text = this->text.fArray; +} + +StringCharacterIterator::~StringCharacterIterator() { +} + +StringCharacterIterator& +StringCharacterIterator::operator=(const StringCharacterIterator& that) { + UCharCharacterIterator::operator=(that); + text = that.text; + // we had set the input parameter's array, now we need to set our copy's array + UCharCharacterIterator::text = this->text.fArray; + return *this; +} + +UBool +StringCharacterIterator::operator==(const ForwardCharacterIterator& that) const { + if (this == &that) { + return TRUE; + } + + // do not call UCharCharacterIterator::operator==() + // because that checks for array pointer equality + // while we compare UnicodeString objects + + if (getDynamicClassID() != that.getDynamicClassID()) { + return FALSE; + } + + StringCharacterIterator& realThat = (StringCharacterIterator&)that; + + return text == realThat.text + && pos == realThat.pos + && begin == realThat.begin + && end == realThat.end; +} + +CharacterIterator* +StringCharacterIterator::clone() const { + return new StringCharacterIterator(*this); +} + +void +StringCharacterIterator::setText(const UnicodeString& newText) { + text = newText; + UCharCharacterIterator::setText(text.fArray, text.length()); +} + +void +StringCharacterIterator::getText(UnicodeString& result) { + result = text; +} +U_NAMESPACE_END diff --git a/Build/source/libs/icu/icu-xetex/common/serv.cpp b/Build/source/libs/icu/icu-xetex/common/serv.cpp new file mode 100644 index 00000000000..8dc15b2c0fa --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/serv.cpp @@ -0,0 +1,974 @@ +/** +******************************************************************************* +* Copyright (C) 2001-2007, International Business Machines Corporation. * +* All Rights Reserved. * +******************************************************************************* +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_SERVICE + +#include "serv.h" +#include "umutex.h" + +#undef SERVICE_REFCOUNT + +// in case we use the refcount stuff + +U_NAMESPACE_BEGIN + +/* +****************************************************************** +*/ + +const UChar ICUServiceKey::PREFIX_DELIMITER = 0x002F; /* '/' */ + +ICUServiceKey::ICUServiceKey(const UnicodeString& id) +: _id(id) { +} + +ICUServiceKey::~ICUServiceKey() +{ +} + +const UnicodeString& +ICUServiceKey::getID() const +{ + return _id; +} + +UnicodeString& +ICUServiceKey::canonicalID(UnicodeString& result) const +{ + return result.append(_id); +} + +UnicodeString& +ICUServiceKey::currentID(UnicodeString& result) const +{ + return canonicalID(result); +} + +UnicodeString& +ICUServiceKey::currentDescriptor(UnicodeString& result) const +{ + prefix(result); + result.append(PREFIX_DELIMITER); + return currentID(result); +} + +UBool +ICUServiceKey::fallback() +{ + return FALSE; +} + +UBool +ICUServiceKey::isFallbackOf(const UnicodeString& id) const +{ + return id == _id; +} + +UnicodeString& +ICUServiceKey::prefix(UnicodeString& result) const +{ + return result; +} + +UnicodeString& +ICUServiceKey::parsePrefix(UnicodeString& result) +{ + int32_t n = result.indexOf(PREFIX_DELIMITER); + if (n < 0) { + n = 0; + } + result.remove(n); + return result; +} + +UnicodeString& +ICUServiceKey::parseSuffix(UnicodeString& result) +{ + int32_t n = result.indexOf(PREFIX_DELIMITER); + if (n >= 0) { + result.remove(0, n+1); + } + return result; +} + +#ifdef SERVICE_DEBUG +UnicodeString& +ICUServiceKey::debug(UnicodeString& result) const +{ + debugClass(result); + result.append(" id: "); + result.append(_id); + return result; +} + +UnicodeString& +ICUServiceKey::debugClass(UnicodeString& result) const +{ + return result.append("ICUServiceKey"); +} +#endif + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(ICUServiceKey) + +/* +****************************************************************** +*/ + +SimpleFactory::SimpleFactory(UObject* instanceToAdopt, const UnicodeString& id, UBool visible) +: _instance(instanceToAdopt), _id(id), _visible(visible) +{ +} + +SimpleFactory::~SimpleFactory() +{ + delete _instance; +} + +UObject* +SimpleFactory::create(const ICUServiceKey& key, const ICUService* service, UErrorCode& status) const +{ + if (U_SUCCESS(status)) { + UnicodeString temp; + if (_id == key.currentID(temp)) { + return service->cloneInstance(_instance); + } + } + return NULL; +} + +void +SimpleFactory::updateVisibleIDs(Hashtable& result, UErrorCode& status) const +{ + if (_visible) { + result.put(_id, (void*)this, status); // cast away const + } else { + result.remove(_id); + } +} + +UnicodeString& +SimpleFactory::getDisplayName(const UnicodeString& id, const Locale& /* locale */, UnicodeString& result) const +{ + if (_visible && _id == id) { + result = _id; + } else { + result.setToBogus(); + } + return result; +} + +#ifdef SERVICE_DEBUG +UnicodeString& +SimpleFactory::debug(UnicodeString& toAppendTo) const +{ + debugClass(toAppendTo); + toAppendTo.append(" id: "); + toAppendTo.append(_id); + toAppendTo.append(", visible: "); + toAppendTo.append(_visible ? "T" : "F"); + return toAppendTo; +} + +UnicodeString& +SimpleFactory::debugClass(UnicodeString& toAppendTo) const +{ + return toAppendTo.append("SimpleFactory"); +} +#endif + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(SimpleFactory) + +/* +****************************************************************** +*/ + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(ServiceListener) + +/* +****************************************************************** +*/ + +// Record the actual id for this service in the cache, so we can return it +// even if we succeed later with a different id. +class CacheEntry : public UMemory { +private: + int32_t refcount; + +public: + UnicodeString actualDescriptor; + UObject* service; + + /** + * Releases a reference to the shared resource. + */ + ~CacheEntry() { + delete service; + } + + CacheEntry(const UnicodeString& _actualDescriptor, UObject* _service) + : refcount(1), actualDescriptor(_actualDescriptor), service(_service) { + } + + /** + * Instantiation creates an initial reference, so don't call this + * unless you're creating a new pointer to this. Management of + * that pointer will have to know how to deal with refcounts. + * Return true if the resource has not already been released. + */ + CacheEntry* ref() { + ++refcount; + return this; + } + + /** + * Destructions removes a reference, so don't call this unless + * you're removing pointer to this somewhere. Management of that + * pointer will have to know how to deal with refcounts. Once + * the refcount drops to zero, the resource is released. Return + * false if the resouce has been released. + */ + CacheEntry* unref() { + if ((--refcount) == 0) { + delete this; + return NULL; + } + return this; + } + + /** + * Return TRUE if there is at least one reference to this and the + * resource has not been released. + */ + UBool isShared() const { + return refcount > 1; + } +}; + +// UObjectDeleter for serviceCache +U_CDECL_BEGIN +static void U_CALLCONV +cacheDeleter(void* obj) { + U_NAMESPACE_USE ((CacheEntry*)obj)->unref(); +} + +/** +* Deleter for UObjects +*/ +static void U_CALLCONV +deleteUObject(void *obj) { + U_NAMESPACE_USE delete (UObject*) obj; +} +U_CDECL_END + +/* +****************************************************************** +*/ + +class DNCache : public UMemory { +public: + Hashtable cache; + const Locale locale; + + DNCache(const Locale& _locale) + : cache(), locale(_locale) + { + // cache.setKeyDeleter(uhash_deleteUnicodeString); + } +}; + + +/* +****************************************************************** +*/ + +StringPair* +StringPair::create(const UnicodeString& displayName, + const UnicodeString& id, + UErrorCode& status) +{ + if (U_SUCCESS(status)) { + StringPair* sp = new StringPair(displayName, id); + if (sp == NULL || sp->isBogus()) { + status = U_MEMORY_ALLOCATION_ERROR; + delete sp; + return NULL; + } + return sp; + } + return NULL; +} + +UBool +StringPair::isBogus() const { + return displayName.isBogus() || id.isBogus(); +} + +StringPair::StringPair(const UnicodeString& _displayName, + const UnicodeString& _id) +: displayName(_displayName) +, id(_id) +{ +} + +U_CDECL_BEGIN +static void U_CALLCONV +userv_deleteStringPair(void *obj) { + U_NAMESPACE_USE delete (StringPair*) obj; +} +U_CDECL_END + +/* +****************************************************************** +*/ + +ICUService::ICUService() +: name() +, lock(0) +, timestamp(0) +, factories(NULL) +, serviceCache(NULL) +, idCache(NULL) +, dnCache(NULL) +{ + umtx_init(&lock); +} + +ICUService::ICUService(const UnicodeString& newName) +: name(newName) +, lock(0) +, timestamp(0) +, factories(NULL) +, serviceCache(NULL) +, idCache(NULL) +, dnCache(NULL) +{ + umtx_init(&lock); +} + +ICUService::~ICUService() +{ + { + Mutex mutex(&lock); + clearCaches(); + delete factories; + factories = NULL; + } + umtx_destroy(&lock); +} + +UObject* +ICUService::get(const UnicodeString& descriptor, UErrorCode& status) const +{ + return get(descriptor, NULL, status); +} + +UObject* +ICUService::get(const UnicodeString& descriptor, UnicodeString* actualReturn, UErrorCode& status) const +{ + UObject* result = NULL; + ICUServiceKey* key = createKey(&descriptor, status); + if (key) { + result = getKey(*key, actualReturn, status); + delete key; + } + return result; +} + +UObject* +ICUService::getKey(ICUServiceKey& key, UErrorCode& status) const +{ + return getKey(key, NULL, status); +} + +// this is a vector that subclasses of ICUService can override to further customize the result object +// before returning it. All other public get functions should call this one. + +UObject* +ICUService::getKey(ICUServiceKey& key, UnicodeString* actualReturn, UErrorCode& status) const +{ + return getKey(key, actualReturn, NULL, status); +} + +// make it possible to call reentrantly on systems that don't have reentrant mutexes. +// we can use this simple approach since we know the situation where we're calling +// reentrantly even without knowing the thread. +class XMutex : public UMemory { +public: + inline XMutex(UMTX *mutex, UBool reentering) + : fMutex(mutex) + , fActive(!reentering) + { + if (fActive) umtx_lock(fMutex); + } + inline ~XMutex() { + if (fActive) umtx_unlock(fMutex); + } + +private: + UMTX *fMutex; + UBool fActive; +}; + +struct UVectorDeleter { + UVector* _obj; + UVectorDeleter() : _obj(NULL) {} + ~UVectorDeleter() { delete _obj; } +}; + +// called only by factories, treat as private +UObject* +ICUService::getKey(ICUServiceKey& key, UnicodeString* actualReturn, const ICUServiceFactory* factory, UErrorCode& status) const +{ + if (U_FAILURE(status)) { + return NULL; + } + + if (isDefault()) { + return handleDefault(key, actualReturn, status); + } + + ICUService* ncthis = (ICUService*)this; // cast away semantic const + + CacheEntry* result = NULL; + { + // The factory list can't be modified until we're done, + // otherwise we might update the cache with an invalid result. + // The cache has to stay in synch with the factory list. + // ICU doesn't have monitors so we can't use rw locks, so + // we single-thread everything using this service, for now. + + // if factory is not null, we're calling from within the mutex, + // and since some unix machines don't have reentrant mutexes we + // need to make sure not to try to lock it again. + XMutex mutex(&ncthis->lock, factory != NULL); + + if (serviceCache == NULL) { + ncthis->serviceCache = new Hashtable(status); + if (U_FAILURE(status)) { + delete serviceCache; + return NULL; + } + serviceCache->setValueDeleter(cacheDeleter); + } + + UnicodeString currentDescriptor; + UVectorDeleter cacheDescriptorList; + UBool putInCache = FALSE; + + int32_t startIndex = 0; + int32_t limit = factories->size(); + UBool cacheResult = TRUE; + + if (factory != NULL) { + for (int32_t i = 0; i < limit; ++i) { + if (factory == (const ICUServiceFactory*)factories->elementAt(i)) { + startIndex = i + 1; + break; + } + } + if (startIndex == 0) { + // throw new InternalError("Factory " + factory + "not registered with service: " + this); + status = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + cacheResult = FALSE; + } + + do { + currentDescriptor.remove(); + key.currentDescriptor(currentDescriptor); + result = (CacheEntry*)serviceCache->get(currentDescriptor); + if (result != NULL) { + break; + } + + // first test of cache failed, so we'll have to update + // the cache if we eventually succeed-- that is, if we're + // going to update the cache at all. + putInCache = TRUE; + + int32_t index = startIndex; + while (index < limit) { + ICUServiceFactory* f = (ICUServiceFactory*)factories->elementAt(index++); + UObject* service = f->create(key, this, status); + if (U_FAILURE(status)) { + delete service; + return NULL; + } + if (service != NULL) { + result = new CacheEntry(currentDescriptor, service); + if (result == NULL) { + delete service; + status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + + goto outerEnd; + } + } + + // prepare to load the cache with all additional ids that + // will resolve to result, assuming we'll succeed. We + // don't want to keep querying on an id that's going to + // fallback to the one that succeeded, we want to hit the + // cache the first time next goaround. + if (cacheDescriptorList._obj == NULL) { + cacheDescriptorList._obj = new UVector(uhash_deleteUnicodeString, NULL, 5, status); + if (U_FAILURE(status)) { + return NULL; + } + } + UnicodeString* idToCache = new UnicodeString(currentDescriptor); + if (idToCache == NULL || idToCache->isBogus()) { + status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + + cacheDescriptorList._obj->addElement(idToCache, status); + if (U_FAILURE(status)) { + return NULL; + } + } while (key.fallback()); +outerEnd: + + if (result != NULL) { + if (putInCache && cacheResult) { + serviceCache->put(result->actualDescriptor, result, status); + if (U_FAILURE(status)) { + delete result; + return NULL; + } + + if (cacheDescriptorList._obj != NULL) { + for (int32_t i = cacheDescriptorList._obj->size(); --i >= 0;) { + UnicodeString* desc = (UnicodeString*)cacheDescriptorList._obj->elementAt(i); + serviceCache->put(*desc, result, status); + if (U_FAILURE(status)) { + delete result; + return NULL; + } + + result->ref(); + cacheDescriptorList._obj->removeElementAt(i); + } + } + } + + if (actualReturn != NULL) { + // strip null prefix + if (result->actualDescriptor.indexOf((UChar)0x2f) == 0) { // U+002f=slash (/) + actualReturn->remove(); + actualReturn->append(result->actualDescriptor, + 1, + result->actualDescriptor.length() - 1); + } else { + *actualReturn = result->actualDescriptor; + } + + if (actualReturn->isBogus()) { + status = U_MEMORY_ALLOCATION_ERROR; + delete result; + return NULL; + } + } + + UObject* service = cloneInstance(result->service); + if (putInCache && !cacheResult) { + delete result; + } + return service; + } + } + + return handleDefault(key, actualReturn, status); +} + +UObject* +ICUService::handleDefault(const ICUServiceKey& /* key */, UnicodeString* /* actualIDReturn */, UErrorCode& /* status */) const +{ + return NULL; +} + +UVector& +ICUService::getVisibleIDs(UVector& result, UErrorCode& status) const { + return getVisibleIDs(result, NULL, status); +} + +UVector& +ICUService::getVisibleIDs(UVector& result, const UnicodeString* matchID, UErrorCode& status) const +{ + result.removeAllElements(); + + if (U_FAILURE(status)) { + return result; + } + + ICUService * ncthis = (ICUService*)this; // cast away semantic const + { + Mutex mutex(&ncthis->lock); + const Hashtable* map = getVisibleIDMap(status); + if (map != NULL) { + ICUServiceKey* fallbackKey = createKey(matchID, status); + + for (int32_t pos = -1;;) { + const UHashElement* e = map->nextElement(pos); + if (e == NULL) { + break; + } + + const UnicodeString* id = (const UnicodeString*)e->key.pointer; + if (fallbackKey != NULL) { + if (!fallbackKey->isFallbackOf(*id)) { + continue; + } + } + + UnicodeString* idClone = new UnicodeString(*id); + if (idClone == NULL || idClone->isBogus()) { + delete idClone; + status = U_MEMORY_ALLOCATION_ERROR; + break; + } + result.addElement(idClone, status); + if (U_FAILURE(status)) { + delete idClone; + break; + } + } + delete fallbackKey; + } + } + if (U_FAILURE(status)) { + result.removeAllElements(); + } + return result; +} + +const Hashtable* +ICUService::getVisibleIDMap(UErrorCode& status) const { + if (U_FAILURE(status)) return NULL; + + // must only be called when lock is already held + + ICUService* ncthis = (ICUService*)this; // cast away semantic const + if (idCache == NULL) { + ncthis->idCache = new Hashtable(status); + if (idCache == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + } else if (factories != NULL) { + for (int32_t pos = factories->size(); --pos >= 0;) { + ICUServiceFactory* f = (ICUServiceFactory*)factories->elementAt(pos); + f->updateVisibleIDs(*idCache, status); + } + if (U_FAILURE(status)) { + delete idCache; + ncthis->idCache = NULL; + } + } + } + + return idCache; +} + + +UnicodeString& +ICUService::getDisplayName(const UnicodeString& id, UnicodeString& result) const +{ + return getDisplayName(id, result, Locale::getDefault()); +} + +UnicodeString& +ICUService::getDisplayName(const UnicodeString& id, UnicodeString& result, const Locale& locale) const +{ + { + ICUService* ncthis = (ICUService*)this; // cast away semantic const + UErrorCode status = U_ZERO_ERROR; + Mutex mutex(&ncthis->lock); + const Hashtable* map = getVisibleIDMap(status); + if (map != NULL) { + ICUServiceFactory* f = (ICUServiceFactory*)map->get(id); + if (f != NULL) { + f->getDisplayName(id, locale, result); + return result; + } + + // fallback + UErrorCode status = U_ZERO_ERROR; + ICUServiceKey* fallbackKey = createKey(&id, status); + while (fallbackKey->fallback()) { + UnicodeString us; + fallbackKey->currentID(us); + f = (ICUServiceFactory*)map->get(us); + if (f != NULL) { + f->getDisplayName(id, locale, result); + delete fallbackKey; + return result; + } + } + delete fallbackKey; + } + } + result.setToBogus(); + return result; +} + +UVector& +ICUService::getDisplayNames(UVector& result, UErrorCode& status) const +{ + return getDisplayNames(result, Locale::getDefault(), NULL, status); +} + + +UVector& +ICUService::getDisplayNames(UVector& result, const Locale& locale, UErrorCode& status) const +{ + return getDisplayNames(result, locale, NULL, status); +} + +UVector& +ICUService::getDisplayNames(UVector& result, + const Locale& locale, + const UnicodeString* matchID, + UErrorCode& status) const +{ + result.removeAllElements(); + result.setDeleter(userv_deleteStringPair); + if (U_SUCCESS(status)) { + ICUService* ncthis = (ICUService*)this; // cast away semantic const + Mutex mutex(&ncthis->lock); + + if (dnCache != NULL && dnCache->locale != locale) { + delete dnCache; + ncthis->dnCache = NULL; + } + + if (dnCache == NULL) { + const Hashtable* m = getVisibleIDMap(status); + if (m != NULL) { + ncthis->dnCache = new DNCache(locale); + if (dnCache == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + return result; + } + + int32_t pos = 0; + const UHashElement* entry = NULL; + while ((entry = m->nextElement(pos)) != NULL) { + const UnicodeString* id = (const UnicodeString*)entry->key.pointer; + ICUServiceFactory* f = (ICUServiceFactory*)entry->value.pointer; + UnicodeString dname; + f->getDisplayName(*id, locale, dname); + if (dname.isBogus()) { + status = U_MEMORY_ALLOCATION_ERROR; + } else { + dnCache->cache.put(dname, (void*)id, status); // share pointer with visibleIDMap + if (U_SUCCESS(status)) { + continue; + } + } + delete dnCache; + ncthis->dnCache = NULL; + return result; + } + } + } + } + + ICUServiceKey* matchKey = createKey(matchID, status); + int32_t pos = 0; + const UHashElement *entry = NULL; + while ((entry = dnCache->cache.nextElement(pos)) != NULL) { + const UnicodeString* id = (const UnicodeString*)entry->value.pointer; + if (matchKey != NULL && !matchKey->isFallbackOf(*id)) { + continue; + } + const UnicodeString* dn = (const UnicodeString*)entry->key.pointer; + StringPair* sp = StringPair::create(*id, *dn, status); + result.addElement(sp, status); + if (U_FAILURE(status)) { + result.removeAllElements(); + break; + } + } + delete matchKey; + + return result; +} + +URegistryKey +ICUService::registerInstance(UObject* objToAdopt, const UnicodeString& id, UErrorCode& status) +{ + return registerInstance(objToAdopt, id, TRUE, status); +} + +URegistryKey +ICUService::registerInstance(UObject* objToAdopt, const UnicodeString& id, UBool visible, UErrorCode& status) +{ + ICUServiceKey* key = createKey(&id, status); + if (key != NULL) { + UnicodeString canonicalID; + key->canonicalID(canonicalID); + delete key; + + ICUServiceFactory* f = createSimpleFactory(objToAdopt, canonicalID, visible, status); + if (f != NULL) { + return registerFactory(f, status); + } + } + delete objToAdopt; + return NULL; +} + +ICUServiceFactory* +ICUService::createSimpleFactory(UObject* objToAdopt, const UnicodeString& id, UBool visible, UErrorCode& status) +{ + if (U_SUCCESS(status)) { + if ((objToAdopt != NULL) && (!id.isBogus())) { + return new SimpleFactory(objToAdopt, id, visible); + } + status = U_ILLEGAL_ARGUMENT_ERROR; + } + return NULL; +} + +URegistryKey +ICUService::registerFactory(ICUServiceFactory* factoryToAdopt, UErrorCode& status) +{ + if (U_SUCCESS(status) && factoryToAdopt != NULL) { + Mutex mutex(&lock); + + if (factories == NULL) { + factories = new UVector(deleteUObject, NULL, status); + if (U_FAILURE(status)) { + delete factories; + return NULL; + } + } + factories->insertElementAt(factoryToAdopt, 0, status); + if (U_SUCCESS(status)) { + clearCaches(); + } else { + delete factoryToAdopt; + factoryToAdopt = NULL; + } + } + + if (factoryToAdopt != NULL) { + notifyChanged(); + } + + return (URegistryKey)factoryToAdopt; +} + +UBool +ICUService::unregister(URegistryKey rkey, UErrorCode& status) +{ + ICUServiceFactory *factory = (ICUServiceFactory*)rkey; + UBool result = FALSE; + if (factory != NULL && factories != NULL) { + Mutex mutex(&lock); + + if (factories->removeElement(factory)) { + clearCaches(); + result = TRUE; + } else { + status = U_ILLEGAL_ARGUMENT_ERROR; + delete factory; + } + } + if (result) { + notifyChanged(); + } + return result; +} + +void +ICUService::reset() +{ + { + Mutex mutex(&lock); + reInitializeFactories(); + clearCaches(); + } + notifyChanged(); +} + +void +ICUService::reInitializeFactories() +{ + if (factories != NULL) { + factories->removeAllElements(); + } +} + +UBool +ICUService::isDefault() const +{ + return countFactories() == 0; +} + +ICUServiceKey* +ICUService::createKey(const UnicodeString* id, UErrorCode& status) const +{ + return (U_FAILURE(status) || id == NULL) ? NULL : new ICUServiceKey(*id); +} + +void +ICUService::clearCaches() +{ + // callers synchronize before use + ++timestamp; + delete dnCache; + dnCache = NULL; + delete idCache; + idCache = NULL; + delete serviceCache; serviceCache = NULL; +} + +void +ICUService::clearServiceCache() +{ + // callers synchronize before use + delete serviceCache; serviceCache = NULL; +} + +UBool +ICUService::acceptsListener(const EventListener& l) const +{ + return l.getDynamicClassID() == ServiceListener::getStaticClassID(); +} + +void +ICUService::notifyListener(EventListener& l) const +{ + ((ServiceListener&)l).serviceChanged(*this); +} + +UnicodeString& +ICUService::getName(UnicodeString& result) const +{ + return result.append(name); +} + +int32_t +ICUService::countFactories() const +{ + return factories == NULL ? 0 : factories->size(); +} + +int32_t +ICUService::getTimestamp() const +{ + return timestamp; +} + +U_NAMESPACE_END + +/* UCONFIG_NO_SERVICE */ +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/serv.h b/Build/source/libs/icu/icu-xetex/common/serv.h new file mode 100644 index 00000000000..2e498fb4a05 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/serv.h @@ -0,0 +1,996 @@ +/** + ******************************************************************************* + * Copyright (C) 2001-2007, International Business Machines Corporation. * + * All Rights Reserved. * + ******************************************************************************* + */ + +#ifndef ICUSERV_H +#define ICUSERV_H + +#include "unicode/utypes.h" + +#if UCONFIG_NO_SERVICE + +U_NAMESPACE_BEGIN + +/* + * Allow the declaration of APIs with pointers to ICUService + * even when service is removed from the build. + */ +class ICUService; + +U_NAMESPACE_END + +#else + +#include "unicode/unistr.h" +#include "unicode/locid.h" +#include "unicode/umisc.h" + +#include "hash.h" +#include "uvector.h" +#include "servnotf.h" + +class ICUServiceTest; + +U_NAMESPACE_BEGIN + +class ICUServiceKey; +class ICUServiceFactory; +class SimpleFactory; +class ServiceListener; +class ICUService; + +class DNCache; + +/******************************************************************* + * ICUServiceKey + */ + +/** + *

ICUServiceKeys are used to communicate with factories to + * generate an instance of the service. ICUServiceKeys define how + * ids are canonicalized, provide both a current id and a current + * descriptor to use in querying the cache and factories, and + * determine the fallback strategy.

+ * + *

ICUServiceKeys provide both a currentDescriptor and a currentID. + * The descriptor contains an optional prefix, followed by '/' + * and the currentID. Factories that handle complex keys, + * for example number format factories that generate multiple + * kinds of formatters for the same locale, use the descriptor + * to provide a fully unique identifier for the service object, + * while using the currentID (in this case, the locale string), + * as the visible IDs that can be localized.

+ * + *

The default implementation of ICUServiceKey has no fallbacks and + * has no custom descriptors.

+ */ +class U_COMMON_API ICUServiceKey : public UObject { + private: + const UnicodeString _id; + + protected: + static const UChar PREFIX_DELIMITER; + + public: + + /** + *

Construct a key from an id.

+ * + * @param id the ID from which to construct the key. + */ + ICUServiceKey(const UnicodeString& id); + + /** + *

Virtual destructor.

+ */ + virtual ~ICUServiceKey(); + + /** + *

Return the original ID used to construct this key.

+ * + * @return the ID used to construct this key. + */ + virtual const UnicodeString& getID() const; + + /** + *

Return the canonical version of the original ID. This implementation + * appends the original ID to result. Result is returned as a convenience.

+ * + * @param result the output parameter to which the id will be appended. + * @return the modified result. + */ + virtual UnicodeString& canonicalID(UnicodeString& result) const; + + /** + *

Return the (canonical) current ID. This implementation appends + * the canonical ID to result. Result is returned as a convenience.

+ * + * @param result the output parameter to which the current id will be appended. + * @return the modified result. + */ + virtual UnicodeString& currentID(UnicodeString& result) const; + + /** + *

Return the current descriptor. This implementation appends + * the current descriptor to result. Result is returned as a convenience.

+ * + *

The current descriptor is used to fully + * identify an instance of the service in the cache. A + * factory may handle all descriptors for an ID, or just a + * particular descriptor. The factory can either parse the + * descriptor or use custom API on the key in order to + * instantiate the service.

+ * + * @param result the output parameter to which the current id will be appended. + * @return the modified result. + */ + virtual UnicodeString& currentDescriptor(UnicodeString& result) const; + + /** + *

If the key has a fallback, modify the key and return true, + * otherwise return false. The current ID will change if there + * is a fallback. No currentIDs should be repeated, and fallback + * must eventually return false. This implementation has no fallbacks + * and always returns false.

+ * + * @return TRUE if the ICUServiceKey changed to a valid fallback value. + */ + virtual UBool fallback(); + + /** + *

Return TRUE if a key created from id matches, or would eventually + * fallback to match, the canonical ID of this ICUServiceKey.

+ * + * @param id the id to test. + * @return TRUE if this ICUServiceKey's canonical ID is a fallback of id. + */ + virtual UBool isFallbackOf(const UnicodeString& id) const; + + /** + *

Return the prefix. This implementation leaves result unchanged. + * Result is returned as a convenience.

+ * + * @param result the output parameter to which the prefix will be appended. + * @return the modified result. + */ + virtual UnicodeString& prefix(UnicodeString& result) const; + + /** + *

A utility to parse the prefix out of a descriptor string. Only + * the (undelimited) prefix, if any, remains in result. Result is returned as a + * convenience.

+ * + * @param result an input/output parameter that on entry is a descriptor, and + * on exit is the prefix of that descriptor. + * @return the modified result. + */ + static UnicodeString& parsePrefix(UnicodeString& result); + + /** + *

A utility to parse the suffix out of a descriptor string. Only + * the (undelimited) suffix, if any, remains in result. Result is returned as a + * convenience.

+ * + * @param result an input/output parameter that on entry is a descriptor, and + * on exit is the suffix of that descriptor. + * @return the modified result. + */ + static UnicodeString& parseSuffix(UnicodeString& result); + +public: + /** + * UObject RTTI boilerplate. + */ + static UClassID U_EXPORT2 getStaticClassID(); + + /** + * UObject RTTI boilerplate. + */ + virtual UClassID getDynamicClassID() const; + +#ifdef SERVICE_DEBUG + public: + virtual UnicodeString& debug(UnicodeString& result) const; + virtual UnicodeString& debugClass(UnicodeString& result) const; +#endif + +}; + + /******************************************************************* + * ICUServiceFactory + */ + + /** + *

An implementing ICUServiceFactory generates the service objects maintained by the + * service. A factory generates a service object from a key, + * updates id->factory mappings, and returns the display name for + * a supported id.

+ */ +class U_COMMON_API ICUServiceFactory : public UObject { + public: + + /** + *

Create a service object from the key, if this factory + * supports the key. Otherwise, return NULL.

+ * + *

If the factory supports the key, then it can call + * the service's getKey(ICUServiceKey, String[], ICUServiceFactory) method + * passing itself as the factory to get the object that + * the service would have created prior to the factory's + * registration with the service. This can change the + * key, so any information required from the key should + * be extracted before making such a callback.

+ * + * @param key the service key. + * @param service the service with which this factory is registered. + * @param status the error code status. + * @return the service object, or NULL if the factory does not support the key. + */ + virtual UObject* create(const ICUServiceKey& key, const ICUService* service, UErrorCode& status) const = 0; + + /** + *

Update result to reflect the IDs (not descriptors) that this + * factory publicly handles. Result contains mappings from ID to + * factory. On entry it will contain all (visible) mappings from + * previously-registered factories.

+ * + *

This function, together with getDisplayName, are used to + * support ICUService::getDisplayNames. The factory determines + * which IDs (of those it supports) it will make visible, and of + * those, which it will provide localized display names for. In + * most cases it will register mappings from all IDs it supports + * to itself.

+ * + * @param result the mapping table to update. + * @param status the error code status. + */ + virtual void updateVisibleIDs(Hashtable& result, UErrorCode& status) const = 0; + + /** + *

Return, in result, the display name of the id in the provided locale. + * This is an id, not a descriptor. If the id is + * not visible, sets result to bogus. If the + * incoming result is bogus, it remains bogus. Result is returned as a + * convenience. Results are not defined if id is not one supported by this + * factory.

+ * + * @param id a visible id supported by this factory. + * @param locale the locale for which to generate the corresponding localized display name. + * @param result output parameter to hold the display name. + * @return result. + */ + virtual UnicodeString& getDisplayName(const UnicodeString& id, const Locale& locale, UnicodeString& result) const = 0; +}; + +/* + ****************************************************************** + */ + + /** + *

A default implementation of factory. This provides default + * implementations for subclasses, and implements a singleton + * factory that matches a single ID and returns a single + * (possibly deferred-initialized) instance. This implements + * updateVisibleIDs to add a mapping from its ID to itself + * if visible is true, or to remove any existing mapping + * for its ID if visible is false. No localization of display + * names is performed.

+ */ +class U_COMMON_API SimpleFactory : public ICUServiceFactory { + protected: + UObject* _instance; + const UnicodeString _id; + const UBool _visible; + + public: + /** + *

Construct a SimpleFactory that maps a single ID to a single + * service instance. If visible is TRUE, the ID will be visible. + * The instance must not be NULL. The SimpleFactory will adopt + * the instance, which must not be changed subsequent to this call.

+ * + * @param instanceToAdopt the service instance to adopt. + * @param id the ID to assign to this service instance. + * @param visible if TRUE, the ID will be visible. + */ + SimpleFactory(UObject* instanceToAdopt, const UnicodeString& id, UBool visible = TRUE); + + /** + *

Destructor.

+ */ + virtual ~SimpleFactory(); + + /** + *

This implementation returns a clone of the service instance if the factory's ID is equal to + * the key's currentID. Service and prefix are ignored.

+ * + * @param key the service key. + * @param service the service with which this factory is registered. + * @param status the error code status. + * @return the service object, or NULL if the factory does not support the key. + */ + virtual UObject* create(const ICUServiceKey& key, const ICUService* service, UErrorCode& status) const; + + /** + *

This implementation adds a mapping from ID -> this to result if visible is TRUE, + * otherwise it removes ID from result.

+ * + * @param result the mapping table to update. + * @param status the error code status. + */ + virtual void updateVisibleIDs(Hashtable& result, UErrorCode& status) const; + + /** + *

This implementation returns the factory ID if it equals id and visible is TRUE, + * otherwise it returns the empty string. (This implementation provides + * no localized id information.)

+ * + * @param id a visible id supported by this factory. + * @param locale the locale for which to generate the corresponding localized display name. + * @param result output parameter to hold the display name. + * @return result. + */ + virtual UnicodeString& getDisplayName(const UnicodeString& id, const Locale& locale, UnicodeString& result) const; + +public: + /** + * UObject RTTI boilerplate. + */ + static UClassID U_EXPORT2 getStaticClassID(); + + /** + * UObject RTTI boilerplate. + */ + virtual UClassID getDynamicClassID() const; + +#ifdef SERVICE_DEBUG + public: + virtual UnicodeString& debug(UnicodeString& toAppendTo) const; + virtual UnicodeString& debugClass(UnicodeString& toAppendTo) const; +#endif + +}; + +/* + ****************************************************************** + */ + +/** + *

ServiceListener is the listener that ICUService provides by default. + * ICUService will notifiy this listener when factories are added to + * or removed from the service. Subclasses can provide + * different listener interfaces that extend EventListener, and modify + * acceptsListener and notifyListener as appropriate.

+ */ +class U_COMMON_API ServiceListener : public EventListener { +public: + /** + *

This method is called when the service changes. At the time of the + * call this listener is registered with the service. It must + * not modify the notifier in the context of this call.

+ * + * @param service the service that changed. + */ + virtual void serviceChanged(const ICUService& service) const = 0; + +public: + /** + * UObject RTTI boilerplate. + */ + static UClassID U_EXPORT2 getStaticClassID(); + + /** + * UObject RTTI boilerplate. + */ + virtual UClassID getDynamicClassID() const; + +}; + +/* + ****************************************************************** + */ + +/** + *

A StringPair holds a displayName/ID pair. ICUService uses it + * as the array elements returned by getDisplayNames. + */ +class U_COMMON_API StringPair : public UMemory { +public: + /** + *

The display name of the pair.

+ */ + const UnicodeString displayName; + + /** + *

The ID of the pair.

+ */ + const UnicodeString id; + + /** + *

Creates a string pair from a displayName and an ID.

+ * + * @param displayName the displayName. + * @param id the ID. + * @param status the error code status. + * @return a StringPair if the creation was successful, otherwise NULL. + */ + static StringPair* create(const UnicodeString& displayName, + const UnicodeString& id, + UErrorCode& status); + + /** + *

Return TRUE if either string of the pair is bogus.

+ * @return TRUE if either string of the pair is bogus. + */ + UBool isBogus() const; + +private: + StringPair(const UnicodeString& displayName, const UnicodeString& id); +}; + +/******************************************************************* + * ICUService + */ + + /** + *

A Service provides access to service objects that implement a + * particular service, e.g. transliterators. Users provide a String + * id (for example, a locale string) to the service, and get back an + * object for that id. Service objects can be any kind of object. A + * new service object is returned for each query. The caller is + * responsible for deleting it.

+ * + *

Services 'canonicalize' the query ID and use the canonical ID to + * query for the service. The service also defines a mechanism to + * 'fallback' the ID multiple times. Clients can optionally request + * the actual ID that was matched by a query when they use an ID to + * retrieve a service object.

+ * + *

Service objects are instantiated by ICUServiceFactory objects + * registered with the service. The service queries each + * ICUServiceFactory in turn, from most recently registered to + * earliest registered, until one returns a service object. If none + * responds with a service object, a fallback ID is generated, and the + * process repeats until a service object is returned or until the ID + * has no further fallbacks.

+ * + *

In ICU 2.4, UObject (the base class of service instances) does + * not define a polymorphic clone function. ICUService uses clones to + * manage ownership. Thus, for now, ICUService defines an abstract + * method, cloneInstance, that clients must implement to create clones + * of the service instances. This may change in future releases of + * ICU.

+ * + *

ICUServiceFactories can be dynamically registered and + * unregistered with the service. When registered, an + * ICUServiceFactory is installed at the head of the factory list, and + * so gets 'first crack' at any keys or fallback keys. When + * unregistered, it is removed from the service and can no longer be + * located through it. Service objects generated by this factory and + * held by the client are unaffected.

+ * + *

If a service has variants (e.g., the different variants of + * BreakIterator) an ICUServiceFactory can use the prefix of the + * ICUServiceKey to determine the variant of a service to generate. + * If it does not support all variants, it can request + * previously-registered factories to handle the ones it does not + * support.

+ * + *

ICUService uses ICUServiceKeys to query factories and perform + * fallback. The ICUServiceKey defines the canonical form of the ID, + * and implements the fallback strategy. Custom ICUServiceKeys can be + * defined that parse complex IDs into components that + * ICUServiceFactories can more easily use. The ICUServiceKey can + * cache the results of this parsing to save repeated effort. + * ICUService provides convenience APIs that take UnicodeStrings and + * generate default ICUServiceKeys for use in querying.

+ * + *

ICUService provides API to get the list of IDs publicly + * supported by the service (although queries aren't restricted to + * this list). This list contains only 'simple' IDs, and not fully + * unique IDs. ICUServiceFactories are associated with each simple ID + * and the responsible factory can also return a human-readable + * localized version of the simple ID, for use in user interfaces. + * ICUService can also provide an array of the all the localized + * visible IDs and their corresponding internal IDs.

+ * + *

ICUService implements ICUNotifier, so that clients can register + * to receive notification when factories are added or removed from + * the service. ICUService provides a default EventListener + * subinterface, ServiceListener, which can be registered with the + * service. When the service changes, the ServiceListener's + * serviceChanged method is called with the service as the + * argument.

+ * + *

The ICUService API is both rich and generic, and it is expected + * that most implementations will statically 'wrap' ICUService to + * present a more appropriate API-- for example, to declare the type + * of the objects returned from get, to limit the factories that can + * be registered with the service, or to define their own listener + * interface with a custom callback method. They might also customize + * ICUService by overriding it, for example, to customize the + * ICUServiceKey and fallback strategy. ICULocaleService is a + * subclass of ICUService that uses Locale names as IDs and uses + * ICUServiceKeys that implement the standard resource bundle fallback + * strategy. Most clients will wish to subclass it instead of + * ICUService.

+ */ +class U_COMMON_API ICUService : public ICUNotifier { + protected: + /** + * Name useful for debugging. + */ + const UnicodeString name; + + private: + + /** + * single lock used by this service. + */ + UMTX lock; + + /** + * Timestamp so iterators can be fail-fast. + */ + uint32_t timestamp; + + /** + * All the factories registered with this service. + */ + UVector* factories; + + /** + * The service cache. + */ + Hashtable* serviceCache; + + /** + * The ID cache. + */ + Hashtable* idCache; + + /** + * The name cache. + */ + DNCache* dnCache; + + /** + * Constructor. + */ + public: + /** + *

Construct a new ICUService.

+ */ + ICUService(); + + /** + *

Construct with a name (useful for debugging).

+ * + * @param name a name to use in debugging. + */ + ICUService(const UnicodeString& name); + + /** + *

Destructor.

+ */ + virtual ~ICUService(); + + /** + *

Return the name of this service. This will be the empty string if none was assigned. + * Returns result as a convenience.

+ * + * @param result an output parameter to contain the name of this service. + * @return the name of this service. + */ + UnicodeString& getName(UnicodeString& result) const; + + /** + *

Convenience override for get(ICUServiceKey&, UnicodeString*). This uses + * createKey to create a key for the provided descriptor.

+ * + * @param descriptor the descriptor. + * @param status the error code status. + * @return the service instance, or NULL. + */ + UObject* get(const UnicodeString& descriptor, UErrorCode& status) const; + + /** + *

Convenience override for get(ICUServiceKey&, UnicodeString*). This uses + * createKey to create a key from the provided descriptor.

+ * + * @param descriptor the descriptor. + * @param actualReturn a pointer to a UnicodeString to hold the matched descriptor, or NULL. + * @param status the error code status. + * @return the service instance, or NULL. + */ + UObject* get(const UnicodeString& descriptor, UnicodeString* actualReturn, UErrorCode& status) const; + + /** + *

Convenience override for get(ICUServiceKey&, UnicodeString*).

+ * + * @param key the key. + * @param status the error code status. + * @return the service instance, or NULL. + */ + UObject* getKey(ICUServiceKey& key, UErrorCode& status) const; + + /** + *

Given a key, return a service object, and, if actualReturn + * is not NULL, the descriptor with which it was found in the + * first element of actualReturn. If no service object matches + * this key, returns NULL and leaves actualReturn unchanged.

+ * + *

This queries the cache using the key's descriptor, and if no + * object in the cache matches, tries the key on each + * registered factory, in order. If none generates a service + * object for the key, repeats the process with each fallback of + * the key, until either a factory returns a service object, or the key + * has no fallback. If no object is found, the result of handleDefault + * is returned.

+ * + *

Subclasses can override this method to further customize the + * result before returning it. + * + * @param key the key. + * @param actualReturn a pointer to a UnicodeString to hold the matched descriptor, or NULL. + * @param status the error code status. + * @return the service instance, or NULL. + */ + virtual UObject* getKey(ICUServiceKey& key, UnicodeString* actualReturn, UErrorCode& status) const; + + /** + *

This version of getKey is only called by ICUServiceFactories within the scope + * of a previous getKey call, to determine what previously-registered factories would + * have returned. For details, see getKey(ICUServiceKey&, UErrorCode&). Subclasses + * should not call it directly, but call through one of the other get functions.

+ * + * @param key the key. + * @param actualReturn a pointer to a UnicodeString to hold the matched descriptor, or NULL. + * @param factory the factory making the recursive call. + * @param status the error code status. + * @return the service instance, or NULL. + */ + UObject* getKey(ICUServiceKey& key, UnicodeString* actualReturn, const ICUServiceFactory* factory, UErrorCode& status) const; + + /** + *

Convenience override for getVisibleIDs(String) that passes null + * as the fallback, thus returning all visible IDs.

+ * + * @param result a vector to hold the returned IDs. + * @param status the error code status. + * @return the result vector. + */ + UVector& getVisibleIDs(UVector& result, UErrorCode& status) const; + + /** + *

Return a snapshot of the visible IDs for this service. This + * list will not change as ICUServiceFactories are added or removed, but the + * supported IDs will, so there is no guarantee that all and only + * the IDs in the returned list will be visible and supported by the + * service in subsequent calls.

+ * + *

The IDs are returned as pointers to UnicodeStrings. The + * caller owns the IDs. Previous contents of result are discarded before + * new elements, if any, are added.

+ * + *

matchID is passed to createKey to create a key. If the key + * is not NULL, its isFallbackOf method is used to filter out IDs + * that don't match the key or have it as a fallback.

+ * + * @param result a vector to hold the returned IDs. + * @param matchID an ID used to filter the result, or NULL if all IDs are desired. + * @param status the error code status. + * @return the result vector. + */ + UVector& getVisibleIDs(UVector& result, const UnicodeString* matchID, UErrorCode& status) const; + + /** + *

Convenience override for getDisplayName(const UnicodeString&, const Locale&, UnicodeString&) that + * uses the current default locale.

+ * + * @param id the ID for which to retrieve the localized displayName. + * @param result an output parameter to hold the display name. + * @return the modified result. + */ + UnicodeString& getDisplayName(const UnicodeString& id, UnicodeString& result) const; + + /** + *

Given a visible ID, return the display name in the requested locale. + * If there is no directly supported ID corresponding to this ID, result is + * set to bogus.

+ * + * @param id the ID for which to retrieve the localized displayName. + * @param result an output parameter to hold the display name. + * @param locale the locale in which to localize the ID. + * @return the modified result. + */ + UnicodeString& getDisplayName(const UnicodeString& id, UnicodeString& result, const Locale& locale) const; + + /** + *

Convenience override of getDisplayNames(const Locale&, const UnicodeString*) that + * uses the current default Locale as the locale and NULL for + * the matchID.

+ * + * @param result a vector to hold the returned displayName/id StringPairs. + * @param status the error code status. + * @return the modified result vector. + */ + UVector& getDisplayNames(UVector& result, UErrorCode& status) const; + + /** + *

Convenience override of getDisplayNames(const Locale&, const UnicodeString*) that + * uses NULL for the matchID.

+ * + * @param result a vector to hold the returned displayName/id StringPairs. + * @param locale the locale in which to localize the ID. + * @param status the error code status. + * @return the modified result vector. + */ + UVector& getDisplayNames(UVector& result, const Locale& locale, UErrorCode& status) const; + + /** + *

Return a snapshot of the mapping from display names to visible + * IDs for this service. This set will not change as factories + * are added or removed, but the supported IDs will, so there is + * no guarantee that all and only the IDs in the returned map will + * be visible and supported by the service in subsequent calls, + * nor is there any guarantee that the current display names match + * those in the result.

+ * + *

The names are returned as pointers to StringPairs, which + * contain both the displayName and the corresponding ID. The + * caller owns the StringPairs. Previous contents of result are + * discarded before new elements, if any, are added.

+ * + *

matchID is passed to createKey to create a key. If the key + * is not NULL, its isFallbackOf method is used to filter out IDs + * that don't match the key or have it as a fallback.

+ * + * @param result a vector to hold the returned displayName/id StringPairs. + * @param locale the locale in which to localize the ID. + * @param matchID an ID used to filter the result, or NULL if all IDs are desired. + * @param status the error code status. + * @return the result vector. */ + UVector& getDisplayNames(UVector& result, + const Locale& locale, + const UnicodeString* matchID, + UErrorCode& status) const; + + /** + *

A convenience override of registerInstance(UObject*, const UnicodeString&, UBool) + * that defaults visible to TRUE.

+ * + * @param objToAdopt the object to register and adopt. + * @param id the ID to assign to this object. + * @param status the error code status. + * @return a registry key that can be passed to unregister to unregister + * (and discard) this instance. + */ + URegistryKey registerInstance(UObject* objToAdopt, const UnicodeString& id, UErrorCode& status); + + /** + *

Register a service instance with the provided ID. The ID will be + * canonicalized. The canonicalized ID will be returned by + * getVisibleIDs if visible is TRUE. The service instance will be adopted and + * must not be modified subsequent to this call.

+ * + *

This issues a serviceChanged notification to registered listeners.

+ * + *

This implementation wraps the object using + * createSimpleFactory, and calls registerFactory.

+ * + * @param objToAdopt the object to register and adopt. + * @param id the ID to assign to this object. + * @param visible TRUE if getVisibleIDs is to return this ID. + * @param status the error code status. + * @return a registry key that can be passed to unregister() to unregister + * (and discard) this instance. + */ + virtual URegistryKey registerInstance(UObject* objToAdopt, const UnicodeString& id, UBool visible, UErrorCode& status); + + /** + *

Register an ICUServiceFactory. Returns a registry key that + * can be used to unregister the factory. The factory + * must not be modified subsequent to this call. The service owns + * all registered factories. In case of an error, the factory is + * deleted.

+ * + *

This issues a serviceChanged notification to registered listeners.

+ * + *

The default implementation accepts all factories.

+ * + * @param factoryToAdopt the factory to register and adopt. + * @param status the error code status. + * @return a registry key that can be passed to unregister to unregister + * (and discard) this factory. + */ + virtual URegistryKey registerFactory(ICUServiceFactory* factoryToAdopt, UErrorCode& status); + + /** + *

Unregister a factory using a registry key returned by + * registerInstance or registerFactory. After a successful call, + * the factory will be removed from the service factory list and + * deleted, and the key becomes invalid.

+ * + *

This issues a serviceChanged notification to registered + * listeners.

+ * + * @param rkey the registry key. + * @param status the error code status. + * @return TRUE if the call successfully unregistered the factory. + */ + virtual UBool unregister(URegistryKey rkey, UErrorCode& status); + + /** + *

Reset the service to the default factories. The factory + * lock is acquired and then reInitializeFactories is called.

+ * + *

This issues a serviceChanged notification to registered listeners.

+ */ + virtual void reset(void); + + /** + *

Return TRUE if the service is in its default state.

+ * + *

The default implementation returns TRUE if there are no + * factories registered.

+ */ + virtual UBool isDefault(void) const; + + /** + *

Create a key from an ID. If ID is NULL, returns NULL.

+ * + *

The default implementation creates an ICUServiceKey instance. + * Subclasses can override to define more useful keys appropriate + * to the factories they accept.

+ * + * @param a pointer to the ID for which to create a default ICUServiceKey. + * @param status the error code status. + * @return the ICUServiceKey corresponding to ID, or NULL. + */ + virtual ICUServiceKey* createKey(const UnicodeString* id, UErrorCode& status) const; + + /** + *

Clone object so that caller can own the copy. In ICU2.4, UObject doesn't define + * clone, so we need an instance-aware method that knows how to do this. + * This is public so factories can call it, but should really be protected.

+ * + * @param instance the service instance to clone. + * @return a clone of the passed-in instance, or NULL if cloning was unsuccessful. + */ + virtual UObject* cloneInstance(UObject* instance) const = 0; + + + /************************************************************************ + * Subclassing API + */ + + protected: + + /** + *

Create a factory that wraps a single service object. Called by registerInstance.

+ * + *

The default implementation returns an instance of SimpleFactory.

+ * + * @param instanceToAdopt the service instance to adopt. + * @param id the ID to assign to this service instance. + * @param visible if TRUE, the ID will be visible. + * @param status the error code status. + * @return an instance of ICUServiceFactory that maps this instance to the provided ID. + */ + virtual ICUServiceFactory* createSimpleFactory(UObject* instanceToAdopt, const UnicodeString& id, UBool visible, UErrorCode& status); + + /** + *

Reinitialize the factory list to its default state. After this call, isDefault() + * must return TRUE.

+ * + *

This issues a serviceChanged notification to registered listeners.

+ * + *

The default implementation clears the factory list. + * Subclasses can override to provide other default initialization + * of the factory list. Subclasses must not call this method + * directly, since it must only be called while holding write + * access to the factory list.

+ */ + virtual void reInitializeFactories(void); + + /** + *

Default handler for this service if no factory in the factory list + * handled the key passed to getKey.

+ * + *

The default implementation returns NULL.

+ * + * @param key the key. + * @param actualReturn a pointer to a UnicodeString to hold the matched descriptor, or NULL. + * @param status the error code status. + * @return the service instance, or NULL. + */ + virtual UObject* handleDefault(const ICUServiceKey& key, UnicodeString* actualReturn, UErrorCode& status) const; + + /** + *

Clear caches maintained by this service.

+ * + *

Subclasses can override if they implement additional caches + * that need to be cleared when the service changes. Subclasses + * should generally not call this method directly, as it must only + * be called while synchronized on the factory lock.

+ */ + virtual void clearCaches(void); + + /** + *

Return true if the listener is accepted.

+ * + *

The default implementation accepts the listener if it is + * a ServiceListener. Subclasses can override this to accept + * different listeners.

+ * + * @param l the listener to test. + * @return TRUE if the service accepts the listener. + */ + virtual UBool acceptsListener(const EventListener& l) const; + + /** + *

Notify the listener of a service change.

+ * + *

The default implementation assumes a ServiceListener. + * If acceptsListener has been overridden to accept different + * listeners, this should be overridden as well.

+ * + * @param l the listener to notify. + */ + virtual void notifyListener(EventListener& l) const; + + /************************************************************************ + * Utilities for subclasses. + */ + + /** + *

Clear only the service cache.

+ * + *

This can be called by subclasses when a change affects the service + * cache but not the ID caches, e.g., when the default locale changes + * the resolution of IDs also changes, requiring the cache to be + * flushed, but not the visible IDs themselves.

+ */ + void clearServiceCache(void); + + /** + *

Return a map from visible IDs to factories. + * This must only be called when the mutex is held.

+ * + * @param status the error code status. + * @return a Hashtable containing mappings from visible + * IDs to factories. + */ + const Hashtable* getVisibleIDMap(UErrorCode& status) const; + + /** + *

Allow subclasses to read the time stamp.

+ * + * @return the timestamp. + */ + int32_t getTimestamp(void) const; + + /** + *

Return the number of registered factories.

+ * + * @return the number of factories registered at the time of the call. + */ + int32_t countFactories(void) const; + +private: + + friend class ::ICUServiceTest; // give tests access to countFactories. +}; + +U_NAMESPACE_END + + /* UCONFIG_NO_SERVICE */ +#endif + + /* ICUSERV_H */ +#endif + diff --git a/Build/source/libs/icu/icu-xetex/common/servlk.cpp b/Build/source/libs/icu/icu-xetex/common/servlk.cpp new file mode 100644 index 00000000000..b6204148458 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/servlk.cpp @@ -0,0 +1,187 @@ +/** + ******************************************************************************* + * Copyright (C) 2001-2004, International Business Machines Corporation and * + * others. All Rights Reserved. * + ******************************************************************************* + * + ******************************************************************************* + */ +#include "unicode/utypes.h" + +#if !UCONFIG_NO_SERVICE + +#include "unicode/resbund.h" +#include "uresimp.h" +#include "cmemory.h" +#include "servloc.h" +#include "ustrfmt.h" +#include "uhash.h" +#include "charstr.h" +#include "ucln_cmn.h" +#include "uassert.h" + +#define UNDERSCORE_CHAR ((UChar)0x005f) +#define AT_SIGN_CHAR ((UChar)64) +#define PERIOD_CHAR ((UChar)46) + +U_NAMESPACE_BEGIN + +LocaleKey* +LocaleKey::createWithCanonicalFallback(const UnicodeString* primaryID, + const UnicodeString* canonicalFallbackID, + UErrorCode& status) +{ + return LocaleKey::createWithCanonicalFallback(primaryID, canonicalFallbackID, KIND_ANY, status); +} + +LocaleKey* +LocaleKey::createWithCanonicalFallback(const UnicodeString* primaryID, + const UnicodeString* canonicalFallbackID, + int32_t kind, + UErrorCode& status) +{ + if (primaryID == NULL || U_FAILURE(status)) { + return NULL; + } + UnicodeString canonicalPrimaryID; + LocaleUtility::canonicalLocaleString(primaryID, canonicalPrimaryID); + return new LocaleKey(*primaryID, canonicalPrimaryID, canonicalFallbackID, kind); +} + +LocaleKey::LocaleKey(const UnicodeString& primaryID, + const UnicodeString& canonicalPrimaryID, + const UnicodeString* canonicalFallbackID, + int32_t kind) + : ICUServiceKey(primaryID) + , _kind(kind) + , _primaryID(canonicalPrimaryID) + , _fallbackID() + , _currentID() +{ + _fallbackID.setToBogus(); + if (_primaryID.length() != 0) { + if (canonicalFallbackID != NULL && _primaryID != *canonicalFallbackID) { + _fallbackID = *canonicalFallbackID; + } + } + + _currentID = _primaryID; +} + +LocaleKey::~LocaleKey() {} + +UnicodeString& +LocaleKey::prefix(UnicodeString& result) const { + if (_kind != KIND_ANY) { + UChar buffer[64]; + uprv_itou(buffer, 64, _kind, 10, 0); + UnicodeString temp(buffer); + result.append(temp); + } + return result; +} + +int32_t +LocaleKey::kind() const { + return _kind; +} + +UnicodeString& +LocaleKey::canonicalID(UnicodeString& result) const { + return result.append(_primaryID); +} + +UnicodeString& +LocaleKey::currentID(UnicodeString& result) const { + if (!_currentID.isBogus()) { + result.append(_currentID); + } + return result; +} + +UnicodeString& +LocaleKey::currentDescriptor(UnicodeString& result) const { + if (!_currentID.isBogus()) { + prefix(result).append(PREFIX_DELIMITER).append(_currentID); + } else { + result.setToBogus(); + } + return result; +} + +Locale& +LocaleKey::canonicalLocale(Locale& result) const { + return LocaleUtility::initLocaleFromName(_primaryID, result); +} + +Locale& +LocaleKey::currentLocale(Locale& result) const { + return LocaleUtility::initLocaleFromName(_currentID, result); +} + +UBool +LocaleKey::fallback() { + if (!_currentID.isBogus()) { + int x = _currentID.lastIndexOf(UNDERSCORE_CHAR); + if (x != -1) { + _currentID.remove(x); // truncate current or fallback, whichever we're pointing to + return TRUE; + } + + if (!_fallbackID.isBogus()) { + _currentID = _fallbackID; + _fallbackID.setToBogus(); + return TRUE; + } + + if (_currentID.length() > 0) { + _currentID.remove(0); // completely truncate + return TRUE; + } + + _currentID.setToBogus(); + } + + return FALSE; +} + +UBool +LocaleKey::isFallbackOf(const UnicodeString& id) const { + UnicodeString temp(id); + parseSuffix(temp); + return temp.indexOf(_primaryID) == 0 && + (temp.length() == _primaryID.length() || + temp.charAt(_primaryID.length()) == UNDERSCORE_CHAR); +} + +#ifdef SERVICE_DEBUG +UnicodeString& +LocaleKey::debug(UnicodeString& result) const +{ + ICUServiceKey::debug(result); + result.append(" kind: "); + result.append(_kind); + result.append(" primaryID: "); + result.append(_primaryID); + result.append(" fallbackID: "); + result.append(_fallbackID); + result.append(" currentID: "); + result.append(_currentID); + return result; +} + +UnicodeString& +LocaleKey::debugClass(UnicodeString& result) const +{ + return result.append("LocaleKey "); +} +#endif + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(LocaleKey) + +U_NAMESPACE_END + +/* !UCONFIG_NO_SERVICE */ +#endif + + diff --git a/Build/source/libs/icu/icu-xetex/common/servlkf.cpp b/Build/source/libs/icu/icu-xetex/common/servlkf.cpp new file mode 100644 index 00000000000..c455080fdd9 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/servlkf.cpp @@ -0,0 +1,151 @@ +/** + ******************************************************************************* + * Copyright (C) 2001-2005, International Business Machines Corporation and * + * others. All Rights Reserved. * + ******************************************************************************* + * + ******************************************************************************* + */ +#include "unicode/utypes.h" + +#if !UCONFIG_NO_SERVICE + +#include "unicode/resbund.h" +#include "uresimp.h" +#include "cmemory.h" +#include "servloc.h" +#include "ustrfmt.h" +#include "uhash.h" +#include "charstr.h" +#include "ucln_cmn.h" +#include "uassert.h" + +#define UNDERSCORE_CHAR ((UChar)0x005f) +#define AT_SIGN_CHAR ((UChar)64) +#define PERIOD_CHAR ((UChar)46) + + +U_NAMESPACE_BEGIN + +LocaleKeyFactory::LocaleKeyFactory(int32_t coverage) + : _name() + , _coverage(coverage) +{ +} + +LocaleKeyFactory::LocaleKeyFactory(int32_t coverage, const UnicodeString& name) + : _name(name) + , _coverage(coverage) +{ +} + +LocaleKeyFactory::~LocaleKeyFactory() { +} + +UObject* +LocaleKeyFactory::create(const ICUServiceKey& key, const ICUService* service, UErrorCode& status) const { + if (handlesKey(key, status)) { + const LocaleKey& lkey = (const LocaleKey&)key; + int32_t kind = lkey.kind(); + Locale loc; + lkey.currentLocale(loc); + + return handleCreate(loc, kind, service, status); + } + return NULL; +} + +UBool +LocaleKeyFactory::handlesKey(const ICUServiceKey& key, UErrorCode& status) const { + const Hashtable* supported = getSupportedIDs(status); + if (supported) { + UnicodeString id; + key.currentID(id); + return supported->get(id) != NULL; + } + return FALSE; +} + +void +LocaleKeyFactory::updateVisibleIDs(Hashtable& result, UErrorCode& status) const { + const Hashtable* supported = getSupportedIDs(status); + if (supported) { + UBool visible = (_coverage & 0x1) == 0; + + const UHashElement* elem = NULL; + int32_t pos = 0; + while ((elem = supported->nextElement(pos)) != NULL) { + const UnicodeString& id = *((const UnicodeString*)elem->key.pointer); + if (!visible) { + result.remove(id); + } else { + result.put(id, (void*)this, status); // this is dummy non-void marker used for set semantics + if (U_FAILURE(status)) { + break; + } + } + } + } +} + +UnicodeString& +LocaleKeyFactory::getDisplayName(const UnicodeString& id, const Locale& locale, UnicodeString& result) const { + if ((_coverage & 0x1) == 0) { + //UErrorCode status = U_ZERO_ERROR; + // assume if this is called on us, we support some fallback of this id + // if (isSupportedID(id, status)) { + Locale loc; + LocaleUtility::initLocaleFromName(id, loc); + return loc.getDisplayName(locale, result); + // } + } + result.setToBogus(); + return result; +} + +UObject* +LocaleKeyFactory::handleCreate(const Locale& /* loc */, + int32_t /* kind */, + const ICUService* /* service */, + UErrorCode& /* status */) const { + return NULL; +} + +//UBool +//LocaleKeyFactory::isSupportedID(const UnicodeString& id, UErrorCode& status) const { +// const Hashtable* ids = getSupportedIDs(status); +// return ids && ids->get(id); +//} + +const Hashtable* +LocaleKeyFactory::getSupportedIDs(UErrorCode& /* status */) const { + return NULL; +} + +#ifdef SERVICE_DEBUG +UnicodeString& +LocaleKeyFactory::debug(UnicodeString& result) const +{ + debugClass(result); + result.append(", name: "); + result.append(_name); + result.append(", coverage: "); + result.append(_coverage); + return result; +} + +UnicodeString& +LocaleKeyFactory::debugClass(UnicodeString& result) const +{ + return result.append("LocaleKeyFactory"); +} +#endif + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(LocaleKeyFactory) + +U_NAMESPACE_END + +/* !UCONFIG_NO_SERVICE */ +#endif + + diff --git a/Build/source/libs/icu/icu-xetex/common/servloc.h b/Build/source/libs/icu/icu-xetex/common/servloc.h new file mode 100644 index 00000000000..d08b09e36ea --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/servloc.h @@ -0,0 +1,550 @@ +/** + ******************************************************************************* + * Copyright (C) 2001-2005, International Business Machines Corporation and * + * others. All Rights Reserved. * + ******************************************************************************* + * + ******************************************************************************* + */ +#ifndef ICULSERV_H +#define ICULSERV_H + +#include "unicode/utypes.h" + +#if UCONFIG_NO_SERVICE + +U_NAMESPACE_BEGIN + +/* + * Allow the declaration of APIs with pointers to ICUService + * even when service is removed from the build. + */ +class ICULocaleService; + +U_NAMESPACE_END + +#else + +#include "unicode/unistr.h" +#include "unicode/locid.h" +#include "unicode/strenum.h" + +#include "hash.h" +#include "uvector.h" + +#include "serv.h" +#include "locutil.h" + +U_NAMESPACE_BEGIN + +class ICULocaleService; + +class LocaleKey; +class LocaleKeyFactory; +class SimpleLocaleKeyFactory; +class ServiceListener; + +/* + ****************************************************************** + */ + +/** + * A subclass of Key that implements a locale fallback mechanism. + * The first locale to search for is the locale provided by the + * client, and the fallback locale to search for is the current + * default locale. If a prefix is present, the currentDescriptor + * includes it before the locale proper, separated by "/". This + * is the default key instantiated by ICULocaleService.

+ * + *

Canonicalization adjusts the locale string so that the + * section before the first understore is in lower case, and the rest + * is in upper case, with no trailing underscores.

+ */ + +class U_COMMON_API LocaleKey : public ICUServiceKey { + private: + int32_t _kind; + UnicodeString _primaryID; + UnicodeString _fallbackID; + UnicodeString _currentID; + + public: + enum { + KIND_ANY = -1 + }; + + /** + * Create a LocaleKey with canonical primary and fallback IDs. + */ + static LocaleKey* createWithCanonicalFallback(const UnicodeString* primaryID, + const UnicodeString* canonicalFallbackID, + UErrorCode& status); + + /** + * Create a LocaleKey with canonical primary and fallback IDs. + */ + static LocaleKey* createWithCanonicalFallback(const UnicodeString* primaryID, + const UnicodeString* canonicalFallbackID, + int32_t kind, + UErrorCode& status); + + protected: + /** + * PrimaryID is the user's requested locale string, + * canonicalPrimaryID is this string in canonical form, + * fallbackID is the current default locale's string in + * canonical form. + */ + LocaleKey(const UnicodeString& primaryID, + const UnicodeString& canonicalPrimaryID, + const UnicodeString* canonicalFallbackID, + int32_t kind); + + public: + /** + * Append the prefix associated with the kind, or nothing if the kind is KIND_ANY. + */ + virtual UnicodeString& prefix(UnicodeString& result) const; + + /** + * Return the kind code associated with this key. + */ + virtual int32_t kind() const; + + /** + * Return the canonicalID. + */ + virtual UnicodeString& canonicalID(UnicodeString& result) const; + + /** + * Return the currentID. + */ + virtual UnicodeString& currentID(UnicodeString& result) const; + + /** + * Return the (canonical) current descriptor, or null if no current id. + */ + virtual UnicodeString& currentDescriptor(UnicodeString& result) const; + + /** + * Convenience method to return the locale corresponding to the (canonical) original ID. + */ + virtual Locale& canonicalLocale(Locale& result) const; + + /** + * Convenience method to return the locale corresponding to the (canonical) current ID. + */ + virtual Locale& currentLocale(Locale& result) const; + + /** + * If the key has a fallback, modify the key and return true, + * otherwise return false.

+ * + *

First falls back through the primary ID, then through + * the fallbackID. The final fallback is the empty string, + * unless the primary id was the empty string, in which case + * there is no fallback. + */ + virtual UBool fallback(); + + /** + * Return true if a key created from id matches, or would eventually + * fallback to match, the canonical ID of this key. + */ + virtual UBool isFallbackOf(const UnicodeString& id) const; + + public: + /** + * UObject boilerplate. + */ + static UClassID U_EXPORT2 getStaticClassID(); + + virtual UClassID getDynamicClassID() const; + + /** + * Destructor. + */ + virtual ~LocaleKey(); + +#ifdef SERVICE_DEBUG + public: + virtual UnicodeString& debug(UnicodeString& result) const; + virtual UnicodeString& debugClass(UnicodeString& result) const; +#endif + +}; + +/* + ****************************************************************** + */ + +/** + * A subclass of ICUServiceFactory that uses LocaleKeys, and is able to + * 'cover' more specific locales with more general locales that it + * supports. + * + *

Coverage may be either of the values VISIBLE or INVISIBLE. + * + *

'Visible' indicates that the specific locale(s) supported by + * the factory are registered in getSupportedIDs, 'Invisible' + * indicates that they are not. + * + *

Localization of visible ids is handled + * by the handling factory, regardless of kind. + */ +class U_COMMON_API LocaleKeyFactory : public ICUServiceFactory { +protected: + const UnicodeString _name; + const int32_t _coverage; + +public: + enum { + /** + * Coverage value indicating that the factory makes + * its locales visible, and does not cover more specific + * locales. + */ + VISIBLE = 0, + + /** + * Coverage value indicating that the factory does not make + * its locales visible, and does not cover more specific + * locales. + */ + INVISIBLE = 1 + }; + + /** + * Destructor. + */ + virtual ~LocaleKeyFactory(); + +protected: + /** + * Constructor used by subclasses. + */ + LocaleKeyFactory(int32_t coverage); + + /** + * Constructor used by subclasses. + */ + LocaleKeyFactory(int32_t coverage, const UnicodeString& name); + + /** + * Implement superclass abstract method. This checks the currentID of + * the key against the supported IDs, and passes the canonicalLocale and + * kind off to handleCreate (which subclasses must implement). + */ +public: + virtual UObject* create(const ICUServiceKey& key, const ICUService* service, UErrorCode& status) const; + +protected: + virtual UBool handlesKey(const ICUServiceKey& key, UErrorCode& status) const; + +public: + /** + * Override of superclass method. This adjusts the result based + * on the coverage rule for this factory. + */ + virtual void updateVisibleIDs(Hashtable& result, UErrorCode& status) const; + + /** + * Return a localized name for the locale represented by id. + */ + virtual UnicodeString& getDisplayName(const UnicodeString& id, const Locale& locale, UnicodeString& result) const; + +protected: + /** + * Utility method used by create(ICUServiceKey, ICUService). Subclasses can implement + * this instead of create. The default returns NULL. + */ + virtual UObject* handleCreate(const Locale& loc, int32_t kind, const ICUService* service, UErrorCode& status) const; + + /** + * Return true if this id is one the factory supports (visible or + * otherwise). + */ + // virtual UBool isSupportedID(const UnicodeString& id, UErrorCode& status) const; + + /** + * Return the set of ids that this factory supports (visible or + * otherwise). This can be called often and might need to be + * cached if it is expensive to create. + */ + virtual const Hashtable* getSupportedIDs(UErrorCode& status) const; + +public: + /** + * UObject boilerplate. + */ + static UClassID U_EXPORT2 getStaticClassID(); + + virtual UClassID getDynamicClassID() const; + +#ifdef SERVICE_DEBUG + public: + virtual UnicodeString& debug(UnicodeString& result) const; + virtual UnicodeString& debugClass(UnicodeString& result) const; +#endif + +}; + +/* + ****************************************************************** + */ + +/** + * A LocaleKeyFactory that just returns a single object for a kind/locale. + */ + +class U_COMMON_API SimpleLocaleKeyFactory : public LocaleKeyFactory { + private: + UObject* _obj; + UnicodeString _id; + const int32_t _kind; + + public: + SimpleLocaleKeyFactory(UObject* objToAdopt, + const UnicodeString& locale, + int32_t kind, + int32_t coverage); + + SimpleLocaleKeyFactory(UObject* objToAdopt, + const Locale& locale, + int32_t kind, + int32_t coverage); + + /** + * Destructor. + */ + virtual ~SimpleLocaleKeyFactory(); + + /** + * Override of superclass method. Returns the service object if kind/locale match. Service is not used. + */ + virtual UObject* create(const ICUServiceKey& key, const ICUService* service, UErrorCode& status) const; + + /** + * Override of superclass method. This adjusts the result based + * on the coverage rule for this factory. + */ + virtual void updateVisibleIDs(Hashtable& result, UErrorCode& status) const; + + protected: + /** + * Return true if this id is equal to the locale name. + */ + //virtual UBool isSupportedID(const UnicodeString& id, UErrorCode& status) const; + + +public: + /** + * UObject boilerplate. + */ + static UClassID U_EXPORT2 getStaticClassID(); + + virtual UClassID getDynamicClassID() const; + +#ifdef SERVICE_DEBUG + public: + virtual UnicodeString& debug(UnicodeString& result) const; + virtual UnicodeString& debugClass(UnicodeString& result) const; +#endif + +}; + +/* + ****************************************************************** + */ + +/** + * A LocaleKeyFactory that creates a service based on the ICU locale data. + * This is a base class for most ICU factories. Subclasses instantiate it + * with a constructor that takes a bundle name, which determines the supported + * IDs. Subclasses then override handleCreate to create the actual service + * object. The default implementation returns a resource bundle. + */ +class U_COMMON_API ICUResourceBundleFactory : public LocaleKeyFactory +{ + protected: + UnicodeString _bundleName; + + public: + /** + * Convenience constructor that uses the main ICU bundle name. + */ + ICUResourceBundleFactory(); + + /** + * A service factory based on ICU resource data in resources with + * the given name. This should be a 'path' that can be passed to + * ures_openAvailableLocales, such as U_ICUDATA or U_ICUDATA_COLL. + * The empty string is equivalent to U_ICUDATA. + */ + ICUResourceBundleFactory(const UnicodeString& bundleName); + + /** + * Destructor + */ + virtual ~ICUResourceBundleFactory(); + +protected: + /** + * Return the supported IDs. This is the set of all locale names in ICULocaleData. + */ + virtual const Hashtable* getSupportedIDs(UErrorCode& status) const; + + /** + * Create the service. The default implementation returns the resource bundle + * for the locale, ignoring kind, and service. + */ + virtual UObject* handleCreate(const Locale& loc, int32_t kind, const ICUService* service, UErrorCode& status) const; + +public: + /** + * UObject boilerplate. + */ + static UClassID U_EXPORT2 getStaticClassID(); + virtual UClassID getDynamicClassID() const; + + +#ifdef SERVICE_DEBUG + public: + virtual UnicodeString& debug(UnicodeString& result) const; + virtual UnicodeString& debugClass(UnicodeString& result) const; +#endif + +}; + +/* + ****************************************************************** + */ + +class U_COMMON_API ICULocaleService : public ICUService +{ + private: + Locale fallbackLocale; + UnicodeString fallbackLocaleName; + UMTX llock; + + public: + /** + * Construct an ICULocaleService. + */ + ICULocaleService(); + + /** + * Construct an ICULocaleService with a name (useful for debugging). + */ + ICULocaleService(const UnicodeString& name); + + /** + * Destructor. + */ + virtual ~ICULocaleService(); + +#if 0 + // redeclare because of overload resolution rules? + // no, causes ambiguities since both UnicodeString and Locale have constructors that take a const char* + // need some compiler flag to remove warnings + UObject* get(const UnicodeString& descriptor, UErrorCode& status) const { + return ICUService::get(descriptor, status); + } + + UObject* get(const UnicodeString& descriptor, UnicodeString* actualReturn, UErrorCode& status) const { + return ICUService::get(descriptor, actualReturn, status); + } +#endif + + /** + * Convenience override for callers using locales. This calls + * get(Locale, int, Locale[]) with KIND_ANY for kind and null for + * actualReturn. + */ + UObject* get(const Locale& locale, UErrorCode& status) const; + + /** + * Convenience override for callers using locales. This calls + * get(Locale, int, Locale[]) with a null actualReturn. + */ + UObject* get(const Locale& locale, int32_t kind, UErrorCode& status) const; + + /** + * Convenience override for callers using locales. This calls + * get(Locale, String, Locale[]) with a null kind. + */ + UObject* get(const Locale& locale, Locale* actualReturn, UErrorCode& status) const; + + /** + * Convenience override for callers using locales. This uses + * createKey(Locale.toString(), kind) to create a key, calls getKey, and then + * if actualReturn is not null, returns the actualResult from + * getKey (stripping any prefix) into a Locale. + */ + UObject* get(const Locale& locale, int32_t kind, Locale* actualReturn, UErrorCode& status) const; + + /** + * Convenience override for callers using locales. This calls + * registerObject(Object, Locale, int32_t kind, int coverage) + * passing KIND_ANY for the kind, and VISIBLE for the coverage. + */ + virtual URegistryKey registerInstance(UObject* objToAdopt, const Locale& locale, UErrorCode& status); + + /** + * Convenience function for callers using locales. This calls + * registerObject(Object, Locale, int kind, int coverage) + * passing VISIBLE for the coverage. + */ + virtual URegistryKey registerInstance(UObject* objToAdopt, const Locale& locale, int32_t kind, UErrorCode& status); + + /** + * Convenience function for callers using locales. This instantiates + * a SimpleLocaleKeyFactory, and registers the factory. + */ + virtual URegistryKey registerInstance(UObject* objToAdopt, const Locale& locale, int32_t kind, int32_t coverage, UErrorCode& status); + + + /** + * (Stop compiler from complaining about hidden overrides.) + * Since both UnicodeString and Locale have constructors that take const char*, adding a public + * method that takes UnicodeString causes ambiguity at call sites that use const char*. + * We really need a flag that is understood by all compilers that will suppress the warning about + * hidden overrides. + */ + virtual URegistryKey registerInstance(UObject* objToAdopt, const UnicodeString& locale, UBool visible, UErrorCode& status); + + /** + * Convenience method for callers using locales. This returns the standard + * service ID enumeration. + */ + virtual StringEnumeration* getAvailableLocales(void) const; + + protected: + + /** + * Return the name of the current fallback locale. If it has changed since this was + * last accessed, the service cache is cleared. + */ + const UnicodeString& validateFallbackLocale() const; + + /** + * Override superclass createKey method. + */ + virtual ICUServiceKey* createKey(const UnicodeString* id, UErrorCode& status) const; + + /** + * Additional createKey that takes a kind. + */ + virtual ICUServiceKey* createKey(const UnicodeString* id, int32_t kind, UErrorCode& status) const; + + friend class ServiceEnumeration; +}; + +U_NAMESPACE_END + + /* UCONFIG_NO_SERVICE */ +#endif + + /* ICULSERV_H */ +#endif + diff --git a/Build/source/libs/icu/icu-xetex/common/servls.cpp b/Build/source/libs/icu/icu-xetex/common/servls.cpp new file mode 100644 index 00000000000..b39e72ed62b --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/servls.cpp @@ -0,0 +1,297 @@ +/** + ******************************************************************************* + * Copyright (C) 2001-2004, International Business Machines Corporation and * + * others. All Rights Reserved. * + ******************************************************************************* + * + ******************************************************************************* + */ +#include "unicode/utypes.h" + +#if !UCONFIG_NO_SERVICE + +#include "unicode/resbund.h" +#include "uresimp.h" +#include "cmemory.h" +#include "servloc.h" +#include "ustrfmt.h" +#include "uhash.h" +#include "charstr.h" +#include "ucln_cmn.h" +#include "uassert.h" + +#define UNDERSCORE_CHAR ((UChar)0x005f) +#define AT_SIGN_CHAR ((UChar)64) +#define PERIOD_CHAR ((UChar)46) + +U_NAMESPACE_BEGIN + +ICULocaleService::ICULocaleService() + : fallbackLocale(Locale::getDefault()) + , llock(0) +{ + umtx_init(&llock); +} + +ICULocaleService::ICULocaleService(const UnicodeString& dname) + : ICUService(dname) + , fallbackLocale(Locale::getDefault()) + , llock(0) +{ + umtx_init(&llock); +} + +ICULocaleService::~ICULocaleService() +{ + umtx_destroy(&llock); +} + +UObject* +ICULocaleService::get(const Locale& locale, UErrorCode& status) const +{ + return get(locale, LocaleKey::KIND_ANY, NULL, status); +} + +UObject* +ICULocaleService::get(const Locale& locale, int32_t kind, UErrorCode& status) const +{ + return get(locale, kind, NULL, status); +} + +UObject* +ICULocaleService::get(const Locale& locale, Locale* actualReturn, UErrorCode& status) const +{ + return get(locale, LocaleKey::KIND_ANY, actualReturn, status); +} + +UObject* +ICULocaleService::get(const Locale& locale, int32_t kind, Locale* actualReturn, UErrorCode& status) const +{ + UObject* result = NULL; + if (U_FAILURE(status)) { + return result; + } + + UnicodeString locName(locale.getName(), -1, US_INV); + if (locName.isBogus()) { + status = U_MEMORY_ALLOCATION_ERROR; + } else { + ICUServiceKey* key = createKey(&locName, kind, status); + if (key) { + if (actualReturn == NULL) { + result = getKey(*key, status); + } else { + UnicodeString temp; + result = getKey(*key, &temp, status); + + if (result != NULL) { + key->parseSuffix(temp); + LocaleUtility::initLocaleFromName(temp, *actualReturn); + } + } + delete key; + } + } + return result; +} + + +URegistryKey +ICULocaleService::registerInstance(UObject* objToAdopt, const UnicodeString& locale, + UBool visible, UErrorCode& status) +{ + Locale loc; + LocaleUtility::initLocaleFromName(locale, loc); + return registerInstance(objToAdopt, loc, LocaleKey::KIND_ANY, + visible ? LocaleKeyFactory::VISIBLE : LocaleKeyFactory::INVISIBLE, status); +} + +URegistryKey +ICULocaleService::registerInstance(UObject* objToAdopt, const Locale& locale, UErrorCode& status) +{ + return registerInstance(objToAdopt, locale, LocaleKey::KIND_ANY, LocaleKeyFactory::VISIBLE, status); +} + +URegistryKey +ICULocaleService::registerInstance(UObject* objToAdopt, const Locale& locale, int32_t kind, UErrorCode& status) +{ + return registerInstance(objToAdopt, locale, kind, LocaleKeyFactory::VISIBLE, status); +} + +URegistryKey +ICULocaleService::registerInstance(UObject* objToAdopt, const Locale& locale, int32_t kind, int32_t coverage, UErrorCode& status) +{ + ICUServiceFactory * factory = new SimpleLocaleKeyFactory(objToAdopt, locale, kind, coverage); + if (factory != NULL) { + return registerFactory(factory, status); + } + delete objToAdopt; + return NULL; +} + +#if 0 +URegistryKey +ICULocaleService::registerInstance(UObject* objToAdopt, const UnicodeString& locale, UErrorCode& status) +{ + return registerInstance(objToAdopt, locale, LocaleKey::KIND_ANY, LocaleKeyFactory::VISIBLE, status); +} + +URegistryKey +ICULocaleService::registerInstance(UObject* objToAdopt, const UnicodeString& locale, UBool visible, UErrorCode& status) +{ + return registerInstance(objToAdopt, locale, LocaleKey::KIND_ANY, + visible ? LocaleKeyFactory::VISIBLE : LocaleKeyFactory::INVISIBLE, + status); +} + +URegistryKey +ICULocaleService::registerInstance(UObject* objToAdopt, const UnicodeString& locale, int32_t kind, int32_t coverage, UErrorCode& status) +{ + ICUServiceFactory * factory = new SimpleLocaleKeyFactory(objToAdopt, locale, kind, coverage); + if (factory != NULL) { + return registerFactory(factory, status); + } + delete objToAdopt; + return NULL; +} +#endif + +class ServiceEnumeration : public StringEnumeration { +private: + const ICULocaleService* _service; + int32_t _timestamp; + UVector _ids; + int32_t _pos; + +private: + ServiceEnumeration(const ICULocaleService* service, UErrorCode &status) + : _service(service) + , _timestamp(service->getTimestamp()) + , _ids(uhash_deleteUnicodeString, NULL, status) + , _pos(0) + { + _service->getVisibleIDs(_ids, status); + } + + ServiceEnumeration(const ServiceEnumeration &other, UErrorCode &status) + : _service(other._service) + , _timestamp(other._timestamp) + , _ids(uhash_deleteUnicodeString, NULL, status) + , _pos(0) + { + if(U_SUCCESS(status)) { + int32_t i, length; + + length = other._ids.size(); + for(i = 0; i < length; ++i) { + _ids.addElement(((UnicodeString *)other._ids.elementAt(i))->clone(), status); + } + + if(U_SUCCESS(status)) { + _pos = other._pos; + } + } + } + +public: + static ServiceEnumeration* create(const ICULocaleService* service) { + UErrorCode status = U_ZERO_ERROR; + ServiceEnumeration* result = new ServiceEnumeration(service, status); + if (U_SUCCESS(status)) { + return result; + } + delete result; + return NULL; + } + + virtual ~ServiceEnumeration() {} + + virtual StringEnumeration *clone() const { + UErrorCode status = U_ZERO_ERROR; + ServiceEnumeration *cl = new ServiceEnumeration(*this, status); + if(U_FAILURE(status)) { + delete cl; + cl = NULL; + } + return cl; + } + + UBool upToDate(UErrorCode& status) const { + if (U_SUCCESS(status)) { + if (_timestamp == _service->getTimestamp()) { + return TRUE; + } + status = U_ENUM_OUT_OF_SYNC_ERROR; + } + return FALSE; + } + + virtual int32_t count(UErrorCode& status) const { + return upToDate(status) ? _ids.size() : 0; + } + + virtual const UnicodeString* snext(UErrorCode& status) { + if (upToDate(status) && (_pos < _ids.size())) { + return (const UnicodeString*)_ids[_pos++]; + } + return NULL; + } + + virtual void reset(UErrorCode& status) { + if (status == U_ENUM_OUT_OF_SYNC_ERROR) { + status = U_ZERO_ERROR; + } + if (U_SUCCESS(status)) { + _timestamp = _service->getTimestamp(); + _pos = 0; + _service->getVisibleIDs(_ids, status); + } + } + +public: + static UClassID U_EXPORT2 getStaticClassID(void); + virtual UClassID getDynamicClassID(void) const; +}; + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(ServiceEnumeration) + +StringEnumeration* +ICULocaleService::getAvailableLocales(void) const +{ + return ServiceEnumeration::create(this); +} + +const UnicodeString& +ICULocaleService::validateFallbackLocale() const +{ + const Locale& loc = Locale::getDefault(); + ICULocaleService* ncThis = (ICULocaleService*)this; + { + Mutex mutex(&ncThis->llock); + if (loc != fallbackLocale) { + ncThis->fallbackLocale = loc; + LocaleUtility::initNameFromLocale(loc, ncThis->fallbackLocaleName); + ncThis->clearServiceCache(); + } + } + return fallbackLocaleName; +} + +ICUServiceKey* +ICULocaleService::createKey(const UnicodeString* id, UErrorCode& status) const +{ + return LocaleKey::createWithCanonicalFallback(id, &validateFallbackLocale(), status); +} + +ICUServiceKey* +ICULocaleService::createKey(const UnicodeString* id, int32_t kind, UErrorCode& status) const +{ + return LocaleKey::createWithCanonicalFallback(id, &validateFallbackLocale(), kind, status); +} + +U_NAMESPACE_END + +/* !UCONFIG_NO_SERVICE */ +#endif + + diff --git a/Build/source/libs/icu/icu-xetex/common/servnotf.cpp b/Build/source/libs/icu/icu-xetex/common/servnotf.cpp new file mode 100644 index 00000000000..6adf52ece2e --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/servnotf.cpp @@ -0,0 +1,118 @@ +/** + ******************************************************************************* + * Copyright (C) 2001-2006, International Business Machines Corporation and * + * others. All Rights Reserved. * + ******************************************************************************* + */ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_SERVICE + +#include "servnotf.h" +#ifdef NOTIFIER_DEBUG +#include +#endif + +U_NAMESPACE_BEGIN + +EventListener::~EventListener() {} +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(EventListener) + +ICUNotifier::ICUNotifier(void) +: notifyLock(0), listeners(NULL) +{ + umtx_init(¬ifyLock); +} + +ICUNotifier::~ICUNotifier(void) { + { + Mutex lmx(¬ifyLock); + delete listeners; + listeners = NULL; + } + umtx_destroy(¬ifyLock); +} + + +void +ICUNotifier::addListener(const EventListener* l, UErrorCode& status) +{ + if (U_SUCCESS(status)) { + if (l == NULL) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + if (acceptsListener(*l)) { + Mutex lmx(¬ifyLock); + if (listeners == NULL) { + listeners = new UVector(5, status); + } else { + for (int i = 0, e = listeners->size(); i < e; ++i) { + const EventListener* el = (const EventListener*)(listeners->elementAt(i)); + if (l == el) { + return; + } + } + } + + listeners->addElement((void*)l, status); // cast away const + } +#ifdef NOTIFIER_DEBUG + else { + fprintf(stderr, "Listener invalid for this notifier."); + exit(1); + } +#endif + } +} + +void +ICUNotifier::removeListener(const EventListener *l, UErrorCode& status) +{ + if (U_SUCCESS(status)) { + if (l == NULL) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + { + Mutex lmx(¬ifyLock); + if (listeners != NULL) { + // identity equality check + for (int i = 0, e = listeners->size(); i < e; ++i) { + const EventListener* el = (const EventListener*)listeners->elementAt(i); + if (l == el) { + listeners->removeElementAt(i); + if (listeners->size() == 0) { + delete listeners; + listeners = NULL; + } + return; + } + } + } + } + } +} + +void +ICUNotifier::notifyChanged(void) +{ + if (listeners != NULL) { + Mutex lmx(¬ifyLock); + if (listeners != NULL) { + for (int i = 0, e = listeners->size(); i < e; ++i) { + EventListener* el = (EventListener*)listeners->elementAt(i); + notifyListener(*el); + } + } + } +} + +U_NAMESPACE_END + +/* UCONFIG_NO_SERVICE */ +#endif + diff --git a/Build/source/libs/icu/icu-xetex/common/servnotf.h b/Build/source/libs/icu/icu-xetex/common/servnotf.h new file mode 100644 index 00000000000..19c2d6908ee --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/servnotf.h @@ -0,0 +1,124 @@ +/** + ******************************************************************************* + * Copyright (C) 2001-2004, International Business Machines Corporation and * + * others. All Rights Reserved. * + ******************************************************************************* + */ +#ifndef ICUNOTIF_H +#define ICUNOTIF_H + +#include "unicode/utypes.h" + +#if UCONFIG_NO_SERVICE + +U_NAMESPACE_BEGIN + +/* + * Allow the declaration of APIs with pointers to BreakIterator + * even when break iteration is removed from the build. + */ +class ICUNotifier; + +U_NAMESPACE_END + +#else + +#include "unicode/uobject.h" +#include "unicode/unistr.h" + +#include "mutex.h" +#include "uvector.h" + +U_NAMESPACE_BEGIN + +class U_COMMON_API EventListener : public UObject { +public: + virtual ~EventListener(); + +public: + static UClassID U_EXPORT2 getStaticClassID(); + + virtual UClassID getDynamicClassID() const; + +public: +#ifdef SERVICE_DEBUG + virtual UnicodeString& debug(UnicodeString& result) const { + return debugClass(result); + } + + virtual UnicodeString& debugClass(UnicodeString& result) const { + return result.append("Key"); + } +#endif +}; + +/** + *

Abstract implementation of a notification facility. Clients add + * EventListeners with addListener and remove them with removeListener. + * Notifiers call notifyChanged when they wish to notify listeners. + * This queues the listener list on the notification thread, which + * eventually dequeues the list and calls notifyListener on each + * listener in the list.

+ * + *

Subclasses override acceptsListener and notifyListener + * to add type-safe notification. AcceptsListener should return + * true if the listener is of the appropriate type; ICUNotifier + * itself will ensure the listener is non-null and that the + * identical listener is not already registered with the Notifier. + * NotifyListener should cast the listener to the appropriate + * type and call the appropriate method on the listener. + */ + +class U_COMMON_API ICUNotifier : public UMemory { +private: UMTX notifyLock; +private: UVector* listeners; + +public: + ICUNotifier(void); + + virtual ~ICUNotifier(void); + + /** + * Add a listener to be notified when notifyChanged is called. + * The listener must not be null. AcceptsListener must return + * true for the listener. Attempts to concurrently + * register the identical listener more than once will be + * silently ignored. + */ + virtual void addListener(const EventListener* l, UErrorCode& status); + + /** + * Stop notifying this listener. The listener must + * not be null. Attemps to remove a listener that is + * not registered will be silently ignored. + */ + virtual void removeListener(const EventListener* l, UErrorCode& status); + + /** + * ICU doesn't spawn its own threads. All listeners are notified in + * the thread of the caller. Misbehaved listeners can therefore + * indefinitely block the calling thread. Callers should beware of + * deadlock situations. + */ + virtual void notifyChanged(void); + +protected: + /** + * Subclasses implement this to return TRUE if the listener is + * of the appropriate type. + */ + virtual UBool acceptsListener(const EventListener& l) const = 0; + + /** + * Subclasses implement this to notify the listener. + */ + virtual void notifyListener(EventListener& l) const = 0; +}; + +U_NAMESPACE_END + +/* UCONFIG_NO_SERVICE */ +#endif + +/* ICUNOTIF_H */ +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/servrbf.cpp b/Build/source/libs/icu/icu-xetex/common/servrbf.cpp new file mode 100644 index 00000000000..3a0227f76af --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/servrbf.cpp @@ -0,0 +1,94 @@ +/** + ******************************************************************************* + * Copyright (C) 2001-2005, International Business Machines Corporation and * + * others. All Rights Reserved. * + ******************************************************************************* + * + ******************************************************************************* + */ +#include "unicode/utypes.h" + +#if !UCONFIG_NO_SERVICE + +#include "unicode/resbund.h" +#include "uresimp.h" +#include "cmemory.h" +#include "servloc.h" +#include "ustrfmt.h" +#include "uhash.h" +#include "charstr.h" +#include "ucln_cmn.h" +#include "uassert.h" + +#define UNDERSCORE_CHAR ((UChar)0x005f) +#define AT_SIGN_CHAR ((UChar)64) +#define PERIOD_CHAR ((UChar)46) + +U_NAMESPACE_BEGIN + +ICUResourceBundleFactory::ICUResourceBundleFactory() + : LocaleKeyFactory(VISIBLE) + , _bundleName() +{ +} + +ICUResourceBundleFactory::ICUResourceBundleFactory(const UnicodeString& bundleName) + : LocaleKeyFactory(VISIBLE) + , _bundleName(bundleName) +{ +} + +ICUResourceBundleFactory::~ICUResourceBundleFactory() {} + +const Hashtable* +ICUResourceBundleFactory::getSupportedIDs(UErrorCode& status) const +{ + if (U_SUCCESS(status)) { + return LocaleUtility::getAvailableLocaleNames(_bundleName); + } + return NULL; +} + +UObject* +ICUResourceBundleFactory::handleCreate(const Locale& loc, int32_t /* kind */, const ICUService* /* service */, UErrorCode& status) const +{ + if (U_SUCCESS(status)) { + // _bundleName is a package name + // and should only contain invariant characters + // ??? is it always true that the max length of the bundle name is 19? + // who made this change? -- dlf + char pkg[20]; + int32_t length; + length=_bundleName.extract(0, INT32_MAX, pkg, (int32_t)sizeof(pkg), US_INV); + if(length>=(int32_t)sizeof(pkg)) { + return NULL; + } + return new ResourceBundle(pkg, loc, status); + } + return NULL; +} + +#ifdef SERVICE_DEBUG +UnicodeString& +ICUResourceBundleFactory::debug(UnicodeString& result) const +{ + LocaleKeyFactory::debug(result); + result.append(", bundle: "); + return result.append(_bundleName); +} + +UnicodeString& +ICUResourceBundleFactory::debugClass(UnicodeString& result) const +{ + return result.append("ICUResourceBundleFactory"); +} +#endif + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(ICUResourceBundleFactory) + +U_NAMESPACE_END + +/* !UCONFIG_NO_SERVICE */ +#endif + + diff --git a/Build/source/libs/icu/icu-xetex/common/servslkf.cpp b/Build/source/libs/icu/icu-xetex/common/servslkf.cpp new file mode 100644 index 00000000000..b8afaaadc1b --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/servslkf.cpp @@ -0,0 +1,122 @@ +/** + ******************************************************************************* + * Copyright (C) 2001-2005, International Business Machines Corporation and * + * others. All Rights Reserved. * + ******************************************************************************* + * + ******************************************************************************* + */ +#include "unicode/utypes.h" + +#if !UCONFIG_NO_SERVICE + +#include "unicode/resbund.h" +#include "uresimp.h" +#include "cmemory.h" +#include "servloc.h" +#include "ustrfmt.h" +#include "uhash.h" +#include "charstr.h" +#include "ucln_cmn.h" +#include "uassert.h" + +#define UNDERSCORE_CHAR ((UChar)0x005f) +#define AT_SIGN_CHAR ((UChar)64) +#define PERIOD_CHAR ((UChar)46) + +U_NAMESPACE_BEGIN + +/* + ****************************************************************** + */ + +SimpleLocaleKeyFactory::SimpleLocaleKeyFactory(UObject* objToAdopt, + const UnicodeString& locale, + int32_t kind, + int32_t coverage) + : LocaleKeyFactory(coverage) + , _obj(objToAdopt) + , _id(locale) + , _kind(kind) +{ +} + +SimpleLocaleKeyFactory::SimpleLocaleKeyFactory(UObject* objToAdopt, + const Locale& locale, + int32_t kind, + int32_t coverage) + : LocaleKeyFactory(coverage) + , _obj(objToAdopt) + , _id() + , _kind(kind) +{ + LocaleUtility::initNameFromLocale(locale, _id); +} + +SimpleLocaleKeyFactory::~SimpleLocaleKeyFactory() +{ + delete _obj; + _obj = NULL; +} + +UObject* +SimpleLocaleKeyFactory::create(const ICUServiceKey& key, const ICUService* service, UErrorCode& status) const +{ + if (U_SUCCESS(status)) { + const LocaleKey& lkey = (const LocaleKey&)key; + if (_kind == LocaleKey::KIND_ANY || _kind == lkey.kind()) { + UnicodeString keyID; + lkey.currentID(keyID); + if (_id == keyID) { + return service->cloneInstance(_obj); + } + } + } + return NULL; +} + +//UBool +//SimpleLocaleKeyFactory::isSupportedID(const UnicodeString& id, UErrorCode& /* status */) const +//{ +// return id == _id; +//} + +void +SimpleLocaleKeyFactory::updateVisibleIDs(Hashtable& result, UErrorCode& status) const +{ + if (U_SUCCESS(status)) { + if (_coverage & 0x1) { + result.remove(_id); + } else { + result.put(_id, (void*)this, status); + } + } +} + +#ifdef SERVICE_DEBUG +UnicodeString& +SimpleLocaleKeyFactory::debug(UnicodeString& result) const +{ + LocaleKeyFactory::debug(result); + result.append(", id: "); + result.append(_id); + result.append(", kind: "); + result.append(_kind); + return result; +} + +UnicodeString& +SimpleLocaleKeyFactory::debugClass(UnicodeString& result) const +{ + return result.append("SimpleLocaleKeyFactory"); +} +#endif + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(SimpleLocaleKeyFactory) + +U_NAMESPACE_END + +/* !UCONFIG_NO_SERVICE */ +#endif + + diff --git a/Build/source/libs/icu/icu-xetex/common/sprpimpl.h b/Build/source/libs/icu/icu-xetex/common/sprpimpl.h new file mode 100644 index 00000000000..1422cc3acd5 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/sprpimpl.h @@ -0,0 +1,129 @@ +/* + ******************************************************************************* + * + * Copyright (C) 2003-2006, International Business Machines + * Corporation and others. All Rights Reserved. + * + ******************************************************************************* + * file name: sprpimpl.h + * encoding: US-ASCII + * tab size: 8 (not used) + * indentation:4 + * + * created on: 2003feb1 + * created by: Ram Viswanadha + */ + +#ifndef SPRPIMPL_H +#define SPRPIMPL_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_IDNA + +#include "unicode/ustring.h" +#include "unicode/parseerr.h" +#include "unicode/usprep.h" +#include "unicode/udata.h" +#include "utrie.h" +#include "udataswp.h" +#include "ubidi_props.h" + +#define _SPREP_DATA_TYPE "spp" + +enum UStringPrepType{ + USPREP_UNASSIGNED = 0x0000 , + USPREP_MAP = 0x0001 , + USPREP_PROHIBITED = 0x0002 , + USPREP_DELETE = 0x0003 , + USPREP_TYPE_LIMIT = 0x0004 +}; + +typedef enum UStringPrepType UStringPrepType; + +#ifdef USPREP_TYPE_NAMES_ARRAY +static const char* usprepTypeNames[] ={ + "UNASSIGNED" , + "MAP" , + "PROHIBITED" , + "DELETE", + "TYPE_LIMIT" +}; +#endif + +enum{ + _SPREP_NORMALIZATION_ON = 0x0001, + _SPREP_CHECK_BIDI_ON = 0x0002 +}; + +enum{ + _SPREP_TYPE_THRESHOLD = 0xFFF0, + _SPREP_MAX_INDEX_VALUE = 0x3FBF, /*16139*/ + _SPREP_MAX_INDEX_TOP_LENGTH = 0x0003 +}; + +/* indexes[] value names */ +enum { + _SPREP_INDEX_TRIE_SIZE = 0, /* number of bytes in StringPrep trie */ + _SPREP_INDEX_MAPPING_DATA_SIZE = 1, /* The array that contains the mapping */ + _SPREP_NORM_CORRECTNS_LAST_UNI_VERSION = 2, /* The index of Unicode version of last entry in NormalizationCorrections.txt */ + _SPREP_ONE_UCHAR_MAPPING_INDEX_START = 3, /* The starting index of 1 UChar mapping index in the mapping data array */ + _SPREP_TWO_UCHARS_MAPPING_INDEX_START = 4, /* The starting index of 2 UChars mapping index in the mapping data array */ + _SPREP_THREE_UCHARS_MAPPING_INDEX_START = 5, /* The starting index of 3 UChars mapping index in the mapping data array */ + _SPREP_FOUR_UCHARS_MAPPING_INDEX_START = 6, /* The starting index of 4 UChars mapping index in the mapping data array */ + _SPREP_OPTIONS = 7, /* Bit set of options to turn on in the profile */ + _SPREP_INDEX_TOP=16 /* changing this requires a new formatVersion */ +}; + +typedef struct UStringPrepKey UStringPrepKey; + + +struct UStringPrepKey{ + char* name; + char* path; +}; + +struct UStringPrepProfile{ + int32_t indexes[_SPREP_INDEX_TOP]; + UTrie sprepTrie; + const uint16_t* mappingData; + UDataMemory* sprepData; + const UBiDiProps *bdp; /* used only if checkBiDi is set */ + int32_t refCount; + UBool isDataLoaded; + UBool doNFKC; + UBool checkBiDi; +}; + +/** + * Helper function for populating the UParseError struct + * @internal + */ +U_CAPI void U_EXPORT2 +uprv_syntaxError(const UChar* rules, + int32_t pos, + int32_t rulesLen, + UParseError* parseError); + + +/** + * Swap StringPrep .spp profile data. See udataswp.h. + * @internal + */ +U_CAPI int32_t U_EXPORT2 +usprep_swap(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode); + +#endif /* #if !UCONFIG_NO_IDNA */ + +#endif + +/* + * Hey, Emacs, please set the following: + * + * Local Variables: + * indent-tabs-mode: nil + * End: + * + */ diff --git a/Build/source/libs/icu/icu-xetex/common/triedict.cpp b/Build/source/libs/icu/icu-xetex/common/triedict.cpp new file mode 100644 index 00000000000..bb771f12913 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/triedict.cpp @@ -0,0 +1,1407 @@ +/** + ******************************************************************************* + * Copyright (C) 2006-2007, International Business Machines Corporation * + * and others. All Rights Reserved. * + ******************************************************************************* + */ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_BREAK_ITERATION + +#include "triedict.h" +#include "unicode/chariter.h" +#include "unicode/uchriter.h" +#include "unicode/strenum.h" +#include "unicode/uenum.h" +#include "unicode/udata.h" +#include "cmemory.h" +#include "udataswp.h" +#include "uvector.h" +#include "uvectr32.h" +#include "uarrsort.h" + +//#define DEBUG_TRIE_DICT 1 + +#ifdef DEBUG_TRIE_DICT +#include +#include +#include +#endif + +U_NAMESPACE_BEGIN + +/******************************************************************* + * TrieWordDictionary + */ + +TrieWordDictionary::TrieWordDictionary() { +} + +TrieWordDictionary::~TrieWordDictionary() { +} + +/******************************************************************* + * MutableTrieDictionary + */ + +// Node structure for the ternary, uncompressed trie +struct TernaryNode : public UMemory { + UChar ch; // UTF-16 code unit + uint16_t flags; // Flag word + TernaryNode *low; // Less-than link + TernaryNode *equal; // Equal link + TernaryNode *high; // Greater-than link + + TernaryNode(UChar uc); + ~TernaryNode(); +}; + +enum MutableTrieNodeFlags { + kEndsWord = 0x0001 // This node marks the end of a valid word +}; + +inline +TernaryNode::TernaryNode(UChar uc) { + ch = uc; + flags = 0; + low = NULL; + equal = NULL; + high = NULL; +} + +// Not inline since it's recursive +TernaryNode::~TernaryNode() { + delete low; + delete equal; + delete high; +} + +MutableTrieDictionary::MutableTrieDictionary( UChar median, UErrorCode &status ) { + // Start the trie off with something. Having the root node already present + // cuts a special case out of the search/insertion functions. + // Making it a median character cuts the worse case for searches from + // 4x a balanced trie to 2x a balanced trie. It's best to choose something + // that starts a word that is midway in the list. + fTrie = new TernaryNode(median); + if (fTrie == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + } + fIter = utext_openUChars(NULL, NULL, 0, &status); + if (U_SUCCESS(status) && fIter == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + } +} + +MutableTrieDictionary::MutableTrieDictionary( UErrorCode &status ) { + fTrie = NULL; + fIter = utext_openUChars(NULL, NULL, 0, &status); + if (U_SUCCESS(status) && fIter == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + } +} + +MutableTrieDictionary::~MutableTrieDictionary() { + delete fTrie; + utext_close(fIter); +} + +int32_t +MutableTrieDictionary::search( UText *text, + int32_t maxLength, + int32_t *lengths, + int &count, + int limit, + TernaryNode *&parent, + UBool &pMatched ) const { + // TODO: current implementation works in UTF-16 space + const TernaryNode *up = NULL; + const TernaryNode *p = fTrie; + int mycount = 0; + pMatched = TRUE; + int i; + + UChar uc = utext_current32(text); + for (i = 0; i < maxLength && p != NULL; ++i) { + while (p != NULL) { + if (uc < p->ch) { + up = p; + p = p->low; + } + else if (uc == p->ch) { + break; + } + else { + up = p; + p = p->high; + } + } + if (p == NULL) { + pMatched = FALSE; + break; + } + // Must be equal to get here + if (limit > 0 && (p->flags & kEndsWord)) { + lengths[mycount++] = i+1; + --limit; + } + up = p; + p = p->equal; + uc = utext_next32(text); + uc = utext_current32(text); + } + + // Note that there is no way to reach here with up == 0 unless + // maxLength is 0 coming in. + parent = (TernaryNode *)up; + count = mycount; + return i; +} + +void +MutableTrieDictionary::addWord( const UChar *word, + int32_t length, + UErrorCode &status ) { +#if 0 + if (length <= 0) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } +#endif + TernaryNode *parent; + UBool pMatched; + int count; + fIter = utext_openUChars(fIter, word, length, &status); + + int matched; + matched = search(fIter, length, NULL, count, 0, parent, pMatched); + + while (matched++ < length) { + UChar32 uc = utext_next32(fIter); // TODO: supplemetary support? + U_ASSERT(uc != U_SENTINEL); + TernaryNode *newNode = new TernaryNode(uc); + if (newNode == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + if (pMatched) { + parent->equal = newNode; + } + else { + pMatched = TRUE; + if (uc < parent->ch) { + parent->low = newNode; + } + else { + parent->high = newNode; + } + } + parent = newNode; + } + + parent->flags |= kEndsWord; +} + +#if 0 +void +MutableTrieDictionary::addWords( UEnumeration *words, + UErrorCode &status ) { + int32_t length; + const UChar *word; + while ((word = uenum_unext(words, &length, &status)) && U_SUCCESS(status)) { + addWord(word, length, status); + } +} +#endif + +int32_t +MutableTrieDictionary::matches( UText *text, + int32_t maxLength, + int32_t *lengths, + int &count, + int limit ) const { + TernaryNode *parent; + UBool pMatched; + return search(text, maxLength, lengths, count, limit, parent, pMatched); +} + +// Implementation of iteration for MutableTrieDictionary +class MutableTrieEnumeration : public StringEnumeration { +private: + UStack fNodeStack; // Stack of nodes to process + UVector32 fBranchStack; // Stack of which branch we are working on + TernaryNode *fRoot; // Root node + enum StackBranch { + kLessThan, + kEqual, + kGreaterThan, + kDone + }; + +public: + static UClassID U_EXPORT2 getStaticClassID(void); + virtual UClassID getDynamicClassID(void) const; +public: + MutableTrieEnumeration(TernaryNode *root, UErrorCode &status) + : fNodeStack(status), fBranchStack(status) { + fRoot = root; + fNodeStack.push(root, status); + fBranchStack.push(kLessThan, status); + unistr.remove(); + } + + virtual ~MutableTrieEnumeration() { + } + + virtual StringEnumeration *clone() const { + UErrorCode status = U_ZERO_ERROR; + return new MutableTrieEnumeration(fRoot, status); + } + + virtual const UnicodeString *snext(UErrorCode &status) { + if (fNodeStack.empty() || U_FAILURE(status)) { + return NULL; + } + TernaryNode *node = (TernaryNode *) fNodeStack.peek(); + StackBranch where = (StackBranch) fBranchStack.peeki(); + while (!fNodeStack.empty() && U_SUCCESS(status)) { + UBool emit; + UBool equal; + + switch (where) { + case kLessThan: + if (node->low != NULL) { + fBranchStack.setElementAt(kEqual, fBranchStack.size()-1); + node = (TernaryNode *) fNodeStack.push(node->low, status); + where = (StackBranch) fBranchStack.push(kLessThan, status); + break; + } + case kEqual: + emit = (node->flags & kEndsWord) != 0; + equal = (node->equal != NULL); + // If this node should be part of the next emitted string, append + // the UChar to the string, and make sure we pop it when we come + // back to this node. The character should only be in the string + // for as long as we're traversing the equal subtree of this node + if (equal || emit) { + unistr.append(node->ch); + fBranchStack.setElementAt(kGreaterThan, fBranchStack.size()-1); + } + if (equal) { + node = (TernaryNode *) fNodeStack.push(node->equal, status); + where = (StackBranch) fBranchStack.push(kLessThan, status); + } + if (emit) { + return &unistr; + } + if (equal) { + break; + } + case kGreaterThan: + // If this node's character is in the string, remove it. + if (node->equal != NULL || (node->flags & kEndsWord)) { + unistr.truncate(unistr.length()-1); + } + if (node->high != NULL) { + fBranchStack.setElementAt(kDone, fBranchStack.size()-1); + node = (TernaryNode *) fNodeStack.push(node->high, status); + where = (StackBranch) fBranchStack.push(kLessThan, status); + break; + } + case kDone: + fNodeStack.pop(); + fBranchStack.popi(); + node = (TernaryNode *) fNodeStack.peek(); + where = (StackBranch) fBranchStack.peeki(); + break; + default: + return NULL; + } + } + return NULL; + } + + // Very expensive, but this should never be used. + virtual int32_t count(UErrorCode &status) const { + MutableTrieEnumeration counter(fRoot, status); + int32_t result = 0; + while (counter.snext(status) != NULL && U_SUCCESS(status)) { + ++result; + } + return result; + } + + virtual void reset(UErrorCode &status) { + fNodeStack.removeAllElements(); + fBranchStack.removeAllElements(); + fNodeStack.push(fRoot, status); + fBranchStack.push(kLessThan, status); + unistr.remove(); + } +}; + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(MutableTrieEnumeration) + +StringEnumeration * +MutableTrieDictionary::openWords( UErrorCode &status ) const { + if (U_FAILURE(status)) { + return NULL; + } + return new MutableTrieEnumeration(fTrie, status); +} + +/******************************************************************* + * CompactTrieDictionary + */ + +struct CompactTrieHeader { + uint32_t size; // Size of the data in bytes + uint32_t magic; // Magic number (including version) + uint16_t nodeCount; // Number of entries in offsets[] + uint16_t root; // Node number of the root node + uint32_t offsets[1]; // Offsets to nodes from start of data +}; + +// Note that to avoid platform-specific alignment issues, all members of the node +// structures should be the same size, or should contain explicit padding to +// natural alignment boundaries. + +// We can't use a bitfield for the flags+count field, because the layout of those +// is not portable. 12 bits of count allows for up to 4096 entries in a node. +struct CompactTrieNode { + uint16_t flagscount; // Count of sub-entries, plus flags +}; + +enum CompactTrieNodeFlags { + kVerticalNode = 0x1000, // This is a vertical node + kParentEndsWord = 0x2000, // The node whose equal link points to this ends a word + kReservedFlag1 = 0x4000, + kReservedFlag2 = 0x8000, + kCountMask = 0x0FFF, // The count portion of flagscount + kFlagMask = 0xF000 // The flags portion of flagscount +}; + +// The two node types are distinguished by the kVerticalNode flag. + +struct CompactTrieHorizontalEntry { + uint16_t ch; // UChar + uint16_t equal; // Equal link node index +}; + +// We don't use inheritance here because C++ does not guarantee that the +// base class comes first in memory!! + +struct CompactTrieHorizontalNode { + uint16_t flagscount; // Count of sub-entries, plus flags + CompactTrieHorizontalEntry entries[1]; +}; + +struct CompactTrieVerticalNode { + uint16_t flagscount; // Count of sub-entries, plus flags + uint16_t equal; // Equal link node index + uint16_t chars[1]; // Code units +}; + +// {'Dic', 1}, version 1 +#define COMPACT_TRIE_MAGIC_1 0x44696301 + +CompactTrieDictionary::CompactTrieDictionary(UDataMemory *dataObj, + UErrorCode &status ) +: fUData(dataObj) +{ + fData = (const CompactTrieHeader *) udata_getMemory(dataObj); + fOwnData = FALSE; + if (fData->magic != COMPACT_TRIE_MAGIC_1) { + status = U_ILLEGAL_ARGUMENT_ERROR; + fData = NULL; + } +} +CompactTrieDictionary::CompactTrieDictionary( const void *data, + UErrorCode &status ) +: fUData(NULL) +{ + fData = (const CompactTrieHeader *) data; + fOwnData = FALSE; + if (fData->magic != COMPACT_TRIE_MAGIC_1) { + status = U_ILLEGAL_ARGUMENT_ERROR; + fData = NULL; + } +} + +CompactTrieDictionary::CompactTrieDictionary( const MutableTrieDictionary &dict, + UErrorCode &status ) +: fUData(NULL) +{ + fData = compactMutableTrieDictionary(dict, status); + fOwnData = !U_FAILURE(status); +} + +CompactTrieDictionary::~CompactTrieDictionary() { + if (fOwnData) { + uprv_free((void *)fData); + } + if (fUData) { + udata_close(fUData); + } +} + +uint32_t +CompactTrieDictionary::dataSize() const { + return fData->size; +} + +const void * +CompactTrieDictionary::data() const { + return fData; +} + +// This function finds the address of a node for us, given its node ID +static inline const CompactTrieNode * +getCompactNode(const CompactTrieHeader *header, uint16_t node) { + return (const CompactTrieNode *)((const uint8_t *)header + header->offsets[node]); +} + +int32_t +CompactTrieDictionary::matches( UText *text, + int32_t maxLength, + int32_t *lengths, + int &count, + int limit ) const { + // TODO: current implementation works in UTF-16 space + const CompactTrieNode *node = getCompactNode(fData, fData->root); + int mycount = 0; + + UChar uc = utext_current32(text); + int i = 0; + + while (node != NULL) { + // Check if the node we just exited ends a word + if (limit > 0 && (node->flagscount & kParentEndsWord)) { + lengths[mycount++] = i; + --limit; + } + // Check that we haven't exceeded the maximum number of input characters. + // We have to do that here rather than in the while condition so that + // we can check for ending a word, above. + if (i >= maxLength) { + break; + } + + int nodeCount = (node->flagscount & kCountMask); + if (nodeCount == 0) { + // Special terminal node; return now + break; + } + if (node->flagscount & kVerticalNode) { + // Vertical node; check all the characters in it + const CompactTrieVerticalNode *vnode = (const CompactTrieVerticalNode *)node; + for (int j = 0; j < nodeCount && i < maxLength; ++j) { + if (uc != vnode->chars[j]) { + // We hit a non-equal character; return + goto exit; + } + utext_next32(text); + uc = utext_current32(text); + ++i; + } + // To get here we must have come through the whole list successfully; + // go on to the next node. Note that a word cannot end in the middle + // of a vertical node. + node = getCompactNode(fData, vnode->equal); + } + else { + // Horizontal node; do binary search + const CompactTrieHorizontalNode *hnode = (const CompactTrieHorizontalNode *)node; + int low = 0; + int high = nodeCount-1; + int middle; + node = NULL; // If we don't find a match, we'll fall out of the loop + while (high >= low) { + middle = (high+low)/2; + if (uc == hnode->entries[middle].ch) { + // We hit a match; get the next node and next character + node = getCompactNode(fData, hnode->entries[middle].equal); + utext_next32(text); + uc = utext_current32(text); + ++i; + break; + } + else if (uc < hnode->entries[middle].ch) { + high = middle-1; + } + else { + low = middle+1; + } + } + } + } +exit: + count = mycount; + return i; +} + +// Implementation of iteration for CompactTrieDictionary +class CompactTrieEnumeration : public StringEnumeration { +private: + UVector32 fNodeStack; // Stack of nodes to process + UVector32 fIndexStack; // Stack of where in node we are + const CompactTrieHeader *fHeader; // Trie data + +public: + static UClassID U_EXPORT2 getStaticClassID(void); + virtual UClassID getDynamicClassID(void) const; +public: + CompactTrieEnumeration(const CompactTrieHeader *header, UErrorCode &status) + : fNodeStack(status), fIndexStack(status) { + fHeader = header; + fNodeStack.push(header->root, status); + fIndexStack.push(0, status); + unistr.remove(); + } + + virtual ~CompactTrieEnumeration() { + } + + virtual StringEnumeration *clone() const { + UErrorCode status = U_ZERO_ERROR; + return new CompactTrieEnumeration(fHeader, status); + } + + virtual const UnicodeString * snext(UErrorCode &status); + + // Very expensive, but this should never be used. + virtual int32_t count(UErrorCode &status) const { + CompactTrieEnumeration counter(fHeader, status); + int32_t result = 0; + while (counter.snext(status) != NULL && U_SUCCESS(status)) { + ++result; + } + return result; + } + + virtual void reset(UErrorCode &status) { + fNodeStack.removeAllElements(); + fIndexStack.removeAllElements(); + fNodeStack.push(fHeader->root, status); + fIndexStack.push(0, status); + unistr.remove(); + } +}; + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(CompactTrieEnumeration) + +const UnicodeString * +CompactTrieEnumeration::snext(UErrorCode &status) { + if (fNodeStack.empty() || U_FAILURE(status)) { + return NULL; + } + const CompactTrieNode *node = getCompactNode(fHeader, fNodeStack.peeki()); + int where = fIndexStack.peeki(); + while (!fNodeStack.empty() && U_SUCCESS(status)) { + int nodeCount = (node->flagscount & kCountMask); + UBool goingDown = FALSE; + if (nodeCount == 0) { + // Terminal node; go up immediately + fNodeStack.popi(); + fIndexStack.popi(); + node = getCompactNode(fHeader, fNodeStack.peeki()); + where = fIndexStack.peeki(); + } + else if (node->flagscount & kVerticalNode) { + // Vertical node + const CompactTrieVerticalNode *vnode = (const CompactTrieVerticalNode *)node; + if (where == 0) { + // Going down + unistr.append((const UChar *)vnode->chars, (int32_t) nodeCount); + fIndexStack.setElementAt(1, fIndexStack.size()-1); + node = getCompactNode(fHeader, fNodeStack.push(vnode->equal, status)); + where = fIndexStack.push(0, status); + goingDown = TRUE; + } + else { + // Going up + unistr.truncate(unistr.length()-nodeCount); + fNodeStack.popi(); + fIndexStack.popi(); + node = getCompactNode(fHeader, fNodeStack.peeki()); + where = fIndexStack.peeki(); + } + } + else { + // Horizontal node + const CompactTrieHorizontalNode *hnode = (const CompactTrieHorizontalNode *)node; + if (where > 0) { + // Pop previous char + unistr.truncate(unistr.length()-1); + } + if (where < nodeCount) { + // Push on next node + unistr.append((UChar)hnode->entries[where].ch); + fIndexStack.setElementAt(where+1, fIndexStack.size()-1); + node = getCompactNode(fHeader, fNodeStack.push(hnode->entries[where].equal, status)); + where = fIndexStack.push(0, status); + goingDown = TRUE; + } + else { + // Going up + fNodeStack.popi(); + fIndexStack.popi(); + node = getCompactNode(fHeader, fNodeStack.peeki()); + where = fIndexStack.peeki(); + } + } + // Check if the parent of the node we've just gone down to ends a + // word. If so, return it. + if (goingDown && (node->flagscount & kParentEndsWord)) { + return &unistr; + } + } + return NULL; +} + +StringEnumeration * +CompactTrieDictionary::openWords( UErrorCode &status ) const { + if (U_FAILURE(status)) { + return NULL; + } + return new CompactTrieEnumeration(fData, status); +} + +// +// Below here is all code related to converting a ternary trie to a compact trie +// and back again +// + +// Helper classes to construct the compact trie +class BuildCompactTrieNode: public UMemory { + public: + UBool fParentEndsWord; + UBool fVertical; + UBool fHasDuplicate; + int32_t fNodeID; + UnicodeString fChars; + + public: + BuildCompactTrieNode(UBool parentEndsWord, UBool vertical, UStack &nodes, UErrorCode &status) { + fParentEndsWord = parentEndsWord; + fHasDuplicate = FALSE; + fVertical = vertical; + fNodeID = nodes.size(); + nodes.push(this, status); + } + + virtual ~BuildCompactTrieNode() { + } + + virtual uint32_t size() { + return sizeof(uint16_t); + } + + virtual void write(uint8_t *bytes, uint32_t &offset, const UVector32 &/*translate*/) { + // Write flag/count + *((uint16_t *)(bytes+offset)) = (fChars.length() & kCountMask) + | (fVertical ? kVerticalNode : 0) | (fParentEndsWord ? kParentEndsWord : 0 ); + offset += sizeof(uint16_t); + } +}; + +class BuildCompactTrieHorizontalNode: public BuildCompactTrieNode { + public: + UStack fLinks; + + public: + BuildCompactTrieHorizontalNode(UBool parentEndsWord, UStack &nodes, UErrorCode &status) + : BuildCompactTrieNode(parentEndsWord, FALSE, nodes, status), fLinks(status) { + } + + virtual ~BuildCompactTrieHorizontalNode() { + } + + virtual uint32_t size() { + return offsetof(CompactTrieHorizontalNode,entries) + + (fChars.length()*sizeof(CompactTrieHorizontalEntry)); + } + + virtual void write(uint8_t *bytes, uint32_t &offset, const UVector32 &translate) { + BuildCompactTrieNode::write(bytes, offset, translate); + int32_t count = fChars.length(); + for (int32_t i = 0; i < count; ++i) { + CompactTrieHorizontalEntry *entry = (CompactTrieHorizontalEntry *)(bytes+offset); + entry->ch = fChars[i]; + entry->equal = translate.elementAti(((BuildCompactTrieNode *)fLinks[i])->fNodeID); +#ifdef DEBUG_TRIE_DICT + if (entry->equal == 0) { + fprintf(stderr, "ERROR: horizontal link %d, logical node %d maps to physical node zero\n", + i, ((BuildCompactTrieNode *)fLinks[i])->fNodeID); + } +#endif + offset += sizeof(CompactTrieHorizontalEntry); + } + } + + void addNode(UChar ch, BuildCompactTrieNode *link, UErrorCode &status) { + fChars.append(ch); + fLinks.push(link, status); + } +}; + +class BuildCompactTrieVerticalNode: public BuildCompactTrieNode { + public: + BuildCompactTrieNode *fEqual; + + public: + BuildCompactTrieVerticalNode(UBool parentEndsWord, UStack &nodes, UErrorCode &status) + : BuildCompactTrieNode(parentEndsWord, TRUE, nodes, status) { + fEqual = NULL; + } + + virtual ~BuildCompactTrieVerticalNode() { + } + + virtual uint32_t size() { + return offsetof(CompactTrieVerticalNode,chars) + (fChars.length()*sizeof(uint16_t)); + } + + virtual void write(uint8_t *bytes, uint32_t &offset, const UVector32 &translate) { + CompactTrieVerticalNode *node = (CompactTrieVerticalNode *)(bytes+offset); + BuildCompactTrieNode::write(bytes, offset, translate); + node->equal = translate.elementAti(fEqual->fNodeID); + offset += sizeof(node->equal); +#ifdef DEBUG_TRIE_DICT + if (node->equal == 0) { + fprintf(stderr, "ERROR: vertical link, logical node %d maps to physical node zero\n", + fEqual->fNodeID); + } +#endif + fChars.extract(0, fChars.length(), (UChar *)node->chars); + offset += sizeof(uint16_t)*fChars.length(); + } + + void addChar(UChar ch) { + fChars.append(ch); + } + + void setLink(BuildCompactTrieNode *node) { + fEqual = node; + } +}; + +// Forward declaration +static void walkHorizontal(const TernaryNode *node, + BuildCompactTrieHorizontalNode *building, + UStack &nodes, + UErrorCode &status); + +// Convert one node. Uses recursion. + +static BuildCompactTrieNode * +compactOneNode(const TernaryNode *node, UBool parentEndsWord, UStack &nodes, UErrorCode &status) { + if (U_FAILURE(status)) { + return NULL; + } + BuildCompactTrieNode *result = NULL; + UBool horizontal = (node->low != NULL || node->high != NULL); + if (horizontal) { + BuildCompactTrieHorizontalNode *hResult = + new BuildCompactTrieHorizontalNode(parentEndsWord, nodes, status); + if (hResult == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + } + if (U_SUCCESS(status)) { + walkHorizontal(node, hResult, nodes, status); + result = hResult; + } + } + else { + BuildCompactTrieVerticalNode *vResult = + new BuildCompactTrieVerticalNode(parentEndsWord, nodes, status); + if (vResult == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + } + else if (U_SUCCESS(status)) { + UBool endsWord = FALSE; + // Take up nodes until we end a word, or hit a node with < or > links + do { + vResult->addChar(node->ch); + endsWord = (node->flags & kEndsWord) != 0; + node = node->equal; + } + while(node != NULL && !endsWord && node->low == NULL && node->high == NULL); + if (node == NULL) { + if (!endsWord) { + status = U_ILLEGAL_ARGUMENT_ERROR; // Corrupt input trie + } + else { + vResult->setLink((BuildCompactTrieNode *)nodes[1]); + } + } + else { + vResult->setLink(compactOneNode(node, endsWord, nodes, status)); + } + result = vResult; + } + } + return result; +} + +// Walk the set of peers at the same level, to build a horizontal node. +// Uses recursion. + +static void walkHorizontal(const TernaryNode *node, + BuildCompactTrieHorizontalNode *building, + UStack &nodes, + UErrorCode &status) { + while (U_SUCCESS(status) && node != NULL) { + if (node->low != NULL) { + walkHorizontal(node->low, building, nodes, status); + } + BuildCompactTrieNode *link = NULL; + if (node->equal != NULL) { + link = compactOneNode(node->equal, (node->flags & kEndsWord) != 0, nodes, status); + } + else if (node->flags & kEndsWord) { + link = (BuildCompactTrieNode *)nodes[1]; + } + if (U_SUCCESS(status) && link != NULL) { + building->addNode(node->ch, link, status); + } + // Tail recurse manually instead of leaving it to the compiler. + //if (node->high != NULL) { + // walkHorizontal(node->high, building, nodes, status); + //} + node = node->high; + } +} + +U_NAMESPACE_END +U_NAMESPACE_USE +U_CDECL_BEGIN +static int32_t U_CALLCONV +_sortBuildNodes(const void * /*context*/, const void *voidl, const void *voidr) { + BuildCompactTrieNode *left = *(BuildCompactTrieNode **)voidl; + BuildCompactTrieNode *right = *(BuildCompactTrieNode **)voidr; + // Check for comparing a node to itself, to avoid spurious duplicates + if (left == right) { + return 0; + } + // Most significant is type of node. Can never coalesce. + if (left->fVertical != right->fVertical) { + return left->fVertical - right->fVertical; + } + // Next, the "parent ends word" flag. If that differs, we cannot coalesce. + if (left->fParentEndsWord != right->fParentEndsWord) { + return left->fParentEndsWord - right->fParentEndsWord; + } + // Next, the string. If that differs, we can never coalesce. + int32_t result = left->fChars.compare(right->fChars); + if (result != 0) { + return result; + } + // We know they're both the same node type, so branch for the two cases. + if (left->fVertical) { + result = ((BuildCompactTrieVerticalNode *)left)->fEqual->fNodeID + - ((BuildCompactTrieVerticalNode *)right)->fEqual->fNodeID; + } + else { + // We need to compare the links vectors. They should be the + // same size because the strings were equal. + // We compare the node IDs instead of the pointers, to handle + // coalesced nodes. + BuildCompactTrieHorizontalNode *hleft, *hright; + hleft = (BuildCompactTrieHorizontalNode *)left; + hright = (BuildCompactTrieHorizontalNode *)right; + int32_t count = hleft->fLinks.size(); + for (int32_t i = 0; i < count && result == 0; ++i) { + result = ((BuildCompactTrieNode *)(hleft->fLinks[i]))->fNodeID - + ((BuildCompactTrieNode *)(hright->fLinks[i]))->fNodeID; + } + } + // If they are equal to each other, mark them (speeds coalescing) + if (result == 0) { + left->fHasDuplicate = TRUE; + right->fHasDuplicate = TRUE; + } + return result; +} +U_CDECL_END +U_NAMESPACE_BEGIN + +static void coalesceDuplicates(UStack &nodes, UErrorCode &status) { + // We sort the array of nodes to place duplicates next to each other + if (U_FAILURE(status)) { + return; + } + int32_t size = nodes.size(); + void **array = (void **)uprv_malloc(sizeof(void *)*size); + if (array == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + (void) nodes.toArray(array); + + // Now repeatedly identify duplicates until there are no more + int32_t dupes = 0; + long passCount = 0; +#ifdef DEBUG_TRIE_DICT + long totalDupes = 0; +#endif + do { + BuildCompactTrieNode *node; + BuildCompactTrieNode *first = NULL; + BuildCompactTrieNode **p; + BuildCompactTrieNode **pFirst = NULL; + int32_t counter = size - 2; + // Sort the array, skipping nodes 0 and 1. Use quicksort for the first + // pass for speed. For the second and subsequent passes, we use stable + // (insertion) sort for two reasons: + // 1. The array is already mostly ordered, so we get better performance. + // 2. The way we find one and only one instance of a set of duplicates is to + // check that the node ID equals the array index. If we used an unstable + // sort for the second or later passes, it's possible that none of the + // duplicates would wind up with a node ID equal to its array index. + // The sort stability guarantees that, because as we coalesce more and + // more groups, the first element of the resultant group will be one of + // the first elements of the groups being coalesced. + // To use quicksort for the second and subsequent passes, we would have to + // find the minimum of the node numbers in a group, and set all the nodes + // in the group to that node number. + uprv_sortArray(array+2, counter, sizeof(void *), _sortBuildNodes, NULL, (passCount > 0), &status); + dupes = 0; + for (p = (BuildCompactTrieNode **)array + 2; counter > 0; --counter, ++p) { + node = *p; + if (node->fHasDuplicate) { + if (first == NULL) { + first = node; + pFirst = p; + } + else if (_sortBuildNodes(NULL, pFirst, p) != 0) { + // Starting a new run of dupes + first = node; + pFirst = p; + } + else if (node->fNodeID != first->fNodeID) { + // Slave one to the other, note duplicate + node->fNodeID = first->fNodeID; + dupes += 1; + } + } + else { + // This node has no dupes + first = NULL; + pFirst = NULL; + } + } + passCount += 1; +#ifdef DEBUG_TRIE_DICT + totalDupes += dupes; + fprintf(stderr, "Trie node dupe removal, pass %d: %d nodes tagged\n", passCount, dupes); +#endif + } + while (dupes > 0); +#ifdef DEBUG_TRIE_DICT + fprintf(stderr, "Trie node dupe removal complete: %d tagged in %d passes\n", totalDupes, passCount); +#endif + + // We no longer need the temporary array, as the nodes have all been marked appropriately. + uprv_free(array); +} + +U_NAMESPACE_END +U_CDECL_BEGIN +static void U_CALLCONV _deleteBuildNode(void *obj) { + delete (BuildCompactTrieNode *) obj; +} +U_CDECL_END +U_NAMESPACE_BEGIN + +CompactTrieHeader * +CompactTrieDictionary::compactMutableTrieDictionary( const MutableTrieDictionary &dict, + UErrorCode &status ) { + if (U_FAILURE(status)) { + return NULL; + } +#ifdef DEBUG_TRIE_DICT + struct tms timing; + struct tms previous; + (void) ::times(&previous); +#endif + UStack nodes(_deleteBuildNode, NULL, status); // Index of nodes + + // Add node 0, used as the NULL pointer/sentinel. + nodes.addElement((int32_t)0, status); + + // Start by creating the special empty node we use to indicate that the parent + // terminates a word. This must be node 1, because the builder assumes + // that. + if (U_FAILURE(status)) { + return NULL; + } + BuildCompactTrieNode *terminal = new BuildCompactTrieNode(TRUE, FALSE, nodes, status); + if (terminal == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + } + + // This call does all the work of building the new trie structure. The root + // will be node 2. + BuildCompactTrieNode *root = compactOneNode(dict.fTrie, FALSE, nodes, status); +#ifdef DEBUG_TRIE_DICT + (void) ::times(&timing); + fprintf(stderr, "Compact trie built, %d nodes, time user %f system %f\n", + nodes.size(), (double)(timing.tms_utime-previous.tms_utime)/CLK_TCK, + (double)(timing.tms_stime-previous.tms_stime)/CLK_TCK); + previous = timing; +#endif + + // Now coalesce all duplicate nodes. + coalesceDuplicates(nodes, status); +#ifdef DEBUG_TRIE_DICT + (void) ::times(&timing); + fprintf(stderr, "Duplicates coalesced, time user %f system %f\n", + (double)(timing.tms_utime-previous.tms_utime)/CLK_TCK, + (double)(timing.tms_stime-previous.tms_stime)/CLK_TCK); + previous = timing; +#endif + + // Next, build the output trie. + // First we compute all the sizes and build the node ID translation table. + uint32_t totalSize = offsetof(CompactTrieHeader,offsets); + int32_t count = nodes.size(); + int32_t nodeCount = 1; // The sentinel node we already have + BuildCompactTrieNode *node; + int32_t i; + UVector32 translate(count, status); // Should be no growth needed after this + translate.push(0, status); // The sentinel node + + if (U_FAILURE(status)) { + return NULL; + } + + for (i = 1; i < count; ++i) { + node = (BuildCompactTrieNode *)nodes[i]; + if (node->fNodeID == i) { + // Only one node out of each duplicate set is used + if (i >= translate.size()) { + // Logically extend the mapping table + translate.setSize(i+1); + } + translate.setElementAt(nodeCount++, i); + totalSize += node->size(); + } + } + + // Check for overflowing 16 bits worth of nodes. + if (nodeCount > 0x10000) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + + // Add enough room for the offsets. + totalSize += nodeCount*sizeof(uint32_t); +#ifdef DEBUG_TRIE_DICT + (void) ::times(&timing); + fprintf(stderr, "Sizes/mapping done, time user %f system %f\n", + (double)(timing.tms_utime-previous.tms_utime)/CLK_TCK, + (double)(timing.tms_stime-previous.tms_stime)/CLK_TCK); + previous = timing; + fprintf(stderr, "%d nodes, %d unique, %d bytes\n", nodes.size(), nodeCount, totalSize); +#endif + uint8_t *bytes = (uint8_t *)uprv_malloc(totalSize); + if (bytes == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + + CompactTrieHeader *header = (CompactTrieHeader *)bytes; + header->size = totalSize; + header->nodeCount = nodeCount; + header->offsets[0] = 0; // Sentinel + header->root = translate.elementAti(root->fNodeID); +#ifdef DEBUG_TRIE_DICT + if (header->root == 0) { + fprintf(stderr, "ERROR: root node %d translate to physical zero\n", root->fNodeID); + } +#endif + uint32_t offset = offsetof(CompactTrieHeader,offsets)+(nodeCount*sizeof(uint32_t)); + nodeCount = 1; + // Now write the data + for (i = 1; i < count; ++i) { + node = (BuildCompactTrieNode *)nodes[i]; + if (node->fNodeID == i) { + header->offsets[nodeCount++] = offset; + node->write(bytes, offset, translate); + } + } +#ifdef DEBUG_TRIE_DICT + (void) ::times(&timing); + fprintf(stderr, "Trie built, time user %f system %f\n", + (double)(timing.tms_utime-previous.tms_utime)/CLK_TCK, + (double)(timing.tms_stime-previous.tms_stime)/CLK_TCK); + previous = timing; + fprintf(stderr, "Final offset is %d\n", offset); + + // Collect statistics on node types and sizes + int hCount = 0; + int vCount = 0; + size_t hSize = 0; + size_t vSize = 0; + size_t hItemCount = 0; + size_t vItemCount = 0; + uint32_t previousOff = offset; + for (uint16_t nodeIdx = nodeCount-1; nodeIdx >= 2; --nodeIdx) { + const CompactTrieNode *node = getCompactNode(header, nodeIdx); + if (node->flagscount & kVerticalNode) { + vCount += 1; + vItemCount += (node->flagscount & kCountMask); + vSize += previousOff-header->offsets[nodeIdx]; + } + else { + hCount += 1; + hItemCount += (node->flagscount & kCountMask); + hSize += previousOff-header->offsets[nodeIdx]; + } + previousOff = header->offsets[nodeIdx]; + } + fprintf(stderr, "Horizontal nodes: %d total, average %f bytes with %f items\n", hCount, + (double)hSize/hCount, (double)hItemCount/hCount); + fprintf(stderr, "Vertical nodes: %d total, average %f bytes with %f items\n", vCount, + (double)vSize/vCount, (double)vItemCount/vCount); +#endif + + if (U_FAILURE(status)) { + uprv_free(bytes); + header = NULL; + } + else { + header->magic = COMPACT_TRIE_MAGIC_1; + } + return header; +} + +// Forward declaration +static TernaryNode * +unpackOneNode( const CompactTrieHeader *header, const CompactTrieNode *node, UErrorCode &status ); + + +// Convert a horizontal node (or subarray thereof) into a ternary subtrie +static TernaryNode * +unpackHorizontalArray( const CompactTrieHeader *header, const CompactTrieHorizontalEntry *array, + int low, int high, UErrorCode &status ) { + if (U_FAILURE(status) || low > high) { + return NULL; + } + int middle = (low+high)/2; + TernaryNode *result = new TernaryNode(array[middle].ch); + if (result == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + const CompactTrieNode *equal = getCompactNode(header, array[middle].equal); + if (equal->flagscount & kParentEndsWord) { + result->flags |= kEndsWord; + } + result->low = unpackHorizontalArray(header, array, low, middle-1, status); + result->high = unpackHorizontalArray(header, array, middle+1, high, status); + result->equal = unpackOneNode(header, equal, status); + return result; +} + +// Convert one compact trie node into a ternary subtrie +static TernaryNode * +unpackOneNode( const CompactTrieHeader *header, const CompactTrieNode *node, UErrorCode &status ) { + int nodeCount = (node->flagscount & kCountMask); + if (nodeCount == 0 || U_FAILURE(status)) { + // Failure, or terminal node + return NULL; + } + if (node->flagscount & kVerticalNode) { + const CompactTrieVerticalNode *vnode = (const CompactTrieVerticalNode *)node; + TernaryNode *head = NULL; + TernaryNode *previous = NULL; + TernaryNode *latest = NULL; + for (int i = 0; i < nodeCount; ++i) { + latest = new TernaryNode(vnode->chars[i]); + if (latest == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + break; + } + if (head == NULL) { + head = latest; + } + if (previous != NULL) { + previous->equal = latest; + } + previous = latest; + } + if (latest != NULL) { + const CompactTrieNode *equal = getCompactNode(header, vnode->equal); + if (equal->flagscount & kParentEndsWord) { + latest->flags |= kEndsWord; + } + latest->equal = unpackOneNode(header, equal, status); + } + return head; + } + else { + // Horizontal node + const CompactTrieHorizontalNode *hnode = (const CompactTrieHorizontalNode *)node; + return unpackHorizontalArray(header, &hnode->entries[0], 0, nodeCount-1, status); + } +} + +MutableTrieDictionary * +CompactTrieDictionary::cloneMutable( UErrorCode &status ) const { + MutableTrieDictionary *result = new MutableTrieDictionary( status ); + if (result == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + TernaryNode *root = unpackOneNode(fData, getCompactNode(fData, fData->root), status); + if (U_FAILURE(status)) { + delete root; // Clean up + delete result; + return NULL; + } + result->fTrie = root; + return result; +} + +U_NAMESPACE_END + +U_CAPI int32_t U_EXPORT2 +triedict_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData, + UErrorCode *status) { + + if (status == NULL || U_FAILURE(*status)) { + return 0; + } + if(ds==NULL || inData==NULL || length<-1 || (length>0 && outData==NULL)) { + *status=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + // + // Check that the data header is for for dictionary data. + // (Header contents are defined in genxxx.cpp) + // + const UDataInfo *pInfo = (const UDataInfo *)((const uint8_t *)inData+4); + if(!( pInfo->dataFormat[0]==0x54 && /* dataFormat="TrDc" */ + pInfo->dataFormat[1]==0x72 && + pInfo->dataFormat[2]==0x44 && + pInfo->dataFormat[3]==0x63 && + pInfo->formatVersion[0]==1 )) { + udata_printError(ds, "triedict_swap(): data format %02x.%02x.%02x.%02x (format version %02x) is not recognized\n", + pInfo->dataFormat[0], pInfo->dataFormat[1], + pInfo->dataFormat[2], pInfo->dataFormat[3], + pInfo->formatVersion[0]); + *status=U_UNSUPPORTED_ERROR; + return 0; + } + + // + // Swap the data header. (This is the generic ICU Data Header, not the + // CompactTrieHeader). This swap also conveniently gets us + // the size of the ICU d.h., which lets us locate the start + // of the RBBI specific data. + // + int32_t headerSize=udata_swapDataHeader(ds, inData, length, outData, status); + + // + // Get the CompactTrieHeader, and check that it appears to be OK. + // + const uint8_t *inBytes =(const uint8_t *)inData+headerSize; + const CompactTrieHeader *header = (const CompactTrieHeader *)inBytes; + if (ds->readUInt32(header->magic) != COMPACT_TRIE_MAGIC_1 + || ds->readUInt32(header->size) < sizeof(CompactTrieHeader)) + { + udata_printError(ds, "triedict_swap(): CompactTrieHeader is invalid.\n"); + *status=U_UNSUPPORTED_ERROR; + return 0; + } + + // + // Prefight operation? Just return the size + // + uint32_t totalSize = ds->readUInt32(header->size); + int32_t sizeWithUData = (int32_t)totalSize + headerSize; + if (length < 0) { + return sizeWithUData; + } + + // + // Check that length passed in is consistent with length from RBBI data header. + // + if (length < sizeWithUData) { + udata_printError(ds, "triedict_swap(): too few bytes (%d after ICU Data header) for trie data.\n", + totalSize); + *status=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + + // + // Swap the Data. Do the data itself first, then the CompactTrieHeader, because + // we need to reference the header to locate the data, and an + // inplace swap of the header leaves it unusable. + // + uint8_t *outBytes = (uint8_t *)outData + headerSize; + CompactTrieHeader *outputHeader = (CompactTrieHeader *)outBytes; + +#if 0 + // + // If not swapping in place, zero out the output buffer before starting. + // + if (inBytes != outBytes) { + uprv_memset(outBytes, 0, totalSize); + } + + // We need to loop through all the nodes in the offset table, and swap each one. + uint16_t nodeCount = ds->readUInt16(header->nodeCount); + // Skip node 0, which should always be 0. + for (int i = 1; i < nodeCount; ++i) { + uint32_t nodeOff = ds->readUInt32(header->offsets[i]); + const CompactTrieNode *inNode = (const CompactTrieNode *)(inBytes + nodeOff); + CompactTrieNode *outNode = (CompactTrieNode *)(outBytes + nodeOff); + uint16_t flagscount = ds->readUInt16(inNode->flagscount); + uint16_t itemCount = flagscount & kCountMask; + ds->writeUInt16(&outNode->flagscount, flagscount); + if (itemCount > 0) { + if (flagscount & kVerticalNode) { + ds->swapArray16(ds, inBytes+nodeOff+offsetof(CompactTrieVerticalNode,chars), + itemCount*sizeof(uint16_t), + outBytes+nodeOff+offsetof(CompactTrieVerticalNode,chars), status); + uint16_t equal = ds->readUInt16(inBytes+nodeOff+offsetof(CompactTrieVerticalNode,equal); + ds->writeUInt16(outBytes+nodeOff+offsetof(CompactTrieVerticalNode,equal)); + } + else { + const CompactTrieHorizontalNode *inHNode = (const CompactTrieHorizontalNode *)inNode; + CompactTrieHorizontalNode *outHNode = (CompactTrieHorizontalNode *)outNode; + for (int j = 0; j < itemCount; ++j) { + uint16_t word = ds->readUInt16(inHNode->entries[j].ch); + ds->writeUInt16(&outHNode->entries[j].ch, word); + word = ds->readUInt16(inHNode->entries[j].equal); + ds->writeUInt16(&outHNode->entries[j].equal, word); + } + } + } + } +#endif + + // All the data in all the nodes consist of 16 bit items. Swap them all at once. + uint16_t nodeCount = ds->readUInt16(header->nodeCount); + uint32_t nodesOff = offsetof(CompactTrieHeader,offsets)+((uint32_t)nodeCount*sizeof(uint32_t)); + ds->swapArray16(ds, inBytes+nodesOff, totalSize-nodesOff, outBytes+nodesOff, status); + + // Swap the header + ds->writeUInt32(&outputHeader->size, totalSize); + uint32_t magic = ds->readUInt32(header->magic); + ds->writeUInt32(&outputHeader->magic, magic); + ds->writeUInt16(&outputHeader->nodeCount, nodeCount); + uint16_t root = ds->readUInt16(header->root); + ds->writeUInt16(&outputHeader->root, root); + ds->swapArray32(ds, inBytes+offsetof(CompactTrieHeader,offsets), + sizeof(uint32_t)*(int32_t)nodeCount, + outBytes+offsetof(CompactTrieHeader,offsets), status); + + return sizeWithUData; +} + +#endif /* #if !UCONFIG_NO_BREAK_ITERATION */ diff --git a/Build/source/libs/icu/icu-xetex/common/triedict.h b/Build/source/libs/icu/icu-xetex/common/triedict.h new file mode 100644 index 00000000000..b8796617690 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/triedict.h @@ -0,0 +1,346 @@ +/** + ******************************************************************************* + * Copyright (C) 2006, International Business Machines Corporation and others. * + * All Rights Reserved. * + ******************************************************************************* + */ + +#ifndef TRIEDICT_H +#define TRIEDICT_H + +#include "unicode/utypes.h" +#include "unicode/uobject.h" +#include "unicode/utext.h" + +struct UEnumeration; +struct UDataSwapper; +struct UDataMemory; + + /** + *

UDataSwapFn function for use in swapping a compact dictionary.

+ * + * @param ds Pointer to UDataSwapper containing global data about the + * transformation and function pointers for handling primitive + * types. + * @param inData Pointer to the input data to be transformed or examined. + * @param length Length of the data, counting bytes. May be -1 for preflighting. + * If length>=0, then transform the data. + * If length==-1, then only determine the length of the data. + * The length cannot be determined from the data itself for all + * types of data (e.g., not for simple arrays of integers). + * @param outData Pointer to the output data buffer. + * If length>=0 (transformation), then the output buffer must + * have a capacity of at least length. + * If length==-1, then outData will not be used and can be NULL. + * @param pErrorCode ICU UErrorCode parameter, must not be NULL and must + * fulfill U_SUCCESS on input. + * @return The actual length of the data. + * + * @see UDataSwapper + */ + +U_CAPI int32_t U_EXPORT2 +triedict_swap(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode); + +U_NAMESPACE_BEGIN + +class StringEnumeration; +struct CompactTrieHeader; + +/******************************************************************* + * TrieWordDictionary + */ + +/** + *

TrieWordDictionary is an abstract class that represents a word + * dictionary based on a trie. The base protocol is read-only. + * Subclasses may allow writing.

+ */ +class U_COMMON_API TrieWordDictionary : public UMemory { + public: + + /** + *

Default constructor.

+ * + */ + TrieWordDictionary(); + + /** + *

Virtual destructor.

+ */ + virtual ~TrieWordDictionary(); + + /** + *

Find dictionary words that match the text.

+ * + * @param text A UText representing the text. The + * iterator is left after the longest prefix match in the dictionary. + * @param start The current position in text. + * @param maxLength The maximum number of code units to match. + * @param lengths An array that is filled with the lengths of words that matched. + * @param count Filled with the number of elements output in lengths. + * @param limit The size of the lengths array; this limits the number of words output. + * @return The number of characters in text that were matched. + */ + virtual int32_t matches( UText *text, + int32_t maxLength, + int32_t *lengths, + int &count, + int limit ) const = 0; + + /** + *

Return a StringEnumeration for iterating all the words in the dictionary.

+ * + * @param status A status code recording the success of the call. + * @return A StringEnumeration that will iterate through the whole dictionary. + * The caller is responsible for closing it. The order is unspecified. + */ + virtual StringEnumeration *openWords( UErrorCode &status ) const = 0; + +}; + +/******************************************************************* + * MutableTrieDictionary + */ + +/** + *

MutableTrieDictionary is a TrieWordDictionary that allows words to be + * added.

+ */ + +struct TernaryNode; // Forwards declaration + +class U_COMMON_API MutableTrieDictionary : public TrieWordDictionary { + private: + /** + * The root node of the trie + * @internal + */ + + TernaryNode *fTrie; + + /** + * A UText for internal use + * @internal + */ + + UText *fIter; + + friend class CompactTrieDictionary; // For fast conversion + + public: + + /** + *

Constructor.

+ * + * @param median A UChar around which to balance the trie. Ideally, it should + * begin at least one word that is near the median of the set in the dictionary + * @param status A status code recording the success of the call. + */ + MutableTrieDictionary( UChar median, UErrorCode &status ); + + /** + *

Virtual destructor.

+ */ + virtual ~MutableTrieDictionary(); + + /** + *

Find dictionary words that match the text.

+ * + * @param text A UText representing the text. The + * iterator is left after the longest prefix match in the dictionary. + * @param maxLength The maximum number of code units to match. + * @param lengths An array that is filled with the lengths of words that matched. + * @param count Filled with the number of elements output in lengths. + * @param limit The size of the lengths array; this limits the number of words output. + * @return The number of characters in text that were matched. + */ + virtual int32_t matches( UText *text, + int32_t maxLength, + int32_t *lengths, + int &count, + int limit ) const; + + /** + *

Return a StringEnumeration for iterating all the words in the dictionary.

+ * + * @param status A status code recording the success of the call. + * @return A StringEnumeration that will iterate through the whole dictionary. + * The caller is responsible for closing it. The order is unspecified. + */ + virtual StringEnumeration *openWords( UErrorCode &status ) const; + + /** + *

Add one word to the dictionary.

+ * + * @param word A UChar buffer containing the word. + * @param length The length of the word. + * @param status The resultant status + */ + virtual void addWord( const UChar *word, + int32_t length, + UErrorCode &status); + +#if 0 + /** + *

Add all strings from a UEnumeration to the dictionary.

+ * + * @param words A UEnumeration that will return the desired words. + * @param status The resultant status + */ + virtual void addWords( UEnumeration *words, UErrorCode &status ); +#endif + +protected: + /** + *

Search the dictionary for matches.

+ * + * @param text A UText representing the text. The + * iterator is left after the longest prefix match in the dictionary. + * @param maxLength The maximum number of code units to match. + * @param lengths An array that is filled with the lengths of words that matched. + * @param count Filled with the number of elements output in lengths. + * @param limit The size of the lengths array; this limits the number of words output. + * @param parent The parent of the current node + * @param pMatched The returned parent node matched the input + * @return The number of characters in text that were matched. + */ + virtual int32_t search( UText *text, + int32_t maxLength, + int32_t *lengths, + int &count, + int limit, + TernaryNode *&parent, + UBool &pMatched ) const; + +private: + /** + *

Private constructor. The root node it not allocated.

+ * + * @param status A status code recording the success of the call. + */ + MutableTrieDictionary( UErrorCode &status ); +}; + +/******************************************************************* + * CompactTrieDictionary + */ + +/** + *

CompactTrieDictionary is a TrieWordDictionary that has been compacted + * to save space.

+ */ +class U_COMMON_API CompactTrieDictionary : public TrieWordDictionary { + private: + /** + * The root node of the trie + */ + + const CompactTrieHeader *fData; + + /** + * A UBool indicating whether or not we own the fData. + */ + + UBool fOwnData; + + UDataMemory *fUData; + public: + /** + *

Construct a dictionary from a UDataMemory.

+ * + * @param data A pointer to a UDataMemory, which is adopted + * @param status A status code giving the result of the constructor + */ + CompactTrieDictionary(UDataMemory *dataObj, UErrorCode &status); + + /** + *

Construct a dictionary from raw saved data.

+ * + * @param data A pointer to the raw data, which is still owned by the caller + * @param status A status code giving the result of the constructor + */ + CompactTrieDictionary(const void *dataObj, UErrorCode &status); + + /** + *

Construct a dictionary from a MutableTrieDictionary.

+ * + * @param dict The dictionary to use as input. + * @param status A status code recording the success of the call. + */ + CompactTrieDictionary( const MutableTrieDictionary &dict, UErrorCode &status ); + + /** + *

Virtual destructor.

+ */ + virtual ~CompactTrieDictionary(); + + /** + *

Find dictionary words that match the text.

+ * + * @param text A UText representing the text. The + * iterator is left after the longest prefix match in the dictionary. + * @param maxLength The maximum number of code units to match. + * @param lengths An array that is filled with the lengths of words that matched. + * @param count Filled with the number of elements output in lengths. + * @param limit The size of the lengths array; this limits the number of words output. + * @return The number of characters in text that were matched. + */ + virtual int32_t matches( UText *text, + int32_t rangeEnd, + int32_t *lengths, + int &count, + int limit ) const; + + /** + *

Return a StringEnumeration for iterating all the words in the dictionary.

+ * + * @param status A status code recording the success of the call. + * @return A StringEnumeration that will iterate through the whole dictionary. + * The caller is responsible for closing it. The order is unspecified. + */ + virtual StringEnumeration *openWords( UErrorCode &status ) const; + + /** + *

Return the size of the compact data.

+ * + * @return The size of the dictionary's compact data. + */ + virtual uint32_t dataSize() const; + + /** + *

Return a void * pointer to the compact data, platform-endian.

+ * + * @return The data for the compact dictionary, suitable for passing to the + * constructor. + */ + virtual const void *data() const; + + /** + *

Return a MutableTrieDictionary clone of this dictionary.

+ * + * @param status A status code recording the success of the call. + * @return A MutableTrieDictionary with the same data as this dictionary + */ + virtual MutableTrieDictionary *cloneMutable( UErrorCode &status ) const; + + private: + + /** + *

Convert a MutableTrieDictionary into a compact data blob.

+ * + * @param dict The dictionary to convert. + * @param status A status code recording the success of the call. + * @return A single data blob starting with a CompactTrieHeader. + */ + static CompactTrieHeader *compactMutableTrieDictionary( const MutableTrieDictionary &dict, + UErrorCode &status ); + +}; + +U_NAMESPACE_END + + /* TRIEDICT_H */ +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/uarrsort.c b/Build/source/libs/icu/icu-xetex/common/uarrsort.c new file mode 100644 index 00000000000..8bc967ce161 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uarrsort.c @@ -0,0 +1,236 @@ +/* +******************************************************************************* +* +* Copyright (C) 2003, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: uarrsort.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2003aug04 +* created by: Markus W. Scherer +* +* Internal function for sorting arrays. +*/ + +#include "unicode/utypes.h" +#include "cmemory.h" +#include "uarrsort.h" + +enum { + MIN_QSORT=9, /* from Knuth */ + STACK_ITEM_SIZE=200 +}; + +/* UComparator convenience implementations ---------------------------------- */ + +U_CAPI int32_t U_EXPORT2 +uprv_uint16Comparator(const void *context, const void *left, const void *right) { + return (int32_t)*(const uint16_t *)left - (int32_t)*(const uint16_t *)right; +} + +U_CAPI int32_t U_EXPORT2 +uprv_int32Comparator(const void *context, const void *left, const void *right) { + return *(const int32_t *)left - *(const int32_t *)right; +} + +U_CAPI int32_t U_EXPORT2 +uprv_uint32Comparator(const void *context, const void *left, const void *right) { + uint32_t l=*(const uint32_t *)left, r=*(const uint32_t *)right; + + /* compare directly because (l-r) would overflow the int32_t result */ + if(lr */ { + return 1; + } +} + +/* Straight insertion sort from Knuth vol. III, pg. 81 ---------------------- */ + +static void +doInsertionSort(char *array, int32_t start, int32_t limit, int32_t itemSize, + UComparator *cmp, const void *context, void *pv) { + int32_t i, j; + + for(j=start+1; jstart; --i) { + if(/* v>=array[i-1] */ cmp(context, pv, array+(i-1)*itemSize)>=0) { + break; + } + + /* array[i]=array[i-1]; */ + uprv_memcpy(array+i*itemSize, array+(i-1)*itemSize, itemSize); + } + + if(i!=j) { + /* array[i]=v; */ + uprv_memcpy(array+i*itemSize, pv, itemSize); + } + } +} + +static void +insertionSort(char *array, int32_t length, int32_t itemSize, + UComparator *cmp, const void *context, UErrorCode *pErrorCode) { + UAlignedMemory v[STACK_ITEM_SIZE/sizeof(UAlignedMemory)+1]; + void *pv; + + /* allocate an intermediate item variable (v) */ + if(itemSize<=STACK_ITEM_SIZE) { + pv=v; + } else { + pv=uprv_malloc(itemSize); + if(pv==NULL) { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return; + } + } + + doInsertionSort(array, 0, length, itemSize, cmp, context, pv); + + if(pv!=v) { + uprv_free(pv); + } +} + +/* QuickSort ---------------------------------------------------------------- */ + +/* + * This implementation is semi-recursive: + * It recurses for the smaller sub-array to shorten the recursion depth, + * and loops for the larger sub-array. + * + * Loosely after QuickSort algorithms in + * Niklaus Wirth + * Algorithmen und Datenstrukturen mit Modula-2 + * B.G. Teubner Stuttgart + * 4. Auflage 1986 + * ISBN 3-519-02260-5 + */ +static void +subQuickSort(char *array, int32_t start, int32_t limit, int32_t itemSize, + UComparator *cmp, const void *context, + void *px, void *pw) { + int32_t left, right; + + /* start and left are inclusive, limit and right are exclusive */ + do { + if((start+MIN_QSORT)>=limit) { + doInsertionSort(array, start, limit, itemSize, cmp, context, px); + break; + } + + left=start; + right=limit; + + /* x=array[middle] */ + uprv_memcpy(px, array+((start+limit)/2)*itemSize, itemSize); + + do { + while(/* array[left]0 && array==NULL) || length<0 || itemSize<=0 || cmp==NULL) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + if(length<=1) { + return; + } else if(length0 if left>right + * + * @internal + */ +typedef int32_t U_CALLCONV +UComparator(const void *context, const void *left, const void *right); +U_CDECL_END + +/** + * Array sorting function. + * Uses a UComparator for comparing array items to each other, and simple + * memory copying to move items. + * + * @param array The array to be sorted. + * @param length The number of items in the array. + * @param itemSize The size in bytes of each array item. + * @param cmp UComparator function used to compare two items each. + * @param context Application-specific pointer, passed through to the UComparator. + * @param sortStable If true, a stable sorting algorithm must be used. + * @param pErrorCode ICU in/out UErrorCode parameter. + * + * @internal + */ +U_CAPI void U_EXPORT2 +uprv_sortArray(void *array, int32_t length, int32_t itemSize, + UComparator *cmp, const void *context, + UBool sortStable, UErrorCode *pErrorCode); + +/** + * Convenience UComparator implementation for uint16_t arrays. + * @internal + */ +U_CAPI int32_t U_EXPORT2 +uprv_uint16Comparator(const void *context, const void *left, const void *right); + +/** + * Convenience UComparator implementation for int32_t arrays. + * @internal + */ +U_CAPI int32_t U_EXPORT2 +uprv_int32Comparator(const void *context, const void *left, const void *right); + +/** + * Convenience UComparator implementation for uint32_t arrays. + * @internal + */ +U_CAPI int32_t U_EXPORT2 +uprv_uint32Comparator(const void *context, const void *left, const void *right); + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/uassert.h b/Build/source/libs/icu/icu-xetex/common/uassert.h new file mode 100644 index 00000000000..9b1a14168fb --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uassert.h @@ -0,0 +1,32 @@ +/* +****************************************************************************** +* +* Copyright (C) 2002-2003, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* +* File uassert.h +* +* Contains U_ASSERT macro +* +* By default, U_ASSERT just wraps the C library assert macro. +* By changing the definition here, the assert behavior for ICU can be changed +* without affecting other non-ICU uses of the C library assert(). +* +****************************************************************************** +*/ + +#ifndef U_ASSERT_H +#define U_ASSERT_H +/* utypes.h is included to get the proper define for uint8_t */ +#include "unicode/utypes.h" +#if U_RELEASE +#define U_ASSERT(exp) +#else +#include +#define U_ASSERT(exp) assert(exp) +#endif +#endif + + diff --git a/Build/source/libs/icu/icu-xetex/common/ubidi.c b/Build/source/libs/icu/icu-xetex/common/ubidi.c new file mode 100644 index 00000000000..edd693a0443 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ubidi.c @@ -0,0 +1,2224 @@ +/* +****************************************************************************** +* +* Copyright (C) 1999-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: ubidi.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 1999jul27 +* created by: Markus W. Scherer, updated by Matitiahu Allouche +*/ + +#include "cmemory.h" +#include "unicode/utypes.h" +#include "unicode/ustring.h" +#include "unicode/uchar.h" +#include "unicode/ubidi.h" +#include "ubidi_props.h" +#include "ubidiimp.h" +#include "uassert.h" + +/* + * General implementation notes: + * + * Throughout the implementation, there are comments like (W2) that refer to + * rules of the BiDi algorithm in its version 5, in this example to the second + * rule of the resolution of weak types. + * + * For handling surrogate pairs, where two UChar's form one "abstract" (or UTF-32) + * character according to UTF-16, the second UChar gets the directional property of + * the entire character assigned, while the first one gets a BN, a boundary + * neutral, type, which is ignored by most of the algorithm according to + * rule (X9) and the implementation suggestions of the BiDi algorithm. + * + * Later, adjustWSLevels() will set the level for each BN to that of the + * following character (UChar), which results in surrogate pairs getting the + * same level on each of their surrogates. + * + * In a UTF-8 implementation, the same thing could be done: the last byte of + * a multi-byte sequence would get the "real" property, while all previous + * bytes of that sequence would get BN. + * + * It is not possible to assign all those parts of a character the same real + * property because this would fail in the resolution of weak types with rules + * that look at immediately surrounding types. + * + * As a related topic, this implementation does not remove Boundary Neutral + * types from the input, but ignores them wherever this is relevant. + * For example, the loop for the resolution of the weak types reads + * types until it finds a non-BN. + * Also, explicit embedding codes are neither changed into BN nor removed. + * They are only treated the same way real BNs are. + * As stated before, adjustWSLevels() takes care of them at the end. + * For the purpose of conformance, the levels of all these codes + * do not matter. + * + * Note that this implementation never modifies the dirProps + * after the initial setup. + * + * + * In this implementation, the resolution of weak types (Wn), + * neutrals (Nn), and the assignment of the resolved level (In) + * are all done in one single loop, in resolveImplicitLevels(). + * Changes of dirProp values are done on the fly, without writing + * them back to the dirProps array. + * + * + * This implementation contains code that allows to bypass steps of the + * algorithm that are not needed on the specific paragraph + * in order to speed up the most common cases considerably, + * like text that is entirely LTR, or RTL text without numbers. + * + * Most of this is done by setting a bit for each directional property + * in a flags variable and later checking for whether there are + * any LTR characters or any RTL characters, or both, whether + * there are any explicit embedding codes, etc. + * + * If the (Xn) steps are performed, then the flags are re-evaluated, + * because they will then not contain the embedding codes any more + * and will be adjusted for override codes, so that subsequently + * more bypassing may be possible than what the initial flags suggested. + * + * If the text is not mixed-directional, then the + * algorithm steps for the weak type resolution are not performed, + * and all levels are set to the paragraph level. + * + * If there are no explicit embedding codes, then the (Xn) steps + * are not performed. + * + * If embedding levels are supplied as a parameter, then all + * explicit embedding codes are ignored, and the (Xn) steps + * are not performed. + * + * White Space types could get the level of the run they belong to, + * and are checked with a test of (flags&MASK_EMBEDDING) to + * consider if the paragraph direction should be considered in + * the flags variable. + * + * If there are no White Space types in the paragraph, then + * (L1) is not necessary in adjustWSLevels(). + */ + +/* to avoid some conditional statements, use tiny constant arrays */ +static const Flags flagLR[2]={ DIRPROP_FLAG(L), DIRPROP_FLAG(R) }; +static const Flags flagE[2]={ DIRPROP_FLAG(LRE), DIRPROP_FLAG(RLE) }; +static const Flags flagO[2]={ DIRPROP_FLAG(LRO), DIRPROP_FLAG(RLO) }; + +#define DIRPROP_FLAG_LR(level) flagLR[(level)&1] +#define DIRPROP_FLAG_E(level) flagE[(level)&1] +#define DIRPROP_FLAG_O(level) flagO[(level)&1] + +/* UBiDi object management -------------------------------------------------- */ + +U_CAPI UBiDi * U_EXPORT2 +ubidi_open(void) +{ + UErrorCode errorCode=U_ZERO_ERROR; + return ubidi_openSized(0, 0, &errorCode); +} + +U_CAPI UBiDi * U_EXPORT2 +ubidi_openSized(int32_t maxLength, int32_t maxRunCount, UErrorCode *pErrorCode) { + UBiDi *pBiDi; + + /* check the argument values */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return NULL; + } else if(maxLength<0 || maxRunCount<0) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return NULL; /* invalid arguments */ + } + + /* allocate memory for the object */ + pBiDi=(UBiDi *)uprv_malloc(sizeof(UBiDi)); + if(pBiDi==NULL) { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + + /* reset the object, all pointers NULL, all flags FALSE, all sizes 0 */ + uprv_memset(pBiDi, 0, sizeof(UBiDi)); + + /* get BiDi properties */ + pBiDi->bdp=ubidi_getSingleton(pErrorCode); + if(U_FAILURE(*pErrorCode)) { + uprv_free(pBiDi); + return NULL; + } + + /* allocate memory for arrays as requested */ + if(maxLength>0) { + if( !getInitialDirPropsMemory(pBiDi, maxLength) || + !getInitialLevelsMemory(pBiDi, maxLength) + ) { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + } + } else { + pBiDi->mayAllocateText=TRUE; + } + + if(maxRunCount>0) { + if(maxRunCount==1) { + /* use simpleRuns[] */ + pBiDi->runsSize=sizeof(Run); + } else if(!getInitialRunsMemory(pBiDi, maxRunCount)) { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + } + } else { + pBiDi->mayAllocateRuns=TRUE; + } + + if(U_SUCCESS(*pErrorCode)) { + return pBiDi; + } else { + ubidi_close(pBiDi); + return NULL; + } +} + +/* + * We are allowed to allocate memory if memory==NULL or + * mayAllocate==TRUE for each array that we need. + * We also try to grow and shrink memory as needed if we + * allocate it. + * + * Assume sizeNeeded>0. + * If *pMemory!=NULL, then assume *pSize>0. + * + * ### this realloc() may unnecessarily copy the old data, + * which we know we don't need any more; + * is this the best way to do this?? + */ +U_CFUNC UBool +ubidi_getMemory(BidiMemoryForAllocation *bidiMem, int32_t *pSize, UBool mayAllocate, int32_t sizeNeeded) { + void **pMemory = (void **)bidiMem; + /* check for existing memory */ + if(*pMemory==NULL) { + /* we need to allocate memory */ + if(mayAllocate && (*pMemory=uprv_malloc(sizeNeeded))!=NULL) { + *pSize=sizeNeeded; + return TRUE; + } else { + return FALSE; + } + } else { + /* there is some memory, is it enough or too much? */ + if(sizeNeeded>*pSize && !mayAllocate) { + /* not enough memory, and we must not allocate */ + return FALSE; + } else if(sizeNeeded!=*pSize && mayAllocate) { + /* FOOD FOR THOUGHT: in hope to improve performance, we should + * try never shrinking memory, only growing it when required. + */ + /* we may try to grow or shrink */ + void *memory; + + if((memory=uprv_realloc(*pMemory, sizeNeeded))!=NULL) { + *pMemory=memory; + *pSize=sizeNeeded; + return TRUE; + } else { + /* we failed to grow */ + return FALSE; + } + } else { + /* we have at least enough memory and must not allocate */ + return TRUE; + } + } +} + +U_CAPI void U_EXPORT2 +ubidi_close(UBiDi *pBiDi) { + if(pBiDi!=NULL) { + pBiDi->pParaBiDi=NULL; /* in case one tries to reuse this block */ + if(pBiDi->dirPropsMemory!=NULL) { + uprv_free(pBiDi->dirPropsMemory); + } + if(pBiDi->levelsMemory!=NULL) { + uprv_free(pBiDi->levelsMemory); + } + if(pBiDi->runsMemory!=NULL) { + uprv_free(pBiDi->runsMemory); + } + if(pBiDi->parasMemory!=NULL) { + uprv_free(pBiDi->parasMemory); + } + if(pBiDi->insertPoints.points!=NULL) { + uprv_free(pBiDi->insertPoints.points); + } + + uprv_free(pBiDi); + } +} + +/* set to approximate "inverse BiDi" ---------------------------------------- */ + +U_CAPI void U_EXPORT2 +ubidi_setInverse(UBiDi *pBiDi, UBool isInverse) { + if(pBiDi!=NULL) { + pBiDi->isInverse=isInverse; + pBiDi->reorderingMode = isInverse ? UBIDI_REORDER_INVERSE_NUMBERS_AS_L + : UBIDI_REORDER_DEFAULT; + } +} + +U_CAPI UBool U_EXPORT2 +ubidi_isInverse(UBiDi *pBiDi) { + if(pBiDi!=NULL) { + return pBiDi->isInverse; + } else { + return FALSE; + } +} + +/* FOOD FOR THOUGHT: currently the reordering modes are a mixture of + * algorithm for direct BiDi, algorithm for inverse BiDi and the bizarre + * concept of RUNS_ONLY which is a double operation. + * It could be advantageous to divide this into 3 concepts: + * a) Operation: direct / inverse / RUNS_ONLY + * b) Direct algorithm: default / NUMBERS_SPECIAL / GROUP_NUMBERS_WITH_R + * c) Inverse algorithm: default / INVERSE_LIKE_DIRECT / NUMBERS_SPECIAL + * This would allow combinations not possible today like RUNS_ONLY with + * NUMBERS_SPECIAL. + * Also allow to set INSERT_MARKS for the direct step of RUNS_ONLY and + * REMOVE_CONTROLS for the inverse step. + * Not all combinations would be supported, and probably not all do make sense. + * This would need to document which ones are supported and what are the + * fallbacks for unsupported combinations. + */ +U_CAPI void U_EXPORT2 +ubidi_setReorderingMode(UBiDi *pBiDi, UBiDiReorderingMode reorderingMode) { + if ((pBiDi!=NULL) && (reorderingMode >= UBIDI_REORDER_DEFAULT) + && (reorderingMode < UBIDI_REORDER_COUNT)) { + pBiDi->reorderingMode = reorderingMode; + pBiDi->isInverse = (UBool)(reorderingMode == UBIDI_REORDER_INVERSE_NUMBERS_AS_L); + } +} + +U_CAPI UBiDiReorderingMode U_EXPORT2 +ubidi_getReorderingMode(UBiDi *pBiDi) { + if (pBiDi!=NULL) { + return pBiDi->reorderingMode; + } else { + return UBIDI_REORDER_DEFAULT; + } +} + +U_CAPI void U_EXPORT2 +ubidi_setReorderingOptions(UBiDi *pBiDi, uint32_t reorderingOptions) { + if (reorderingOptions & UBIDI_OPTION_REMOVE_CONTROLS) { + reorderingOptions&=~UBIDI_OPTION_INSERT_MARKS; + } + if (pBiDi!=NULL) { + pBiDi->reorderingOptions=reorderingOptions; + } +} + +U_CAPI uint32_t U_EXPORT2 +ubidi_getReorderingOptions(UBiDi *pBiDi) { + if (pBiDi!=NULL) { + return pBiDi->reorderingOptions; + } else { + return 0; + } +} + +/* perform (P2)..(P3) ------------------------------------------------------- */ + +/* + * Get the directional properties for the text, + * calculate the flags bit-set, and + * determine the paragraph level if necessary. + */ +static void +getDirProps(UBiDi *pBiDi) { + const UChar *text=pBiDi->text; + DirProp *dirProps=pBiDi->dirPropsMemory; /* pBiDi->dirProps is const */ + + int32_t i=0, i0, i1, length=pBiDi->originalLength; + Flags flags=0; /* collect all directionalities in the text */ + UChar32 uchar; + DirProp dirProp=0, paraDirDefault=0;/* initialize to avoid compiler warnings */ + UBool isDefaultLevel=IS_DEFAULT_LEVEL(pBiDi->paraLevel); + /* for inverse BiDi, the default para level is set to RTL if there is a + strong R or AL character at either end of the text */ + UBool isDefaultLevelInverse=isDefaultLevel && (UBool) + (pBiDi->reorderingMode==UBIDI_REORDER_INVERSE_LIKE_DIRECT || + pBiDi->reorderingMode==UBIDI_REORDER_INVERSE_FOR_NUMBERS_SPECIAL); + int32_t lastArabicPos=-1; + int32_t controlCount=0; + UBool removeBiDiControls = (UBool)(pBiDi->reorderingOptions & + UBIDI_OPTION_REMOVE_CONTROLS); + + typedef enum { + NOT_CONTEXTUAL, /* 0: not contextual paraLevel */ + LOOKING_FOR_STRONG, /* 1: looking for first strong char */ + FOUND_STRONG_CHAR /* 2: found first strong char */ + } State; + State state; + int32_t paraStart=0; /* index of first char in paragraph */ + DirProp paraDir; /* == CONTEXT_RTL within paragraphs + starting with strong R char */ + DirProp lastStrongDir=0; /* for default level & inverse BiDi */ + int32_t lastStrongLTR=0; /* for STREAMING option */ + + if(pBiDi->reorderingOptions & UBIDI_OPTION_STREAMING) { + pBiDi->length=0; + lastStrongLTR=0; + } + if(isDefaultLevel) { + paraDirDefault=pBiDi->paraLevel&1 ? CONTEXT_RTL : 0; + paraDir=paraDirDefault; + lastStrongDir=paraDirDefault; + state=LOOKING_FOR_STRONG; + } else { + state=NOT_CONTEXTUAL; + paraDir=0; + } + /* count paragraphs and determine the paragraph level (P2..P3) */ + /* + * see comment in ubidi.h: + * the DEFAULT_XXX values are designed so that + * their bit 0 alone yields the intended default + */ + for( /* i=0 above */ ; ii0) { /* set previous code units' properties to BN */ + flags|=DIRPROP_FLAG(BN); + do { + dirProps[--i1]=(DirProp)(BN|paraDir); + } while(i1>i0); + } + if(state==LOOKING_FOR_STRONG) { + if(dirProp==L) { + state=FOUND_STRONG_CHAR; + if(paraDir) { + paraDir=0; + for(i1=paraStart; i1reorderingOptions & UBIDI_OPTION_STREAMING) { + pBiDi->length=i; /* i is index to next character */ + } + if(isDefaultLevelInverse && (lastStrongDir==CONTEXT_RTL) &&(paraDir!=lastStrongDir)) { + for( ; paraStartparaCount++; + } + if(isDefaultLevel) { + state=LOOKING_FOR_STRONG; + paraStart=i; /* i is index to next character */ + paraDir=paraDirDefault; + lastStrongDir=paraDirDefault; + } + } + } + if(removeBiDiControls && IS_BIDI_CONTROL_CHAR(uchar)) { + controlCount++; + } + } + if(isDefaultLevelInverse && (lastStrongDir==CONTEXT_RTL) &&(paraDir!=lastStrongDir)) { + for(i1=paraStart; i1paraLevel=GET_PARALEVEL(pBiDi, 0); + } + if(pBiDi->reorderingOptions & UBIDI_OPTION_STREAMING) { + if((lastStrongLTR>pBiDi->length) && + (GET_PARALEVEL(pBiDi, lastStrongLTR)==0)) { + pBiDi->length = lastStrongLTR; + } + if(pBiDi->lengthoriginalLength) { + pBiDi->paraCount--; + } + } + /* The following line does nothing new for contextual paraLevel, but is + needed for absolute paraLevel. */ + flags|=DIRPROP_FLAG_LR(pBiDi->paraLevel); + + if(pBiDi->orderParagraphsLTR && (flags&DIRPROP_FLAG(B))) { + flags|=DIRPROP_FLAG(L); + } + + pBiDi->controlCount = controlCount; + pBiDi->flags=flags; + pBiDi->lastArabicPos=lastArabicPos; +} + +/* perform (X1)..(X9) ------------------------------------------------------- */ + +/* determine if the text is mixed-directional or single-directional */ +static UBiDiDirection +directionFromFlags(UBiDi *pBiDi) { + Flags flags=pBiDi->flags; + /* if the text contains AN and neutrals, then some neutrals may become RTL */ + if(!(flags&MASK_RTL || ((flags&DIRPROP_FLAG(AN)) && (flags&MASK_POSSIBLE_N)))) { + return UBIDI_LTR; + } else if(!(flags&MASK_LTR)) { + return UBIDI_RTL; + } else { + return UBIDI_MIXED; + } +} + +/* + * Resolve the explicit levels as specified by explicit embedding codes. + * Recalculate the flags to have them reflect the real properties + * after taking the explicit embeddings into account. + * + * The BiDi algorithm is designed to result in the same behavior whether embedding + * levels are externally specified (from "styled text", supposedly the preferred + * method) or set by explicit embedding codes (LRx, RLx, PDF) in the plain text. + * That is why (X9) instructs to remove all explicit codes (and BN). + * However, in a real implementation, this removal of these codes and their index + * positions in the plain text is undesirable since it would result in + * reallocated, reindexed text. + * Instead, this implementation leaves the codes in there and just ignores them + * in the subsequent processing. + * In order to get the same reordering behavior, positions with a BN or an + * explicit embedding code just get the same level assigned as the last "real" + * character. + * + * Some implementations, not this one, then overwrite some of these + * directionality properties at "real" same-level-run boundaries by + * L or R codes so that the resolution of weak types can be performed on the + * entire paragraph at once instead of having to parse it once more and + * perform that resolution on same-level-runs. + * This limits the scope of the implicit rules in effectively + * the same way as the run limits. + * + * Instead, this implementation does not modify these codes. + * On one hand, the paragraph has to be scanned for same-level-runs, but + * on the other hand, this saves another loop to reset these codes, + * or saves making and modifying a copy of dirProps[]. + * + * + * Note that (Pn) and (Xn) changed significantly from version 4 of the BiDi algorithm. + * + * + * Handling the stack of explicit levels (Xn): + * + * With the BiDi stack of explicit levels, + * as pushed with each LRE, RLE, LRO, and RLO and popped with each PDF, + * the explicit level must never exceed UBIDI_MAX_EXPLICIT_LEVEL==61. + * + * In order to have a correct push-pop semantics even in the case of overflows, + * there are two overflow counters: + * - countOver60 is incremented with each LRx at level 60 + * - from level 60, one RLx increases the level to 61 + * - countOver61 is incremented with each LRx and RLx at level 61 + * + * Popping levels with PDF must work in the opposite order so that level 61 + * is correct at the correct point. Underflows (too many PDFs) must be checked. + * + * This implementation assumes that UBIDI_MAX_EXPLICIT_LEVEL is odd. + */ +static UBiDiDirection +resolveExplicitLevels(UBiDi *pBiDi) { + const DirProp *dirProps=pBiDi->dirProps; + UBiDiLevel *levels=pBiDi->levels; + const UChar *text=pBiDi->text; + + int32_t i=0, length=pBiDi->length; + Flags flags=pBiDi->flags; /* collect all directionalities in the text */ + DirProp dirProp; + UBiDiLevel level=GET_PARALEVEL(pBiDi, 0); + + UBiDiDirection direction; + int32_t paraIndex=0; + + /* determine if the text is mixed-directional or single-directional */ + direction=directionFromFlags(pBiDi); + + /* we may not need to resolve any explicit levels, but for multiple + paragraphs we want to loop on all chars to set the para boundaries */ + if((direction!=UBIDI_MIXED) && (pBiDi->paraCount==1)) { + /* not mixed directionality: levels don't matter - trailingWSStart will be 0 */ + } else if((pBiDi->paraCount==1) && + (!(flags&MASK_EXPLICIT) || + (pBiDi->reorderingMode > UBIDI_REORDER_LAST_LOGICAL_TO_VISUAL))) { + /* mixed, but all characters are at the same embedding level */ + /* or we are in "inverse BiDi" */ + /* and we don't have contextual multiple paragraphs with some B char */ + /* set all levels to the paragraph level */ + for(i=0; i=UBIDI_MAX_EXPLICIT_LEVEL */ + uint32_t countOver60=0, countOver61=0; /* count overflows of explicit levels */ + + /* recalculate the flags */ + flags=0; + + for(i=0; i0) { + --countOver61; + } else if(countOver60>0 && (embeddingLevel&~UBIDI_LEVEL_OVERRIDE)!=UBIDI_MAX_EXPLICIT_LEVEL) { + /* handle LRx overflows from level 60 */ + --countOver60; + } else if(stackTop>0) { + /* this is the pop operation; it also pops level 61 while countOver60>0 */ + --stackTop; + embeddingLevel=stack[stackTop]; + /* } else { (underflow) */ + } + flags|=DIRPROP_FLAG(BN); + break; + case B: + stackTop=0; + countOver60=countOver61=0; + level=GET_PARALEVEL(pBiDi, i); + if((i+1)paras[paraIndex++]=i+1; + } + } + flags|=DIRPROP_FLAG(B); + break; + case BN: + /* BN, LRE, RLE, and PDF are supposed to be removed (X9) */ + /* they will get their levels set correctly in adjustWSLevels() */ + flags|=DIRPROP_FLAG(BN); + break; + default: + /* all other types get the "real" level */ + if(level!=embeddingLevel) { + level=embeddingLevel; + if(level&UBIDI_LEVEL_OVERRIDE) { + flags|=DIRPROP_FLAG_O(level)|DIRPROP_FLAG_MULTI_RUNS; + } else { + flags|=DIRPROP_FLAG_E(level)|DIRPROP_FLAG_MULTI_RUNS; + } + } + if(!(level&UBIDI_LEVEL_OVERRIDE)) { + flags|=DIRPROP_FLAG(dirProp); + } + break; + } + + /* + * We need to set reasonable levels even on BN codes and + * explicit codes because we will later look at same-level runs (X10). + */ + levels[i]=level; + } + if(flags&MASK_EMBEDDING) { + flags|=DIRPROP_FLAG_LR(pBiDi->paraLevel); + } + if(pBiDi->orderParagraphsLTR && (flags&DIRPROP_FLAG(B))) { + flags|=DIRPROP_FLAG(L); + } + + /* subsequently, ignore the explicit codes and BN (X9) */ + + /* again, determine if the text is mixed-directional or single-directional */ + pBiDi->flags=flags; + direction=directionFromFlags(pBiDi); + } + + return direction; +} + +/* + * Use a pre-specified embedding levels array: + * + * Adjust the directional properties for overrides (->LEVEL_OVERRIDE), + * ignore all explicit codes (X9), + * and check all the preset levels. + * + * Recalculate the flags to have them reflect the real properties + * after taking the explicit embeddings into account. + */ +static UBiDiDirection +checkExplicitLevels(UBiDi *pBiDi, UErrorCode *pErrorCode) { + const DirProp *dirProps=pBiDi->dirProps; + DirProp dirProp; + UBiDiLevel *levels=pBiDi->levels; + const UChar *text=pBiDi->text; + + int32_t i, length=pBiDi->length; + Flags flags=0; /* collect all directionalities in the text */ + UBiDiLevel level; + uint32_t paraIndex=0; + + for(i=0; iparas[paraIndex++]=i+1; + } + } + } + if(flags&MASK_EMBEDDING) { + flags|=DIRPROP_FLAG_LR(pBiDi->paraLevel); + } + + /* determine if the text is mixed-directional or single-directional */ + pBiDi->flags=flags; + return directionFromFlags(pBiDi); +} + +/****************************************************************** + The Properties state machine table +******************************************************************* + + All table cells are 8 bits: + bits 0..4: next state + bits 5..7: action to perform (if > 0) + + Cells may be of format "n" where n represents the next state + (except for the rightmost column). + Cells may also be of format "s(x,y)" where x represents an action + to perform and y represents the next state. + +******************************************************************* + Definitions and type for properties state table +******************************************************************* +*/ +#define IMPTABPROPS_COLUMNS 14 +#define IMPTABPROPS_RES (IMPTABPROPS_COLUMNS - 1) +#define GET_STATEPROPS(cell) ((cell)&0x1f) +#define GET_ACTIONPROPS(cell) ((cell)>>5) +#define s(action, newState) ((uint8_t)(newState+(action<<5))) + +static const uint8_t groupProp[] = /* dirProp regrouped */ +{ +/* L R EN ES ET AN CS B S WS ON LRE LRO AL RLE RLO PDF NSM BN */ + 0, 1, 2, 7, 8, 3, 9, 6, 5, 4, 4, 10, 10, 12, 10, 10, 10, 11, 10 +}; + +/* #undef symbols that may have been defined by ctype.h on IRIX or HP-UX */ +#undef _L +#undef _S +#undef _B + +enum { _L=0, _R=1, _EN=2, _AN=3, _ON=4, _S=5, _B=6 }; /* reduced dirProp */ + +/****************************************************************** + + PROPERTIES STATE TABLE + + In table impTabProps, + - the ON column regroups ON and WS + - the BN column regroups BN, LRE, RLE, LRO, RLO, PDF + - the Res column is the reduced property assigned to a run + + Action 1: process current run1, init new run1 + 2: init new run2 + 3: process run1, process run2, init new run1 + 4: process run1, set run1=run2, init new run2 + + Notes: + 1) This table is used in resolveImplicitLevels(). + 2) This table triggers actions when there is a change in the Bidi + property of incoming characters (action 1). + 3) Most such property sequences are processed immediately (in + fact, passed to processPropertySeq(). + 4) However, numbers are assembled as one sequence. This means + that undefined situations (like CS following digits, until + it is known if the next char will be a digit) are held until + following chars define them. + Example: digits followed by CS, then comes another CS or ON; + the digits will be processed, then the CS assigned + as the start of an ON sequence (action 3). + 5) There are cases where more than one sequence must be + processed, for instance digits followed by CS followed by L: + the digits must be processed as one sequence, and the CS + must be processed as an ON sequence, all this before starting + assembling chars for the opening L sequence. + + +*/ +static const uint8_t impTabProps[][IMPTABPROPS_COLUMNS] = +{ +/* L , R , EN , AN , ON , S , B , ES , ET , CS , BN , NSM , AL , Res */ +/* 0 Init */ { 1 , 2 , 4 , 5 , 7 , 15 , 17 , 7 , 9 , 7 , 0 , 7 , 3 , _ON }, +/* 1 L */ { 1 , s(1,2), s(1,4), s(1,5), s(1,7),s(1,15),s(1,17), s(1,7), s(1,9), s(1,7), 1 , 1 , s(1,3), _L }, +/* 2 R */ { s(1,1), 2 , s(1,4), s(1,5), s(1,7),s(1,15),s(1,17), s(1,7), s(1,9), s(1,7), 2 , 2 , s(1,3), _R }, +/* 3 AL */ { s(1,1), s(1,2), s(1,6), s(1,6), s(1,8),s(1,16),s(1,17), s(1,8), s(1,8), s(1,8), 3 , 3 , 3 , _R }, +/* 4 EN */ { s(1,1), s(1,2), 4 , s(1,5), s(1,7),s(1,15),s(1,17),s(2,10), 11 ,s(2,10), 4 , 4 , s(1,3), _EN }, +/* 5 AN */ { s(1,1), s(1,2), s(1,4), 5 , s(1,7),s(1,15),s(1,17), s(1,7), s(1,9),s(2,12), 5 , 5 , s(1,3), _AN }, +/* 6 AL:EN/AN */ { s(1,1), s(1,2), 6 , 6 , s(1,8),s(1,16),s(1,17), s(1,8), s(1,8),s(2,13), 6 , 6 , s(1,3), _AN }, +/* 7 ON */ { s(1,1), s(1,2), s(1,4), s(1,5), 7 ,s(1,15),s(1,17), 7 ,s(2,14), 7 , 7 , 7 , s(1,3), _ON }, +/* 8 AL:ON */ { s(1,1), s(1,2), s(1,6), s(1,6), 8 ,s(1,16),s(1,17), 8 , 8 , 8 , 8 , 8 , s(1,3), _ON }, +/* 9 ET */ { s(1,1), s(1,2), 4 , s(1,5), 7 ,s(1,15),s(1,17), 7 , 9 , 7 , 9 , 9 , s(1,3), _ON }, +/*10 EN+ES/CS */ { s(3,1), s(3,2), 4 , s(3,5), s(4,7),s(3,15),s(3,17), s(4,7),s(4,14), s(4,7), 10 , s(4,7), s(3,3), _EN }, +/*11 EN+ET */ { s(1,1), s(1,2), 4 , s(1,5), s(1,7),s(1,15),s(1,17), s(1,7), 11 , s(1,7), 11 , 11 , s(1,3), _EN }, +/*12 AN+CS */ { s(3,1), s(3,2), s(3,4), 5 , s(4,7),s(3,15),s(3,17), s(4,7),s(4,14), s(4,7), 12 , s(4,7), s(3,3), _AN }, +/*13 AL:EN/AN+CS */ { s(3,1), s(3,2), 6 , 6 , s(4,8),s(3,16),s(3,17), s(4,8), s(4,8), s(4,8), 13 , s(4,8), s(3,3), _AN }, +/*14 ON+ET */ { s(1,1), s(1,2), s(4,4), s(1,5), 7 ,s(1,15),s(1,17), 7 , 14 , 7 , 14 , 14 , s(1,3), _ON }, +/*15 S */ { s(1,1), s(1,2), s(1,4), s(1,5), s(1,7), 15 ,s(1,17), s(1,7), s(1,9), s(1,7), 15 , s(1,7), s(1,3), _S }, +/*16 AL:S */ { s(1,1), s(1,2), s(1,6), s(1,6), s(1,8), 16 ,s(1,17), s(1,8), s(1,8), s(1,8), 16 , s(1,8), s(1,3), _S }, +/*17 B */ { s(1,1), s(1,2), s(1,4), s(1,5), s(1,7),s(1,15), 17 , s(1,7), s(1,9), s(1,7), 17 , s(1,7), s(1,3), _B } +}; + +/* we must undef macro s because the levels table have a different + * structure (4 bits for action and 4 bits for next state. + */ +#undef s + +/****************************************************************** + The levels state machine tables +******************************************************************* + + All table cells are 8 bits: + bits 0..3: next state + bits 4..7: action to perform (if > 0) + + Cells may be of format "n" where n represents the next state + (except for the rightmost column). + Cells may also be of format "s(x,y)" where x represents an action + to perform and y represents the next state. + + This format limits each table to 16 states each and to 15 actions. + +******************************************************************* + Definitions and type for levels state tables +******************************************************************* +*/ +#define IMPTABLEVELS_COLUMNS (_B + 2) +#define IMPTABLEVELS_RES (IMPTABLEVELS_COLUMNS - 1) +#define GET_STATE(cell) ((cell)&0x0f) +#define GET_ACTION(cell) ((cell)>>4) +#define s(action, newState) ((uint8_t)(newState+(action<<4))) + +typedef uint8_t ImpTab[][IMPTABLEVELS_COLUMNS]; +typedef uint8_t ImpAct[]; + +/* FOOD FOR THOUGHT: each ImpTab should have its associated ImpAct, + * instead of having a pair of ImpTab and a pair of ImpAct. + */ +typedef struct ImpTabPair { + const void * pImpTab[2]; + const void * pImpAct[2]; +} ImpTabPair; + +/****************************************************************** + + LEVELS STATE TABLES + + In all levels state tables, + - state 0 is the initial state + - the Res column is the increment to add to the text level + for this property sequence. + + The impAct arrays for each table of a pair map the local action + numbers of the table to the total list of actions. For instance, + action 2 in a given table corresponds to the action number which + appears in entry [2] of the impAct array for that table. + The first entry of all impAct arrays must be 0. + + Action 1: init conditional sequence + 2: prepend conditional sequence to current sequence + 3: set ON sequence to new level - 1 + 4: init EN/AN/ON sequence + 5: fix EN/AN/ON sequence followed by R + 6: set previous level sequence to level 2 + + Notes: + 1) These tables are used in processPropertySeq(). The input + is property sequences as determined by resolveImplicitLevels. + 2) Most such property sequences are processed immediately + (levels are assigned). + 3) However, some sequences cannot be assigned a final level till + one or more following sequences are received. For instance, + ON following an R sequence within an even-level paragraph. + If the following sequence is R, the ON sequence will be + assigned basic run level+1, and so will the R sequence. + 4) S is generally handled like ON, since its level will be fixed + to paragraph level in adjustWSLevels(). + +*/ + +static const ImpTab impTabL_DEFAULT = /* Even paragraph level */ +/* In this table, conditional sequences receive the higher possible level + until proven otherwise. +*/ +{ +/* L , R , EN , AN , ON , S , B , Res */ +/* 0 : init */ { 0 , 1 , 0 , 2 , 0 , 0 , 0 , 0 }, +/* 1 : R */ { 0 , 1 , 3 , 3 , s(1,4), s(1,4), 0 , 1 }, +/* 2 : AN */ { 0 , 1 , 0 , 2 , s(1,5), s(1,5), 0 , 2 }, +/* 3 : R+EN/AN */ { 0 , 1 , 3 , 3 , s(1,4), s(1,4), 0 , 2 }, +/* 4 : R+ON */ { s(2,0), 1 , 3 , 3 , 4 , 4 , s(2,0), 1 }, +/* 5 : AN+ON */ { s(2,0), 1 , s(2,0), 2 , 5 , 5 , s(2,0), 1 } +}; +static const ImpTab impTabR_DEFAULT = /* Odd paragraph level */ +/* In this table, conditional sequences receive the lower possible level + until proven otherwise. +*/ +{ +/* L , R , EN , AN , ON , S , B , Res */ +/* 0 : init */ { 1 , 0 , 2 , 2 , 0 , 0 , 0 , 0 }, +/* 1 : L */ { 1 , 0 , 1 , 3 , s(1,4), s(1,4), 0 , 1 }, +/* 2 : EN/AN */ { 1 , 0 , 2 , 2 , 0 , 0 , 0 , 1 }, +/* 3 : L+AN */ { 1 , 0 , 1 , 3 , 5 , 5 , 0 , 1 }, +/* 4 : L+ON */ { s(2,1), 0 , s(2,1), 3 , 4 , 4 , 0 , 0 }, +/* 5 : L+AN+ON */ { 1 , 0 , 1 , 3 , 5 , 5 , 0 , 0 } +}; +static const ImpAct impAct0 = {0,1,2,3,4,5,6}; +static const ImpTabPair impTab_DEFAULT = {{&impTabL_DEFAULT, + &impTabR_DEFAULT}, + {&impAct0, &impAct0}}; + +static const ImpTab impTabL_NUMBERS_SPECIAL = /* Even paragraph level */ +/* In this table, conditional sequences receive the higher possible level + until proven otherwise. +*/ +{ +/* L , R , EN , AN , ON , S , B , Res */ +/* 0 : init */ { 0 , 2 , 1 , 1 , 0 , 0 , 0 , 0 }, +/* 1 : L+EN/AN */ { 0 , 2 , 1 , 1 , 0 , 0 , 0 , 2 }, +/* 2 : R */ { 0 , 2 , 4 , 4 , s(1,3), 0 , 0 , 1 }, +/* 3 : R+ON */ { s(2,0), 2 , 4 , 4 , 3 , 3 , s(2,0), 1 }, +/* 4 : R+EN/AN */ { 0 , 2 , 4 , 4 , s(1,3), s(1,3), 0 , 2 } + }; +static const ImpTabPair impTab_NUMBERS_SPECIAL = {{&impTabL_NUMBERS_SPECIAL, + &impTabR_DEFAULT}, + {&impAct0, &impAct0}}; + +static const ImpTab impTabL_GROUP_NUMBERS_WITH_R = +/* In this table, EN/AN+ON sequences receive levels as if associated with R + until proven that there is L or sor/eor on both sides. AN is handled like EN. +*/ +{ +/* L , R , EN , AN , ON , S , B , Res */ +/* 0 init */ { 0 , 3 , s(1,1), s(1,1), 0 , 0 , 0 , 0 }, +/* 1 EN/AN */ { s(2,0), 3 , 1 , 1 , 2 , s(2,0), s(2,0), 2 }, +/* 2 EN/AN+ON */ { s(2,0), 3 , 1 , 1 , 2 , s(2,0), s(2,0), 1 }, +/* 3 R */ { 0 , 3 , 5 , 5 , s(1,4), 0 , 0 , 1 }, +/* 4 R+ON */ { s(2,0), 3 , 5 , 5 , 4 , s(2,0), s(2,0), 1 }, +/* 5 R+EN/AN */ { 0 , 3 , 5 , 5 , s(1,4), 0 , 0 , 2 } +}; +static const ImpTab impTabR_GROUP_NUMBERS_WITH_R = +/* In this table, EN/AN+ON sequences receive levels as if associated with R + until proven that there is L on both sides. AN is handled like EN. +*/ +{ +/* L , R , EN , AN , ON , S , B , Res */ +/* 0 init */ { 2 , 0 , 1 , 1 , 0 , 0 , 0 , 0 }, +/* 1 EN/AN */ { 2 , 0 , 1 , 1 , 0 , 0 , 0 , 1 }, +/* 2 L */ { 2 , 0 , s(1,4), s(1,4), s(1,3), 0 , 0 , 1 }, +/* 3 L+ON */ { s(2,2), 0 , 4 , 4 , 3 , 0 , 0 , 0 }, +/* 4 L+EN/AN */ { s(2,2), 0 , 4 , 4 , 3 , 0 , 0 , 1 } +}; +static const ImpTabPair impTab_GROUP_NUMBERS_WITH_R = { + {&impTabL_GROUP_NUMBERS_WITH_R, + &impTabR_GROUP_NUMBERS_WITH_R}, + {&impAct0, &impAct0}}; + + +static const ImpTab impTabL_INVERSE_NUMBERS_AS_L = +/* This table is identical to the Default LTR table except that EN and AN are + handled like L. +*/ +{ +/* L , R , EN , AN , ON , S , B , Res */ +/* 0 : init */ { 0 , 1 , 0 , 0 , 0 , 0 , 0 , 0 }, +/* 1 : R */ { 0 , 1 , 0 , 0 , s(1,4), s(1,4), 0 , 1 }, +/* 2 : AN */ { 0 , 1 , 0 , 0 , s(1,5), s(1,5), 0 , 2 }, +/* 3 : R+EN/AN */ { 0 , 1 , 0 , 0 , s(1,4), s(1,4), 0 , 2 }, +/* 4 : R+ON */ { s(2,0), 1 , s(2,0), s(2,0), 4 , 4 , s(2,0), 1 }, +/* 5 : AN+ON */ { s(2,0), 1 , s(2,0), s(2,0), 5 , 5 , s(2,0), 1 } +}; +static const ImpTab impTabR_INVERSE_NUMBERS_AS_L = +/* This table is identical to the Default RTL table except that EN and AN are + handled like L. +*/ +{ +/* L , R , EN , AN , ON , S , B , Res */ +/* 0 : init */ { 1 , 0 , 1 , 1 , 0 , 0 , 0 , 0 }, +/* 1 : L */ { 1 , 0 , 1 , 1 , s(1,4), s(1,4), 0 , 1 }, +/* 2 : EN/AN */ { 1 , 0 , 1 , 1 , 0 , 0 , 0 , 1 }, +/* 3 : L+AN */ { 1 , 0 , 1 , 1 , 5 , 5 , 0 , 1 }, +/* 4 : L+ON */ { s(2,1), 0 , s(2,1), s(2,1), 4 , 4 , 0 , 0 }, +/* 5 : L+AN+ON */ { 1 , 0 , 1 , 1 , 5 , 5 , 0 , 0 } +}; +static const ImpTabPair impTab_INVERSE_NUMBERS_AS_L = { + {&impTabL_INVERSE_NUMBERS_AS_L, + &impTabR_INVERSE_NUMBERS_AS_L}, + {&impAct0, &impAct0}}; + +static const ImpTab impTabR_INVERSE_LIKE_DIRECT = /* Odd paragraph level */ +/* In this table, conditional sequences receive the lower possible level + until proven otherwise. +*/ +{ +/* L , R , EN , AN , ON , S , B , Res */ +/* 0 : init */ { 1 , 0 , 2 , 2 , 0 , 0 , 0 , 0 }, +/* 1 : L */ { 1 , 0 , 1 , 2 , s(1,3), s(1,3), 0 , 1 }, +/* 2 : EN/AN */ { 1 , 0 , 2 , 2 , 0 , 0 , 0 , 1 }, +/* 3 : L+ON */ { s(2,1), s(3,0), 6 , 4 , 3 , 3 , s(3,0), 0 }, +/* 4 : L+ON+AN */ { s(2,1), s(3,0), 6 , 4 , 5 , 5 , s(3,0), 3 }, +/* 5 : L+AN+ON */ { s(2,1), s(3,0), 6 , 4 , 5 , 5 , s(3,0), 2 }, +/* 6 : L+ON+EN */ { s(2,1), s(3,0), 6 , 4 , 3 , 3 , s(3,0), 1 } +}; +static const ImpAct impAct1 = {0,1,11,12}; +/* FOOD FOR THOUGHT: in LTR table below, check case "JKL 123abc" + */ +static const ImpTabPair impTab_INVERSE_LIKE_DIRECT = { + {&impTabL_DEFAULT, + &impTabR_INVERSE_LIKE_DIRECT}, + {&impAct0, &impAct1}}; + +static const ImpTab impTabL_INVERSE_LIKE_DIRECT_WITH_MARKS = +/* The case handled in this table is (visually): R EN L +*/ +{ +/* L , R , EN , AN , ON , S , B , Res */ +/* 0 : init */ { 0 , s(6,3), 0 , 1 , 0 , 0 , 0 , 0 }, +/* 1 : L+AN */ { 0 , s(6,3), 0 , 1 , s(1,2), s(3,0), 0 , 4 }, +/* 2 : L+AN+ON */ { s(2,0), s(6,3), s(2,0), 1 , 2 , s(3,0), s(2,0), 3 }, +/* 3 : R */ { 0 , s(6,3), s(5,5), s(5,6), s(1,4), s(3,0), 0 , 3 }, +/* 4 : R+ON */ { s(3,0), s(4,3), s(5,5), s(5,6), 4 , s(3,0), s(3,0), 3 }, +/* 5 : R+EN */ { s(3,0), s(4,3), 5 , s(5,6), s(1,4), s(3,0), s(3,0), 4 }, +/* 6 : R+AN */ { s(3,0), s(4,3), s(5,5), 6 , s(1,4), s(3,0), s(3,0), 4 } +}; +static const ImpTab impTabR_INVERSE_LIKE_DIRECT_WITH_MARKS = +/* The cases handled in this table are (visually): R EN L + R L AN L +*/ +{ +/* L , R , EN , AN , ON , S , B , Res */ +/* 0 : init */ { s(1,3), 0 , 1 , 1 , 0 , 0 , 0 , 0 }, +/* 1 : R+EN/AN */ { s(2,3), 0 , 1 , 1 , 2 , s(4,0), 0 , 1 }, +/* 2 : R+EN/AN+ON */ { s(2,3), 0 , 1 , 1 , 2 , s(4,0), 0 , 0 }, +/* 3 : L */ { 3 , 0 , 3 , s(3,6), s(1,4), s(4,0), 0 , 1 }, +/* 4 : L+ON */ { s(5,3), s(4,0), 5 , s(3,6), 4 , s(4,0), s(4,0), 0 }, +/* 5 : L+ON+EN */ { s(5,3), s(4,0), 5 , s(3,6), 4 , s(4,0), s(4,0), 1 }, +/* 6 : L+AN */ { s(5,3), s(4,0), 6 , 6 , 4 , s(4,0), s(4,0), 3 } +}; +static const ImpAct impAct2 = {0,1,7,8,9,10}; +static const ImpTabPair impTab_INVERSE_LIKE_DIRECT_WITH_MARKS = { + {&impTabL_INVERSE_LIKE_DIRECT_WITH_MARKS, + &impTabR_INVERSE_LIKE_DIRECT_WITH_MARKS}, + {&impAct0, &impAct2}}; + +static const ImpTabPair impTab_INVERSE_FOR_NUMBERS_SPECIAL = { + {&impTabL_NUMBERS_SPECIAL, + &impTabR_INVERSE_LIKE_DIRECT}, + {&impAct0, &impAct1}}; + +static const ImpTab impTabL_INVERSE_FOR_NUMBERS_SPECIAL_WITH_MARKS = +/* The case handled in this table is (visually): R EN L +*/ +{ +/* L , R , EN , AN , ON , S , B , Res */ +/* 0 : init */ { 0 , s(6,2), 1 , 1 , 0 , 0 , 0 , 0 }, +/* 1 : L+EN/AN */ { 0 , s(6,2), 1 , 1 , 0 , s(3,0), 0 , 4 }, +/* 2 : R */ { 0 , s(6,2), s(5,4), s(5,4), s(1,3), s(3,0), 0 , 3 }, +/* 3 : R+ON */ { s(3,0), s(4,2), s(5,4), s(5,4), 3 , s(3,0), s(3,0), 3 }, +/* 4 : R+EN/AN */ { s(3,0), s(4,2), 4 , 4 , s(1,3), s(3,0), s(3,0), 4 } +}; +static const ImpTabPair impTab_INVERSE_FOR_NUMBERS_SPECIAL_WITH_MARKS = { + {&impTabL_INVERSE_FOR_NUMBERS_SPECIAL_WITH_MARKS, + &impTabR_INVERSE_LIKE_DIRECT_WITH_MARKS}, + {&impAct0, &impAct2}}; + +#undef s + +typedef struct { + const ImpTab * pImpTab; /* level table pointer */ + const ImpAct * pImpAct; /* action map array */ + int32_t startON; /* start of ON sequence */ + int32_t startL2EN; /* start of level 2 sequence */ + int32_t lastStrongRTL; /* index of last found R or AL */ + int32_t state; /* current state */ + UBiDiLevel runLevel; /* run level before implicit solving */ +} LevState; + +/*------------------------------------------------------------------------*/ + +static void +addPoint(UBiDi *pBiDi, int32_t pos, int32_t flag) + /* param pos: position where to insert + param flag: one of LRM_BEFORE, LRM_AFTER, RLM_BEFORE, RLM_AFTER + */ +{ +#define FIRSTALLOC 10 + Point point; + InsertPoints * pInsertPoints=&(pBiDi->insertPoints); + + if (pInsertPoints->capacity == 0) + { + pInsertPoints->points=uprv_malloc(sizeof(Point)*FIRSTALLOC); + if (pInsertPoints->points == NULL) + { + pInsertPoints->errorCode=U_MEMORY_ALLOCATION_ERROR; + return; + } + pInsertPoints->capacity=FIRSTALLOC; + } + if (pInsertPoints->size >= pInsertPoints->capacity) /* no room for new point */ + { + void * savePoints=pInsertPoints->points; + pInsertPoints->points=uprv_realloc(pInsertPoints->points, + pInsertPoints->capacity*2*sizeof(Point)); + if (pInsertPoints->points == NULL) + { + pInsertPoints->points=savePoints; + pInsertPoints->errorCode=U_MEMORY_ALLOCATION_ERROR; + return; + } + else pInsertPoints->capacity*=2; + } + point.pos=pos; + point.flag=flag; + pInsertPoints->points[pInsertPoints->size]=point; + pInsertPoints->size++; +#undef FIRSTALLOC +} + +/* perform rules (Wn), (Nn), and (In) on a run of the text ------------------ */ + +/* + * This implementation of the (Wn) rules applies all rules in one pass. + * In order to do so, it needs a look-ahead of typically 1 character + * (except for W5: sequences of ET) and keeps track of changes + * in a rule Wp that affect a later Wq (ppImpTab; + const ImpAct * pImpAct=pLevState->pImpAct; + UBiDiLevel * levels=pBiDi->levels; + UBiDiLevel level, addLevel; + InsertPoints * pInsertPoints; + int32_t start0, k; + + start0=start; /* save original start position */ + oldStateSeq=(uint8_t)pLevState->state; + cell=(*pImpTab)[oldStateSeq][_prop]; + pLevState->state=GET_STATE(cell); /* isolate the new state */ + actionSeq=(*pImpAct)[GET_ACTION(cell)]; /* isolate the action */ + addLevel=(*pImpTab)[pLevState->state][IMPTABLEVELS_RES]; + + if(actionSeq) { + switch(actionSeq) { + case 1: /* init ON seq */ + pLevState->startON=start0; + break; + + case 2: /* prepend ON seq to current seq */ + start=pLevState->startON; + break; + + case 3: /* L or S after possible relevant EN/AN */ + /* check if we had EN after R/AL */ + if (pLevState->startL2EN >= 0) { + addPoint(pBiDi, pLevState->startL2EN, LRM_BEFORE); + } + pLevState->startL2EN=-1; /* not within previous if since could also be -2 */ + /* check if we had any relevant EN/AN after R/AL */ + pInsertPoints=&(pBiDi->insertPoints); + if ((pInsertPoints->capacity == 0) || + (pInsertPoints->size <= pInsertPoints->confirmed)) + { + /* nothing, just clean up */ + pLevState->lastStrongRTL=-1; + /* check if we have a pending conditional segment */ + level=(*pImpTab)[oldStateSeq][IMPTABLEVELS_RES]; + if ((level & 1) && (pLevState->startON > 0)) { /* after ON */ + start=pLevState->startON; /* reset to basic run level */ + } + if (_prop == _S) /* add LRM before S */ + { + addPoint(pBiDi, start0, LRM_BEFORE); + pInsertPoints->confirmed=pInsertPoints->size; + } + break; + } + /* reset previous RTL cont to level for LTR text */ + for (k=pLevState->lastStrongRTL+1; kconfirmed=pInsertPoints->size; + pLevState->lastStrongRTL=-1; + if (_prop == _S) /* add LRM before S */ + { + addPoint(pBiDi, start0, LRM_BEFORE); + pInsertPoints->confirmed=pInsertPoints->size; + } + break; + + case 4: /* R/AL after possible relevant EN/AN */ + /* just clean up */ + pInsertPoints=&(pBiDi->insertPoints); + if (pInsertPoints->capacity > 0) + /* remove all non confirmed insert points */ + pInsertPoints->size=pInsertPoints->confirmed; + pLevState->startON=-1; + pLevState->startL2EN=-1; + pLevState->lastStrongRTL=limit - 1; + break; + + case 5: /* EN/AN after R/AL + possible cont */ + /* check for real AN */ + if ((_prop == _AN) && (NO_CONTEXT_RTL(pBiDi->dirProps[start0]) == AN) && + (pBiDi->reorderingMode!=UBIDI_REORDER_INVERSE_FOR_NUMBERS_SPECIAL)) + { + /* real AN */ + if (pLevState->startL2EN == -1) /* if no relevant EN already found */ + { + /* just note the righmost digit as a strong RTL */ + pLevState->lastStrongRTL=limit - 1; + break; + } + if (pLevState->startL2EN >= 0) /* after EN, no AN */ + { + addPoint(pBiDi, pLevState->startL2EN, LRM_BEFORE); + pLevState->startL2EN=-2; + } + /* note AN */ + addPoint(pBiDi, start0, LRM_BEFORE); + break; + } + /* if first EN/AN after R/AL */ + if (pLevState->startL2EN == -1) { + pLevState->startL2EN=start0; + } + break; + + case 6: /* note location of latest R/AL */ + pLevState->lastStrongRTL=limit - 1; + pLevState->startON=-1; + break; + + case 7: /* L after R+ON/EN/AN */ + /* include possible adjacent number on the left */ + for (k=start0-1; k>=0 && !(levels[k]&1); k--); + if(k>=0) { + addPoint(pBiDi, k, RLM_BEFORE); /* add RLM before */ + pInsertPoints=&(pBiDi->insertPoints); + pInsertPoints->confirmed=pInsertPoints->size; /* confirm it */ + } + pLevState->startON=start0; + break; + + case 8: /* AN after L */ + /* AN numbers between L text on both sides may be trouble. */ + /* tentatively bracket with LRMs; will be confirmed if followed by L */ + addPoint(pBiDi, start0, LRM_BEFORE); /* add LRM before */ + addPoint(pBiDi, start0, LRM_AFTER); /* add LRM after */ + break; + + case 9: /* R after L+ON/EN/AN */ + /* false alert, infirm LRMs around previous AN */ + pInsertPoints=&(pBiDi->insertPoints); + pInsertPoints->size=pInsertPoints->confirmed; + if (_prop == _S) /* add RLM before S */ + { + addPoint(pBiDi, start0, RLM_BEFORE); + pInsertPoints->confirmed=pInsertPoints->size; + } + break; + + case 10: /* L after L+ON/AN */ + level=pLevState->runLevel + addLevel; + for(k=pLevState->startON; kinsertPoints); + pInsertPoints->confirmed=pInsertPoints->size; /* confirm inserts */ + pLevState->startON=start0; + break; + + case 11: /* L after L+ON+EN/AN/ON */ + level=pLevState->runLevel; + for(k=start0-1; k>=pLevState->startON; k--) { + if(levels[k]==level+3) { + while(levels[k]==level+3) { + levels[k--]-=2; + } + while(levels[k]==level) { + k--; + } + } + if(levels[k]==level+2) { + levels[k]=level; + continue; + } + levels[k]=level+1; + } + break; + + case 12: /* R after L+ON+EN/AN/ON */ + level=pLevState->runLevel+1; + for(k=start0-1; k>=pLevState->startON; k--) { + if(levels[k]>level) { + levels[k]-=2; + } + } + break; + + default: /* we should never get here */ + U_ASSERT(FALSE); + break; + } + } + if((addLevel) || (start < start0)) { + level=pLevState->runLevel + addLevel; + for(k=start; kdirProps; + + LevState levState; + int32_t i, start1, start2; + uint8_t oldStateImp, stateImp, actionImp; + uint8_t gprop, resProp, cell; + UBool inverseRTL; + DirProp nextStrongProp=R; + int32_t nextStrongPos=-1; + + /* check for RTL inverse BiDi mode */ + /* FOOD FOR THOUGHT: in case of RTL inverse BiDi, it would make sense to + * loop on the text characters from end to start. + * This would need a different properties state table (at least different + * actions) and different levels state tables (maybe very similar to the + * LTR corresponding ones. + */ + inverseRTL=(UBool) + ((startlastArabicPos) && (GET_PARALEVEL(pBiDi, start) & 1) && + (pBiDi->reorderingMode==UBIDI_REORDER_INVERSE_LIKE_DIRECT || + pBiDi->reorderingMode==UBIDI_REORDER_INVERSE_FOR_NUMBERS_SPECIAL)); + /* initialize for levels state table */ + levState.startL2EN=-1; /* used for INVERSE_LIKE_DIRECT_WITH_MARKS */ + levState.lastStrongRTL=-1; /* used for INVERSE_LIKE_DIRECT_WITH_MARKS */ + levState.state=0; + levState.runLevel=pBiDi->levels[start]; + levState.pImpTab=(const ImpTab*)((pBiDi->pImpTabPair)->pImpTab)[levState.runLevel&1]; + levState.pImpAct=(const ImpAct*)((pBiDi->pImpTabPair)->pImpAct)[levState.runLevel&1]; + processPropertySeq(pBiDi, &levState, sor, start, start); + /* initialize for property state table */ + if(dirProps[start]==NSM) { + stateImp = 1 + sor; + } else { + stateImp=0; + } + start1=start; + start2=start; + + for(i=start; i<=limit; i++) { + if(i>=limit) { + gprop=eor; + } else { + DirProp prop, prop1; + prop=NO_CONTEXT_RTL(dirProps[i]); + if(inverseRTL) { + if(prop==AL) { + /* AL before EN does not make it AN */ + prop=R; + } else if(prop==EN) { + if(nextStrongPos<=i) { + /* look for next strong char (L/R/AL) */ + int32_t j; + nextStrongProp=R; /* set default */ + nextStrongPos=limit; + for(j=i+1; jdirProps; + UBiDiLevel *levels=pBiDi->levels; + int32_t i; + + if(pBiDi->flags&MASK_WS) { + UBool orderParagraphsLTR=pBiDi->orderParagraphsLTR; + Flags flag; + + i=pBiDi->trailingWSStart; + while(i>0) { + /* reset a sequence of WS/BN before eop and B/S to the paragraph paraLevel */ + while(i>0 && (flag=DIRPROP_FLAG_NC(dirProps[--i]))&MASK_WS) { + if(orderParagraphsLTR&&(flag&DIRPROP_FLAG(B))) { + levels[i]=0; + } else { + levels[i]=GET_PARALEVEL(pBiDi, i); + } + } + + /* reset BN to the next character's paraLevel until B/S, which restarts above loop */ + /* here, i+1 is guaranteed to be 0) { + flag=DIRPROP_FLAG_NC(dirProps[--i]); + if(flag&MASK_BN_EXPLICIT) { + levels[i]=levels[i+1]; + } else if(orderParagraphsLTR&&(flag&DIRPROP_FLAG(B))) { + levels[i]=0; + break; + } else if(flag&MASK_B_S) { + levels[i]=GET_PARALEVEL(pBiDi, i); + break; + } + } + } + } +} + +#define BIDI_MIN(x, y) ((x)<(y) ? (x) : (y)) +#define BIDI_ABS(x) ((x)>=0 ? (x) : (-(x))) +static void +setParaRunsOnly(UBiDi *pBiDi, const UChar *text, int32_t length, + UBiDiLevel paraLevel, UErrorCode *pErrorCode) { + void *runsOnlyMemory; + int32_t *visualMap; + UChar *visualText; + int32_t saveLength, saveTrailingWSStart; + const UBiDiLevel *levels; + UBiDiLevel *saveLevels; + UBiDiDirection saveDirection; + UBool saveMayAllocateText; + Run *runs; + int32_t visualLength, i, j, visualStart, logicalStart, + runCount, runLength, addedRuns, insertRemove, + start, limit, step, indexOddBit, logicalPos, + index, index1; + uint32_t saveOptions; + + pBiDi->reorderingMode=UBIDI_REORDER_DEFAULT; + if(length==0) { + ubidi_setPara(pBiDi, text, length, paraLevel, NULL, pErrorCode); + goto cleanup3; + } + /* obtain memory for mapping table and visual text */ + runsOnlyMemory=uprv_malloc(length*(sizeof(int32_t)+sizeof(UChar)+sizeof(UBiDiLevel))); + if(runsOnlyMemory==NULL) { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + goto cleanup3; + } + visualMap=runsOnlyMemory; + visualText=(UChar *)&visualMap[length]; + saveLevels=(UBiDiLevel *)&visualText[length]; + saveOptions=pBiDi->reorderingOptions; + if(saveOptions & UBIDI_OPTION_INSERT_MARKS) { + pBiDi->reorderingOptions&=~UBIDI_OPTION_INSERT_MARKS; + pBiDi->reorderingOptions|=UBIDI_OPTION_REMOVE_CONTROLS; + } + paraLevel&=1; /* accept only 0 or 1 */ + ubidi_setPara(pBiDi, text, length, paraLevel, NULL, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + goto cleanup3; + } + /* we cannot access directly pBiDi->levels since it is not yet set if + * direction is not MIXED + */ + levels=ubidi_getLevels(pBiDi, pErrorCode); + uprv_memcpy(saveLevels, levels, pBiDi->length*sizeof(UBiDiLevel)); + saveTrailingWSStart=pBiDi->trailingWSStart; + saveLength=pBiDi->length; + saveDirection=pBiDi->direction; + + /* FOOD FOR THOUGHT: instead of writing the visual text, we could use + * the visual map and the dirProps array to drive the second call + * to ubidi_setPara (but must make provision for possible removal of + * BiDi controls. Alternatively, only use the dirProps array via + * customized classifier callback. + */ + visualLength=ubidi_writeReordered(pBiDi, visualText, length, + UBIDI_DO_MIRRORING, pErrorCode); + ubidi_getVisualMap(pBiDi, visualMap, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + goto cleanup2; + } + pBiDi->reorderingOptions=saveOptions; + + pBiDi->reorderingMode=UBIDI_REORDER_INVERSE_LIKE_DIRECT; + paraLevel^=1; + /* Because what we did with reorderingOptions, visualText may be shorter + * than the original text. But we don't want the levels memory to be + * reallocated shorter than the original length, since we need to restore + * the levels as after the first call to ubidi_setpara() before returning. + * We will force mayAllocateText to FALSE before the second call to + * ubidi_setpara(), and will restore it afterwards. + */ + saveMayAllocateText=pBiDi->mayAllocateText; + pBiDi->mayAllocateText=FALSE; + ubidi_setPara(pBiDi, visualText, visualLength, paraLevel, NULL, pErrorCode); + pBiDi->mayAllocateText=saveMayAllocateText; + ubidi_getRuns(pBiDi, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + goto cleanup1; + } + /* check if some runs must be split, count how many splits */ + addedRuns=0; + runCount=pBiDi->runCount; + runs=pBiDi->runs; + visualStart=0; + for(i=0; irunsMemory[0]=runs[0]; + } + runs=pBiDi->runs=pBiDi->runsMemory; + pBiDi->runCount+=addedRuns; + } else { + goto cleanup1; + } + } + /* split runs which are not consecutive in source text */ + for(i=runCount-1; i>=0; i--) { + runLength= i==0 ? runs[0].visualLimit : + runs[i].visualLimit-runs[i-1].visualLimit; + logicalStart=runs[i].logicalStart; + indexOddBit=GET_ODD_BIT(logicalStart); + logicalStart=GET_INDEX(logicalStart); + if(runLength<2) { + if(addedRuns) { + runs[i+addedRuns]=runs[i]; + } + logicalPos=visualMap[logicalStart]; + runs[i+addedRuns].logicalStart=MAKE_INDEX_ODD_PAIR(logicalPos, + saveLevels[logicalPos]^indexOddBit); + continue; + } + if(indexOddBit) { + start=logicalStart; + limit=logicalStart+runLength-1; + step=1; + } else { + start=logicalStart+runLength-1; + limit=logicalStart; + step=-1; + } + for(j=start; j!=limit; j+=step) { + index=visualMap[j]; + index1=visualMap[j+step]; + if((BIDI_ABS(index-index1)!=1) || (saveLevels[index]!=saveLevels[index1])) { + logicalPos=BIDI_MIN(visualMap[start], index); + runs[i+addedRuns].logicalStart=MAKE_INDEX_ODD_PAIR(logicalPos, + saveLevels[logicalPos]^indexOddBit); + runs[i+addedRuns].visualLimit=runs[i].visualLimit; + runs[i].visualLimit-=BIDI_ABS(j-start)+1; + insertRemove=runs[i].insertRemove&(LRM_AFTER|RLM_AFTER); + runs[i+addedRuns].insertRemove=insertRemove; + runs[i].insertRemove&=~insertRemove; + start=j+step; + addedRuns--; + } + } + if(addedRuns) { + runs[i+addedRuns]=runs[i]; + } + logicalPos=BIDI_MIN(visualMap[start], visualMap[limit]); + runs[i+addedRuns].logicalStart=MAKE_INDEX_ODD_PAIR(logicalPos, + saveLevels[logicalPos]^indexOddBit); + } + + cleanup1: + /* restore initial paraLevel */ + pBiDi->paraLevel^=1; + cleanup2: + /* restore real text */ + pBiDi->text=text; + pBiDi->length=saveLength; + pBiDi->originalLength=length; + pBiDi->direction=saveDirection; + /* the saved levels should never excess levelsSize, but we check anyway */ + if(saveLength>pBiDi->levelsSize) { + saveLength=pBiDi->levelsSize; + } + uprv_memcpy(pBiDi->levels, saveLevels, saveLength*sizeof(UBiDiLevel)); + pBiDi->trailingWSStart=saveTrailingWSStart; + /* free memory for mapping table and visual text */ + uprv_free(runsOnlyMemory); + if(pBiDi->runCount>1) { + pBiDi->direction=UBIDI_MIXED; + } + cleanup3: + pBiDi->reorderingMode=UBIDI_REORDER_RUNS_ONLY; +} + +/* ubidi_setPara ------------------------------------------------------------ */ + +U_CAPI void U_EXPORT2 +ubidi_setPara(UBiDi *pBiDi, const UChar *text, int32_t length, + UBiDiLevel paraLevel, UBiDiLevel *embeddingLevels, + UErrorCode *pErrorCode) { + UBiDiDirection direction; + + /* check the argument values */ + RETURN_VOID_IF_NULL_OR_FAILING_ERRCODE(pErrorCode); + if(pBiDi==NULL || text==NULL || length<-1 || + (paraLevel>UBIDI_MAX_EXPLICIT_LEVEL && paraLevelreorderingMode==UBIDI_REORDER_RUNS_ONLY) { + setParaRunsOnly(pBiDi, text, length, paraLevel, pErrorCode); + return; + } + + /* initialize the UBiDi structure */ + pBiDi->pParaBiDi=NULL; /* mark unfinished setPara */ + pBiDi->text=text; + pBiDi->length=pBiDi->originalLength=pBiDi->resultLength=length; + pBiDi->paraLevel=paraLevel; + pBiDi->direction=UBIDI_LTR; + pBiDi->paraCount=1; + + pBiDi->dirProps=NULL; + pBiDi->levels=NULL; + pBiDi->runs=NULL; + pBiDi->insertPoints.size=0; /* clean up from last call */ + pBiDi->insertPoints.confirmed=0; /* clean up from last call */ + + /* + * Save the original paraLevel if contextual; otherwise, set to 0. + */ + if(IS_DEFAULT_LEVEL(paraLevel)) { + pBiDi->defaultParaLevel=paraLevel; + } else { + pBiDi->defaultParaLevel=0; + } + + if(length==0) { + /* + * For an empty paragraph, create a UBiDi object with the paraLevel and + * the flags and the direction set but without allocating zero-length arrays. + * There is nothing more to do. + */ + if(IS_DEFAULT_LEVEL(paraLevel)) { + pBiDi->paraLevel&=1; + pBiDi->defaultParaLevel=0; + } + if(paraLevel&1) { + pBiDi->flags=DIRPROP_FLAG(R); + pBiDi->direction=UBIDI_RTL; + } else { + pBiDi->flags=DIRPROP_FLAG(L); + pBiDi->direction=UBIDI_LTR; + } + + pBiDi->runCount=0; + pBiDi->paraCount=0; + pBiDi->pParaBiDi=pBiDi; /* mark successful setPara */ + return; + } + + pBiDi->runCount=-1; + + /* + * Get the directional properties, + * the flags bit-set, and + * determine the paragraph level if necessary. + */ + if(getDirPropsMemory(pBiDi, length)) { + pBiDi->dirProps=pBiDi->dirPropsMemory; + getDirProps(pBiDi); + } else { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return; + } + /* the processed length may have changed if UBIDI_OPTION_STREAMING */ + length= pBiDi->length; + pBiDi->trailingWSStart=length; /* the levels[] will reflect the WS run */ + /* allocate paras memory */ + if(pBiDi->paraCount>1) { + if(getInitialParasMemory(pBiDi, pBiDi->paraCount)) { + pBiDi->paras=pBiDi->parasMemory; + pBiDi->paras[pBiDi->paraCount-1]=length; + } else { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return; + } + } else { + /* initialize paras for single paragraph */ + pBiDi->paras=pBiDi->simpleParas; + pBiDi->simpleParas[0]=length; + } + + /* are explicit levels specified? */ + if(embeddingLevels==NULL) { + /* no: determine explicit levels according to the (Xn) rules */\ + if(getLevelsMemory(pBiDi, length)) { + pBiDi->levels=pBiDi->levelsMemory; + direction=resolveExplicitLevels(pBiDi); + } else { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return; + } + } else { + /* set BN for all explicit codes, check that all levels are 0 or paraLevel..UBIDI_MAX_EXPLICIT_LEVEL */ + pBiDi->levels=embeddingLevels; + direction=checkExplicitLevels(pBiDi, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return; + } + } + + /* + * The steps after (X9) in the UBiDi algorithm are performed only if + * the paragraph text has mixed directionality! + */ + pBiDi->direction=direction; + switch(direction) { + case UBIDI_LTR: + /* make sure paraLevel is even */ + pBiDi->paraLevel=(UBiDiLevel)((pBiDi->paraLevel+1)&~1); + + /* all levels are implicitly at paraLevel (important for ubidi_getLevels()) */ + pBiDi->trailingWSStart=0; + break; + case UBIDI_RTL: + /* make sure paraLevel is odd */ + pBiDi->paraLevel|=1; + + /* all levels are implicitly at paraLevel (important for ubidi_getLevels()) */ + pBiDi->trailingWSStart=0; + break; + default: + /* + * Choose the right implicit state table + */ + switch(pBiDi->reorderingMode) { + case UBIDI_REORDER_DEFAULT: + pBiDi->pImpTabPair=&impTab_DEFAULT; + break; + case UBIDI_REORDER_NUMBERS_SPECIAL: + pBiDi->pImpTabPair=&impTab_NUMBERS_SPECIAL; + break; + case UBIDI_REORDER_GROUP_NUMBERS_WITH_R: + pBiDi->pImpTabPair=&impTab_GROUP_NUMBERS_WITH_R; + break; + case UBIDI_REORDER_INVERSE_NUMBERS_AS_L: + pBiDi->pImpTabPair=&impTab_INVERSE_NUMBERS_AS_L; + break; + case UBIDI_REORDER_INVERSE_LIKE_DIRECT: + if (pBiDi->reorderingOptions & UBIDI_OPTION_INSERT_MARKS) { + pBiDi->pImpTabPair=&impTab_INVERSE_LIKE_DIRECT_WITH_MARKS; + } else { + pBiDi->pImpTabPair=&impTab_INVERSE_LIKE_DIRECT; + } + break; + case UBIDI_REORDER_INVERSE_FOR_NUMBERS_SPECIAL: + if (pBiDi->reorderingOptions & UBIDI_OPTION_INSERT_MARKS) { + pBiDi->pImpTabPair=&impTab_INVERSE_FOR_NUMBERS_SPECIAL_WITH_MARKS; + } else { + pBiDi->pImpTabPair=&impTab_INVERSE_FOR_NUMBERS_SPECIAL; + } + break; + default: + /* we should never get here */ + U_ASSERT(FALSE); + break; + } + /* + * If there are no external levels specified and there + * are no significant explicit level codes in the text, + * then we can treat the entire paragraph as one run. + * Otherwise, we need to perform the following rules on runs of + * the text with the same embedding levels. (X10) + * "Significant" explicit level codes are ones that actually + * affect non-BN characters. + * Examples for "insignificant" ones are empty embeddings + * LRE-PDF, LRE-RLE-PDF-PDF, etc. + */ + if(embeddingLevels==NULL && pBiDi->paraCount<=1 && + !(pBiDi->flags&DIRPROP_FLAG_MULTI_RUNS)) { + resolveImplicitLevels(pBiDi, 0, length, + GET_LR_FROM_LEVEL(GET_PARALEVEL(pBiDi, 0)), + GET_LR_FROM_LEVEL(GET_PARALEVEL(pBiDi, length-1))); + } else { + /* sor, eor: start and end types of same-level-run */ + UBiDiLevel *levels=pBiDi->levels; + int32_t start, limit=0; + UBiDiLevel level, nextLevel; + DirProp sor, eor; + + /* determine the first sor and set eor to it because of the loop body (sor=eor there) */ + level=GET_PARALEVEL(pBiDi, 0); + nextLevel=levels[0]; + if(level0) && (NO_CONTEXT_RTL(pBiDi->dirProps[start-1])==B)) { + /* except if this is a new paragraph, then set sor = para level */ + sor=GET_LR_FROM_LEVEL(GET_PARALEVEL(pBiDi, start)); + } else { + sor=eor; + } + + /* search for the limit of this run */ + while(++limitinsertPoints.errorCode)) + { + *pErrorCode=pBiDi->insertPoints.errorCode; + return; + } + /* reset the embedding levels for some non-graphic characters (L1), (X9) */ + adjustWSLevels(pBiDi); + break; + } + /* add RLM for inverse Bidi with contextual orientation resolving + * to RTL which would not round-trip otherwise + */ + if((pBiDi->defaultParaLevel>0) && + (pBiDi->reorderingOptions & UBIDI_OPTION_INSERT_MARKS) && + ((pBiDi->reorderingMode==UBIDI_REORDER_INVERSE_LIKE_DIRECT) || + (pBiDi->reorderingMode==UBIDI_REORDER_INVERSE_FOR_NUMBERS_SPECIAL))) { + int32_t i, j, start, last; + DirProp dirProp; + for(i=0; iparaCount; i++) { + last=pBiDi->paras[i]-1; + if((pBiDi->dirProps[last] & CONTEXT_RTL)==0) { + continue; /* LTR paragraph */ + } + start= i==0 ? 0 : pBiDi->paras[i - 1]; + for(j=last; j>=start; j--) { + dirProp=NO_CONTEXT_RTL(pBiDi->dirProps[j]); + if(dirProp==L) { + if(jdirProps[last])==B) { + last--; + } + } + addPoint(pBiDi, last, RLM_BEFORE); + break; + } + if(DIRPROP_FLAG(dirProp) & MASK_R_AL) { + break; + } + } + } + } + + if(pBiDi->reorderingOptions & UBIDI_OPTION_REMOVE_CONTROLS) { + pBiDi->resultLength -= pBiDi->controlCount; + } else { + pBiDi->resultLength += pBiDi->insertPoints.size; + } + pBiDi->pParaBiDi=pBiDi; /* mark successful setPara */ +} + +U_CAPI void U_EXPORT2 +ubidi_orderParagraphsLTR(UBiDi *pBiDi, UBool orderParagraphsLTR) { + if(pBiDi!=NULL) { + pBiDi->orderParagraphsLTR=orderParagraphsLTR; + } +} + +U_CAPI UBool U_EXPORT2 +ubidi_isOrderParagraphsLTR(UBiDi *pBiDi) { + if(pBiDi!=NULL) { + return pBiDi->orderParagraphsLTR; + } else { + return FALSE; + } +} + +U_CAPI UBiDiDirection U_EXPORT2 +ubidi_getDirection(const UBiDi *pBiDi) { + if(IS_VALID_PARA_OR_LINE(pBiDi)) { + return pBiDi->direction; + } else { + return UBIDI_LTR; + } +} + +U_CAPI const UChar * U_EXPORT2 +ubidi_getText(const UBiDi *pBiDi) { + if(IS_VALID_PARA_OR_LINE(pBiDi)) { + return pBiDi->text; + } else { + return NULL; + } +} + +U_CAPI int32_t U_EXPORT2 +ubidi_getLength(const UBiDi *pBiDi) { + if(IS_VALID_PARA_OR_LINE(pBiDi)) { + return pBiDi->originalLength; + } else { + return 0; + } +} + +U_CAPI int32_t U_EXPORT2 +ubidi_getProcessedLength(const UBiDi *pBiDi) { + if(IS_VALID_PARA_OR_LINE(pBiDi)) { + return pBiDi->length; + } else { + return 0; + } +} + +U_CAPI int32_t U_EXPORT2 +ubidi_getResultLength(const UBiDi *pBiDi) { + if(IS_VALID_PARA_OR_LINE(pBiDi)) { + return pBiDi->resultLength; + } else { + return 0; + } +} + +/* paragraphs API functions ------------------------------------------------- */ + +U_CAPI UBiDiLevel U_EXPORT2 +ubidi_getParaLevel(const UBiDi *pBiDi) { + if(IS_VALID_PARA_OR_LINE(pBiDi)) { + return pBiDi->paraLevel; + } else { + return 0; + } +} + +U_CAPI int32_t U_EXPORT2 +ubidi_countParagraphs(UBiDi *pBiDi) { + if(!IS_VALID_PARA_OR_LINE(pBiDi)) { + return 0; + } else { + return pBiDi->paraCount; + } +} + +U_CAPI void U_EXPORT2 +ubidi_getParagraphByIndex(const UBiDi *pBiDi, int32_t paraIndex, + int32_t *pParaStart, int32_t *pParaLimit, + UBiDiLevel *pParaLevel, UErrorCode *pErrorCode) { + int32_t paraStart; + + /* check the argument values */ + RETURN_VOID_IF_NULL_OR_FAILING_ERRCODE(pErrorCode); + RETURN_VOID_IF_NOT_VALID_PARA_OR_LINE(pBiDi, *pErrorCode); + RETURN_VOID_IF_BAD_RANGE(paraIndex, 0, pBiDi->paraCount, *pErrorCode); + + pBiDi=pBiDi->pParaBiDi; /* get Para object if Line object */ + if(paraIndex) { + paraStart=pBiDi->paras[paraIndex-1]; + } else { + paraStart=0; + } + if(pParaStart!=NULL) { + *pParaStart=paraStart; + } + if(pParaLimit!=NULL) { + *pParaLimit=pBiDi->paras[paraIndex]; + } + if(pParaLevel!=NULL) { + *pParaLevel=GET_PARALEVEL(pBiDi, paraStart); + } +} + +U_CAPI int32_t U_EXPORT2 +ubidi_getParagraph(const UBiDi *pBiDi, int32_t charIndex, + int32_t *pParaStart, int32_t *pParaLimit, + UBiDiLevel *pParaLevel, UErrorCode *pErrorCode) { + uint32_t paraIndex; + + /* check the argument values */ + /* pErrorCode will be checked by the call to ubidi_getParagraphByIndex */ + RETURN_IF_NULL_OR_FAILING_ERRCODE(pErrorCode, -1); + RETURN_IF_NOT_VALID_PARA_OR_LINE(pBiDi, *pErrorCode, -1); + pBiDi=pBiDi->pParaBiDi; /* get Para object if Line object */ + RETURN_IF_BAD_RANGE(charIndex, 0, pBiDi->length, *pErrorCode, -1); + + for(paraIndex=0; charIndex>=pBiDi->paras[paraIndex]; paraIndex++); + ubidi_getParagraphByIndex(pBiDi, paraIndex, pParaStart, pParaLimit, pParaLevel, pErrorCode); + return paraIndex; +} + +U_CAPI void U_EXPORT2 +ubidi_setClassCallback(UBiDi *pBiDi, UBiDiClassCallback *newFn, + const void *newContext, UBiDiClassCallback **oldFn, + const void **oldContext, UErrorCode *pErrorCode) +{ + RETURN_VOID_IF_NULL_OR_FAILING_ERRCODE(pErrorCode); + if(pBiDi==NULL) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return; + } + if( oldFn ) + { + *oldFn = pBiDi->fnClassCallback; + } + if( oldContext ) + { + *oldContext = pBiDi->coClassCallback; + } + pBiDi->fnClassCallback = newFn; + pBiDi->coClassCallback = newContext; +} + +U_CAPI void U_EXPORT2 +ubidi_getClassCallback(UBiDi *pBiDi, UBiDiClassCallback **fn, const void **context) +{ + if(pBiDi==NULL) { + return; + } + if( fn ) + { + *fn = pBiDi->fnClassCallback; + } + if( context ) + { + *context = pBiDi->coClassCallback; + } +} + +U_CAPI UCharDirection U_EXPORT2 +ubidi_getCustomizedClass(UBiDi *pBiDi, UChar32 c) +{ + UCharDirection dir; + + if( pBiDi->fnClassCallback == NULL || + (dir = (*pBiDi->fnClassCallback)(pBiDi->coClassCallback, c)) == U_BIDI_CLASS_DEFAULT ) + { + return ubidi_getClass(pBiDi->bdp, c); + } else { + return dir; + } +} + diff --git a/Build/source/libs/icu/icu-xetex/common/ubidi_props.c b/Build/source/libs/icu/icu-xetex/common/ubidi_props.c new file mode 100644 index 00000000000..010a3db05d4 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ubidi_props.c @@ -0,0 +1,520 @@ +/* +******************************************************************************* +* +* Copyright (C) 2004-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: ubidi_props.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2004dec30 +* created by: Markus W. Scherer +* +* Low-level Unicode bidi/shaping properties access. +*/ + +#include "unicode/utypes.h" +#include "unicode/uset.h" +#include "unicode/udata.h" /* UDataInfo */ +#include "ucmndata.h" /* DataHeader */ +#include "udatamem.h" +#include "umutex.h" +#include "uassert.h" +#include "cmemory.h" +#include "utrie.h" +#include "ubidi_props.h" +#include "ucln_cmn.h" + +struct UBiDiProps { + UDataMemory *mem; + const int32_t *indexes; + const uint32_t *mirrors; + const uint8_t *jgArray; + + UTrie trie; + uint8_t formatVersion[4]; +}; + +/* data loading etc. -------------------------------------------------------- */ + +#define UBIDI_HARDCODE_DATA 1 + +#if UBIDI_HARDCODE_DATA + +/* ubidi_props_data.c is machine-generated by genbidi --csource */ +#include "ubidi_props_data.c" + +#else + +static UBool U_CALLCONV +isAcceptable(void *context, + const char *type, const char *name, + const UDataInfo *pInfo) { + if( + pInfo->size>=20 && + pInfo->isBigEndian==U_IS_BIG_ENDIAN && + pInfo->charsetFamily==U_CHARSET_FAMILY && + pInfo->dataFormat[0]==UBIDI_FMT_0 && /* dataFormat="BiDi" */ + pInfo->dataFormat[1]==UBIDI_FMT_1 && + pInfo->dataFormat[2]==UBIDI_FMT_2 && + pInfo->dataFormat[3]==UBIDI_FMT_3 && + pInfo->formatVersion[0]==1 && + pInfo->formatVersion[2]==UTRIE_SHIFT && + pInfo->formatVersion[3]==UTRIE_INDEX_SHIFT + ) { + UBiDiProps *bdp=(UBiDiProps *)context; + uprv_memcpy(bdp->formatVersion, pInfo->formatVersion, 4); + return TRUE; + } else { + return FALSE; + } +} + +static UBiDiProps * +ubidi_openData(UBiDiProps *bdpProto, + const uint8_t *bin, int32_t length, UErrorCode *pErrorCode) { + UBiDiProps *bdp; + int32_t size; + + bdpProto->indexes=(const int32_t *)bin; + if( (length>=0 && length<16*4) || + bdpProto->indexes[UBIDI_IX_INDEX_TOP]<16 + ) { + /* length or indexes[] too short for minimum indexes[] length of 16 */ + *pErrorCode=U_INVALID_FORMAT_ERROR; + return NULL; + } + size=bdpProto->indexes[UBIDI_IX_INDEX_TOP]*4; + if(length>=0) { + if(length>=size && length>=bdpProto->indexes[UBIDI_IX_LENGTH]) { + length-=size; + } else { + /* length too short for indexes[] or for the whole data length */ + *pErrorCode=U_INVALID_FORMAT_ERROR; + return NULL; + } + } + bin+=size; + /* from here on, assume that the sizes of the items fit into the total length */ + + /* unserialize the trie, after indexes[] */ + size=bdpProto->indexes[UBIDI_IX_TRIE_SIZE]; + utrie_unserialize(&bdpProto->trie, bin, size, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return NULL; + } + bin+=size; + + /* get mirrors[] */ + size=4*bdpProto->indexes[UBIDI_IX_MIRROR_LENGTH]; + bdpProto->mirrors=(const uint32_t *)bin; + bin+=size; + + /* get jgArray[] */ + size=bdpProto->indexes[UBIDI_IX_JG_LIMIT]-bdpProto->indexes[UBIDI_IX_JG_START]; + bdpProto->jgArray=bin; + bin+=size; + + /* allocate, copy, and return the new UBiDiProps */ + bdp=(UBiDiProps *)uprv_malloc(sizeof(UBiDiProps)); + if(bdp==NULL) { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return NULL; + } else { + uprv_memcpy(bdp, bdpProto, sizeof(UBiDiProps)); + return bdp; + } +} + +U_CFUNC UBiDiProps * +ubidi_openProps(UErrorCode *pErrorCode) { + UBiDiProps bdpProto={ NULL }, *bdp; + + bdpProto.mem=udata_openChoice(NULL, UBIDI_DATA_TYPE, UBIDI_DATA_NAME, isAcceptable, &bdpProto, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return NULL; + } + + bdp=ubidi_openData( + &bdpProto, + udata_getMemory(bdpProto.mem), + udata_getLength(bdpProto.mem), + pErrorCode); + if(U_FAILURE(*pErrorCode)) { + udata_close(bdpProto.mem); + return NULL; + } else { + return bdp; + } +} + +U_CFUNC UBiDiProps * +ubidi_openBinary(const uint8_t *bin, int32_t length, UErrorCode *pErrorCode) { + UBiDiProps bdpProto={ NULL }; + const DataHeader *hdr; + + if(U_FAILURE(*pErrorCode)) { + return NULL; + } + if(bin==NULL) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + + /* check the header */ + if(length>=0 && length<20) { + *pErrorCode=U_INVALID_FORMAT_ERROR; + return NULL; + } + hdr=(const DataHeader *)bin; + if( + !(hdr->dataHeader.magic1==0xda && hdr->dataHeader.magic2==0x27 && + hdr->info.isBigEndian==U_IS_BIG_ENDIAN && + isAcceptable(&bdpProto, UBIDI_DATA_TYPE, UBIDI_DATA_NAME, &hdr->info)) + ) { + *pErrorCode=U_INVALID_FORMAT_ERROR; + return NULL; + } + + bin+=hdr->dataHeader.headerSize; + if(length>=0) { + length-=hdr->dataHeader.headerSize; + } + return ubidi_openData(&bdpProto, bin, length, pErrorCode); +} + +#endif + +U_CFUNC void +ubidi_closeProps(UBiDiProps *bdp) { + if(bdp!=NULL) { +#if !UBIDI_HARDCODE_DATA + udata_close(bdp->mem); +#endif + uprv_free(bdp); + } +} + +/* UBiDiProps singleton ----------------------------------------------------- */ + +static UBiDiProps *gBdpDummy=NULL; +#if !UBIDI_HARDCODE_DATA +static UBiDiProps *gBdp=NULL; +static UErrorCode gErrorCode=U_ZERO_ERROR; +static int8_t gHaveData=0; +#endif + +static UBool U_CALLCONV +ubidi_cleanup(void) { + ubidi_closeProps(gBdpDummy); + gBdpDummy=NULL; +#if !UBIDI_HARDCODE_DATA + ubidi_closeProps(gBdp); + gBdp=NULL; + gErrorCode=U_ZERO_ERROR; + gHaveData=0; +#endif + return TRUE; +} + +U_CFUNC const UBiDiProps * +ubidi_getSingleton(UErrorCode *pErrorCode) { +#if UBIDI_HARDCODE_DATA + if(U_FAILURE(*pErrorCode)) { + return NULL; + } + return &ubidi_props_singleton; +#else + int8_t haveData; + + if(U_FAILURE(*pErrorCode)) { + return NULL; + } + + UMTX_CHECK(NULL, gHaveData, haveData); + + if(haveData>0) { + /* data was loaded */ + return gBdp; + } else if(haveData<0) { + /* data loading failed */ + *pErrorCode=gErrorCode; + return NULL; + } else /* haveData==0 */ { + /* load the data */ + UBiDiProps *bdp=ubidi_openProps(pErrorCode); + if(U_FAILURE(*pErrorCode)) { + gHaveData=-1; + gErrorCode=*pErrorCode; + return NULL; + } + + /* set the static variables */ + umtx_lock(NULL); + if(gBdp==NULL) { + gBdp=bdp; + bdp=NULL; + gHaveData=1; + ucln_common_registerCleanup(UCLN_COMMON_UBIDI, ubidi_cleanup); + } + umtx_unlock(NULL); + + ubidi_closeProps(bdp); + return gBdp; + } +#endif +} + +U_CAPI const UBiDiProps * +ubidi_getDummy(UErrorCode *pErrorCode) { + UBiDiProps *bdp; + + if(U_FAILURE(*pErrorCode)) { + return NULL; + } + + UMTX_CHECK(NULL, gBdpDummy, bdp); + + if(bdp!=NULL) { + /* the dummy object was already created */ + return bdp; + } else /* bdp==NULL */ { + /* create the dummy object */ + int32_t *indexes; + + bdp=(UBiDiProps *)uprv_malloc(sizeof(UBiDiProps)+UBIDI_IX_TOP*4+UTRIE_DUMMY_SIZE); + if(bdp==NULL) { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + uprv_memset(bdp, 0, sizeof(UBiDiProps)+UBIDI_IX_TOP*4); + + bdp->indexes=indexes=(int32_t *)(bdp+1); + indexes[UBIDI_IX_INDEX_TOP]=UBIDI_IX_TOP; + + indexes[UBIDI_IX_TRIE_SIZE]= + utrie_unserializeDummy(&bdp->trie, indexes+UBIDI_IX_TOP, UTRIE_DUMMY_SIZE, 0, 0, TRUE, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + uprv_free(bdp); + return NULL; + } + + bdp->formatVersion[0]=1; + bdp->formatVersion[2]=UTRIE_SHIFT; + bdp->formatVersion[3]=UTRIE_INDEX_SHIFT; + + /* set the static variables */ + umtx_lock(NULL); + if(gBdpDummy==NULL) { + gBdpDummy=bdp; + bdp=NULL; + ucln_common_registerCleanup(UCLN_COMMON_UBIDI, ubidi_cleanup); + } + umtx_unlock(NULL); + + uprv_free(bdp); + return gBdpDummy; + } +} + +/* set of property starts for UnicodeSet ------------------------------------ */ + +static UBool U_CALLCONV +_enumPropertyStartsRange(const void *context, UChar32 start, UChar32 limit, uint32_t value) { + /* add the start code point to the USet */ + const USetAdder *sa=(const USetAdder *)context; + sa->add(sa->set, start); + return TRUE; +} + +U_CFUNC void +ubidi_addPropertyStarts(const UBiDiProps *bdp, const USetAdder *sa, UErrorCode *pErrorCode) { + int32_t i, length; + UChar32 c, start, limit; + + const uint8_t *jgArray; + uint8_t prev, jg; + + if(U_FAILURE(*pErrorCode)) { + return; + } + + /* add the start code point of each same-value range of the trie */ + utrie_enum(&bdp->trie, NULL, _enumPropertyStartsRange, sa); + + /* add the code points from the bidi mirroring table */ + length=bdp->indexes[UBIDI_IX_MIRROR_LENGTH]; + for(i=0; imirrors[i]); + sa->addRange(sa->set, c, c+1); + } + + /* add the code points from the Joining_Group array where the value changes */ + start=bdp->indexes[UBIDI_IX_JG_START]; + limit=bdp->indexes[UBIDI_IX_JG_LIMIT]; + jgArray=bdp->jgArray; + prev=0; + while(startadd(sa->set, start); + prev=jg; + } + ++start; + } + if(prev!=0) { + /* add the limit code point if the last value was not 0 (it is now start==limit) */ + sa->add(sa->set, limit); + } + + /* add code points with hardcoded properties, plus the ones following them */ + + /* (none right now) */ +} + +/* data access primitives --------------------------------------------------- */ + +/* UTRIE_GET16() itself validates c */ +#define GET_PROPS(bdp, c, result) \ + UTRIE_GET16(&(bdp)->trie, c, result); + +/* property access functions ------------------------------------------------ */ + +U_CFUNC int32_t +ubidi_getMaxValue(const UBiDiProps *bdp, UProperty which) { + int32_t max; + + if(bdp==NULL) { + return -1; + } + + max=bdp->indexes[UBIDI_MAX_VALUES_INDEX]; + switch(which) { + case UCHAR_BIDI_CLASS: + return (max&UBIDI_CLASS_MASK); + case UCHAR_JOINING_GROUP: + return (max&UBIDI_MAX_JG_MASK)>>UBIDI_MAX_JG_SHIFT; + case UCHAR_JOINING_TYPE: + return (max&UBIDI_JT_MASK)>>UBIDI_JT_SHIFT; + default: + return -1; /* undefined */ + } +} + +U_CAPI UCharDirection +ubidi_getClass(const UBiDiProps *bdp, UChar32 c) { + uint32_t props; + GET_PROPS(bdp, c, props); + return (UCharDirection)UBIDI_GET_CLASS(props); +} + +U_CFUNC UBool +ubidi_isMirrored(const UBiDiProps *bdp, UChar32 c) { + uint32_t props; + GET_PROPS(bdp, c, props); + return (UBool)UBIDI_GET_FLAG(props, UBIDI_IS_MIRRORED_SHIFT); +} + +U_CFUNC UChar32 +ubidi_getMirror(const UBiDiProps *bdp, UChar32 c) { + uint32_t props; + int32_t delta; + + GET_PROPS(bdp, c, props); + delta=((int16_t)props)>>UBIDI_MIRROR_DELTA_SHIFT; + if(delta!=UBIDI_ESC_MIRROR_DELTA) { + return c+delta; + } else { + /* look for mirror code point in the mirrors[] table */ + const uint32_t *mirrors; + uint32_t m; + int32_t i, length; + UChar32 c2; + + mirrors=bdp->mirrors; + length=bdp->indexes[UBIDI_IX_MIRROR_LENGTH]; + + /* linear search */ + for(i=0; i>UBIDI_JT_SHIFT); +} + +U_CFUNC UJoiningGroup +ubidi_getJoiningGroup(const UBiDiProps *bdp, UChar32 c) { + UChar32 start, limit; + + start=bdp->indexes[UBIDI_IX_JG_START]; + limit=bdp->indexes[UBIDI_IX_JG_LIMIT]; + if(start<=c && cjgArray[c-start]; + } else { + return U_JG_NO_JOINING_GROUP; + } +} + +/* public API (see uchar.h) ------------------------------------------------- */ + +U_CFUNC UCharDirection +u_charDirection(UChar32 c) { + UErrorCode errorCode=U_ZERO_ERROR; + const UBiDiProps *bdp=ubidi_getSingleton(&errorCode); + if(bdp!=NULL) { + return ubidi_getClass(bdp, c); + } else { + return U_LEFT_TO_RIGHT; + } +} + +U_CFUNC UBool +u_isMirrored(UChar32 c) { + UErrorCode errorCode=U_ZERO_ERROR; + const UBiDiProps *bdp=ubidi_getSingleton(&errorCode); + return (UBool)(bdp!=NULL && ubidi_isMirrored(bdp, c)); +} + +U_CFUNC UChar32 +u_charMirror(UChar32 c) { + UErrorCode errorCode=U_ZERO_ERROR; + const UBiDiProps *bdp=ubidi_getSingleton(&errorCode); + if(bdp!=NULL) { + return ubidi_getMirror(bdp, c); + } else { + return c; + } +} diff --git a/Build/source/libs/icu/icu-xetex/common/ubidi_props.h b/Build/source/libs/icu/icu-xetex/common/ubidi_props.h new file mode 100644 index 00000000000..d6da311f9e5 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ubidi_props.h @@ -0,0 +1,160 @@ +/* +******************************************************************************* +* +* Copyright (C) 2004-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: ubidi_props.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2004dec30 +* created by: Markus W. Scherer +* +* Low-level Unicode bidi/shaping properties access. +*/ + +#ifndef __UBIDI_PROPS_H__ +#define __UBIDI_PROPS_H__ + +#include "unicode/utypes.h" +#include "unicode/uset.h" +#include "uset_imp.h" +#include "udataswp.h" + +U_CDECL_BEGIN + +/* library API -------------------------------------------------------------- */ + +struct UBiDiProps; +typedef struct UBiDiProps UBiDiProps; + +U_CFUNC UBiDiProps * +ubidi_openProps(UErrorCode *pErrorCode); + +U_CFUNC UBiDiProps * +ubidi_openBinary(const uint8_t *bin, int32_t length, UErrorCode *pErrorCode); + +U_CFUNC void +ubidi_closeProps(UBiDiProps *bdp); + + +U_CFUNC const UBiDiProps * +ubidi_getSingleton(UErrorCode *pErrorCode); + +/** + * Get a singleton dummy object, one that works with no real data. + * This can be used when the real data is not available. + * Using the dummy can reduce checks for available data after an initial failure. + */ +U_CAPI const UBiDiProps * +ubidi_getDummy(UErrorCode *pErrorCode); + + +U_CAPI int32_t +ubidi_swap(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode); + +U_CFUNC void +ubidi_addPropertyStarts(const UBiDiProps *bdp, const USetAdder *sa, UErrorCode *pErrorCode); + +/* property access functions */ + +U_CFUNC int32_t +ubidi_getMaxValue(const UBiDiProps *bdp, UProperty which); + +U_CAPI UCharDirection +ubidi_getClass(const UBiDiProps *bdp, UChar32 c); + +U_CFUNC UBool +ubidi_isMirrored(const UBiDiProps *bdp, UChar32 c); + +U_CFUNC UChar32 +ubidi_getMirror(const UBiDiProps *bdp, UChar32 c); + +U_CFUNC UBool +ubidi_isBidiControl(const UBiDiProps *bdp, UChar32 c); + +U_CFUNC UBool +ubidi_isJoinControl(const UBiDiProps *bdp, UChar32 c); + +U_CFUNC UJoiningType +ubidi_getJoiningType(const UBiDiProps *bdp, UChar32 c); + +U_CFUNC UJoiningGroup +ubidi_getJoiningGroup(const UBiDiProps *bdp, UChar32 c); + +/* file definitions --------------------------------------------------------- */ + +#define UBIDI_DATA_NAME "ubidi" +#define UBIDI_DATA_TYPE "icu" + +/* format "BiDi" */ +#define UBIDI_FMT_0 0x42 +#define UBIDI_FMT_1 0x69 +#define UBIDI_FMT_2 0x44 +#define UBIDI_FMT_3 0x69 + +/* indexes into indexes[] */ +enum { + UBIDI_IX_INDEX_TOP, + UBIDI_IX_LENGTH, + UBIDI_IX_TRIE_SIZE, + UBIDI_IX_MIRROR_LENGTH, + + UBIDI_IX_JG_START, + UBIDI_IX_JG_LIMIT, + + UBIDI_MAX_VALUES_INDEX=15, + UBIDI_IX_TOP=16 +}; + +/* definitions for 16-bit bidi/shaping properties word ---------------------- */ + +enum { + /* UBIDI_CLASS_SHIFT=0, */ /* bidi class: 5 bits (4..0) */ + UBIDI_JT_SHIFT=5, /* joining type: 3 bits (7..5) */ + + /* UBIDI__SHIFT=8, reserved: 2 bits (9..8) */ + + UBIDI_JOIN_CONTROL_SHIFT=10, + UBIDI_BIDI_CONTROL_SHIFT=11, + + UBIDI_IS_MIRRORED_SHIFT=12, /* 'is mirrored' */ + UBIDI_MIRROR_DELTA_SHIFT=13, /* bidi mirroring delta: 3 bits (15..13) */ + + UBIDI_MAX_JG_SHIFT=16 /* max JG value in indexes[UBIDI_MAX_VALUES_INDEX] bits 23..16 */ +}; + +#define UBIDI_CLASS_MASK 0x0000001f +#define UBIDI_JT_MASK 0x000000e0 + +#define UBIDI_MAX_JG_MASK 0x00ff0000 + +#define UBIDI_GET_CLASS(props) ((props)&UBIDI_CLASS_MASK) +#define UBIDI_GET_FLAG(props, shift) (((props)>>(shift))&1) + +enum { + UBIDI_ESC_MIRROR_DELTA=-4, + UBIDI_MIN_MIRROR_DELTA=-3, + UBIDI_MAX_MIRROR_DELTA=3 +}; + +/* definitions for 32-bit mirror table entry -------------------------------- */ + +enum { + /* the source Unicode code point takes 21 bits (20..0) */ + UBIDI_MIRROR_INDEX_SHIFT=21, + UBIDI_MAX_MIRROR_INDEX=0x7ff +}; + +#define UBIDI_GET_MIRROR_CODE_POINT(m) (UChar32)((m)&0x1fffff) + +#define UBIDI_GET_MIRROR_INDEX(m) ((m)>>UBIDI_MIRROR_INDEX_SHIFT) + +U_CDECL_END + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/ubidi_props_data.c b/Build/source/libs/icu/icu-xetex/common/ubidi_props_data.c new file mode 100644 index 00000000000..ea30b7e0cde --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ubidi_props_data.c @@ -0,0 +1,549 @@ +/* + * Copyright (C) 1999-2007, International Business Machines + * Corporation and others. All Rights Reserved. + * + * file name: ubidi_props_data.c + * + * machine-generated on: 2007-08-15 + */ + +static const UVersionInfo ubidi_props_dataVersion={5,0,0,0}; + +static const int32_t ubidi_props_indexes[UBIDI_IX_TOP]={0x10,0x3f04,0x3d10,0x1a,0x622,0x76e,0,0,0,0,0,0,0,0,0,0x3500b2}; + +static const uint16_t ubidi_props_trieIndex[7808]={ +0x250,0x258,0x260,0x268,0x270,0x278,0x280,0x288,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x28e,0x296,0x29e,0x2a6,0x2a6,0x2a6,0x2aa,0x2b2,0x248,0x248,0x2b5, +0x248,0x248,0x248,0x248,0x2bd,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x2c3,0x2c8,0x2d0,0x2d2, +0x2da,0x2e2,0x2ea,0x2f2,0x2f8,0x2ff,0x307,0x30f,0x317,0x31f,0x325,0x32c,0x330,0x337,0x33f,0x345, +0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x34d,0x34e,0x356,0x35e,0x366,0x34e,0x36e,0x376, +0x34d,0x34e,0x37e,0x382,0x34d,0x34e,0x38a,0x392,0x366,0x397,0x39f,0x248,0x3a4,0x248,0x3ac,0x3b0, +0x248,0x3b7,0x3bf,0x248,0x248,0x3c5,0x3cd,0x3d5,0x248,0x248,0x3dd,0x248,0x248,0x248,0x3e3,0x248, +0x248,0x3e9,0x3f1,0x248,0x248,0x3f5,0x3fd,0x248,0x401,0x408,0x248,0x410,0x418,0x41f,0x3a3,0x248, +0x248,0x427,0x401,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x42e,0x248,0x436,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x43e,0x248,0x248,0x248,0x446,0x446,0x372,0x372,0x248,0x44c,0x454,0x436, +0x45c,0x248,0x248,0x248,0x248,0x364,0x248,0x248,0x248,0x464,0x46c,0x248,0x248,0x248,0x46e,0x476, +0x47e,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x486,0x489,0x3a4,0x491,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x499,0x3b7, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x49c,0x4a4,0x4a8, +0x4b0,0x4b6,0x4bd,0x4c5,0x4cd,0x4d5,0x4db,0x4df,0x4e7,0x4ef,0x4f7,0x248,0x4ff,0x476,0x476,0x476, +0x507,0x50f,0x517,0x51f,0x524,0x52c,0x534,0x53a,0x542,0x54a,0x248,0x550,0x557,0x476,0x476,0x55d, +0x476,0x565,0x56d,0x476,0x575,0x248,0x248,0x473,0x476,0x476,0x476,0x476,0x476,0x476,0x476,0x476, +0x476,0x476,0x476,0x476,0x57d,0x585,0x248,0x248,0x58d,0x593,0x598,0x5a0,0x57f,0x5a6,0x5ae,0x5b6, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x476,0x476,0x476,0x476,0x5be,0x5c5,0x5cd,0x5d5, +0x5dd,0x5e5,0x5ed,0x5f4,0x5fc,0x604,0x60b,0x613,0x61b,0x623,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x62a,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x632,0x248,0x248,0x248,0x63a,0x476,0x476,0x479,0x476,0x476,0x476,0x476,0x476,0x476,0x641,0x647, +0x64f,0x657,0x248,0x248,0x65f,0x666,0x248,0x287,0x248,0x248,0x248,0x248,0x248,0x248,0x47a,0x248, +0x667,0x248,0x4ff,0x66f,0x248,0x677,0x67f,0x248,0x248,0x248,0x248,0x683,0x248,0x248,0x46e,0x286, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x476,0x476, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x4ff,0x476,0x565,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x61b,0x46d,0x248,0x248,0x248,0x248,0x248,0x248, +0x68b,0x692,0x248,0x695,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x785,0x788,0x248,0x790,0x248,0x790,0x248,0x790,0x248,0x790,0x248,0x790,0x248,0x790,0x248,0x790, +0x248,0x790,0x248,0x790,0x248,0x790,0x248,0x790,0x248,0x790,0x798,0x790,0x248,0x790,0x248,0x790, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x69b,0x6a3,0x6a7,0x330,0x330,0x330,0x330,0x330, +0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x6ab,0x330,0x330,0x330,0x330,0x6b3,0x6b7, +0x6bf,0x6c7,0x6cb,0x6d3,0x330,0x330,0x330,0x6d7,0x6df,0x260,0x6e7,0x6ef,0x248,0x248,0x248,0x6f7, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x6ff,0x248,0x476,0x476,0x56d,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x707,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2, +0x70f,0x713,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2, +0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2, +0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2,0x2d2, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x71b,0x723,0x729,0x248,0x248, +0x476,0x476,0x731,0x248,0x248,0x248,0x248,0x248,0x476,0x476,0x739,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x73f,0x248,0x746,0x248,0x742,0x248,0x749,0x248,0x751,0x755, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248, +0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x248,0x75d, +0x765,0x76d,0x76d,0x76d,0x775,0x775,0x775,0x775,0x2a6,0x2a6,0x2a6,0x2a6,0x2a6,0x2a6,0x2a6,0x77d, +0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775, +0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775, +0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775,0x775, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,8,7,8,9,7,0x12,0x12, +0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,7,7,7,8, +9,0xa,0xa,4,4,4,0xa,0xa,0x300a,0xf00a,0xa,3,6,3,6,6, +2,2,2,2,2,2,2,2,2,2,6,0xa,0x500a,0xa,0xd00a,0xa, +0xa,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0x500a,0xa,0xd00a,0xa,0xa, +0xa,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0x500a,0xa,0xd00a,0xa,0x12, +0x12,0x12,0x12,0x12,0x12,7,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, +0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, +6,0xa,4,4,4,4,0xa,0xa,0xa,0xa,0,0x900a,0xa,0xb2,0xa,0xa, +4,4,2,2,0xa,0,0xa,0xa,0xa,2,0,0x900a,0xa,0xa,0xa,0xa, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0xa,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0xa,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0xa,0xa,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0,0xa,0xa,0xa,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0,0,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1, +0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1, +0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0xa,0xa,0,0, +0,0,0,0,0,0,0xa,0,0,0,0,0,0xa,0xa,0,0xa, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0xa,0,0,0,0,0, +0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0,0xb1,0xb1,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0xa,0,0,0,0,0,1,0xb1,0xb1,0xb1, +0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1, +0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,1,0xb1, +1,0xb1,0xb1,1,0xb1,0xb1,1,0xb1,1,1,1,1,1,1,1,1, +1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, +1,1,1,1,1,1,1,1,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd, +0xd,0xd,0xd,0xd,6,0xd,0xa,0xa,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xd,0xd, +0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0x8d,0x8d,0x8d,0x8d,0x4d,0x8d, +0x4d,0x8d,0x4d,0x4d,0x4d,0x4d,0x4d,0x8d,0x8d,0x8d,0x8d,0x4d,0x4d,0x4d,0x4d,0x4d, +0x4d,0x4d,0x4d,0xd,0xd,0xd,0xd,0xd,0x2d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d, +0x8d,0x4d,0x4d,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1, +0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xd,5,5,5,5,5,5,5,5, +5,5,4,5,5,0xd,0x4d,0x4d,0xb1,0x8d,0x8d,0x8d,0xd,0x8d,0x8d,0x8d, +0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d, +0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d, +0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d, +0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x8d,0x4d,0x4d,0x8d, +0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x4d,0x8d,0x4d,0x8d,0x4d,0x4d,0x8d,0x8d, +0xd,0x8d,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xd,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1, +0xb1,0xd,0xd,0xb1,0xb1,0xa,0xb1,0xb1,0xb1,0xb1,0x8d,0x8d,2,2,2,2, +2,2,2,2,2,2,0x4d,0x4d,0x4d,0xd,0xd,0x4d,0xd,0xd,0xd,0xd, +0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xb2,0x8d,0xb1,0x4d,0x4d, +0x4d,0x8d,0x8d,0x8d,0x8d,0x8d,0x4d,0x4d,0x4d,0x4d,0x8d,0x4d,0x4d,0x4d,0x4d,0x4d, +0x4d,0x4d,0x4d,0x4d,0x8d,0x4d,0x8d,0x4d,0x8d,0x4d,0x4d,0x8d,0xb1,0xb1,0xb1,0xb1, +0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xd, +0xd,0x8d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x8d,0x8d,0x8d, +0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x8d,0x8d,0x4d,0xd,0xd, +0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd, +0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd, +0xd,0xd,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xd,0xd,0xd, +0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,1,1,1,1, +1,1,1,1,1,1,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41, +0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0xb1, +0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,1,1,0xa,0xa,0xa,0xa,0x21,1, +1,1,1,1,0,0xb1,0xb1,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0xb1,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1, +0xb1,0,0,0,0,0xb1,0,0,0,0xb1,0xb1,0xb1,0xb1,0,0,0, +0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0xb1,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0,0,0, +0,0,0,0,0,0xb1,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,4,4,0,0,0,0, +0,0,0,0,0,0,0,0,0,0xb1,0xb1,0,0,0,0,0xb1, +0xb1,0,0,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0xb1,0xb1,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0,0xb1, +0xb1,0,0,0,0,0xb1,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0,0,0,0, +0,0,0,0,0,0,0,0,0,4,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0xb1,0,0,0xb1,0,0xb1,0xb1,0xb1, +0,0,0,0,0,0,0,0,0,0xb1,0,0,0,0,0,0, +0,0,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0xb1,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,4,0xa,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0,0,0, +0,0,0xb1,0xb1,0xb1,0,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0, +0,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0xb1,0,0,0xa0,0,0,0,0,0,0,0xa0,0,0,0,0,0, +0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0xb1,0xb1,0,0,0,0,0,0,0,0, +0,0,0,0,0,0xa,0xa,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0, +0,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0xb1,0,0,0,0,0,0,0,0xb1,0xb1, +0xb1,0,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0xb1,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0, +0,0,0,4,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1, +0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0xb1,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0xb1, +0xb1,0,0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1, +0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0xb1,0,0xb1,0,0xb1,0x300a,0xf00a,0x300a,0xf00a,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0, +0xb1,0xb1,0xb1,0xb1,0xb1,0,0xb1,0xb1,0,0,0,0,0,0,0,0, +0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1, +0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1, +0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0,0xb1,0, +0,0,0xb1,0xb1,0,0xb1,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0xb1,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa, +0xa,0xa,0,0,0,0,0,0,9,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0x300a,0xf00a,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0xa0,0xa0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0, +0,0,0,0,0,0,0xb1,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1, +0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,4,0,0xb1,0,0, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xb1,0xb1,0xb1,9,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0xb1,0xb1,0xb1,0,0,0,0,0xb1,0xb1,0x11,0x11,0x11,0,0,0,0, +0,0,0xb1,0,0,0,0,0,0,0xb1,0xb1,0xb1,0,0,0,0, +0xa,0,0,0,0xa,0xa,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xb1, +0xb1,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0xb1,0,0xb1,0xb1,0xb1,0xb1,0xb1,0, +0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xb1, +0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0, +0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0xa,0,0xa, +0xa,0xa,0,0,0,0,0,0,0,0,0,0,0,0xa,0xa,0xa, +0,0,0,0,0,0,0,0,0,0,0,0,0,0xa,0xa,0xa, +0,0,0,0,0,0,0,0,0,0,0,0,0,0xa,0xa,0, +9,9,9,9,9,9,9,9,9,9,9,0xb2,0x412,0x432,0x8a0,0x8a1, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa, +9,7,0x8ab,0x8ae,0x8b0,0x8ac,0x8af,6,4,4,4,4,4,0xa,0xa,0xa, +0xa,0x300a,0xf00a,0xa,0xa,0xa,0xa,0xa,6,0x300a,0xf00a,0xa,0xa,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa, +0xa,0xa,0xa,9,0xb2,0xb2,0xb2,0xb2,0x12,0x12,0x12,0x12,0x12,0x12,0xb2,0xb2, +0xb2,0xb2,0xb2,0xb2,2,0,0,0,2,2,2,2,2,2,3,3, +0xa,0x300a,0xf00a,0,2,2,2,2,2,2,2,2,2,2,3,3, +0xa,0x300a,0xf00a,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,4,4,4,4,4,4,4,4,4,4,4,4, +4,4,4,4,4,4,4,4,4,4,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1, +0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0xa,0xa,0,0xa, +0xa,0xa,0xa,0,0xa,0xa,0,0,0,0,0,0,0,0,0,0, +0xa,0,0xa,0xa,0xa,0,0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa, +0,0xa,0,0xa,0,0xa,0,0,0,0,4,0,0,0,0,0, +0,0,0,0,0,0,0xa,0xa,0,0,0,0,0x100a,0xa,0xa,0xa, +0xa,0,0,0,0,0,0xa,0xa,0xa,0xa,0,0,0,0,0,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0x100a,0x100a,0x100a, +0x100a,0xa,0xa,0xa,0x700a,0x700a,0x700a,0xb00a,0xb00a,0xb00a,0xa,0xa,0xa,0x100a,3,4, +0xa,0x900a,0x100a,0xa,0xa,0xa,0x100a,0x100a,0x100a,0x100a,0xa,0x100a,0x100a,0x100a,0x100a,0xa, +0x100a,0xa,0x100a,0xa,0xa,0xa,0xa,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a, +0xa,0xa,0xa,0xa,0xa,0x100a,0xa,0x100a,0x300a,0xf00a,0x100a,0x100a,0x100a,0x100a,0x100a,0x900a, +0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0xa,0xa,0xa,0xa,0xa,0x300a,0xf00a, +0x300a,0xf00a,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0x100a,0x100a,0xa,0x100a,0xa, +0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0xa,0xa,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a, +0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x100a,0xa,0xa,0x300a, +0xf00a,0x300a,0xf00a,0xa,0xa,0xa,0xa,0xa,0x900a,0xa,0xa,0xa,0xa,0xa,0xa,0xa, +0xa,0xa,0x300a,0xf00a,0xa,0xa,0x900a,0x100a,0x900a,0x900a,0x100a,0x900a,0x100a,0x100a,0x100a,0x100a, +0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x100a,0xa,0xa,0xa,0xa,0xa,0x100a,0x100a, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0x300a,0xf00a,0x300a,0xf00a,0x900a,0xa,0xa, +0x300a,0xf00a,0xa,0xa,0xa,0xa,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a, +0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0xa,0xa,0x300a,0xf00a,0x900a,0x900a,0x900a,0x100a,0x900a,0x900a, +0x100a,0x100a,0x900a,0x900a,0x900a,0x900a,0x900a,0x100a,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa, +0x300a,0xf00a,0x300a,0xf00a,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0x100a,0x100a,0xa,0xa,0xa,0xa,0xa,0xa, +0xa,0x300a,0xf00a,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa, +0xa,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,2,2,2,2, +2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, +0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa, +0xa,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa, +0,0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0xa,0xa,0xa,0xa,0,0xa,0xa,0xa,0xa,0,0, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0xa,0,0xa, +0xa,0xa,0xa,0,0,0,0xa,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0, +0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a, +0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0x100a,0xa,0xa,0x300a,0xf00a,0x300a,0xf00a,0xa, +0xa,0xa,0xa,0,0,0,0,0,0xa,0xa,0xa,0x100a,0x100a,0x300a,0xf00a,0xa, +0xa,0xa,0xa,0xa,0x100a,0x300a,0xf00a,0xa,0xa,0xa,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a, +0x300a,0xf00a,0x300a,0xf00a,0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0x300a,0xf00a,0x300a,0xf00a,0x300a, +0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x700a,0x300a,0xf00a,0xb00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a, +0xf00a,0xa,0xa,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a, +0x100a,0x100a,0x100a,0x100a,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0x900a,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0x300a,0xf00a,0x100a,0x100a,0x300a,0xf00a,0xa,0xa,0xa,0x100a,0xa,0xa, +0xa,0xa,0x100a,0x300a,0xf00a,0x300a,0xf00a,0xa,0x300a,0xf00a,0xa,0xa,0x300a,0xf00a,0x300a,0xf00a, +0x100a,0xa,0xa,0xa,0xa,0x100a,0xa,0x100a,0x100a,0x100a,0xa,0xa,0x100a,0x100a,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0x100a,0x900a,0x100a,0x100a,0x300a,0xf00a,0xa,0xa, +0x300a,0xf00a,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0x100a,0x100a, +0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a, +0x100a,0xa,0x100a,0x100a,0x100a,0x100a,0xa,0xa,0x100a,0xa,0x100a,0xa,0xa,0x100a,0xa,0x300a, +0xf00a,0x300a,0xf00a,0xa,0xa,0xa,0xa,0xa,0x300a,0xf00a,0xa,0xa,0xa,0xa,0xa,0xa, +0x300a,0xf00a,0x100a,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0x100a,0x100a,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0x300a,0xf00a,0xa,0xa,0xa,0xa,0x100a,0x100a,0x100a,0x100a,0xa,0x100a, +0x100a,0xa,0xa,0x100a,0x100a,0xa,0xa,0xa,0xa,0x300a,0xf00a,0x100a,0x100a,0x300a,0xf00a,0x300a, +0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x300a,0xf00a,0x100a,0x100a,0x100a, +0x100a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x100a,0x100a,0x100a, +0x100a,0x300a,0xf00a,0x100a,0xa,0xa,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0xa,0x300a, +0xf00a,0x100a,0x100a,0x300a,0xf00a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x300a,0xf00a,0x300a,0xf00a,0x300a, +0xf00a,0x300a,0xf00a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a, +0xf00a,0x300a,0xf00a,0xa,0xa,0xa,0xa,0xa,0x100a,0xa,0x900a,0xa,0xa,0xa,0x100a,0x900a, +0x900a,0x900a,0x100a,0xa,0xa,0xa,0xa,0xa,0x300a,0xf00a,0x100a,0xa,0xa,0xa,0xa,0x100a, +0xa,0xa,0xa,0x300a,0xf00a,0x300a,0xf00a,0x100a,0xa,0x100a,0xa,0xa,0xa,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0,0,0xa,0xa,0xa,0xa, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0xa,0xa,0xa, +0xa,0xa,0xa,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0x300a,0xf00a,0x300a,0xf00a,0xa,0xa, +0xa,0x300a,0xf00a,0xa,0x300a,0xf00a,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa, +0,0,0,0,0x300a,0xf00a,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa, +0xa,0xa,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0,9,0xa,0xa,0xa, +0xa,0,0,0,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0xa,0xa, +0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0xa,0xa,0xa,0xa,0xa,0,0,0, +0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xa,0,0,0, +0,0,0xa,0xa,0,0,0,0,0,0xa,0xa,0xa,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0xb1,0xb1,0xa,0xa,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0xa,0xa,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0xa,0xa,0xa,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0, +0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0xa,0xa,0xa,0xa,0,0,0,0,0,0,0,0x11,0, +0,0,0xb1,0,0,0,0,0xb1,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0, +0xa,0xa,0xa,0xa,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,1,0xb1,1,1,1,1,1, +1,1,1,1,1,3,1,1,1,1,1,1,1,1,1,1, +1,1,1,1,1,1,1,1,1,1,1,1,0xd,0xd,0xd,0xd, +0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd, +0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xa,0xa,0xd,0xd,0xd,0xd, +0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0x12,0x12,0x12,0x12, +0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0xd,0xd,0xd,0xd, +0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xa,0xd,0xd,0xb1,0xb1,0xb1,0xb1, +0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xa,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1, +0,0,0,0,0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,6,0xa,6,0, +0xa,6,0xa,0xa,0xa,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,4,0xa,0xa,3,3, +0x300a,0xf00a,0xa,0,0xa,4,4,0xa,0,0,0,0,0xd,0xd,0xd,0xd, +0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd, +0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xb2,0,0xa,0xa,4, +4,4,0xa,0xa,0x300a,0xf00a,0xa,3,6,3,6,6,2,2,2,2, +2,2,2,2,2,2,6,0xa,0x500a,0xa,0xd00a,0xa,0xa,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0x500a,0xa,0xd00a,0xa,0x300a,0xf00a,0xa,0x300a,0xf00a, +0xa,0xa,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,4,4,0xa,0xa, +0xa,4,4,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0x12,0x12,0x12,0x12, +0x12,0x12,0x12,0x12,0x12,0xaa,0xaa,0xaa,0xa,0xa,0x12,0x12,0,0xa,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1, +1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, +1,1,1,1,1,1,1,1,1,1,1,0xa,1,0xb1,0xb1,0xb1, +1,0xb1,0xb1,1,1,1,1,1,0xb1,0xb1,0xb1,0xb1,1,1,1,1, +1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, +1,1,1,1,0xb1,0xb1,0xb1,1,1,1,1,0xb1,0,0,0,0, +0,0,0,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0xb2, +0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0, +0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xb1,0xb1, +0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0xa,0xa,0xb1,0xb1,0xb1,0xa,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa, +0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0x1000,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0x1000,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0x1000,0,0,0,0,0,0,0,0, +0,0,2,2,2,2,2,2,2,2,2,2,2,2,2,2, +2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, +2,2,2,2,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0x12,0x12,0x12,0xb2,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, +0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, +0x12,0x12,0x12,0x12,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2, +0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2, +0xb2,0xb2,0xb2,0xb2,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, +0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, +0x12,0x12,0x12,0x12,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1, +0xb1,0xb1,0xb1,0xb1,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, +0x12,0x12,0x12,0x12,0x820,0,0x840,0x860,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0x880,0x8a0,0,0,0,0,0,0,0,0,0,0x8c0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0x8c0, +0x8e0,0x900,0x900,0x900,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +}; + +static const uint32_t ubidi_props_mirrors[26]={ +0x2000ab,0xbb,0x2a02215,0x1202243,0x2802298,0x2c022a6,0x30022a8,0x2e022a9,0x32022ab,0x6022cd,0x1e022f2,0x20022f3,0x22022f4,0x24022f6,0x26022f7,0x14022fa, +0x16022fb,0x18022fc,0x1a022fd,0x1c022fe,0x8029b8,0x4029f5,0xa02ade,0xe02ae3,0xc02ae4,0x1002ae5 +}; + +static const uint8_t ubidi_props_jgArray[332]={ +3,3,0x2c,3,0x2d,3,4,0x2a,4,4,0xd,0xd,0xd,6,6,0x1f, +0x1f,0x23,0x23,0x21,0x21,0x28,0x28,1,1,0,0,0,0,0,0,9, +0x1d,0x13,0x16,0x18,0x1a,0x10,0x2c,0x2d,0x2d,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,4,0x1d,0,3, +3,3,0,3,0x2c,0x2c,0x2d,4,4,4,4,4,4,4,4,0xd, +0xd,0xd,0xd,0xd,0xd,0xd,6,6,6,6,6,6,6,6,6,0x1f, +0x1f,0x1f,0x1f,0x1f,0x1f,0x1f,0x1f,0x1f,0x23,0x23,0x23,0x21,0x21,0x28,1,9, +9,9,9,9,9,0x1d,0x1d,0xb,0x26,0xb,0x13,0x13,0x13,0xb,0xb,0xb, +0xb,0xb,0xb,0x16,0x16,0x16,0x16,0x1a,0x1a,0x1a,0x1a,0x1a,0x15,0xd,0x2a,0x11, +0x11,0xe,0x2c,0x2c,0x2c,0x2c,0x2c,0x2c,0x2c,0x2c,0x2d,0x2f,0x2d,0x2c,0x2d,0x2d, +0x2e,0x2e,0,0x2a,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,6,0x1f,0,0, +0,0,0,0,0,0,0,0,0x23,0x21,1,0,0,0x15,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0, +5,0xc,0xc,7,7,0xf,0x27,0x32,0x12,0x2b,0x2b,0x30,0x31,0x14,0x17,0x19, +0x1b,0x24,0xa,8,0x1c,0x20,0x22,0x1e,7,0x25,0x29,5,0xc,7,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0x35,0x34,0x33,4,4, +4,4,4,4,4,0xd,0xd,6,6,0x1f,0x23,1,1,1,9,9, +0xb,0xb,0xb,0x18,0x18,0x1a,0x1a,0x1a,0x16,0x1f,0x1f,0x23 +}; + +static const UBiDiProps ubidi_props_singleton={ + NULL, + ubidi_props_indexes, + ubidi_props_mirrors, + ubidi_props_jgArray, + { + ubidi_props_trieIndex, + NULL, + utrie_defaultGetFoldingOffset, + 2336, + 5472, + 0, + TRUE + }, + { 1,0,5,2 } +}; diff --git a/Build/source/libs/icu/icu-xetex/common/ubidiimp.h b/Build/source/libs/icu/icu-xetex/common/ubidiimp.h new file mode 100644 index 00000000000..1f1dcaa04f7 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ubidiimp.h @@ -0,0 +1,385 @@ +/* +****************************************************************************** +* +* Copyright (C) 1999-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: ubidiimp.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 1999aug06 +* created by: Markus W. Scherer, updated by Matitiahu Allouche +*/ + +#ifndef UBIDIIMP_H +#define UBIDIIMP_H + +/* set import/export definitions */ +#ifdef U_COMMON_IMPLEMENTATION + +#include "unicode/utypes.h" +#include "unicode/uchar.h" +#include "ubidi_props.h" + +/* miscellaneous definitions ---------------------------------------------- */ + +typedef uint8_t DirProp; +typedef uint32_t Flags; + +/* Comparing the description of the BiDi algorithm with this implementation + is easier with the same names for the BiDi types in the code as there. + See UCharDirection in uchar.h . +*/ +enum { + L= U_LEFT_TO_RIGHT, + R= U_RIGHT_TO_LEFT, + EN= U_EUROPEAN_NUMBER, + ES= U_EUROPEAN_NUMBER_SEPARATOR, + ET= U_EUROPEAN_NUMBER_TERMINATOR, + AN= U_ARABIC_NUMBER, + CS= U_COMMON_NUMBER_SEPARATOR, + B= U_BLOCK_SEPARATOR, + S= U_SEGMENT_SEPARATOR, + WS= U_WHITE_SPACE_NEUTRAL, + ON= U_OTHER_NEUTRAL, + LRE=U_LEFT_TO_RIGHT_EMBEDDING, + LRO=U_LEFT_TO_RIGHT_OVERRIDE, + AL= U_RIGHT_TO_LEFT_ARABIC, + RLE=U_RIGHT_TO_LEFT_EMBEDDING, + RLO=U_RIGHT_TO_LEFT_OVERRIDE, + PDF=U_POP_DIRECTIONAL_FORMAT, + NSM=U_DIR_NON_SPACING_MARK, + BN= U_BOUNDARY_NEUTRAL, + dirPropCount +}; + +/* + * Sometimes, bit values are more appropriate + * to deal with directionality properties. + * Abbreviations in these macro names refer to names + * used in the BiDi algorithm. + */ +#define DIRPROP_FLAG(dir) (1UL<<(dir)) + +/* special flag for multiple runs from explicit embedding codes */ +#define DIRPROP_FLAG_MULTI_RUNS (1UL<<31) + +/* are there any characters that are LTR or RTL? */ +#define MASK_LTR (DIRPROP_FLAG(L)|DIRPROP_FLAG(EN)|DIRPROP_FLAG(AN)|DIRPROP_FLAG(LRE)|DIRPROP_FLAG(LRO)) +#define MASK_RTL (DIRPROP_FLAG(R)|DIRPROP_FLAG(AL)|DIRPROP_FLAG(RLE)|DIRPROP_FLAG(RLO)) +#define MASK_R_AL (DIRPROP_FLAG(R)|DIRPROP_FLAG(AL)) + +/* explicit embedding codes */ +#define MASK_LRX (DIRPROP_FLAG(LRE)|DIRPROP_FLAG(LRO)) +#define MASK_RLX (DIRPROP_FLAG(RLE)|DIRPROP_FLAG(RLO)) +#define MASK_OVERRIDE (DIRPROP_FLAG(LRO)|DIRPROP_FLAG(RLO)) + +#define MASK_EXPLICIT (MASK_LRX|MASK_RLX|DIRPROP_FLAG(PDF)) +#define MASK_BN_EXPLICIT (DIRPROP_FLAG(BN)|MASK_EXPLICIT) + +/* paragraph and segment separators */ +#define MASK_B_S (DIRPROP_FLAG(B)|DIRPROP_FLAG(S)) + +/* all types that are counted as White Space or Neutral in some steps */ +#define MASK_WS (MASK_B_S|DIRPROP_FLAG(WS)|MASK_BN_EXPLICIT) +#define MASK_N (DIRPROP_FLAG(ON)|MASK_WS) + +/* all types that are included in a sequence of European Terminators for (W5) */ +#define MASK_ET_NSM_BN (DIRPROP_FLAG(ET)|DIRPROP_FLAG(NSM)|MASK_BN_EXPLICIT) + +/* types that are neutrals or could becomes neutrals in (Wn) */ +#define MASK_POSSIBLE_N (DIRPROP_FLAG(CS)|DIRPROP_FLAG(ES)|DIRPROP_FLAG(ET)|MASK_N) + +/* + * These types may be changed to "e", + * the embedding type (L or R) of the run, + * in the BiDi algorithm (N2) + */ +#define MASK_EMBEDDING (DIRPROP_FLAG(NSM)|MASK_POSSIBLE_N) + +/* the dirProp's L and R are defined to 0 and 1 values in UCharDirection */ +#define GET_LR_FROM_LEVEL(level) ((DirProp)((level)&1)) + +#define IS_DEFAULT_LEVEL(level) ((level)>=0xfe) + +/* + * The following bit is ORed to the property of characters in paragraphs + * with contextual RTL direction when paraLevel is contextual. + */ +#define CONTEXT_RTL 0x80 +#define NO_CONTEXT_RTL(dir) ((dir)&~CONTEXT_RTL) +/* + * The following is a variant of DIRPROP_FLAG which ignores the CONTEXT_RTL bit. + */ +#define DIRPROP_FLAG_NC(dir) (1UL<<(NO_CONTEXT_RTL(dir))) + +#define GET_PARALEVEL(ubidi, index) \ + (UBiDiLevel)((ubidi)->defaultParaLevel ? (ubidi)->dirProps[index]>>7 \ + : (ubidi)->paraLevel) + +/* Paragraph type for multiple paragraph support ---------------------------- */ +typedef int32_t Para; + +#define CR 0x000D +#define LF 0x000A + +/* Run structure for reordering --------------------------------------------- */ +enum { + LRM_BEFORE=1, + LRM_AFTER=2, + RLM_BEFORE=4, + RLM_AFTER=8 +}; + +typedef struct Run { + int32_t logicalStart, /* first character of the run; b31 indicates even/odd level */ + visualLimit, /* last visual position of the run +1 */ + insertRemove; /* if >0, flags for inserting LRM/RLM before/after run, + if <0, count of bidi controls within run */ +} Run; + +/* in a Run, logicalStart will get this bit set if the run level is odd */ +#define INDEX_ODD_BIT (1UL<<31) + +#define MAKE_INDEX_ODD_PAIR(index, level) ((index)|((int32_t)(level)<<31)) +#define ADD_ODD_BIT_FROM_LEVEL(x, level) ((x)|=((int32_t)(level)<<31)) +#define REMOVE_ODD_BIT(x) ((x)&=~INDEX_ODD_BIT) + +#define GET_INDEX(x) ((x)&~INDEX_ODD_BIT) +#define GET_ODD_BIT(x) ((uint32_t)(x)>>31) +#define IS_ODD_RUN(x) ((UBool)(((x)&INDEX_ODD_BIT)!=0)) +#define IS_EVEN_RUN(x) ((UBool)(((x)&INDEX_ODD_BIT)==0)) + +U_CFUNC UBool +ubidi_getRuns(UBiDi *pBiDi, UErrorCode *pErrorCode); + +/** BiDi control code points */ +enum { + ZWNJ_CHAR=0x200c, + ZWJ_CHAR, + LRM_CHAR, + RLM_CHAR, + LRE_CHAR=0x202a, + RLE_CHAR, + PDF_CHAR, + LRO_CHAR, + RLO_CHAR +}; + +#define IS_BIDI_CONTROL_CHAR(c) (((uint32_t)(c)&0xfffffffc)==ZWNJ_CHAR || (uint32_t)((c)-LRE_CHAR)<5) + +/* InsertPoints structure for noting where to put BiDi marks ---------------- */ + +typedef struct Point { + int32_t pos; /* position in text */ + int32_t flag; /* flag for LRM/RLM, before/after */ +} Point; + +typedef struct InsertPoints { + int32_t capacity; /* number of points allocated */ + int32_t size; /* number of points used */ + int32_t confirmed; /* number of points confirmed */ + UErrorCode errorCode; /* for eventual memory shortage */ + Point *points; /* pointer to array of points */ +} InsertPoints; + + +/* UBiDi structure ----------------------------------------------------------- */ + +struct UBiDi { + /* pointer to parent paragraph object (pointer to self if this object is + * a paragraph object); set to NULL in a newly opened object; set to a + * real value after a successful execution of ubidi_setPara or ubidi_setLine + */ + const UBiDi * pParaBiDi; + + const UBiDiProps *bdp; + + /* alias pointer to the current text */ + const UChar *text; + + /* length of the current text */ + int32_t originalLength; + + /* if the UBIDI_OPTION_STREAMING option is set, this is the length + * of text actually processed by ubidi_setPara, which may be shorter than + * the original length. + * Otherwise, it is identical to the original length. + */ + int32_t length; + + /* if the UBIDI_OPTION_REMOVE_CONTROLS option is set, and/or + * marks are allowed to be inserted in one of the reordering mode, the + * length of the result string may be different from the processed length. + */ + int32_t resultLength; + + /* memory sizes in bytes */ + int32_t dirPropsSize, levelsSize, parasSize, runsSize; + + /* allocated memory */ + DirProp *dirPropsMemory; + UBiDiLevel *levelsMemory; + Para *parasMemory; + Run *runsMemory; + + /* indicators for whether memory may be allocated after ubidi_open() */ + UBool mayAllocateText, mayAllocateRuns; + + /* arrays with one value per text-character */ + const DirProp *dirProps; + UBiDiLevel *levels; + + /* are we performing an approximation of the "inverse BiDi" algorithm? */ + UBool isInverse; + + /* are we using the basic algorithm or its variation? */ + UBiDiReorderingMode reorderingMode; + + /* UBIDI_REORDER_xxx values must be ordered so that all the regular + * logical to visual modes come first, and all inverse BiDi modes + * come last. + */ + #define UBIDI_REORDER_LAST_LOGICAL_TO_VISUAL UBIDI_REORDER_NUMBERS_SPECIAL + + /* bitmask for reordering options */ + uint32_t reorderingOptions; + + /* must block separators receive level 0? */ + UBool orderParagraphsLTR; + + /* the paragraph level */ + UBiDiLevel paraLevel; + /* original paraLevel when contextual */ + /* must be one of UBIDI_DEFAULT_xxx or 0 if not contextual */ + UBiDiLevel defaultParaLevel; + + /* the following is set in ubidi_setPara, used in processPropertySeq */ + const struct ImpTabPair * pImpTabPair; /* pointer to levels state table pair */ + + /* the overall paragraph or line directionality - see UBiDiDirection */ + UBiDiDirection direction; + + /* flags is a bit set for which directional properties are in the text */ + Flags flags; + + /* lastArabicPos is index to the last AL in the text, -1 if none */ + int32_t lastArabicPos; + + /* characters after trailingWSStart are WS and are */ + /* implicitly at the paraLevel (rule (L1)) - levels may not reflect that */ + int32_t trailingWSStart; + + /* fields for paragraph handling */ + int32_t paraCount; /* set in getDirProps() */ + Para *paras; /* limits of paragraphs, filled in + ResolveExplicitLevels() or CheckExplicitLevels() */ + + /* for single paragraph text, we only need a tiny array of paras (no malloc()) */ + Para simpleParas[1]; + + /* fields for line reordering */ + int32_t runCount; /* ==-1: runs not set up yet */ + Run *runs; + + /* for non-mixed text, we only need a tiny array of runs (no malloc()) */ + Run simpleRuns[1]; + + /* for inverse Bidi with insertion of directional marks */ + InsertPoints insertPoints; + + /* for option UBIDI_OPTION_REMOVE_CONTROLS */ + int32_t controlCount; + + /* for Bidi class callback */ + UBiDiClassCallback *fnClassCallback; /* action pointer */ + const void *coClassCallback; /* context pointer */ +}; + +#define IS_VALID_PARA(x) ((x) && ((x)->pParaBiDi==(x))) +#define IS_VALID_PARA_OR_LINE(x) ((x) && ((x)->pParaBiDi==(x) || (((x)->pParaBiDi) && (x)->pParaBiDi->pParaBiDi==(x)->pParaBiDi))) + +typedef union { + DirProp *dirPropsMemory; + UBiDiLevel *levelsMemory; + Para *parasMemory; + Run *runsMemory; +} BidiMemoryForAllocation; + +/* Macros for initial checks at function entry */ +#define RETURN_IF_NULL_OR_FAILING_ERRCODE(pErrcode, retvalue) \ + if((pErrcode)==NULL || U_FAILURE(*pErrcode)) return retvalue +#define RETURN_IF_NOT_VALID_PARA(bidi, errcode, retvalue) \ + if(!IS_VALID_PARA(bidi)) { \ + errcode=U_INVALID_STATE_ERROR; \ + return retvalue; \ + } +#define RETURN_IF_NOT_VALID_PARA_OR_LINE(bidi, errcode, retvalue) \ + if(!IS_VALID_PARA_OR_LINE(bidi)) { \ + errcode=U_INVALID_STATE_ERROR; \ + return retvalue; \ + } +#define RETURN_IF_BAD_RANGE(arg, start, limit, errcode, retvalue) \ + if((arg)<(start) || (arg)>=(limit)) { \ + (errcode)=U_ILLEGAL_ARGUMENT_ERROR; \ + return retvalue; \ + } + +#define RETURN_VOID_IF_NULL_OR_FAILING_ERRCODE(pErrcode) \ + if((pErrcode)==NULL || U_FAILURE(*pErrcode)) return +#define RETURN_VOID_IF_NOT_VALID_PARA(bidi, errcode) \ + if(!IS_VALID_PARA(bidi)) { \ + errcode=U_INVALID_STATE_ERROR; \ + return; \ + } +#define RETURN_VOID_IF_NOT_VALID_PARA_OR_LINE(bidi, errcode) \ + if(!IS_VALID_PARA_OR_LINE(bidi)) { \ + errcode=U_INVALID_STATE_ERROR; \ + return; \ + } +#define RETURN_VOID_IF_BAD_RANGE(arg, start, limit, errcode) \ + if((arg)<(start) || (arg)>=(limit)) { \ + (errcode)=U_ILLEGAL_ARGUMENT_ERROR; \ + return; \ + } + +/* helper function to (re)allocate memory if allowed */ +U_CFUNC UBool +ubidi_getMemory(BidiMemoryForAllocation *pMemory, int32_t *pSize, UBool mayAllocate, int32_t sizeNeeded); + +/* helper macros for each allocated array in UBiDi */ +#define getDirPropsMemory(pBiDi, length) \ + ubidi_getMemory((BidiMemoryForAllocation *)&(pBiDi)->dirPropsMemory, &(pBiDi)->dirPropsSize, \ + (pBiDi)->mayAllocateText, (length)) + +#define getLevelsMemory(pBiDi, length) \ + ubidi_getMemory((BidiMemoryForAllocation *)&(pBiDi)->levelsMemory, &(pBiDi)->levelsSize, \ + (pBiDi)->mayAllocateText, (length)) + +#define getRunsMemory(pBiDi, length) \ + ubidi_getMemory((BidiMemoryForAllocation *)&(pBiDi)->runsMemory, &(pBiDi)->runsSize, \ + (pBiDi)->mayAllocateRuns, (length)*sizeof(Run)) + +/* additional macros used by ubidi_open() - always allow allocation */ +#define getInitialDirPropsMemory(pBiDi, length) \ + ubidi_getMemory((BidiMemoryForAllocation *)&(pBiDi)->dirPropsMemory, &(pBiDi)->dirPropsSize, \ + TRUE, (length)) + +#define getInitialLevelsMemory(pBiDi, length) \ + ubidi_getMemory((BidiMemoryForAllocation *)&(pBiDi)->levelsMemory, &(pBiDi)->levelsSize, \ + TRUE, (length)) + +#define getInitialParasMemory(pBiDi, length) \ + ubidi_getMemory((BidiMemoryForAllocation *)&(pBiDi)->parasMemory, &(pBiDi)->parasSize, \ + TRUE, (length)*sizeof(Para)) + +#define getInitialRunsMemory(pBiDi, length) \ + ubidi_getMemory((BidiMemoryForAllocation *)&(pBiDi)->runsMemory, &(pBiDi)->runsSize, \ + TRUE, (length)*sizeof(Run)) + +#endif + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/ubidiln.c b/Build/source/libs/icu/icu-xetex/common/ubidiln.c new file mode 100644 index 00000000000..8cb079c2c3c --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ubidiln.c @@ -0,0 +1,1346 @@ +/* +****************************************************************************** +* +* Copyright (C) 1999-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: ubidiln.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 1999aug06 +* created by: Markus W. Scherer, updated by Matitiahu Allouche +*/ + +#include "cmemory.h" +#include "unicode/utypes.h" +#include "unicode/ustring.h" +#include "unicode/uchar.h" +#include "unicode/ubidi.h" +#include "ubidiimp.h" +#include "uassert.h" + +/* + * General remarks about the functions in this file: + * + * These functions deal with the aspects of potentially mixed-directional + * text in a single paragraph or in a line of a single paragraph + * which has already been processed according to + * the Unicode 3.0 BiDi algorithm as defined in + * http://www.unicode.org/unicode/reports/tr9/ , version 13, + * also described in The Unicode Standard, Version 4.0.1 . + * + * This means that there is a UBiDi object with a levels + * and a dirProps array. + * paraLevel and direction are also set. + * Only if the length of the text is zero, then levels==dirProps==NULL. + * + * The overall directionality of the paragraph + * or line is used to bypass the reordering steps if possible. + * Even purely RTL text does not need reordering there because + * the ubidi_getLogical/VisualIndex() functions can compute the + * index on the fly in such a case. + * + * The implementation of the access to same-level-runs and of the reordering + * do attempt to provide better performance and less memory usage compared to + * a direct implementation of especially rule (L2) with an array of + * one (32-bit) integer per text character. + * + * Here, the levels array is scanned as soon as necessary, and a vector of + * same-level-runs is created. Reordering then is done on this vector. + * For each run of text positions that were resolved to the same level, + * only 8 bytes are stored: the first text position of the run and the visual + * position behind the run after reordering. + * One sign bit is used to hold the directionality of the run. + * This is inefficient if there are many very short runs. If the average run + * length is <2, then this uses more memory. + * + * In a further attempt to save memory, the levels array is never changed + * after all the resolution rules (Xn, Wn, Nn, In). + * Many functions have to consider the field trailingWSStart: + * if it is less than length, then there is an implicit trailing run + * at the paraLevel, + * which is not reflected in the levels array. + * This allows a line UBiDi object to use the same levels array as + * its paragraph parent object. + * + * When a UBiDi object is created for a line of a paragraph, then the + * paragraph's levels and dirProps arrays are reused by way of setting + * a pointer into them, not by copying. This again saves memory and forbids to + * change the now shared levels for (L1). + */ + +/* handle trailing WS (L1) -------------------------------------------------- */ + +/* + * setTrailingWSStart() sets the start index for a trailing + * run of WS in the line. This is necessary because we do not modify + * the paragraph's levels array that we just point into. + * Using trailingWSStart is another form of performing (L1). + * + * To make subsequent operations easier, we also include the run + * before the WS if it is at the paraLevel - we merge the two here. + * + * This function is called only from ubidi_setLine(), so pBiDi->paraLevel is + * set correctly for the line even when contextual multiple paragraphs. + */ +static void +setTrailingWSStart(UBiDi *pBiDi) { + /* pBiDi->direction!=UBIDI_MIXED */ + + const DirProp *dirProps=pBiDi->dirProps; + UBiDiLevel *levels=pBiDi->levels; + int32_t start=pBiDi->length; + UBiDiLevel paraLevel=pBiDi->paraLevel; + + /* If the line is terminated by a block separator, all preceding WS etc... + are already set to paragraph level. + Setting trailingWSStart to pBidi->length will avoid changing the + level of B chars from 0 to paraLevel in ubidi_getLevels when + orderParagraphsLTR==TRUE. + */ + if(NO_CONTEXT_RTL(dirProps[start-1])==B) { + pBiDi->trailingWSStart=start; /* currently == pBiDi->length */ + return; + } + /* go backwards across all WS, BN, explicit codes */ + while(start>0 && DIRPROP_FLAG_NC(dirProps[start-1])&MASK_WS) { + --start; + } + + /* if the WS run can be merged with the previous run then do so here */ + while(start>0 && levels[start-1]==paraLevel) { + --start; + } + + pBiDi->trailingWSStart=start; +} + +/* ubidi_setLine ------------------------------------------------------------ */ + +U_CAPI void U_EXPORT2 +ubidi_setLine(const UBiDi *pParaBiDi, + int32_t start, int32_t limit, + UBiDi *pLineBiDi, + UErrorCode *pErrorCode) { + int32_t length; + + /* check the argument values */ + RETURN_VOID_IF_NULL_OR_FAILING_ERRCODE(pErrorCode); + RETURN_VOID_IF_NOT_VALID_PARA(pParaBiDi, *pErrorCode); + RETURN_VOID_IF_BAD_RANGE(start, 0, limit, *pErrorCode); + RETURN_VOID_IF_BAD_RANGE(limit, 0, pParaBiDi->length+1, *pErrorCode); + if(pLineBiDi==NULL) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return; + } + if(ubidi_getParagraph(pParaBiDi, start, NULL, NULL, NULL, pErrorCode) != + ubidi_getParagraph(pParaBiDi, limit-1, NULL, NULL, NULL, pErrorCode)) { + /* the line crosses a paragraph boundary */ + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + /* set the values in pLineBiDi from its pParaBiDi parent */ + pLineBiDi->pParaBiDi=NULL; /* mark unfinished setLine */ + pLineBiDi->text=pParaBiDi->text+start; + length=pLineBiDi->length=limit-start; + pLineBiDi->resultLength=pLineBiDi->originalLength=length; + pLineBiDi->paraLevel=GET_PARALEVEL(pParaBiDi, start); + pLineBiDi->paraCount=pParaBiDi->paraCount; + pLineBiDi->runs=NULL; + pLineBiDi->flags=0; + pLineBiDi->reorderingMode=pParaBiDi->reorderingMode; + pLineBiDi->reorderingOptions=pParaBiDi->reorderingOptions; + pLineBiDi->controlCount=0; + if(pParaBiDi->controlCount>0) { + int32_t j; + for(j=start; jtext[j])) { + pLineBiDi->controlCount++; + } + } + pLineBiDi->resultLength-=pLineBiDi->controlCount; + } + + pLineBiDi->dirProps=pParaBiDi->dirProps+start; + pLineBiDi->levels=pParaBiDi->levels+start; + pLineBiDi->runCount=-1; + + if(pParaBiDi->direction!=UBIDI_MIXED) { + /* the parent is already trivial */ + pLineBiDi->direction=pParaBiDi->direction; + + /* + * The parent's levels are all either + * implicitly or explicitly ==paraLevel; + * do the same here. + */ + if(pParaBiDi->trailingWSStart<=start) { + pLineBiDi->trailingWSStart=0; + } else if(pParaBiDi->trailingWSStarttrailingWSStart=pParaBiDi->trailingWSStart-start; + } else { + pLineBiDi->trailingWSStart=length; + } + } else { + const UBiDiLevel *levels=pLineBiDi->levels; + int32_t i, trailingWSStart; + UBiDiLevel level; + + setTrailingWSStart(pLineBiDi); + trailingWSStart=pLineBiDi->trailingWSStart; + + /* recalculate pLineBiDi->direction */ + if(trailingWSStart==0) { + /* all levels are at paraLevel */ + pLineBiDi->direction=(UBiDiDirection)(pLineBiDi->paraLevel&1); + } else { + /* get the level of the first character */ + level=(UBiDiLevel)(levels[0]&1); + + /* if there is anything of a different level, then the line is mixed */ + if(trailingWSStartparaLevel&1)!=level) { + /* the trailing WS is at paraLevel, which differs from levels[0] */ + pLineBiDi->direction=UBIDI_MIXED; + } else { + /* see if levels[1..trailingWSStart-1] have the same direction as levels[0] and paraLevel */ + i=1; + for(;;) { + if(i==trailingWSStart) { + /* the direction values match those in level */ + pLineBiDi->direction=(UBiDiDirection)level; + break; + } else if((levels[i]&1)!=level) { + pLineBiDi->direction=UBIDI_MIXED; + break; + } + ++i; + } + } + } + + switch(pLineBiDi->direction) { + case UBIDI_LTR: + /* make sure paraLevel is even */ + pLineBiDi->paraLevel=(UBiDiLevel)((pLineBiDi->paraLevel+1)&~1); + + /* all levels are implicitly at paraLevel (important for ubidi_getLevels()) */ + pLineBiDi->trailingWSStart=0; + break; + case UBIDI_RTL: + /* make sure paraLevel is odd */ + pLineBiDi->paraLevel|=1; + + /* all levels are implicitly at paraLevel (important for ubidi_getLevels()) */ + pLineBiDi->trailingWSStart=0; + break; + default: + break; + } + } + pLineBiDi->pParaBiDi=pParaBiDi; /* mark successful setLine */ + return; +} + +U_CAPI UBiDiLevel U_EXPORT2 +ubidi_getLevelAt(const UBiDi *pBiDi, int32_t charIndex) { + /* return paraLevel if in the trailing WS run, otherwise the real level */ + if(!IS_VALID_PARA_OR_LINE(pBiDi) || charIndex<0 || pBiDi->length<=charIndex) { + return 0; + } else if(pBiDi->direction!=UBIDI_MIXED || charIndex>=pBiDi->trailingWSStart) { + return GET_PARALEVEL(pBiDi, charIndex); + } else { + return pBiDi->levels[charIndex]; + } +} + +U_CAPI const UBiDiLevel * U_EXPORT2 +ubidi_getLevels(UBiDi *pBiDi, UErrorCode *pErrorCode) { + int32_t start, length; + + RETURN_IF_NULL_OR_FAILING_ERRCODE(pErrorCode, NULL); + RETURN_IF_NOT_VALID_PARA_OR_LINE(pBiDi, *pErrorCode, NULL); + if((length=pBiDi->length)<=0) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + if((start=pBiDi->trailingWSStart)==length) { + /* the current levels array reflects the WS run */ + return pBiDi->levels; + } + + /* + * After the previous if(), we know that the levels array + * has an implicit trailing WS run and therefore does not fully + * reflect itself all the levels. + * This must be a UBiDi object for a line, and + * we need to create a new levels array. + */ + if(getLevelsMemory(pBiDi, length)) { + UBiDiLevel *levels=pBiDi->levelsMemory; + + if(start>0 && levels!=pBiDi->levels) { + uprv_memcpy(levels, pBiDi->levels, start); + } + /* pBiDi->paraLevel is ok even if contextual multiple paragraphs, + since pBidi is a line object */ + uprv_memset(levels+start, pBiDi->paraLevel, length-start); + + /* this new levels array is set for the line and reflects the WS run */ + pBiDi->trailingWSStart=length; + return pBiDi->levels=levels; + } else { + /* out of memory */ + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return NULL; + } +} + +U_CAPI void U_EXPORT2 +ubidi_getLogicalRun(const UBiDi *pBiDi, int32_t logicalPosition, + int32_t *pLogicalLimit, UBiDiLevel *pLevel) { + UErrorCode errorCode; + int32_t runCount, visualStart, logicalLimit, logicalFirst, i; + Run iRun; + + errorCode=U_ZERO_ERROR; + RETURN_VOID_IF_BAD_RANGE(logicalPosition, 0, pBiDi->length, errorCode); + /* ubidi_countRuns will check VALID_PARA_OR_LINE */ + runCount=ubidi_countRuns((UBiDi *)pBiDi, &errorCode); + if(U_FAILURE(errorCode)) { + return; + } + /* this is done based on runs rather than on levels since levels have + a special interpretation when UBIDI_REORDER_RUNS_ONLY + */ + visualStart=logicalLimit=0; + iRun=pBiDi->runs[0]; + + for(i=0; iruns[i]; + logicalFirst=GET_INDEX(iRun.logicalStart); + logicalLimit=logicalFirst+iRun.visualLimit-visualStart; + if((logicalPosition>=logicalFirst) && + (logicalPositionreorderingMode==UBIDI_REORDER_RUNS_ONLY) { + *pLevel=(UBiDiLevel)GET_ODD_BIT(iRun.logicalStart); + } + else if(pBiDi->direction!=UBIDI_MIXED || logicalPosition>=pBiDi->trailingWSStart) { + *pLevel=GET_PARALEVEL(pBiDi, logicalPosition); + } else { + *pLevel=pBiDi->levels[logicalPosition]; + } + } +} + +/* runs API functions ------------------------------------------------------- */ + +U_CAPI int32_t U_EXPORT2 +ubidi_countRuns(UBiDi *pBiDi, UErrorCode *pErrorCode) { + RETURN_IF_NULL_OR_FAILING_ERRCODE(pErrorCode, -1); + RETURN_IF_NOT_VALID_PARA_OR_LINE(pBiDi, *pErrorCode, -1); + ubidi_getRuns(pBiDi, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return -1; + } + return pBiDi->runCount; +} + +U_CAPI UBiDiDirection U_EXPORT2 +ubidi_getVisualRun(UBiDi *pBiDi, int32_t runIndex, + int32_t *pLogicalStart, int32_t *pLength) +{ + int32_t start; + UErrorCode errorCode = U_ZERO_ERROR; + RETURN_IF_NOT_VALID_PARA_OR_LINE(pBiDi, errorCode, UBIDI_LTR); + ubidi_getRuns(pBiDi, &errorCode); + if(U_FAILURE(errorCode)) { + return UBIDI_LTR; + } + RETURN_IF_BAD_RANGE(runIndex, 0, pBiDi->runCount, errorCode, UBIDI_LTR); + + start=pBiDi->runs[runIndex].logicalStart; + if(pLogicalStart!=NULL) { + *pLogicalStart=GET_INDEX(start); + } + if(pLength!=NULL) { + if(runIndex>0) { + *pLength=pBiDi->runs[runIndex].visualLimit- + pBiDi->runs[runIndex-1].visualLimit; + } else { + *pLength=pBiDi->runs[0].visualLimit; + } + } + return (UBiDiDirection)GET_ODD_BIT(start); +} + +/* in trivial cases there is only one trivial run; called by ubidi_getRuns() */ +static void +getSingleRun(UBiDi *pBiDi, UBiDiLevel level) { + /* simple, single-run case */ + pBiDi->runs=pBiDi->simpleRuns; + pBiDi->runCount=1; + + /* fill and reorder the single run */ + pBiDi->runs[0].logicalStart=MAKE_INDEX_ODD_PAIR(0, level); + pBiDi->runs[0].visualLimit=pBiDi->length; + pBiDi->runs[0].insertRemove=0; +} + +/* reorder the runs array (L2) ---------------------------------------------- */ + +/* + * Reorder the same-level runs in the runs array. + * Here, runCount>1 and maxLevel>=minLevel>=paraLevel. + * All the visualStart fields=logical start before reordering. + * The "odd" bits are not set yet. + * + * Reordering with this data structure lends itself to some handy shortcuts: + * + * Since each run is moved but not modified, and since at the initial maxLevel + * each sequence of same-level runs consists of only one run each, we + * don't need to do anything there and can predecrement maxLevel. + * In many simple cases, the reordering is thus done entirely in the + * index mapping. + * Also, reordering occurs only down to the lowest odd level that occurs, + * which is minLevel|1. However, if the lowest level itself is odd, then + * in the last reordering the sequence of the runs at this level or higher + * will be all runs, and we don't need the elaborate loop to search for them. + * This is covered by ++minLevel instead of minLevel|=1 followed + * by an extra reorder-all after the reorder-some loop. + * About a trailing WS run: + * Such a run would need special treatment because its level is not + * reflected in levels[] if this is not a paragraph object. + * Instead, all characters from trailingWSStart on are implicitly at + * paraLevel. + * However, for all maxLevel>paraLevel, this run will never be reordered + * and does not need to be taken into account. maxLevel==paraLevel is only reordered + * if minLevel==paraLevel is odd, which is done in the extra segment. + * This means that for the main reordering loop we don't need to consider + * this run and can --runCount. If it is later part of the all-runs + * reordering, then runCount is adjusted accordingly. + */ +static void +reorderLine(UBiDi *pBiDi, UBiDiLevel minLevel, UBiDiLevel maxLevel) { + Run *runs, tempRun; + UBiDiLevel *levels; + int32_t firstRun, endRun, limitRun, runCount; + + /* nothing to do? */ + if(maxLevel<=(minLevel|1)) { + return; + } + + /* + * Reorder only down to the lowest odd level + * and reorder at an odd minLevel in a separate, simpler loop. + * See comments above for why minLevel is always incremented. + */ + ++minLevel; + + runs=pBiDi->runs; + levels=pBiDi->levels; + runCount=pBiDi->runCount; + + /* do not include the WS run at paraLevel<=old minLevel except in the simple loop */ + if(pBiDi->trailingWSStartlength) { + --runCount; + } + + while(--maxLevel>=minLevel) { + firstRun=0; + + /* loop for all sequences of runs */ + for(;;) { + /* look for a sequence of runs that are all at >=maxLevel */ + /* look for the first run of such a sequence */ + while(firstRun=runCount) { + break; /* no more such runs */ + } + + /* look for the limit run of such a sequence (the run behind it) */ + for(limitRun=firstRun; ++limitRun=maxLevel;) {} + + /* Swap the entire sequence of runs from firstRun to limitRun-1. */ + endRun=limitRun-1; + while(firstRuntrailingWSStart==pBiDi->length) { + --runCount; + } + + /* Swap the entire sequence of all runs. (endRun==runCount) */ + while(firstRunruns; + int32_t runCount=pBiDi->runCount, visualStart=0, i, length, logicalStart; + + for(i=0; i=logicalStart) && (logicalIndex<(logicalStart+length))) { + return i; + } + visualStart+=length; + } + /* we should never get here */ + U_ASSERT(FALSE); + *pErrorCode = U_INVALID_STATE_ERROR; + return 0; +} + +/* + * Compute the runs array from the levels array. + * After ubidi_getRuns() returns TRUE, runCount is guaranteed to be >0 + * and the runs are reordered. + * Odd-level runs have visualStart on their visual right edge and + * they progress visually to the left. + * If option UBIDI_OPTION_INSERT_MARKS is set, insertRemove will contain the + * sum of appropriate LRM/RLM_BEFORE/AFTER flags. + * If option UBIDI_OPTION_REMOVE_CONTROLS is set, insertRemove will contain the + * negative number of BiDi control characters within this run. + */ +U_CFUNC UBool +ubidi_getRuns(UBiDi *pBiDi, UErrorCode *pErrorCode) { + /* + * This method returns immediately if the runs are already set. This + * includes the case of length==0 (handled in setPara).. + */ + if (pBiDi->runCount>=0) { + return TRUE; + } + + if(pBiDi->direction!=UBIDI_MIXED) { + /* simple, single-run case - this covers length==0 */ + /* pBiDi->paraLevel is ok even for contextual multiple paragraphs */ + getSingleRun(pBiDi, pBiDi->paraLevel); + } else /* UBIDI_MIXED, length>0 */ { + /* mixed directionality */ + int32_t length=pBiDi->length, limit; + UBiDiLevel *levels=pBiDi->levels; + int32_t i, runCount; + UBiDiLevel level=UBIDI_DEFAULT_LTR; /* initialize with no valid level */ + /* + * If there are WS characters at the end of the line + * and the run preceding them has a level different from + * paraLevel, then they will form their own run at paraLevel (L1). + * Count them separately. + * We need some special treatment for this in order to not + * modify the levels array which a line UBiDi object shares + * with its paragraph parent and its other line siblings. + * In other words, for the trailing WS, it may be + * levels[]!=paraLevel but we have to treat it like it were so. + */ + limit=pBiDi->trailingWSStart; + /* count the runs, there is at least one non-WS run, and limit>0 */ + runCount=0; + for(i=0; i1 || limit1 */ + if(getRunsMemory(pBiDi, runCount)) { + runs=pBiDi->runsMemory; + } else { + return FALSE; + } + + /* set the runs */ + /* FOOD FOR THOUGHT: this could be optimized, e.g.: + * 464->444, 484->444, 575->555, 595->555 + * However, that would take longer. Check also how it would + * interact with BiDi control removal and inserting Marks. + */ + runIndex=0; + + /* search for the run limits and initialize visualLimit values with the run lengths */ + i=0; + do { + /* prepare this run */ + start=i; + level=levels[i]; + if(levelmaxLevel) { + maxLevel=level; + } + + /* look for the run limit */ + while(++iparaLevel is ok even + if contextual multiple paragraphs. */ + if(pBiDi->paraLevelparaLevel; + } + } + + /* set the object fields */ + pBiDi->runs=runs; + pBiDi->runCount=runCount; + + reorderLine(pBiDi, minLevel, maxLevel); + + /* now add the direction flags and adjust the visualLimit's to be just that */ + /* this loop will also handle the trailing WS run */ + limit=0; + for(i=0; iparaLevel is ok even if + contextual multiple paragraphs. */ + if(runIndexparaLevel & 1) != 0)? 0 : runIndex; + + ADD_ODD_BIT_FROM_LEVEL(runs[trailingRun].logicalStart, pBiDi->paraLevel); + } + } + } + + /* handle insert LRM/RLM BEFORE/AFTER run */ + if(pBiDi->insertPoints.size>0) { + Point *point, *start=pBiDi->insertPoints.points, + *limit=start+pBiDi->insertPoints.size; + int32_t runIndex; + for(point=start; pointpos, pErrorCode); + pBiDi->runs[runIndex].insertRemove|=point->flag; + } + } + + /* handle remove BiDi control characters */ + if(pBiDi->controlCount>0) { + int32_t runIndex; + const UChar *start=pBiDi->text, *limit=start+pBiDi->length, *pu; + for(pu=start; puruns[runIndex].insertRemove--; + } + } + } + + return TRUE; +} + +static UBool +prepareReorder(const UBiDiLevel *levels, int32_t length, + int32_t *indexMap, + UBiDiLevel *pMinLevel, UBiDiLevel *pMaxLevel) { + int32_t start; + UBiDiLevel level, minLevel, maxLevel; + + if(levels==NULL || length<=0) { + return FALSE; + } + + /* determine minLevel and maxLevel */ + minLevel=UBIDI_MAX_EXPLICIT_LEVEL+1; + maxLevel=0; + for(start=length; start>0;) { + level=levels[--start]; + if(level>UBIDI_MAX_EXPLICIT_LEVEL+1) { + return FALSE; + } + if(levelmaxLevel) { + maxLevel=level; + } + } + *pMinLevel=minLevel; + *pMaxLevel=maxLevel; + + /* initialize the index map */ + for(start=length; start>0;) { + --start; + indexMap[start]=start; + } + + return TRUE; +} + +/* reorder a line based on a levels array (L2) ------------------------------ */ + +U_CAPI void U_EXPORT2 +ubidi_reorderLogical(const UBiDiLevel *levels, int32_t length, int32_t *indexMap) { + int32_t start, limit, sumOfSosEos; + UBiDiLevel minLevel, maxLevel; + + if(indexMap==NULL || !prepareReorder(levels, length, indexMap, &minLevel, &maxLevel)) { + return; + } + + /* nothing to do? */ + if(minLevel==maxLevel && (minLevel&1)==0) { + return; + } + + /* reorder only down to the lowest odd level */ + minLevel|=1; + + /* loop maxLevel..minLevel */ + do { + start=0; + + /* loop for all sequences of levels to reorder at the current maxLevel */ + for(;;) { + /* look for a sequence of levels that are all at >=maxLevel */ + /* look for the first index of such a sequence */ + while(start=length) { + break; /* no more such sequences */ + } + + /* look for the limit of such a sequence (the index behind it) */ + for(limit=start; ++limit=maxLevel;) {} + + /* + * sos=start of sequence, eos=end of sequence + * + * The closed (inclusive) interval from sos to eos includes all the logical + * and visual indexes within this sequence. They are logically and + * visually contiguous and in the same range. + * + * For each run, the new visual index=sos+eos-old visual index; + * we pre-add sos+eos into sumOfSosEos -> + * new visual index=sumOfSosEos-old visual index; + */ + sumOfSosEos=start+limit-1; + + /* reorder each index in the sequence */ + do { + indexMap[start]=sumOfSosEos-indexMap[start]; + } while(++start=minLevel); +} + +U_CAPI void U_EXPORT2 +ubidi_reorderVisual(const UBiDiLevel *levels, int32_t length, int32_t *indexMap) { + int32_t start, end, limit, temp; + UBiDiLevel minLevel, maxLevel; + + if(indexMap==NULL || !prepareReorder(levels, length, indexMap, &minLevel, &maxLevel)) { + return; + } + + /* nothing to do? */ + if(minLevel==maxLevel && (minLevel&1)==0) { + return; + } + + /* reorder only down to the lowest odd level */ + minLevel|=1; + + /* loop maxLevel..minLevel */ + do { + start=0; + + /* loop for all sequences of levels to reorder at the current maxLevel */ + for(;;) { + /* look for a sequence of levels that are all at >=maxLevel */ + /* look for the first index of such a sequence */ + while(start=length) { + break; /* no more such runs */ + } + + /* look for the limit of such a sequence (the index behind it) */ + for(limit=start; ++limit=maxLevel;) {} + + /* + * Swap the entire interval of indexes from start to limit-1. + * We don't need to swap the levels for the purpose of this + * algorithm: the sequence of levels that we look at does not + * move anyway. + */ + end=limit-1; + while(start=minLevel); +} + +/* API functions for logical<->visual mapping ------------------------------- */ + +U_CAPI int32_t U_EXPORT2 +ubidi_getVisualIndex(UBiDi *pBiDi, int32_t logicalIndex, UErrorCode *pErrorCode) { + int32_t visualIndex=UBIDI_MAP_NOWHERE; + RETURN_IF_NULL_OR_FAILING_ERRCODE(pErrorCode, -1); + RETURN_IF_NOT_VALID_PARA_OR_LINE(pBiDi, *pErrorCode, -1); + RETURN_IF_BAD_RANGE(logicalIndex, 0, pBiDi->length, *pErrorCode, -1); + + /* we can do the trivial cases without the runs array */ + switch(pBiDi->direction) { + case UBIDI_LTR: + visualIndex=logicalIndex; + break; + case UBIDI_RTL: + visualIndex=pBiDi->length-logicalIndex-1; + break; + default: + if(!ubidi_getRuns(pBiDi, pErrorCode)) { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return -1; + } else { + Run *runs=pBiDi->runs; + int32_t i, visualStart=0, offset, length; + + /* linear search for the run, search on the visual runs */ + for(i=0; irunCount; ++i) { + length=runs[i].visualLimit-visualStart; + offset=logicalIndex-GET_INDEX(runs[i].logicalStart); + if(offset>=0 && offset=pBiDi->runCount) { + return UBIDI_MAP_NOWHERE; + } + } + } + + if(pBiDi->insertPoints.size>0) { + /* add the number of added marks until the calculated visual index */ + Run *runs=pBiDi->runs; + int32_t i, length, insertRemove; + int32_t visualStart=0, markFound=0; + for(i=0; ; i++, visualStart+=length) { + length=runs[i].visualLimit-visualStart; + insertRemove=runs[i].insertRemove; + if(insertRemove & (LRM_BEFORE|RLM_BEFORE)) { + markFound++; + } + /* is it the run containing the visual index? */ + if(visualIndexcontrolCount>0) { + /* subtract the number of controls until the calculated visual index */ + Run *runs=pBiDi->runs; + int32_t i, j, start, limit, length, insertRemove; + int32_t visualStart=0, controlFound=0; + UChar uchar=pBiDi->text[logicalIndex]; + /* is the logical index pointing to a control ? */ + if(IS_BIDI_CONTROL_CHAR(uchar)) { + return UBIDI_MAP_NOWHERE; + } + /* loop on runs */ + for(i=0; ; i++, visualStart+=length) { + length=runs[i].visualLimit-visualStart; + insertRemove=runs[i].insertRemove; + /* calculated visual index is beyond this run? */ + if(visualIndex>=runs[i].visualLimit) { + controlFound-=insertRemove; + continue; + } + /* calculated visual index must be within current run */ + if(insertRemove==0) { + return visualIndex-controlFound; + } + if(IS_EVEN_RUN(runs[i].logicalStart)) { + /* LTR: check from run start to logical index */ + start=runs[i].logicalStart; + limit=logicalIndex; + } else { + /* RTL: check from logical index to run end */ + start=logicalIndex+1; + limit=GET_INDEX(runs[i].logicalStart)+length; + } + for(j=start; jtext[j]; + if(IS_BIDI_CONTROL_CHAR(uchar)) { + controlFound++; + } + } + return visualIndex-controlFound; + } + } + + return visualIndex; +} + +U_CAPI int32_t U_EXPORT2 +ubidi_getLogicalIndex(UBiDi *pBiDi, int32_t visualIndex, UErrorCode *pErrorCode) { + Run *runs; + int32_t i, runCount, start; + RETURN_IF_NULL_OR_FAILING_ERRCODE(pErrorCode, -1); + RETURN_IF_NOT_VALID_PARA_OR_LINE(pBiDi, *pErrorCode, -1); + RETURN_IF_BAD_RANGE(visualIndex, 0, pBiDi->resultLength, *pErrorCode, -1); + /* we can do the trivial cases without the runs array */ + if(pBiDi->insertPoints.size==0 && pBiDi->controlCount==0) { + if(pBiDi->direction==UBIDI_LTR) { + return visualIndex; + } + else if(pBiDi->direction==UBIDI_RTL) { + return pBiDi->length-visualIndex-1; + } + } + if(!ubidi_getRuns(pBiDi, pErrorCode)) { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return -1; + } + + runs=pBiDi->runs; + runCount=pBiDi->runCount; + if(pBiDi->insertPoints.size>0) { + /* handle inserted LRM/RLM */ + int32_t markFound=0, insertRemove; + int32_t visualStart=0, length; + runs=pBiDi->runs; + /* subtract number of marks until visual index */ + for(i=0; ; i++, visualStart+=length) { + length=runs[i].visualLimit-visualStart; + insertRemove=runs[i].insertRemove; + if(insertRemove&(LRM_BEFORE|RLM_BEFORE)) { + if(visualIndex<=(visualStart+markFound)) { + return UBIDI_MAP_NOWHERE; + } + markFound++; + } + /* is adjusted visual index within this run? */ + if(visualIndex<(runs[i].visualLimit+markFound)) { + visualIndex-=markFound; + break; + } + if(insertRemove&(LRM_AFTER|RLM_AFTER)) { + if(visualIndex==(visualStart+length+markFound)) { + return UBIDI_MAP_NOWHERE; + } + markFound++; + } + } + } + else if(pBiDi->controlCount>0) { + /* handle removed BiDi control characters */ + int32_t controlFound=0, insertRemove, length; + int32_t logicalStart, logicalEnd, visualStart=0, j, k; + UChar uchar; + UBool evenRun; + /* add number of controls until visual index */ + for(i=0; ; i++, visualStart+=length) { + length=runs[i].visualLimit-visualStart; + insertRemove=runs[i].insertRemove; + /* is adjusted visual index beyond current run? */ + if(visualIndex>=(runs[i].visualLimit-controlFound+insertRemove)) { + controlFound-=insertRemove; + continue; + } + /* adjusted visual index is within current run */ + if(insertRemove==0) { + visualIndex+=controlFound; + break; + } + /* count non-control chars until visualIndex */ + logicalStart=runs[i].logicalStart; + evenRun=IS_EVEN_RUN(logicalStart); + REMOVE_ODD_BIT(logicalStart); + logicalEnd=logicalStart+length-1; + for(j=0; jtext[k]; + if(IS_BIDI_CONTROL_CHAR(uchar)) { + controlFound++; + } + if((visualIndex+controlFound)==(visualStart+j)) { + break; + } + } + visualIndex+=controlFound; + break; + } + } + /* handle all cases */ + if(runCount<=10) { + /* linear search for the run */ + for(i=0; visualIndex>=runs[i].visualLimit; ++i) {} + } else { + /* binary search for the run */ + int32_t begin=0, limit=runCount; + + /* the middle if() is guaranteed to find the run, we don't need a loop limit */ + for(;;) { + i=(begin+limit)/2; + if(visualIndex>=runs[i].visualLimit) { + begin=i+1; + } else if(i==0 || visualIndex>=runs[i-1].visualLimit) { + break; + } else { + limit=i; + } + } + } + + start=runs[i].logicalStart; + if(IS_EVEN_RUN(start)) { + /* LTR */ + /* the offset in runs[i] is visualIndex-runs[i-1].visualLimit */ + if(i>0) { + visualIndex-=runs[i-1].visualLimit; + } + return start+visualIndex; + } else { + /* RTL */ + return GET_INDEX(start)+runs[i].visualLimit-visualIndex-1; + } +} + +U_CAPI void U_EXPORT2 +ubidi_getLogicalMap(UBiDi *pBiDi, int32_t *indexMap, UErrorCode *pErrorCode) { + RETURN_VOID_IF_NULL_OR_FAILING_ERRCODE(pErrorCode); + /* ubidi_countRuns() checks for VALID_PARA_OR_LINE */ + ubidi_countRuns(pBiDi, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + /* no op */ + } else if(indexMap==NULL) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + } else { + /* fill a logical-to-visual index map using the runs[] */ + int32_t visualStart, visualLimit, i, j, k; + int32_t logicalStart, logicalLimit; + Run *runs=pBiDi->runs; + if (pBiDi->length<=0) { + return; + } + if (pBiDi->length>pBiDi->resultLength) { + uprv_memset(indexMap, 0xFF, pBiDi->length*sizeof(int32_t)); + } + + visualStart=0; + for(j=0; jrunCount; ++j) { + logicalStart=GET_INDEX(runs[j].logicalStart); + visualLimit=runs[j].visualLimit; + if(IS_EVEN_RUN(runs[j].logicalStart)) { + do { /* LTR */ + indexMap[logicalStart++]=visualStart++; + } while(visualStartinsertPoints.size>0) { + int32_t markFound=0, runCount=pBiDi->runCount; + int32_t length, insertRemove; + visualStart=0; + /* add number of marks found until each index */ + for(i=0; i0) { + logicalStart=GET_INDEX(runs[i].logicalStart); + logicalLimit=logicalStart+length; + for(j=logicalStart; jcontrolCount>0) { + int32_t controlFound=0, runCount=pBiDi->runCount; + int32_t length, insertRemove; + UBool evenRun; + UChar uchar; + visualStart=0; + /* subtract number of controls found until each index */ + for(i=0; itext[k]; + if(IS_BIDI_CONTROL_CHAR(uchar)) { + controlFound++; + indexMap[k]=UBIDI_MAP_NOWHERE; + continue; + } + indexMap[k]-=controlFound; + } + } + } + } +} + +U_CAPI void U_EXPORT2 +ubidi_getVisualMap(UBiDi *pBiDi, int32_t *indexMap, UErrorCode *pErrorCode) { + RETURN_VOID_IF_NULL_OR_FAILING_ERRCODE(pErrorCode); + if(indexMap==NULL) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return; + } + /* ubidi_countRuns() checks for VALID_PARA_OR_LINE */ + ubidi_countRuns(pBiDi, pErrorCode); + if(U_SUCCESS(*pErrorCode)) { + /* fill a visual-to-logical index map using the runs[] */ + Run *runs=pBiDi->runs, *runsLimit=runs+pBiDi->runCount; + int32_t logicalStart, visualStart, visualLimit, *pi=indexMap; + + if (pBiDi->resultLength<=0) { + return; + } + visualStart=0; + for(; runslogicalStart; + visualLimit=runs->visualLimit; + if(IS_EVEN_RUN(logicalStart)) { + do { /* LTR */ + *pi++ = logicalStart++; + } while(++visualStartinsertPoints.size>0) { + int32_t markFound=0, runCount=pBiDi->runCount; + int32_t insertRemove, i, j, k; + runs=pBiDi->runs; + /* count all inserted marks */ + for(i=0; iresultLength; + for(i=runCount-1; i>=0 && markFound>0; i--) { + insertRemove=runs[i].insertRemove; + if(insertRemove&(LRM_AFTER|RLM_AFTER)) { + indexMap[--k]= UBIDI_MAP_NOWHERE; + markFound--; + } + visualStart= i>0 ? runs[i-1].visualLimit : 0; + for(j=runs[i].visualLimit-1; j>=visualStart && markFound>0; j--) { + indexMap[--k]=indexMap[j]; + } + if(insertRemove&(LRM_BEFORE|RLM_BEFORE)) { + indexMap[--k]= UBIDI_MAP_NOWHERE; + markFound--; + } + } + } + else if(pBiDi->controlCount>0) { + int32_t runCount=pBiDi->runCount, logicalEnd; + int32_t insertRemove, length, i, j, k, m; + UChar uchar; + UBool evenRun; + runs=pBiDi->runs; + visualStart=0; + /* move forward indexes by number of preceding controls */ + k=0; + for(i=0; itext[m]; + if(!IS_BIDI_CONTROL_CHAR(uchar)) { + indexMap[k++]=m; + } + } + } + } + } +} + +U_CAPI void U_EXPORT2 +ubidi_invertMap(const int32_t *srcMap, int32_t *destMap, int32_t length) { + if(srcMap!=NULL && destMap!=NULL && length>0) { + const int32_t *pi; + int32_t destLength=-1, count=0; + /* find highest value and count positive indexes in srcMap */ + pi=srcMap+length; + while(pi>srcMap) { + if(*--pi>destLength) { + destLength=*pi; + } + if(*pi>=0) { + count++; + } + } + destLength++; /* add 1 for origin 0 */ + if(count0) { + if(*--pi>=0) { + destMap[*pi]=--length; + } else { + --length; + } + } + } +} diff --git a/Build/source/libs/icu/icu-xetex/common/ubidiwrt.c b/Build/source/libs/icu/icu-xetex/common/ubidiwrt.c new file mode 100644 index 00000000000..34b13711d2c --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ubidiwrt.c @@ -0,0 +1,630 @@ +/* +****************************************************************************** +* +* Copyright (C) 2000-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: ubidiwrt.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 1999aug06 +* created by: Markus W. Scherer, updated by Matitiahu Allouche +* +* This file contains implementations for BiDi functions that use +* the core algorithm and core API to write reordered text. +*/ + +/* set import/export definitions */ +#ifndef U_COMMON_IMPLEMENTATION +# define U_COMMON_IMPLEMENTATION +#endif + +#include "unicode/utypes.h" +#include "unicode/ustring.h" +#include "unicode/uchar.h" +#include "unicode/ubidi.h" +#include "cmemory.h" +#include "ustr_imp.h" +#include "ubidiimp.h" + +/* + * The function implementations in this file are designed + * for UTF-16 and UTF-32, not for UTF-8. + * + * Assumptions that are not true for UTF-8: + * - Any code point always needs the same number of code units + * ("minimum-length-problem" of UTF-8) + * - The BiDi control characters need only one code unit each + * + * Further assumptions for all UTFs: + * - u_charMirror(c) needs the same number of code units as c + */ +#if UTF_SIZE==8 +# error reimplement ubidi_writeReordered() for UTF-8, see comment above +#endif + +#define IS_COMBINING(type) ((1UL<<(type))&(1UL<0); + return srcLength; + } + case UBIDI_DO_MIRRORING: { + /* do mirroring */ + int32_t i=0, j=0; + UChar32 c; + + if(destSize0) { + c=*src++; + if(!IS_BIDI_CONTROL_CHAR(c)) { + --remaining; + } + } + return destSize-remaining; + } + *dest++=c; + } + } while(--srcLength>0); + return destSize-remaining; + } + default: { + /* remove BiDi control characters and do mirroring */ + int32_t remaining=destSize; + int32_t i, j=0; + UChar32 c; + do { + i=0; + UTF_NEXT_CHAR(src, i, srcLength, c); + src+=i; + srcLength-=i; + if(!IS_BIDI_CONTROL_CHAR(c)) { + remaining-=i; + if(remaining<0) { + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + + /* preflight the length */ + while(srcLength>0) { + c=*src++; + if(!IS_BIDI_CONTROL_CHAR(c)) { + --remaining; + } + --srcLength; + } + return destSize-remaining; + } + c=u_charMirror(c); + UTF_APPEND_CHAR_UNSAFE(dest, j, c); + } + } while(srcLength>0); + return j; + } + } /* end of switch */ +} + +static int32_t +doWriteReverse(const UChar *src, int32_t srcLength, + UChar *dest, int32_t destSize, + uint16_t options, + UErrorCode *pErrorCode) { + /* + * RTL run - + * + * RTL runs need to be copied to the destination in reverse order + * of code points, not code units, to keep Unicode characters intact. + * + * The general strategy for this is to read the source text + * in backward order, collect all code units for a code point + * (and optionally following combining characters, see below), + * and copy all these code units in ascending order + * to the destination for this run. + * + * Several options request whether combining characters + * should be kept after their base characters, + * whether BiDi control characters should be removed, and + * whether characters should be replaced by their mirror-image + * equivalent Unicode characters. + */ + int32_t i, j; + UChar32 c; + + /* optimize for several combinations of options */ + switch(options&(UBIDI_REMOVE_BIDI_CONTROLS|UBIDI_DO_MIRRORING|UBIDI_KEEP_BASE_COMBINING)) { + case 0: + /* + * With none of the "complicated" options set, the destination + * run will have the same length as the source run, + * and there is no mirroring and no keeping combining characters + * with their base characters. + */ + if(destSize0); + break; + case UBIDI_KEEP_BASE_COMBINING: + /* + * Here, too, the destination + * run will have the same length as the source run, + * and there is no mirroring. + * We do need to keep combining characters with their base characters. + */ + if(destSize0 && IS_COMBINING(u_charType(c))); + + /* copy this "user character" */ + j=srcLength; + do { + *dest++=src[j++]; + } while(j0); + break; + default: + /* + * With several "complicated" options set, this is the most + * general and the slowest copying of an RTL run. + * We will do mirroring, remove BiDi controls, and + * keep combining characters with their base characters + * as requested. + */ + if(!(options&UBIDI_REMOVE_BIDI_CONTROLS)) { + i=srcLength; + } else { + /* we need to find out the destination length of the run, + which will not include the BiDi control characters */ + int32_t length=srcLength; + UChar ch; + + i=0; + do { + ch=*src++; + if(!IS_BIDI_CONTROL_CHAR(ch)) { + ++i; + } + } while(--length>0); + src-=srcLength; + } + + if(destSize0 && IS_COMBINING(u_charType(c))) { + UTF_PREV_CHAR(src, 0, srcLength, c); + } + } + + if(options&UBIDI_REMOVE_BIDI_CONTROLS && IS_BIDI_CONTROL_CHAR(c)) { + /* do not copy this BiDi control character */ + continue; + } + + /* copy this "user character" */ + j=srcLength; + if(options&UBIDI_DO_MIRRORING) { + /* mirror only the base character */ + int32_t k=0; + c=u_charMirror(c); + UTF_APPEND_CHAR_UNSAFE(dest, k, c); + dest+=k; + j+=k; + } + while(j0); + break; + } /* end of switch */ + + return destSize; +} + +U_CAPI int32_t U_EXPORT2 +ubidi_writeReverse(const UChar *src, int32_t srcLength, + UChar *dest, int32_t destSize, + uint16_t options, + UErrorCode *pErrorCode) { + int32_t destLength; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + /* more error checking */ + if( src==NULL || srcLength<-1 || + destSize<0 || (destSize>0 && dest==NULL)) + { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* do input and output overlap? */ + if( dest!=NULL && + ((src>=dest && src=src && dest0) { + destLength=doWriteReverse(src, srcLength, dest, destSize, options, pErrorCode); + } else { + /* nothing to do */ + destLength=0; + } + + return u_terminateUChars(dest, destSize, destLength, pErrorCode); +} + +U_CAPI int32_t U_EXPORT2 +ubidi_writeReordered(UBiDi *pBiDi, + UChar *dest, int32_t destSize, + uint16_t options, + UErrorCode *pErrorCode) { + const UChar *text; + UChar *saveDest; + int32_t length, destCapacity; + int32_t run, runCount, logicalStart, runLength; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + /* more error checking */ + if( pBiDi==NULL || + (text=pBiDi->text)==NULL || (length=pBiDi->length)<0 || + destSize<0 || (destSize>0 && dest==NULL)) + { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* do input and output overlap? */ + if( dest!=NULL && + ((text>=dest && text=text && destoriginalLength))) + { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + if(length==0) { + /* nothing to do */ + return u_terminateUChars(dest, destSize, 0, pErrorCode); + } + + runCount=ubidi_countRuns(pBiDi, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return 0; + } + + /* destSize shrinks, later destination length=destCapacity-destSize */ + saveDest=dest; + destCapacity=destSize; + + /* + * Option "insert marks" implies UBIDI_INSERT_LRM_FOR_NUMERIC if the + * reordering mode (checked below) is appropriate. + */ + if(pBiDi->reorderingOptions & UBIDI_OPTION_INSERT_MARKS) { + options|=UBIDI_INSERT_LRM_FOR_NUMERIC; + options&=~UBIDI_REMOVE_BIDI_CONTROLS; + } + /* + * Option "remove controls" implies UBIDI_REMOVE_BIDI_CONTROLS + * and cancels UBIDI_INSERT_LRM_FOR_NUMERIC. + */ + if(pBiDi->reorderingOptions & UBIDI_OPTION_REMOVE_CONTROLS) { + options|=UBIDI_REMOVE_BIDI_CONTROLS; + options&=~UBIDI_INSERT_LRM_FOR_NUMERIC; + } + /* + * If we do not perform the "inverse BiDi" algorithm, then we + * don't need to insert any LRMs, and don't need to test for it. + */ + if((pBiDi->reorderingMode != UBIDI_REORDER_INVERSE_NUMBERS_AS_L) && + (pBiDi->reorderingMode != UBIDI_REORDER_INVERSE_LIKE_DIRECT) && + (pBiDi->reorderingMode != UBIDI_REORDER_INVERSE_FOR_NUMBERS_SPECIAL) && + (pBiDi->reorderingMode != UBIDI_REORDER_RUNS_ONLY)) { + options&=~UBIDI_INSERT_LRM_FOR_NUMERIC; + } + /* + * Iterate through all visual runs and copy the run text segments to + * the destination, according to the options. + * + * The tests for where to insert LRMs ignore the fact that there may be + * BN codes or non-BMP code points at the beginning and end of a run; + * they may insert LRMs unnecessarily but the tests are faster this way + * (this would have to be improved for UTF-8). + * + * Note that the only errors that are set by doWriteXY() are buffer overflow + * errors. Ignore them until the end, and continue for preflighting. + */ + if(!(options&UBIDI_OUTPUT_REVERSE)) { + /* forward output */ + if(!(options&UBIDI_INSERT_LRM_FOR_NUMERIC)) { + /* do not insert BiDi controls */ + for(run=0; rundirProps; + const UChar *src; + UChar uc; + UBiDiDirection dir; + int32_t markFlag; + + for(run=0; runruns[run].insertRemove; + if(markFlag<0) { /* BiDi controls count */ + markFlag=0; + } + + if(UBIDI_LTR==dir) { + if((pBiDi->isInverse) && + (/*run>0 &&*/ dirProps[logicalStart]!=L)) { + markFlag |= LRM_BEFORE; + } + if (markFlag & LRM_BEFORE) { + uc=LRM_CHAR; + } + else if (markFlag & RLM_BEFORE) { + uc=RLM_CHAR; + } + else uc=0; + if(uc) { + if(destSize>0) { + *dest++=uc; + } + --destSize; + } + + runLength=doWriteForward(src, runLength, + dest, destSize, + (uint16_t)(options&~UBIDI_DO_MIRRORING), pErrorCode); + dest+=runLength; + destSize-=runLength; + + if((pBiDi->isInverse) && + (/*run0) { + *dest++=uc; + } + --destSize; + } + } else { /* RTL run */ + if((pBiDi->isInverse) && + (/*run>0 &&*/ !(MASK_R_AL&DIRPROP_FLAG(dirProps[logicalStart+runLength-1])))) { + markFlag |= RLM_BEFORE; + } + if (markFlag & LRM_BEFORE) { + uc=LRM_CHAR; + } + else if (markFlag & RLM_BEFORE) { + uc=RLM_CHAR; + } + else uc=0; + if(uc) { + if(destSize>0) { + *dest++=uc; + } + --destSize; + } + + runLength=doWriteReverse(src, runLength, + dest, destSize, + options, pErrorCode); + dest+=runLength; + destSize-=runLength; + + if((pBiDi->isInverse) && + (/*run0) { + *dest++=uc; + } + --destSize; + } + } + } + } + } else { + /* reverse output */ + if(!(options&UBIDI_INSERT_LRM_FOR_NUMERIC)) { + /* do not insert BiDi controls */ + for(run=runCount; --run>=0;) { + if(UBIDI_LTR==ubidi_getVisualRun(pBiDi, run, &logicalStart, &runLength)) { + runLength=doWriteReverse(text+logicalStart, runLength, + dest, destSize, + (uint16_t)(options&~UBIDI_DO_MIRRORING), pErrorCode); + } else { + runLength=doWriteForward(text+logicalStart, runLength, + dest, destSize, + options, pErrorCode); + } + dest+=runLength; + destSize-=runLength; + } + } else { + /* insert BiDi controls for "inverse BiDi" */ + const DirProp *dirProps=pBiDi->dirProps; + const UChar *src; + UBiDiDirection dir; + + for(run=runCount; --run>=0;) { + /* reverse output */ + dir=ubidi_getVisualRun(pBiDi, run, &logicalStart, &runLength); + src=text+logicalStart; + + if(UBIDI_LTR==dir) { + if(/*run0) { + *dest++=LRM_CHAR; + } + --destSize; + } + + runLength=doWriteReverse(src, runLength, + dest, destSize, + (uint16_t)(options&~UBIDI_DO_MIRRORING), pErrorCode); + dest+=runLength; + destSize-=runLength; + + if(/*run>0 &&*/ dirProps[logicalStart]!=L) { + if(destSize>0) { + *dest++=LRM_CHAR; + } + --destSize; + } + } else { + if(/*run0) { + *dest++=RLM_CHAR; + } + --destSize; + } + + runLength=doWriteForward(src, runLength, + dest, destSize, + options, pErrorCode); + dest+=runLength; + destSize-=runLength; + + if(/*run>0 &&*/ !(MASK_R_AL&DIRPROP_FLAG(dirProps[logicalStart+runLength-1]))) { + if(destSize>0) { + *dest++=RLM_CHAR; + } + --destSize; + } + } + } + } + } + + return u_terminateUChars(saveDest, destCapacity, destCapacity-destSize, pErrorCode); +} diff --git a/Build/source/libs/icu/icu-xetex/common/ubrk.cpp b/Build/source/libs/icu/icu-xetex/common/ubrk.cpp new file mode 100644 index 00000000000..89028100283 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ubrk.cpp @@ -0,0 +1,293 @@ +/* +***************************************************************************************** +* Copyright (C) 1996-2007, International Business Machines +* Corporation and others. All Rights Reserved. +***************************************************************************************** +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_BREAK_ITERATION + +#include "unicode/ubrk.h" + +#include "unicode/brkiter.h" +#include "unicode/uloc.h" +#include "unicode/ustring.h" +#include "unicode/uchriter.h" +#include "unicode/rbbi.h" +#include "rbbirb.h" +#include "uassert.h" + +U_NAMESPACE_USE + +//---------------------------------------------------------------------------------------- +// +// ubrk_open Create a canned type of break iterator based on type (word, line, etc.) +// and locale. +// +//---------------------------------------------------------------------------------------- +U_CAPI UBreakIterator* U_EXPORT2 +ubrk_open(UBreakIteratorType type, + const char *locale, + const UChar *text, + int32_t textLength, + UErrorCode *status) +{ + + if(U_FAILURE(*status)) return 0; + + BreakIterator *result = 0; + + switch(type) { + + case UBRK_CHARACTER: + result = BreakIterator::createCharacterInstance(Locale(locale), *status); + break; + + case UBRK_WORD: + result = BreakIterator::createWordInstance(Locale(locale), *status); + break; + + case UBRK_LINE: + result = BreakIterator::createLineInstance(Locale(locale), *status); + break; + + case UBRK_SENTENCE: + result = BreakIterator::createSentenceInstance(Locale(locale), *status); + break; + + case UBRK_TITLE: + result = BreakIterator::createTitleInstance(Locale(locale), *status); + break; + + default: + *status = U_ILLEGAL_ARGUMENT_ERROR; + } + + // check for allocation error + if (U_FAILURE(*status)) { + return 0; + } + if(result == 0) { + *status = U_MEMORY_ALLOCATION_ERROR; + return 0; + } + + + UBreakIterator *uBI = (UBreakIterator *)result; + if (text != NULL) { + ubrk_setText(uBI, text, textLength, status); + } + return uBI; +} + + + +//---------------------------------------------------------------------------------------- +// +// ubrk_openRules open a break iterator from a set of break rules. +// Invokes the rule builder. +// +//---------------------------------------------------------------------------------------- +U_CAPI UBreakIterator* U_EXPORT2 +ubrk_openRules( const UChar *rules, + int32_t rulesLength, + const UChar *text, + int32_t textLength, + UParseError *parseErr, + UErrorCode *status) { + + if (status == NULL || U_FAILURE(*status)){ + return 0; + } + + BreakIterator *result = 0; + UnicodeString ruleString(rules, rulesLength); + result = RBBIRuleBuilder::createRuleBasedBreakIterator(ruleString, *parseErr, *status); + if(U_FAILURE(*status)) { + return 0; + } + + UBreakIterator *uBI = (UBreakIterator *)result; + if (text != NULL) { + ubrk_setText(uBI, text, textLength, status); + } + return uBI; +} + + + + + +U_CAPI UBreakIterator * U_EXPORT2 +ubrk_safeClone( + const UBreakIterator *bi, + void *stackBuffer, + int32_t *pBufferSize, + UErrorCode *status) +{ + if (status == NULL || U_FAILURE(*status)){ + return 0; + } + if (!pBufferSize || !bi){ + *status = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + // Clear any incoming Safe Clone Allocated warning. + // Propagating this through to our return would really + // confuse our caller. + if (*status==U_SAFECLONE_ALLOCATED_WARNING) { + *status = U_ZERO_ERROR; + } + return (UBreakIterator *)(((BreakIterator*)bi)-> + createBufferClone(stackBuffer, *pBufferSize, *status)); +} + + + +U_CAPI void U_EXPORT2 +ubrk_close(UBreakIterator *bi) +{ + BreakIterator *ubi = (BreakIterator*) bi; + if (ubi) { + if (ubi->isBufferClone()) { + ubi->~BreakIterator(); + *(uint32_t *)ubi = 0xdeadbeef; + } else { + delete ubi; + } + } +} + +U_CAPI void U_EXPORT2 +ubrk_setText(UBreakIterator* bi, + const UChar* text, + int32_t textLength, + UErrorCode* status) +{ + BreakIterator *brit = (BreakIterator *)bi; + UText ut = UTEXT_INITIALIZER; + utext_openUChars(&ut, text, textLength, status); + brit->setText(&ut, *status); + // A stack allocated UText wrapping a UCHar * string + // can be dumped without explicitly closing it. +} + + + +U_CAPI void U_EXPORT2 +ubrk_setUText(UBreakIterator *bi, + UText *text, + UErrorCode *status) +{ + RuleBasedBreakIterator *brit = (RuleBasedBreakIterator *)bi; + brit->RuleBasedBreakIterator::setText(text, *status); +} + + + + + +U_CAPI int32_t U_EXPORT2 +ubrk_current(const UBreakIterator *bi) +{ + + return ((RuleBasedBreakIterator*)bi)->RuleBasedBreakIterator::current(); +} + +U_CAPI int32_t U_EXPORT2 +ubrk_next(UBreakIterator *bi) +{ + + return ((RuleBasedBreakIterator*)bi)->RuleBasedBreakIterator::next(); +} + +U_CAPI int32_t U_EXPORT2 +ubrk_previous(UBreakIterator *bi) +{ + + return ((RuleBasedBreakIterator*)bi)->RuleBasedBreakIterator::previous(); +} + +U_CAPI int32_t U_EXPORT2 +ubrk_first(UBreakIterator *bi) +{ + + return ((RuleBasedBreakIterator*)bi)->RuleBasedBreakIterator::first(); +} + +U_CAPI int32_t U_EXPORT2 +ubrk_last(UBreakIterator *bi) +{ + + return ((RuleBasedBreakIterator*)bi)->RuleBasedBreakIterator::last(); +} + +U_CAPI int32_t U_EXPORT2 +ubrk_preceding(UBreakIterator *bi, + int32_t offset) +{ + + return ((RuleBasedBreakIterator*)bi)->RuleBasedBreakIterator::preceding(offset); +} + +U_CAPI int32_t U_EXPORT2 +ubrk_following(UBreakIterator *bi, + int32_t offset) +{ + + return ((RuleBasedBreakIterator*)bi)->RuleBasedBreakIterator::following(offset); +} + +U_CAPI const char* U_EXPORT2 +ubrk_getAvailable(int32_t index) +{ + + return uloc_getAvailable(index); +} + +U_CAPI int32_t U_EXPORT2 +ubrk_countAvailable() +{ + + return uloc_countAvailable(); +} + + +U_CAPI UBool U_EXPORT2 +ubrk_isBoundary(UBreakIterator *bi, int32_t offset) +{ + return ((RuleBasedBreakIterator *)bi)->RuleBasedBreakIterator::isBoundary(offset); +} + + +U_CAPI int32_t U_EXPORT2 +ubrk_getRuleStatus(UBreakIterator *bi) +{ + return ((RuleBasedBreakIterator *)bi)->RuleBasedBreakIterator::getRuleStatus(); +} + +U_CAPI int32_t U_EXPORT2 +ubrk_getRuleStatusVec(UBreakIterator *bi, int32_t *fillInVec, int32_t capacity, UErrorCode *status) +{ + return ((RuleBasedBreakIterator *)bi)->RuleBasedBreakIterator::getRuleStatusVec(fillInVec, capacity, *status); +} + + +U_CAPI const char* U_EXPORT2 +ubrk_getLocaleByType(const UBreakIterator *bi, + ULocDataLocaleType type, + UErrorCode* status) +{ + if (bi == NULL) { + if (U_SUCCESS(*status)) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + } + return NULL; + } + return ((BreakIterator*)bi)->getLocaleID(type, *status); +} + + +#endif /* #if !UCONFIG_NO_BREAK_ITERATION */ diff --git a/Build/source/libs/icu/icu-xetex/common/ubrkimpl.h b/Build/source/libs/icu/icu-xetex/common/ubrkimpl.h new file mode 100644 index 00000000000..e4909712566 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ubrkimpl.h @@ -0,0 +1,13 @@ +/* +********************************************************************** +* Copyright (C) 2006, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +*/ + +#ifndef UBRKIMPL_H +#define UBRKIMPL_H + +#define U_ICUDATA_BRKITR U_ICUDATA_NAME U_TREE_SEPARATOR_STRING "brkitr" + +#endif /*UBRKIMPL_H*/ diff --git a/Build/source/libs/icu/icu-xetex/common/ucase.c b/Build/source/libs/icu/icu-xetex/common/ucase.c new file mode 100644 index 00000000000..2352176be15 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucase.c @@ -0,0 +1,1599 @@ +/* +******************************************************************************* +* +* Copyright (C) 2004-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: ucase.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2004aug30 +* created by: Markus W. Scherer +* +* Low-level Unicode character/string case mapping code. +* Much code moved here (and modified) from uchar.c. +*/ + +#include "unicode/utypes.h" +#include "unicode/uset.h" +#include "unicode/udata.h" /* UDataInfo */ +#include "ucmndata.h" /* DataHeader */ +#include "udatamem.h" +#include "umutex.h" +#include "uassert.h" +#include "cmemory.h" +#include "utrie.h" +#include "ucase.h" +#include "ucln_cmn.h" + +struct UCaseProps { + UDataMemory *mem; + const int32_t *indexes; + const uint16_t *exceptions; + const UChar *unfold; + + UTrie trie; + uint8_t formatVersion[4]; +}; + +/* data loading etc. -------------------------------------------------------- */ + +#define UCASE_HARDCODE_DATA 1 + +#if UCASE_HARDCODE_DATA + +/* ucase_props_data.c is machine-generated by gencase --csource */ +#include "ucase_props_data.c" + +#else + +static UBool U_CALLCONV +isAcceptable(void *context, + const char *type, const char *name, + const UDataInfo *pInfo) { + if( + pInfo->size>=20 && + pInfo->isBigEndian==U_IS_BIG_ENDIAN && + pInfo->charsetFamily==U_CHARSET_FAMILY && + pInfo->dataFormat[0]==UCASE_FMT_0 && /* dataFormat="cAsE" */ + pInfo->dataFormat[1]==UCASE_FMT_1 && + pInfo->dataFormat[2]==UCASE_FMT_2 && + pInfo->dataFormat[3]==UCASE_FMT_3 && + pInfo->formatVersion[0]==1 && + pInfo->formatVersion[2]==UTRIE_SHIFT && + pInfo->formatVersion[3]==UTRIE_INDEX_SHIFT + ) { + UCaseProps *csp=(UCaseProps *)context; + uprv_memcpy(csp->formatVersion, pInfo->formatVersion, 4); + return TRUE; + } else { + return FALSE; + } +} + +static UCaseProps * +ucase_openData(UCaseProps *cspProto, + const uint8_t *bin, int32_t length, UErrorCode *pErrorCode) { + UCaseProps *csp; + int32_t size; + + cspProto->indexes=(const int32_t *)bin; + if( (length>=0 && length<16*4) || + cspProto->indexes[UCASE_IX_INDEX_TOP]<16 + ) { + /* length or indexes[] too short for minimum indexes[] length of 16 */ + *pErrorCode=U_INVALID_FORMAT_ERROR; + return NULL; + } + size=cspProto->indexes[UCASE_IX_INDEX_TOP]*4; + if(length>=0) { + if(length>=size && length>=cspProto->indexes[UCASE_IX_LENGTH]) { + length-=size; + } else { + /* length too short for indexes[] or for the whole data length */ + *pErrorCode=U_INVALID_FORMAT_ERROR; + return NULL; + } + } + bin+=size; + /* from here on, assume that the sizes of the items fit into the total length */ + + /* unserialize the trie, after indexes[] */ + size=cspProto->indexes[UCASE_IX_TRIE_SIZE]; + utrie_unserialize(&cspProto->trie, bin, size, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return NULL; + } + bin+=size; + + /* get exceptions[] */ + size=2*cspProto->indexes[UCASE_IX_EXC_LENGTH]; + cspProto->exceptions=(const uint16_t *)bin; + bin+=size; + + /* get unfold[] */ + size=2*cspProto->indexes[UCASE_IX_UNFOLD_LENGTH]; + if(size!=0) { + cspProto->unfold=(const UChar *)bin; + bin+=size; + } else { + cspProto->unfold=NULL; + } + + /* allocate, copy, and return the new UCaseProps */ + csp=(UCaseProps *)uprv_malloc(sizeof(UCaseProps)); + if(csp==NULL) { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return NULL; + } else { + uprv_memcpy(csp, cspProto, sizeof(UCaseProps)); + return csp; + } +} + +U_CAPI UCaseProps * U_EXPORT2 +ucase_open(UErrorCode *pErrorCode) { + UCaseProps cspProto={ NULL }, *csp; + + cspProto.mem=udata_openChoice(NULL, UCASE_DATA_TYPE, UCASE_DATA_NAME, isAcceptable, &cspProto, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return NULL; + } + + csp=ucase_openData( + &cspProto, + udata_getMemory(cspProto.mem), + udata_getLength(cspProto.mem), + pErrorCode); + if(U_FAILURE(*pErrorCode)) { + udata_close(cspProto.mem); + return NULL; + } else { + return csp; + } +} + +U_CAPI UCaseProps * U_EXPORT2 +ucase_openBinary(const uint8_t *bin, int32_t length, UErrorCode *pErrorCode) { + UCaseProps cspProto={ NULL }; + const DataHeader *hdr; + + if(U_FAILURE(*pErrorCode)) { + return NULL; + } + if(bin==NULL) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + + /* check the header */ + if(length>=0 && length<20) { + *pErrorCode=U_INVALID_FORMAT_ERROR; + return NULL; + } + hdr=(const DataHeader *)bin; + if( + !(hdr->dataHeader.magic1==0xda && hdr->dataHeader.magic2==0x27 && + hdr->info.isBigEndian==U_IS_BIG_ENDIAN && + isAcceptable(&cspProto, UCASE_DATA_TYPE, UCASE_DATA_NAME, &hdr->info)) + ) { + *pErrorCode=U_INVALID_FORMAT_ERROR; + return NULL; + } + + bin+=hdr->dataHeader.headerSize; + if(length>=0) { + length-=hdr->dataHeader.headerSize; + } + return ucase_openData(&cspProto, bin, length, pErrorCode); +} + +#endif + +U_CAPI void U_EXPORT2 +ucase_close(UCaseProps *csp) { + if(csp!=NULL) { +#if !UCASE_HARDCODE_DATA + udata_close(csp->mem); +#endif + uprv_free(csp); + } +} + +/* UCaseProps singleton ----------------------------------------------------- */ + +static UCaseProps *gCsp=NULL, *gCspDummy=NULL; +#if !UCASE_HARDCODE_DATA +static UErrorCode gErrorCode=U_ZERO_ERROR; +static int8_t gHaveData=0; +#endif + +static UBool U_CALLCONV ucase_cleanup(void) { + ucase_close(gCsp); + gCsp=NULL; + ucase_close(gCspDummy); + gCspDummy=NULL; +#if !UCASE_HARDCODE_DATA + gErrorCode=U_ZERO_ERROR; + gHaveData=0; +#endif + return TRUE; +} + +U_CAPI const UCaseProps * U_EXPORT2 +ucase_getSingleton(UErrorCode *pErrorCode) { +#if UCASE_HARDCODE_DATA + if(U_FAILURE(*pErrorCode)) { + return NULL; + } + return &ucase_props_singleton; +#else + int8_t haveData; + + if(U_FAILURE(*pErrorCode)) { + return NULL; + } + + UMTX_CHECK(NULL, gHaveData, haveData); + + if(haveData>0) { + /* data was loaded */ + return gCsp; + } else if(haveData<0) { + /* data loading failed */ + *pErrorCode=gErrorCode; + return NULL; + } else /* haveData==0 */ { + /* load the data */ + UCaseProps *csp=ucase_open(pErrorCode); + if(U_FAILURE(*pErrorCode)) { + gHaveData=-1; + gErrorCode=*pErrorCode; + return NULL; + } + + /* set the static variables */ + umtx_lock(NULL); + if(gCsp==NULL) { + gCsp=csp; + csp=NULL; + gHaveData=1; + ucln_common_registerCleanup(UCLN_COMMON_UCASE, ucase_cleanup); + } + umtx_unlock(NULL); + + ucase_close(csp); + return gCsp; + } +#endif +} + +U_CAPI const UCaseProps * U_EXPORT2 +ucase_getDummy(UErrorCode *pErrorCode) { + UCaseProps *csp; + + if(U_FAILURE(*pErrorCode)) { + return NULL; + } + + UMTX_CHECK(NULL, gCspDummy, csp); + + if(csp!=NULL) { + /* the dummy object was already created */ + return csp; + } else /* csp==NULL */ { + /* create the dummy object */ + int32_t *indexes; + + csp=(UCaseProps *)uprv_malloc(sizeof(UCaseProps)+UCASE_IX_TOP*4+UTRIE_DUMMY_SIZE); + if(csp==NULL) { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + uprv_memset(csp, 0, sizeof(UCaseProps)+UCASE_IX_TOP*4); + + csp->indexes=indexes=(int32_t *)(csp+1); + indexes[UCASE_IX_INDEX_TOP]=UCASE_IX_TOP; + + indexes[UCASE_IX_TRIE_SIZE]= + utrie_unserializeDummy(&csp->trie, indexes+UCASE_IX_TOP, UTRIE_DUMMY_SIZE, 0, 0, TRUE, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + uprv_free(csp); + return NULL; + } + + csp->formatVersion[0]=1; + csp->formatVersion[2]=UTRIE_SHIFT; + csp->formatVersion[3]=UTRIE_INDEX_SHIFT; + + /* set the static variables */ + umtx_lock(NULL); + if(gCspDummy==NULL) { + gCspDummy=csp; + csp=NULL; + ucln_common_registerCleanup(UCLN_COMMON_UCASE, ucase_cleanup); + } + umtx_unlock(NULL); + + uprv_free(csp); + return gCspDummy; + } +} + +/* set of property starts for UnicodeSet ------------------------------------ */ + +static UBool U_CALLCONV +_enumPropertyStartsRange(const void *context, UChar32 start, UChar32 limit, uint32_t value) { + /* add the start code point to the USet */ + const USetAdder *sa=(const USetAdder *)context; + sa->add(sa->set, start); + return TRUE; +} + +U_CFUNC void U_EXPORT2 +ucase_addPropertyStarts(const UCaseProps *csp, const USetAdder *sa, UErrorCode *pErrorCode) { + if(U_FAILURE(*pErrorCode)) { + return; + } + + /* add the start code point of each same-value range of the trie */ + utrie_enum(&csp->trie, NULL, _enumPropertyStartsRange, sa); + + /* add code points with hardcoded properties, plus the ones following them */ + + /* (none right now, see comment below) */ + + /* + * Omit code points with hardcoded specialcasing properties + * because we do not build property UnicodeSets for them right now. + */ +} + +/* data access primitives --------------------------------------------------- */ + +/* UTRIE_GET16() itself validates c */ +#define GET_PROPS(csp, c, result) \ + UTRIE_GET16(&(csp)->trie, c, result); + +#define GET_EXCEPTIONS(csp, props) ((csp)->exceptions+((props)>>UCASE_EXC_SHIFT)) + +#define PROPS_HAS_EXCEPTION(props) ((props)&UCASE_EXCEPTION) + +/* number of bits in an 8-bit integer value */ +static const uint8_t flagsOffset[256]={ + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8 +}; + +#define HAS_SLOT(flags, index) ((flags)&(1<<(index))) +#define SLOT_OFFSET(flags, index) flagsOffset[(flags)&((1<<(index))-1)] + +/* + * Get the value of an optional-value slot where HAS_SLOT(excWord, index). + * + * @param excWord (in) initial exceptions word + * @param index (in) desired slot index + * @param pExc16 (in/out) const uint16_t * after excWord=*pExc16++; + * moved to the last uint16_t of the value, use +1 for beginning of next slot + * @param value (out) int32_t or uint32_t output if hasSlot, otherwise not modified + */ +#define GET_SLOT_VALUE(excWord, index, pExc16, value) \ + if(((excWord)&UCASE_EXC_DOUBLE_SLOTS)==0) { \ + (pExc16)+=SLOT_OFFSET(excWord, index); \ + (value)=*pExc16; \ + } else { \ + (pExc16)+=2*SLOT_OFFSET(excWord, index); \ + (value)=*pExc16++; \ + (value)=((value)<<16)|*pExc16; \ + } + +/* simple case mappings ----------------------------------------------------- */ + +U_CAPI UChar32 U_EXPORT2 +ucase_tolower(const UCaseProps *csp, UChar32 c) { + uint16_t props; + GET_PROPS(csp, c, props); + if(!PROPS_HAS_EXCEPTION(props)) { + if(UCASE_GET_TYPE(props)>=UCASE_UPPER) { + c+=UCASE_GET_DELTA(props); + } + } else { + const uint16_t *pe=GET_EXCEPTIONS(csp, props); + uint16_t excWord=*pe++; + if(HAS_SLOT(excWord, UCASE_EXC_LOWER)) { + GET_SLOT_VALUE(excWord, UCASE_EXC_LOWER, pe, c); + } + } + return c; +} + +U_CAPI UChar32 U_EXPORT2 +ucase_toupper(const UCaseProps *csp, UChar32 c) { + uint16_t props; + GET_PROPS(csp, c, props); + if(!PROPS_HAS_EXCEPTION(props)) { + if(UCASE_GET_TYPE(props)==UCASE_LOWER) { + c+=UCASE_GET_DELTA(props); + } + } else { + const uint16_t *pe=GET_EXCEPTIONS(csp, props); + uint16_t excWord=*pe++; + if(HAS_SLOT(excWord, UCASE_EXC_UPPER)) { + GET_SLOT_VALUE(excWord, UCASE_EXC_UPPER, pe, c); + } + } + return c; +} + +U_CAPI UChar32 U_EXPORT2 +ucase_totitle(const UCaseProps *csp, UChar32 c) { + uint16_t props; + GET_PROPS(csp, c, props); + if(!PROPS_HAS_EXCEPTION(props)) { + if(UCASE_GET_TYPE(props)==UCASE_LOWER) { + c+=UCASE_GET_DELTA(props); + } + } else { + const uint16_t *pe=GET_EXCEPTIONS(csp, props); + uint16_t excWord=*pe++; + int32_t index; + if(HAS_SLOT(excWord, UCASE_EXC_TITLE)) { + index=UCASE_EXC_TITLE; + } else if(HAS_SLOT(excWord, UCASE_EXC_UPPER)) { + index=UCASE_EXC_UPPER; + } else { + return c; + } + GET_SLOT_VALUE(excWord, index, pe, c); + } + return c; +} + +U_CFUNC void U_EXPORT2 +ucase_addCaseClosure(const UCaseProps *csp, UChar32 c, const USetAdder *sa) { + uint16_t props; + + /* + * Hardcode the case closure of i and its relatives and ignore the + * data file data for these characters. + * The Turkic dotless i and dotted I with their case mapping conditions + * and case folding option make the related characters behave specially. + * This code matches their closure behavior to their case folding behavior. + */ + static const UChar + iDot[2]= { 0x69, 0x307 }; + + switch(c) { + case 0x49: + /* regular i and I are in one equivalence class */ + sa->add(sa->set, 0x69); + return; + case 0x69: + sa->add(sa->set, 0x49); + return; + case 0x130: + /* dotted I is in a class with <0069 0307> (for canonical equivalence with <0049 0307>) */ + sa->addString(sa->set, iDot, 2); + return; + case 0x131: + /* dotless i is in a class by itself */ + return; + default: + /* otherwise use the data file data */ + break; + } + + GET_PROPS(csp, c, props); + if(!PROPS_HAS_EXCEPTION(props)) { + if(UCASE_GET_TYPE(props)!=UCASE_NONE) { + /* add the one simple case mapping, no matter what type it is */ + int32_t delta=UCASE_GET_DELTA(props); + if(delta!=0) { + sa->add(sa->set, c+delta); + } + } + } else { + /* + * c has exceptions, so there may be multiple simple and/or + * full case mappings. Add them all. + */ + const uint16_t *pe0, *pe=GET_EXCEPTIONS(csp, props); + const UChar *closure; + uint16_t excWord=*pe++; + int32_t index, closureLength, fullLength, length; + + pe0=pe; + + /* add all simple case mappings */ + for(index=UCASE_EXC_LOWER; index<=UCASE_EXC_TITLE; ++index) { + if(HAS_SLOT(excWord, index)) { + pe=pe0; + GET_SLOT_VALUE(excWord, index, pe, c); + sa->add(sa->set, c); + } + } + + /* get the closure string pointer & length */ + if(HAS_SLOT(excWord, UCASE_EXC_CLOSURE)) { + pe=pe0; + GET_SLOT_VALUE(excWord, UCASE_EXC_CLOSURE, pe, closureLength); + closureLength&=UCASE_CLOSURE_MAX_LENGTH; /* higher bits are reserved */ + closure=(const UChar *)pe+1; /* behind this slot, unless there are full case mappings */ + } else { + closureLength=0; + closure=NULL; + } + + /* add the full case folding */ + if(HAS_SLOT(excWord, UCASE_EXC_FULL_MAPPINGS)) { + pe=pe0; + GET_SLOT_VALUE(excWord, UCASE_EXC_FULL_MAPPINGS, pe, fullLength); + + /* start of full case mapping strings */ + ++pe; + + fullLength&=0xffff; /* bits 16 and higher are reserved */ + + /* skip the lowercase result string */ + pe+=fullLength&UCASE_FULL_LOWER; + fullLength>>=4; + + /* add the full case folding string */ + length=fullLength&0xf; + if(length!=0) { + sa->addString(sa->set, (const UChar *)pe, length); + pe+=length; + } + + /* skip the uppercase and titlecase strings */ + fullLength>>=4; + pe+=fullLength&0xf; + fullLength>>=4; + pe+=fullLength; + + closure=(const UChar *)pe; /* behind full case mappings */ + } + + /* add each code point in the closure string */ + for(index=0; indexadd(sa->set, c); + } + } +} + +/* + * compare s, which has a length, with t, which has a maximum length or is NUL-terminated + * must be length>0 and max>0 and length<=max + */ +static U_INLINE int32_t +strcmpMax(const UChar *s, int32_t length, const UChar *t, int32_t max) { + int32_t c1, c2; + + max-=length; /* we require length<=max, so no need to decrement max in the loop */ + do { + c1=*s++; + c2=*t++; + if(c2==0) { + return 1; /* reached the end of t but not of s */ + } + c1-=c2; + if(c1!=0) { + return c1; /* return difference result */ + } + } while(--length>0); + /* ends with length==0 */ + + if(max==0 || *t==0) { + return 0; /* equal to length of both strings */ + } else { + return -max; /* return lengh difference */ + } +} + +U_CFUNC UBool U_EXPORT2 +ucase_addStringCaseClosure(const UCaseProps *csp, const UChar *s, int32_t length, const USetAdder *sa) { + const UChar *unfold, *p; + int32_t i, start, limit, result, unfoldRows, unfoldRowWidth, unfoldStringWidth; + + if(csp->unfold==NULL || s==NULL) { + return FALSE; /* no reverse case folding data, or no string */ + } + if(length<=1) { + /* the string is too short to find any match */ + /* + * more precise would be: + * if(!u_strHasMoreChar32Than(s, length, 1)) + * but this does not make much practical difference because + * a single supplementary code point would just not be found + */ + return FALSE; + } + + unfold=csp->unfold; + unfoldRows=unfold[UCASE_UNFOLD_ROWS]; + unfoldRowWidth=unfold[UCASE_UNFOLD_ROW_WIDTH]; + unfoldStringWidth=unfold[UCASE_UNFOLD_STRING_WIDTH]; + unfold+=unfoldRowWidth; + + if(length>unfoldStringWidth) { + /* the string is too long to find any match */ + return FALSE; + } + + /* do a binary search for the string */ + start=0; + limit=unfoldRows; + while(startadd(sa->set, c); + ucase_addCaseClosure(csp, c, sa); + } + return TRUE; + } else if(result<0) { + limit=i; + } else /* result>0 */ { + start=i+1; + } + } + + return FALSE; /* string not found */ +} + +/** @return UCASE_NONE, UCASE_LOWER, UCASE_UPPER, UCASE_TITLE */ +U_CAPI int32_t U_EXPORT2 +ucase_getType(const UCaseProps *csp, UChar32 c) { + uint16_t props; + GET_PROPS(csp, c, props); + return UCASE_GET_TYPE(props); +} + +/** @return same as ucase_getType(), or <0 if c is case-ignorable */ +U_CAPI int32_t U_EXPORT2 +ucase_getTypeOrIgnorable(const UCaseProps *csp, UChar32 c) { + int32_t type; + uint16_t props; + GET_PROPS(csp, c, props); + type=UCASE_GET_TYPE(props); + if(type!=UCASE_NONE) { + return type; + } else if( + c==0x307 || + (props&(UCASE_EXCEPTION|UCASE_CASE_IGNORABLE))==UCASE_CASE_IGNORABLE + ) { + return -1; /* case-ignorable */ + } else { + return 0; /* c is neither cased nor case-ignorable */ + } +} + +/** @return UCASE_NO_DOT, UCASE_SOFT_DOTTED, UCASE_ABOVE, UCASE_OTHER_ACCENT */ +static U_INLINE int32_t +getDotType(const UCaseProps *csp, UChar32 c) { + uint16_t props; + GET_PROPS(csp, c, props); + if(!PROPS_HAS_EXCEPTION(props)) { + return props&UCASE_DOT_MASK; + } else { + const uint16_t *pe=GET_EXCEPTIONS(csp, props); + return (*pe>>UCASE_EXC_DOT_SHIFT)&UCASE_DOT_MASK; + } +} + +U_CAPI UBool U_EXPORT2 +ucase_isSoftDotted(const UCaseProps *csp, UChar32 c) { + return (UBool)(getDotType(csp, c)==UCASE_SOFT_DOTTED); +} + +U_CAPI UBool U_EXPORT2 +ucase_isCaseSensitive(const UCaseProps *csp, UChar32 c) { + uint16_t props; + GET_PROPS(csp, c, props); + return (UBool)((props&UCASE_SENSITIVE)!=0); +} + +/* string casing ------------------------------------------------------------ */ + +/* + * These internal functions form the core of string case mappings. + * They map single code points to result code points or strings and take + * all necessary conditions (context, locale ID, options) into account. + * + * They do not iterate over the source or write to the destination + * so that the same functions are useful for non-standard string storage, + * such as in a Replaceable (for Transliterator) or UTF-8/32 strings etc. + * For the same reason, the "surrounding text" context is passed in as a + * UCaseContextIterator which does not make any assumptions about + * the underlying storage. + * + * This section contains helper functions that check for conditions + * in the input text surrounding the current code point + * according to SpecialCasing.txt. + * + * Each helper function gets the index + * - after the current code point if it looks at following text + * - before the current code point if it looks at preceding text + * + * Unicode 3.2 UAX 21 "Case Mappings" defines the conditions as follows: + * + * Final_Sigma + * C is preceded by a sequence consisting of + * a cased letter and a case-ignorable sequence, + * and C is not followed by a sequence consisting of + * an ignorable sequence and then a cased letter. + * + * More_Above + * C is followed by one or more characters of combining class 230 (ABOVE) + * in the combining character sequence. + * + * After_Soft_Dotted + * The last preceding character with combining class of zero before C + * was Soft_Dotted, + * and there is no intervening combining character class 230 (ABOVE). + * + * Before_Dot + * C is followed by combining dot above (U+0307). + * Any sequence of characters with a combining class that is neither 0 nor 230 + * may intervene between the current character and the combining dot above. + * + * The erratum from 2002-10-31 adds the condition + * + * After_I + * The last preceding base character was an uppercase I, and there is no + * intervening combining character class 230 (ABOVE). + * + * (See Jitterbug 2344 and the comments on After_I below.) + * + * Helper definitions in Unicode 3.2 UAX 21: + * + * D1. A character C is defined to be cased + * if it meets any of the following criteria: + * + * - The general category of C is Titlecase Letter (Lt) + * - In [CoreProps], C has one of the properties Uppercase, or Lowercase + * - Given D = NFD(C), then it is not the case that: + * D = UCD_lower(D) = UCD_upper(D) = UCD_title(D) + * (This third criterium does not add any characters to the list + * for Unicode 3.2. Ignored.) + * + * D2. A character C is defined to be case-ignorable + * if it meets either of the following criteria: + * + * - The general category of C is + * Nonspacing Mark (Mn), or Enclosing Mark (Me), or Format Control (Cf), or + * Letter Modifier (Lm), or Symbol Modifier (Sk) + * - C is one of the following characters + * U+0027 APOSTROPHE + * U+00AD SOFT HYPHEN (SHY) + * U+2019 RIGHT SINGLE QUOTATION MARK + * (the preferred character for apostrophe) + * + * D3. A case-ignorable sequence is a sequence of + * zero or more case-ignorable characters. + */ + +enum { + LOC_UNKNOWN, + LOC_ROOT, + LOC_TURKISH, + LOC_LITHUANIAN +}; + +#define is_a(c) ((c)=='a' || (c)=='A') +#define is_e(c) ((c)=='e' || (c)=='E') +#define is_i(c) ((c)=='i' || (c)=='I') +#define is_l(c) ((c)=='l' || (c)=='L') +#define is_r(c) ((c)=='r' || (c)=='R') +#define is_t(c) ((c)=='t' || (c)=='T') +#define is_u(c) ((c)=='u' || (c)=='U') +#define is_z(c) ((c)=='z' || (c)=='Z') + +/* separator? */ +#define is_sep(c) ((c)=='_' || (c)=='-' || (c)==0) + +/** + * Requires non-NULL locale ID but otherwise does the equivalent of + * checking for language codes as if uloc_getLanguage() were called: + * Accepts both 2- and 3-letter codes and accepts case variants. + */ +U_CFUNC int32_t +ucase_getCaseLocale(const char *locale, int32_t *locCache) { + int32_t result; + char c; + + if(locCache!=NULL && (result=*locCache)!=LOC_UNKNOWN) { + return result; + } + + result=LOC_ROOT; + + /* + * This function used to use uloc_getLanguage(), but the current code + * removes the dependency of this low-level code on uloc implementation code + * and is faster because not the whole locale ID has to be + * examined and copied/transformed. + * + * Because this code does not want to depend on uloc, the caller must + * pass in a non-NULL locale, i.e., may need to call uloc_getDefault(). + */ + c=*locale++; + if(is_t(c)) { + /* tr or tur? */ + c=*locale++; + if(is_u(c)) { + c=*locale++; + } + if(is_r(c)) { + c=*locale; + if(is_sep(c)) { + result=LOC_TURKISH; + } + } + } else if(is_a(c)) { + /* az or aze? */ + c=*locale++; + if(is_z(c)) { + c=*locale++; + if(is_e(c)) { + c=*locale; + } + if(is_sep(c)) { + result=LOC_TURKISH; + } + } + } else if(is_l(c)) { + /* lt or lit? */ + c=*locale++; + if(is_i(c)) { + c=*locale++; + } + if(is_t(c)) { + c=*locale; + if(is_sep(c)) { + result=LOC_LITHUANIAN; + } + } + } + + if(locCache!=NULL) { + *locCache=result; + } + return result; +} + +/* Is followed by {case-ignorable}* cased ? (dir determines looking forward/backward) */ +static UBool +isFollowedByCasedLetter(const UCaseProps *csp, UCaseContextIterator *iter, void *context, int8_t dir) { + UChar32 c; + uint16_t props; + + if(iter==NULL) { + return FALSE; + } + + for(/* dir!=0 sets direction */; (c=iter(context, dir))>=0; dir=0) { + GET_PROPS(csp, c, props); + if(UCASE_GET_TYPE(props)!=UCASE_NONE) { + return TRUE; /* followed by cased letter */ + } else if(c==0x307 || (props&(UCASE_EXCEPTION|UCASE_CASE_IGNORABLE))==UCASE_CASE_IGNORABLE) { + /* case-ignorable, continue with the loop */ + } else { + return FALSE; /* not ignorable */ + } + } + + return FALSE; /* not followed by cased letter */ +} + +/* Is preceded by Soft_Dotted character with no intervening cc=230 ? */ +static UBool +isPrecededBySoftDotted(const UCaseProps *csp, UCaseContextIterator *iter, void *context) { + UChar32 c; + int32_t dotType; + int8_t dir; + + if(iter==NULL) { + return FALSE; + } + + for(dir=-1; (c=iter(context, dir))>=0; dir=0) { + dotType=getDotType(csp, c); + if(dotType==UCASE_SOFT_DOTTED) { + return TRUE; /* preceded by TYPE_i */ + } else if(dotType!=UCASE_OTHER_ACCENT) { + return FALSE; /* preceded by different base character (not TYPE_i), or intervening cc==230 */ + } + } + + return FALSE; /* not preceded by TYPE_i */ +} + +/* + * See Jitterbug 2344: + * The condition After_I for Turkic-lowercasing of U+0307 combining dot above + * is checked in ICU 2.0, 2.1, 2.6 but was not in 2.2 & 2.4 because + * we made those releases compatible with Unicode 3.2 which had not fixed + * a related bug in SpecialCasing.txt. + * + * From the Jitterbug 2344 text: + * ... this bug is listed as a Unicode erratum + * from 2002-10-31 at http://www.unicode.org/uni2errata/UnicodeErrata.html + * + * There are two errors in SpecialCasing.txt. + * 1. Missing semicolons on two lines. ... [irrelevant for ICU] + * 2. An incorrect context definition. Correct as follows: + * < 0307; ; 0307; 0307; tr After_Soft_Dotted; # COMBINING DOT ABOVE + * < 0307; ; 0307; 0307; az After_Soft_Dotted; # COMBINING DOT ABOVE + * --- + * > 0307; ; 0307; 0307; tr After_I; # COMBINING DOT ABOVE + * > 0307; ; 0307; 0307; az After_I; # COMBINING DOT ABOVE + * where the context After_I is defined as: + * The last preceding base character was an uppercase I, and there is no + * intervening combining character class 230 (ABOVE). + * + * + * Note that SpecialCasing.txt even in Unicode 3.2 described the condition as: + * + * # When lowercasing, remove dot_above in the sequence I + dot_above, which will turn into i. + * # This matches the behavior of the canonically equivalent I-dot_above + * + * See also the description in this place in older versions of uchar.c (revision 1.100). + * + * Markus W. Scherer 2003-feb-15 + */ + +/* Is preceded by base character 'I' with no intervening cc=230 ? */ +static UBool +isPrecededBy_I(const UCaseProps *csp, UCaseContextIterator *iter, void *context) { + UChar32 c; + int32_t dotType; + int8_t dir; + + if(iter==NULL) { + return FALSE; + } + + for(dir=-1; (c=iter(context, dir))>=0; dir=0) { + if(c==0x49) { + return TRUE; /* preceded by I */ + } + dotType=getDotType(csp, c); + if(dotType!=UCASE_OTHER_ACCENT) { + return FALSE; /* preceded by different base character (not I), or intervening cc==230 */ + } + } + + return FALSE; /* not preceded by I */ +} + +/* Is followed by one or more cc==230 ? */ +static UBool +isFollowedByMoreAbove(const UCaseProps *csp, UCaseContextIterator *iter, void *context) { + UChar32 c; + int32_t dotType; + int8_t dir; + + if(iter==NULL) { + return FALSE; + } + + for(dir=1; (c=iter(context, dir))>=0; dir=0) { + dotType=getDotType(csp, c); + if(dotType==UCASE_ABOVE) { + return TRUE; /* at least one cc==230 following */ + } else if(dotType!=UCASE_OTHER_ACCENT) { + return FALSE; /* next base character, no more cc==230 following */ + } + } + + return FALSE; /* no more cc==230 following */ +} + +/* Is followed by a dot above (without cc==230 in between) ? */ +static UBool +isFollowedByDotAbove(const UCaseProps *csp, UCaseContextIterator *iter, void *context) { + UChar32 c; + int32_t dotType; + int8_t dir; + + if(iter==NULL) { + return FALSE; + } + + for(dir=1; (c=iter(context, dir))>=0; dir=0) { + if(c==0x307) { + return TRUE; + } + dotType=getDotType(csp, c); + if(dotType!=UCASE_OTHER_ACCENT) { + return FALSE; /* next base character or cc==230 in between */ + } + } + + return FALSE; /* no dot above following */ +} + +U_CAPI int32_t U_EXPORT2 +ucase_toFullLower(const UCaseProps *csp, UChar32 c, + UCaseContextIterator *iter, void *context, + const UChar **pString, + const char *locale, int32_t *locCache) { + static const UChar + iDot[2]= { 0x69, 0x307 }, + jDot[2]= { 0x6a, 0x307 }, + iOgonekDot[3]= { 0x12f, 0x307 }, + iDotGrave[3]= { 0x69, 0x307, 0x300 }, + iDotAcute[3]= { 0x69, 0x307, 0x301 }, + iDotTilde[3]= { 0x69, 0x307, 0x303 }; + + UChar32 result; + uint16_t props; + + result=c; + GET_PROPS(csp, c, props); + if(!PROPS_HAS_EXCEPTION(props)) { + if(UCASE_GET_TYPE(props)>=UCASE_UPPER) { + result=c+UCASE_GET_DELTA(props); + } + } else { + const uint16_t *pe=GET_EXCEPTIONS(csp, props), *pe2; + uint16_t excWord=*pe++; + int32_t full; + + pe2=pe; + + if(excWord&UCASE_EXC_CONDITIONAL_SPECIAL) { + /* use hardcoded conditions and mappings */ + int32_t loc=ucase_getCaseLocale(locale, locCache); + + /* + * Test for conditional mappings first + * (otherwise the unconditional default mappings are always taken), + * then test for characters that have unconditional mappings in SpecialCasing.txt, + * then get the UnicodeData.txt mappings. + */ + if( loc==LOC_LITHUANIAN && + /* base characters, find accents above */ + (((c==0x49 || c==0x4a || c==0x12e) && + isFollowedByMoreAbove(csp, iter, context)) || + /* precomposed with accent above, no need to find one */ + (c==0xcc || c==0xcd || c==0x128)) + ) { + /* + # Lithuanian + + # Lithuanian retains the dot in a lowercase i when followed by accents. + + # Introduce an explicit dot above when lowercasing capital I's and J's + # whenever there are more accents above. + # (of the accents used in Lithuanian: grave, acute, tilde above, and ogonek) + + 0049; 0069 0307; 0049; 0049; lt More_Above; # LATIN CAPITAL LETTER I + 004A; 006A 0307; 004A; 004A; lt More_Above; # LATIN CAPITAL LETTER J + 012E; 012F 0307; 012E; 012E; lt More_Above; # LATIN CAPITAL LETTER I WITH OGONEK + 00CC; 0069 0307 0300; 00CC; 00CC; lt; # LATIN CAPITAL LETTER I WITH GRAVE + 00CD; 0069 0307 0301; 00CD; 00CD; lt; # LATIN CAPITAL LETTER I WITH ACUTE + 0128; 0069 0307 0303; 0128; 0128; lt; # LATIN CAPITAL LETTER I WITH TILDE + */ + switch(c) { + case 0x49: /* LATIN CAPITAL LETTER I */ + *pString=iDot; + return 2; + case 0x4a: /* LATIN CAPITAL LETTER J */ + *pString=jDot; + return 2; + case 0x12e: /* LATIN CAPITAL LETTER I WITH OGONEK */ + *pString=iOgonekDot; + return 2; + case 0xcc: /* LATIN CAPITAL LETTER I WITH GRAVE */ + *pString=iDotGrave; + return 3; + case 0xcd: /* LATIN CAPITAL LETTER I WITH ACUTE */ + *pString=iDotAcute; + return 3; + case 0x128: /* LATIN CAPITAL LETTER I WITH TILDE */ + *pString=iDotTilde; + return 3; + default: + return 0; /* will not occur */ + } + /* # Turkish and Azeri */ + } else if(loc==LOC_TURKISH && c==0x130) { + /* + # I and i-dotless; I-dot and i are case pairs in Turkish and Azeri + # The following rules handle those cases. + + 0130; 0069; 0130; 0130; tr # LATIN CAPITAL LETTER I WITH DOT ABOVE + 0130; 0069; 0130; 0130; az # LATIN CAPITAL LETTER I WITH DOT ABOVE + */ + return 0x69; + } else if(loc==LOC_TURKISH && c==0x307 && isPrecededBy_I(csp, iter, context)) { + /* + # When lowercasing, remove dot_above in the sequence I + dot_above, which will turn into i. + # This matches the behavior of the canonically equivalent I-dot_above + + 0307; ; 0307; 0307; tr After_I; # COMBINING DOT ABOVE + 0307; ; 0307; 0307; az After_I; # COMBINING DOT ABOVE + */ + return 0; /* remove the dot (continue without output) */ + } else if(loc==LOC_TURKISH && c==0x49 && !isFollowedByDotAbove(csp, iter, context)) { + /* + # When lowercasing, unless an I is before a dot_above, it turns into a dotless i. + + 0049; 0131; 0049; 0049; tr Not_Before_Dot; # LATIN CAPITAL LETTER I + 0049; 0131; 0049; 0049; az Not_Before_Dot; # LATIN CAPITAL LETTER I + */ + return 0x131; + } else if(c==0x130) { + /* + # Preserve canonical equivalence for I with dot. Turkic is handled below. + + 0130; 0069 0307; 0130; 0130; # LATIN CAPITAL LETTER I WITH DOT ABOVE + */ + *pString=iDot; + return 2; + } else if( c==0x3a3 && + !isFollowedByCasedLetter(csp, iter, context, 1) && + isFollowedByCasedLetter(csp, iter, context, -1) /* -1=preceded */ + ) { + /* greek capital sigma maps depending on surrounding cased letters (see SpecialCasing.txt) */ + /* + # Special case for final form of sigma + + 03A3; 03C2; 03A3; 03A3; Final_Sigma; # GREEK CAPITAL LETTER SIGMA + */ + return 0x3c2; /* greek small final sigma */ + } else { + /* no known conditional special case mapping, use a normal mapping */ + } + } else if(HAS_SLOT(excWord, UCASE_EXC_FULL_MAPPINGS)) { + GET_SLOT_VALUE(excWord, UCASE_EXC_FULL_MAPPINGS, pe, full); + full&=UCASE_FULL_LOWER; + if(full!=0) { + /* set the output pointer to the lowercase mapping */ + *pString=pe+1; + + /* return the string length */ + return full; + } + } + + if(HAS_SLOT(excWord, UCASE_EXC_LOWER)) { + GET_SLOT_VALUE(excWord, UCASE_EXC_LOWER, pe2, result); + } + } + + return (result==c) ? ~result : result; +} + +/* internal */ +static int32_t +toUpperOrTitle(const UCaseProps *csp, UChar32 c, + UCaseContextIterator *iter, void *context, + const UChar **pString, + const char *locale, int32_t *locCache, + UBool upperNotTitle) { + UChar32 result; + uint16_t props; + + result=c; + GET_PROPS(csp, c, props); + if(!PROPS_HAS_EXCEPTION(props)) { + if(UCASE_GET_TYPE(props)==UCASE_LOWER) { + result=c+UCASE_GET_DELTA(props); + } + } else { + const uint16_t *pe=GET_EXCEPTIONS(csp, props), *pe2; + uint16_t excWord=*pe++; + int32_t full, index; + + pe2=pe; + + if(excWord&UCASE_EXC_CONDITIONAL_SPECIAL) { + /* use hardcoded conditions and mappings */ + int32_t loc=ucase_getCaseLocale(locale, locCache); + + if(loc==LOC_TURKISH && c==0x69) { + /* + # Turkish and Azeri + + # I and i-dotless; I-dot and i are case pairs in Turkish and Azeri + # The following rules handle those cases. + + # When uppercasing, i turns into a dotted capital I + + 0069; 0069; 0130; 0130; tr; # LATIN SMALL LETTER I + 0069; 0069; 0130; 0130; az; # LATIN SMALL LETTER I + */ + return 0x130; + } else if(loc==LOC_LITHUANIAN && c==0x307 && isPrecededBySoftDotted(csp, iter, context)) { + /* + # Lithuanian + + # Lithuanian retains the dot in a lowercase i when followed by accents. + + # Remove DOT ABOVE after "i" with upper or titlecase + + 0307; 0307; ; ; lt After_Soft_Dotted; # COMBINING DOT ABOVE + */ + return 0; /* remove the dot (continue without output) */ + } else { + /* no known conditional special case mapping, use a normal mapping */ + } + } else if(HAS_SLOT(excWord, UCASE_EXC_FULL_MAPPINGS)) { + GET_SLOT_VALUE(excWord, UCASE_EXC_FULL_MAPPINGS, pe, full); + + /* start of full case mapping strings */ + ++pe; + + /* skip the lowercase and case-folding result strings */ + pe+=full&UCASE_FULL_LOWER; + full>>=4; + pe+=full&0xf; + full>>=4; + + if(upperNotTitle) { + full&=0xf; + } else { + /* skip the uppercase result string */ + pe+=full&0xf; + full=(full>>4)&0xf; + } + + if(full!=0) { + /* set the output pointer to the result string */ + *pString=pe; + + /* return the string length */ + return full; + } + } + + if(!upperNotTitle && HAS_SLOT(excWord, UCASE_EXC_TITLE)) { + index=UCASE_EXC_TITLE; + } else if(HAS_SLOT(excWord, UCASE_EXC_UPPER)) { + /* here, titlecase is same as uppercase */ + index=UCASE_EXC_UPPER; + } else { + return ~c; + } + GET_SLOT_VALUE(excWord, index, pe2, result); + } + + return (result==c) ? ~result : result; +} + +U_CAPI int32_t U_EXPORT2 +ucase_toFullUpper(const UCaseProps *csp, UChar32 c, + UCaseContextIterator *iter, void *context, + const UChar **pString, + const char *locale, int32_t *locCache) { + return toUpperOrTitle(csp, c, iter, context, pString, locale, locCache, TRUE); +} + +U_CAPI int32_t U_EXPORT2 +ucase_toFullTitle(const UCaseProps *csp, UChar32 c, + UCaseContextIterator *iter, void *context, + const UChar **pString, + const char *locale, int32_t *locCache) { + return toUpperOrTitle(csp, c, iter, context, pString, locale, locCache, FALSE); +} + +/* case folding ------------------------------------------------------------- */ + +/* + * Case folding is similar to lowercasing. + * The result may be a simple mapping, i.e., a single code point, or + * a full mapping, i.e., a string. + * If the case folding for a code point is the same as its simple (1:1) lowercase mapping, + * then only the lowercase mapping is stored. + * + * Some special cases are hardcoded because their conditions cannot be + * parsed and processed from CaseFolding.txt. + * + * Unicode 3.2 CaseFolding.txt specifies for its status field: + +# C: common case folding, common mappings shared by both simple and full mappings. +# F: full case folding, mappings that cause strings to grow in length. Multiple characters are separated by spaces. +# S: simple case folding, mappings to single characters where different from F. +# T: special case for uppercase I and dotted uppercase I +# - For non-Turkic languages, this mapping is normally not used. +# - For Turkic languages (tr, az), this mapping can be used instead of the normal mapping for these characters. +# +# Usage: +# A. To do a simple case folding, use the mappings with status C + S. +# B. To do a full case folding, use the mappings with status C + F. +# +# The mappings with status T can be used or omitted depending on the desired case-folding +# behavior. (The default option is to exclude them.) + + * Unicode 3.2 has 'T' mappings as follows: + +0049; T; 0131; # LATIN CAPITAL LETTER I +0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE + + * while the default mappings for these code points are: + +0049; C; 0069; # LATIN CAPITAL LETTER I +0130; F; 0069 0307; # LATIN CAPITAL LETTER I WITH DOT ABOVE + + * U+0130 has no simple case folding (simple-case-folds to itself). + */ + +/* return the simple case folding mapping for c */ +U_CAPI UChar32 U_EXPORT2 +ucase_fold(const UCaseProps *csp, UChar32 c, uint32_t options) { + uint16_t props; + GET_PROPS(csp, c, props); + if(!PROPS_HAS_EXCEPTION(props)) { + if(UCASE_GET_TYPE(props)>=UCASE_UPPER) { + c+=UCASE_GET_DELTA(props); + } + } else { + const uint16_t *pe=GET_EXCEPTIONS(csp, props); + uint16_t excWord=*pe++; + int32_t index; + if(excWord&UCASE_EXC_CONDITIONAL_FOLD) { + /* special case folding mappings, hardcoded */ + if((options&_FOLD_CASE_OPTIONS_MASK)==U_FOLD_CASE_DEFAULT) { + /* default mappings */ + if(c==0x49) { + /* 0049; C; 0069; # LATIN CAPITAL LETTER I */ + return 0x69; + } else if(c==0x130) { + /* no simple case folding for U+0130 */ + return c; + } + } else { + /* Turkic mappings */ + if(c==0x49) { + /* 0049; T; 0131; # LATIN CAPITAL LETTER I */ + return 0x131; + } else if(c==0x130) { + /* 0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE */ + return 0x69; + } + } + } + if(HAS_SLOT(excWord, UCASE_EXC_FOLD)) { + index=UCASE_EXC_FOLD; + } else if(HAS_SLOT(excWord, UCASE_EXC_LOWER)) { + index=UCASE_EXC_LOWER; + } else { + return c; + } + GET_SLOT_VALUE(excWord, index, pe, c); + } + return c; +} + +/* + * Issue for canonical caseless match (UAX #21): + * Turkic casefolding (using "T" mappings in CaseFolding.txt) does not preserve + * canonical equivalence, unlike default-option casefolding. + * For example, I-grave and I + grave fold to strings that are not canonically + * equivalent. + * For more details, see the comment in unorm_compare() in unorm.cpp + * and the intermediate prototype changes for Jitterbug 2021. + * (For example, revision 1.104 of uchar.c and 1.4 of CaseFolding.txt.) + * + * This did not get fixed because it appears that it is not possible to fix + * it for uppercase and lowercase characters (I-grave vs. i-grave) + * together in a way that they still fold to common result strings. + */ + +U_CAPI int32_t U_EXPORT2 +ucase_toFullFolding(const UCaseProps *csp, UChar32 c, + const UChar **pString, + uint32_t options) { + static const UChar + iDot[2]= { 0x69, 0x307 }; + + UChar32 result; + uint16_t props; + + result=c; + GET_PROPS(csp, c, props); + if(!PROPS_HAS_EXCEPTION(props)) { + if(UCASE_GET_TYPE(props)>=UCASE_UPPER) { + result=c+UCASE_GET_DELTA(props); + } + } else { + const uint16_t *pe=GET_EXCEPTIONS(csp, props), *pe2; + uint16_t excWord=*pe++; + int32_t full, index; + + pe2=pe; + + if(excWord&UCASE_EXC_CONDITIONAL_FOLD) { + /* use hardcoded conditions and mappings */ + if((options&_FOLD_CASE_OPTIONS_MASK)==U_FOLD_CASE_DEFAULT) { + /* default mappings */ + if(c==0x49) { + /* 0049; C; 0069; # LATIN CAPITAL LETTER I */ + return 0x69; + } else if(c==0x130) { + /* 0130; F; 0069 0307; # LATIN CAPITAL LETTER I WITH DOT ABOVE */ + *pString=iDot; + return 2; + } + } else { + /* Turkic mappings */ + if(c==0x49) { + /* 0049; T; 0131; # LATIN CAPITAL LETTER I */ + return 0x131; + } else if(c==0x130) { + /* 0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE */ + return 0x69; + } + } + } else if(HAS_SLOT(excWord, UCASE_EXC_FULL_MAPPINGS)) { + GET_SLOT_VALUE(excWord, UCASE_EXC_FULL_MAPPINGS, pe, full); + + /* start of full case mapping strings */ + ++pe; + + /* skip the lowercase result string */ + pe+=full&UCASE_FULL_LOWER; + full=(full>>4)&0xf; + + if(full!=0) { + /* set the output pointer to the result string */ + *pString=pe; + + /* return the string length */ + return full; + } + } + + if(HAS_SLOT(excWord, UCASE_EXC_FOLD)) { + index=UCASE_EXC_FOLD; + } else if(HAS_SLOT(excWord, UCASE_EXC_LOWER)) { + index=UCASE_EXC_LOWER; + } else { + return ~c; + } + GET_SLOT_VALUE(excWord, index, pe2, result); + } + + return (result==c) ? ~result : result; +} + +/* case mapping properties API ---------------------------------------------- */ + +/* get the UCaseProps singleton, or else its dummy, once and for all */ +static const UCaseProps * +getCaseProps() { + /* + * This lazy intialization with double-checked locking (without mutex protection for + * the initial check) is transiently unsafe under certain circumstances. + * Check the readme and use u_init() if necessary. + */ + + /* the initial check is performed by the GET_CASE_PROPS() macro */ + const UCaseProps *csp; + UErrorCode errorCode=U_ZERO_ERROR; + + csp=ucase_getSingleton(&errorCode); + if(U_FAILURE(errorCode)) { + errorCode=U_ZERO_ERROR; + csp=ucase_getDummy(&errorCode); + if(U_FAILURE(errorCode)) { + return NULL; + } + } + + return csp; +} + +/* + * In ICU 3.0, most Unicode properties were loaded from uprops.icu. + * ICU 3.2 adds ucase.icu for case mapping properties. + * ICU 3.4 adds ubidi.icu for bidi/shaping properties and + * removes case/bidi/shaping properties from uprops.icu. + * + * Loading of uprops.icu was never mutex-protected and required u_init() + * for thread safety. + * In order to maintain performance for all such properties, + * ucase.icu and ubidi.icu are loaded lazily, without mutexing. + * u_init() will try to load them for thread safety, + * but u_init() will not fail if they are missing. + * + * uchar.c maintains a tri-state flag for (not loaded/loaded/failed to load) + * and an error code for load failure. + * Instead, here we try to load at most once. + * If it works, we use the resulting singleton object. + * If it fails, then we get a dummy object, which always works unless + * we are seriously out of memory. + * After the first try, we have a never-changing pointer to either the + * real singleton or the dummy. + * + * This method is used in Unicode properties APIs (uchar.h) that + * do not have a service object and also do not have an error code parameter. + * Other API implementations get the singleton themselves + * (with mutexing), store it in the service object, and report errors. + */ +#define GET_CASE_PROPS() (gCsp!=NULL ? gCsp : getCaseProps()) + +/* public API (see uchar.h) */ + +U_CAPI UBool U_EXPORT2 +u_isULowercase(UChar32 c) { + return (UBool)(UCASE_LOWER==ucase_getType(GET_CASE_PROPS(), c)); +} + +U_CAPI UBool U_EXPORT2 +u_isUUppercase(UChar32 c) { + return (UBool)(UCASE_UPPER==ucase_getType(GET_CASE_PROPS(), c)); +} + +/* Transforms the Unicode character to its lower case equivalent.*/ +U_CAPI UChar32 U_EXPORT2 +u_tolower(UChar32 c) { + return ucase_tolower(GET_CASE_PROPS(), c); +} + +/* Transforms the Unicode character to its upper case equivalent.*/ +U_CAPI UChar32 U_EXPORT2 +u_toupper(UChar32 c) { + return ucase_toupper(GET_CASE_PROPS(), c); +} + +/* Transforms the Unicode character to its title case equivalent.*/ +U_CAPI UChar32 U_EXPORT2 +u_totitle(UChar32 c) { + return ucase_totitle(GET_CASE_PROPS(), c); +} + +/* return the simple case folding mapping for c */ +U_CAPI UChar32 U_EXPORT2 +u_foldCase(UChar32 c, uint32_t options) { + return ucase_fold(GET_CASE_PROPS(), c, options); +} + +U_CFUNC int32_t U_EXPORT2 +ucase_hasBinaryProperty(UChar32 c, UProperty which) { + /* case mapping properties */ + const UCaseProps *csp=GET_CASE_PROPS(); + if(csp==NULL) { + return FALSE; + } + switch(which) { + case UCHAR_LOWERCASE: + return (UBool)(UCASE_LOWER==ucase_getType(csp, c)); + case UCHAR_UPPERCASE: + return (UBool)(UCASE_UPPER==ucase_getType(csp, c)); + case UCHAR_SOFT_DOTTED: + return ucase_isSoftDotted(csp, c); + case UCHAR_CASE_SENSITIVE: + return ucase_isCaseSensitive(csp, c); + default: + return FALSE; + } +} diff --git a/Build/source/libs/icu/icu-xetex/common/ucase.h b/Build/source/libs/icu/icu-xetex/common/ucase.h new file mode 100644 index 00000000000..1c29482d8fc --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucase.h @@ -0,0 +1,376 @@ +/* +******************************************************************************* +* +* Copyright (C) 2004-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: ucase.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2004aug30 +* created by: Markus W. Scherer +* +* Low-level Unicode character/string case mapping code. +*/ + +#ifndef __UCASE_H__ +#define __UCASE_H__ + +#include "unicode/utypes.h" +#include "unicode/uset.h" +#include "uset_imp.h" +#include "udataswp.h" + +U_CDECL_BEGIN + +/* library API -------------------------------------------------------------- */ + +struct UCaseProps; +typedef struct UCaseProps UCaseProps; + +U_CAPI UCaseProps * U_EXPORT2 +ucase_open(UErrorCode *pErrorCode); + +U_CAPI UCaseProps * U_EXPORT2 +ucase_openBinary(const uint8_t *bin, int32_t length, UErrorCode *pErrorCode); + +U_CAPI void U_EXPORT2 +ucase_close(UCaseProps *csp); + + +U_CAPI const UCaseProps * U_EXPORT2 +ucase_getSingleton(UErrorCode *pErrorCode); + +/** + * Get a singleton dummy object, one that works with no real data. + * This can be used when the real data is not available. + * Using the dummy can reduce checks for available data after an initial failure. + */ +U_CAPI const UCaseProps * U_EXPORT2 +ucase_getDummy(UErrorCode *pErrorCode); + + +U_CAPI int32_t U_EXPORT2 +ucase_swap(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode); + +U_CFUNC void U_EXPORT2 +ucase_addPropertyStarts(const UCaseProps *csp, const USetAdder *sa, UErrorCode *pErrorCode); + +/** + * Requires non-NULL locale ID but otherwise does the equivalent of + * checking for language codes as if uloc_getLanguage() were called: + * Accepts both 2- and 3-letter codes and accepts case variants. + */ +U_CFUNC int32_t +ucase_getCaseLocale(const char *locale, int32_t *locCache); + +/** + * Bit mask for getting just the options from a string compare options word + * that are relevant for case-insensitive string comparison. + * See uchar.h. Also include _STRNCMP_STYLE and U_COMPARE_CODE_POINT_ORDER. + * @internal + */ +#define _STRCASECMP_OPTIONS_MASK 0xffff + +/** + * Bit mask for getting just the options from a string compare options word + * that are relevant for case folding (of a single string or code point). + * See uchar.h. + * @internal + */ +#define _FOLD_CASE_OPTIONS_MASK 0xff + +/* single-code point functions */ + +U_CAPI UChar32 U_EXPORT2 +ucase_tolower(const UCaseProps *csp, UChar32 c); + +U_CAPI UChar32 U_EXPORT2 +ucase_toupper(const UCaseProps *csp, UChar32 c); + +U_CAPI UChar32 U_EXPORT2 +ucase_totitle(const UCaseProps *csp, UChar32 c); + +U_CAPI UChar32 U_EXPORT2 +ucase_fold(const UCaseProps *csp, UChar32 c, uint32_t options); + +/** + * Adds all simple case mappings and the full case folding for c to sa, + * and also adds special case closure mappings. + * c itself is not added. + * For example, the mappings + * - for s include long s + * - for sharp s include ss + * - for k include the Kelvin sign + */ +U_CFUNC void U_EXPORT2 +ucase_addCaseClosure(const UCaseProps *csp, UChar32 c, const USetAdder *sa); + +/** + * Maps the string to single code points and adds the associated case closure + * mappings. + * The string is mapped to code points if it is their full case folding string. + * In other words, this performs a reverse full case folding and then + * adds the case closure items of the resulting code points. + * If the string is found and its closure applied, then + * the string itself is added as well as part of its code points' closure. + * It must be length>=0. + * + * @return TRUE if the string was found + */ +U_CFUNC UBool U_EXPORT2 +ucase_addStringCaseClosure(const UCaseProps *csp, const UChar *s, int32_t length, const USetAdder *sa); + +/** @return UCASE_NONE, UCASE_LOWER, UCASE_UPPER, UCASE_TITLE */ +U_CAPI int32_t U_EXPORT2 +ucase_getType(const UCaseProps *csp, UChar32 c); + +/** @return same as ucase_getType(), or <0 if c is case-ignorable */ +U_CAPI int32_t U_EXPORT2 +ucase_getTypeOrIgnorable(const UCaseProps *csp, UChar32 c); + +U_CAPI UBool U_EXPORT2 +ucase_isSoftDotted(const UCaseProps *csp, UChar32 c); + +U_CAPI UBool U_EXPORT2 +ucase_isCaseSensitive(const UCaseProps *csp, UChar32 c); + +/* string case mapping functions */ + +/** + * Iterator function for string case mappings, which need to look at the + * context (surrounding text) of a given character for conditional mappings. + * + * The iterator only needs to go backward or forward away from the + * character in question. It does not use any indexes on this interface. + * It does not support random access or an arbitrary change of + * iteration direction. + * + * The code point being case-mapped itself is never returned by + * this iterator. + * + * @param context A pointer to the iterator's working data. + * @param dir If <0 then start iterating backward from the character; + * if >0 then start iterating forward from the character; + * if 0 then continue iterating in the current direction. + * @return Next code point, or <0 when the iteration is done. + */ +typedef UChar32 U_CALLCONV +UCaseContextIterator(void *context, int8_t dir); + +/** + * Sample struct which may be used by some implementations of + * UCaseContextIterator. + */ +struct UCaseContext { + void *p; + int32_t start, index, limit; + int32_t cpStart, cpLimit; + int8_t dir; + int8_t b1, b2, b3; +}; +typedef struct UCaseContext UCaseContext; + +enum { + /** + * For string case mappings, a single character (a code point) is mapped + * either to itself (in which case in-place mapping functions do nothing), + * or to another single code point, or to a string. + * Aside from the string contents, these are indicated with a single int32_t + * value as follows: + * + * Mapping to self: Negative values (~self instead of -self to support U+0000) + * + * Mapping to another code point: Positive values >UCASE_MAX_STRING_LENGTH + * + * Mapping to a string: The string length (0..UCASE_MAX_STRING_LENGTH) is + * returned. Note that the string result may indeed have zero length. + */ + UCASE_MAX_STRING_LENGTH=0x1f +}; + +/** + * Get the full lowercase mapping for c. + * + * @param csp Case mapping properties. + * @param c Character to be mapped. + * @param iter Character iterator, used for context-sensitive mappings. + * See UCaseContextIterator for details. + * If iter==NULL then a context-independent result is returned. + * @param context Pointer to be passed into iter. + * @param pString If the mapping result is a string, then the pointer is + * written to *pString. + * @param locale Locale ID for locale-dependent mappings. + * @param locCache Initialize to 0; may be used to cache the result of parsing + * the locale ID for subsequent calls. + * Can be NULL. + * @return Output code point or string length, see UCASE_MAX_STRING_LENGTH. + * + * @see UCaseContextIterator + * @see UCASE_MAX_STRING_LENGTH + * @internal + */ +U_CAPI int32_t U_EXPORT2 +ucase_toFullLower(const UCaseProps *csp, UChar32 c, + UCaseContextIterator *iter, void *context, + const UChar **pString, + const char *locale, int32_t *locCache); + +U_CAPI int32_t U_EXPORT2 +ucase_toFullUpper(const UCaseProps *csp, UChar32 c, + UCaseContextIterator *iter, void *context, + const UChar **pString, + const char *locale, int32_t *locCache); + +U_CAPI int32_t U_EXPORT2 +ucase_toFullTitle(const UCaseProps *csp, UChar32 c, + UCaseContextIterator *iter, void *context, + const UChar **pString, + const char *locale, int32_t *locCache); + +U_CAPI int32_t U_EXPORT2 +ucase_toFullFolding(const UCaseProps *csp, UChar32 c, + const UChar **pString, + uint32_t options); + +U_CFUNC int32_t U_EXPORT2 +ucase_hasBinaryProperty(UChar32 c, UProperty which); + + +U_CDECL_BEGIN + +/** + * @internal + */ +typedef int32_t U_CALLCONV +UCaseMapFull(const UCaseProps *csp, UChar32 c, + UCaseContextIterator *iter, void *context, + const UChar **pString, + const char *locale, int32_t *locCache); + +U_CDECL_END + +/* file definitions --------------------------------------------------------- */ + +#define UCASE_DATA_NAME "ucase" +#define UCASE_DATA_TYPE "icu" + +/* format "cAsE" */ +#define UCASE_FMT_0 0x63 +#define UCASE_FMT_1 0x41 +#define UCASE_FMT_2 0x53 +#define UCASE_FMT_3 0x45 + +/* indexes into indexes[] */ +enum { + UCASE_IX_INDEX_TOP, + UCASE_IX_LENGTH, + UCASE_IX_TRIE_SIZE, + UCASE_IX_EXC_LENGTH, + UCASE_IX_UNFOLD_LENGTH, + + UCASE_IX_MAX_FULL_LENGTH=15, + UCASE_IX_TOP=16 +}; + +/* definitions for 16-bit case properties word ------------------------------ */ + +/* 2-bit constants for types of cased characters */ +#define UCASE_TYPE_MASK 3 +enum { + UCASE_NONE, + UCASE_LOWER, + UCASE_UPPER, + UCASE_TITLE +}; + +#define UCASE_GET_TYPE(props) ((props)&UCASE_TYPE_MASK) + +#define UCASE_SENSITIVE 4 +#define UCASE_EXCEPTION 8 + +#define UCASE_DOT_MASK 0x30 +enum { + UCASE_NO_DOT=0, /* normal characters with cc=0 */ + UCASE_SOFT_DOTTED=0x10, /* soft-dotted characters with cc=0 */ + UCASE_ABOVE=0x20, /* "above" accents with cc=230 */ + UCASE_OTHER_ACCENT=0x30 /* other accent character (0>UCASE_DELTA_SHIFT) + +/* case-ignorable uses one of the delta bits, see gencase/store.c */ +#define UCASE_CASE_IGNORABLE 0x40 + +/* exception: bits 15..4 are an unsigned 12-bit index into the exceptions array */ +#define UCASE_EXC_SHIFT 4 +#define UCASE_EXC_MASK 0xfff0 +#define UCASE_MAX_EXCEPTIONS 0x1000 + +/* definitions for 16-bit main exceptions word ------------------------------ */ + +/* first 8 bits indicate values in optional slots */ +enum { + UCASE_EXC_LOWER, + UCASE_EXC_FOLD, + UCASE_EXC_UPPER, + UCASE_EXC_TITLE, + UCASE_EXC_4, /* reserved */ + UCASE_EXC_5, /* reserved */ + UCASE_EXC_CLOSURE, + UCASE_EXC_FULL_MAPPINGS, + UCASE_EXC_ALL_SLOTS /* one past the last slot */ +}; + +/* each slot is 2 uint16_t instead of 1 */ +#define UCASE_EXC_DOUBLE_SLOTS 0x100 + +/* reserved: exception bits 11..9 */ + +/* UCASE_EXC_DOT_MASK=UCASE_DOT_MASK<csp=ucase_getSingleton(pErrorCode); + ucasemap_setLocale(csm, locale, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + uprv_free(csm); + return NULL; + } + + csm->options=options; + return csm; +} + +U_CAPI void U_EXPORT2 +ucasemap_close(UCaseMap *csm) { + if(csm!=NULL) { +#if !UCONFIG_NO_BREAK_ITERATION + ubrk_close(csm->iter); +#endif + uprv_free(csm); + } +} + +U_CAPI const char * U_EXPORT2 +ucasemap_getLocale(const UCaseMap *csm) { + return csm->locale; +} + +U_CAPI uint32_t U_EXPORT2 +ucasemap_getOptions(const UCaseMap *csm) { + return csm->options; +} + +U_CAPI void U_EXPORT2 +ucasemap_setLocale(UCaseMap *csm, const char *locale, UErrorCode *pErrorCode) { + int32_t length; + + if(U_FAILURE(*pErrorCode)) { + return; + } + + length=uloc_getName(locale, csm->locale, (int32_t)sizeof(csm->locale), pErrorCode); + if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR || length==sizeof(csm->locale)) { + *pErrorCode=U_ZERO_ERROR; + /* we only really need the language code for case mappings */ + length=uloc_getLanguage(locale, csm->locale, (int32_t)sizeof(csm->locale), pErrorCode); + } + if(length==sizeof(csm->locale)) { + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + csm->locCache=0; + if(U_SUCCESS(*pErrorCode)) { + ucase_getCaseLocale(csm->locale, &csm->locCache); + } else { + csm->locale[0]=0; + } +} + +U_CAPI void U_EXPORT2 +ucasemap_setOptions(UCaseMap *csm, uint32_t options, UErrorCode *pErrorCode) { + csm->options=options; +} + +#if !UCONFIG_NO_BREAK_ITERATION + +U_CAPI const UBreakIterator * U_EXPORT2 +ucasemap_getBreakIterator(const UCaseMap *csm) { + return csm->iter; +} + +U_CAPI void U_EXPORT2 +ucasemap_setBreakIterator(UCaseMap *csm, UBreakIterator *iterToAdopt, UErrorCode *pErrorCode) { + ubrk_close(csm->iter); + csm->iter=iterToAdopt; +} + +#endif + +/* UTF-8 string case mappings ----------------------------------------------- */ + +/* TODO(markus): Move to a new, separate utf8case.c file. */ + +/* append a full case mapping result, see UCASE_MAX_STRING_LENGTH */ +static U_INLINE int32_t +appendResult(uint8_t *dest, int32_t destIndex, int32_t destCapacity, + int32_t result, const UChar *s) { + UChar32 c; + int32_t length, destLength; + UErrorCode errorCode; + + /* decode the result */ + if(result<0) { + /* (not) original code point */ + c=~result; + length=-1; + } else if(result<=UCASE_MAX_STRING_LENGTH) { + c=U_SENTINEL; + length=result; + } else { + c=result; + length=-1; + } + + if(destIndexindex=csc->cpStart; + csc->dir=dir; + } else if(dir>0) { + /* reset for forward iteration */ + csc->index=csc->cpLimit; + csc->dir=dir; + } else { + /* continue current iteration direction */ + dir=csc->dir; + } + + if(dir<0) { + if(csc->startindex) { + U8_PREV((const uint8_t *)csc->p, csc->start, csc->index, c); + return c; + } + } else { + if(csc->indexlimit) { + U8_NEXT((const uint8_t *)csc->p, csc->index, csc->limit, c); + return c; + } + } + return U_SENTINEL; +} + +/* + * Case-maps [srcStart..srcLimit[ but takes + * context [0..srcLength[ into account. + */ +static int32_t +_caseMap(const UCaseMap *csm, UCaseMapFull *map, + uint8_t *dest, int32_t destCapacity, + const uint8_t *src, UCaseContext *csc, + int32_t srcStart, int32_t srcLimit, + UErrorCode *pErrorCode) { + const UChar *s; + UChar32 c, c2; + int32_t srcIndex, destIndex; + int32_t locCache; + + locCache=csm->locCache; + + /* case mapping loop */ + srcIndex=srcStart; + destIndex=0; + while(srcIndexcpStart=srcIndex; + U8_NEXT(src, srcIndex, srcLimit, c); + csc->cpLimit=srcIndex; + if(c<0) { + int32_t i=csc->cpStart; + while(destIndexcsp, c, utf8_caseContextIterator, csc, &s, csm->locale, &locCache); + if((destIndexdestCapacity) { + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + return destIndex; +} + +#if !UCONFIG_NO_BREAK_ITERATION + +/* + * Internal titlecasing function. + */ +static int32_t +_toTitle(UCaseMap *csm, + uint8_t *dest, int32_t destCapacity, + const uint8_t *src, UCaseContext *csc, + int32_t srcLength, + UErrorCode *pErrorCode) { + UText utext=UTEXT_INITIALIZER; + const UChar *s; + UChar32 c; + int32_t prev, titleStart, titleLimit, index, destIndex, length; + UBool isFirstIndex; + + utext_openUTF8(&utext, (const char *)src, srcLength, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return 0; + } + if(csm->iter==NULL) { + csm->iter=ubrk_open(UBRK_WORD, csm->locale, + NULL, 0, + pErrorCode); + } + ubrk_setUText(csm->iter, &utext, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + utext_close(&utext); + return 0; + } + + /* set up local variables */ + destIndex=0; + prev=0; + isFirstIndex=TRUE; + + /* titlecasing loop */ + while(previter); + } else { + index=ubrk_next(csm->iter); + } + if(index==UBRK_DONE || index>srcLength) { + index=srcLength; + } + + /* + * Unicode 4 & 5 section 3.13 Default Case Operations: + * + * R3 toTitlecase(X): Find the word boundaries based on Unicode Standard Annex + * #29, "Text Boundaries." Between each pair of word boundaries, find the first + * cased character F. If F exists, map F to default_title(F); then map each + * subsequent character C to default_lower(C). + * + * In this implementation, segment [prev..index[ into 3 parts: + * a) uncased characters (copy as-is) [prev..titleStart[ + * b) first case letter (titlecase) [titleStart..titleLimit[ + * c) subsequent characters (lowercase) [titleLimit..index[ + */ + if(prevoptions&U_TITLECASE_NO_BREAK_ADJUSTMENT)==0 && UCASE_NONE==ucase_getType(csm->csp, c)) { + /* Adjust the titlecasing index (titleStart) to the next cased character. */ + for(;;) { + titleStart=titleLimit; + if(titleLimit==index) { + /* + * only uncased characters in [prev..index[ + * stop with titleStart==titleLimit==index + */ + break; + } + U8_NEXT(src, titleLimit, index, c); + if(UCASE_NONE!=ucase_getType(csm->csp, c)) { + break; /* cased letter at [titleStart..titleLimit[ */ + } + } + length=titleStart-prev; + if(length>0) { + if((destIndex+length)<=destCapacity) { + uprv_memcpy(dest+destIndex, src+prev, length); + } + destIndex+=length; + } + } + + if(titleStartcpStart=titleStart; + csc->cpLimit=titleLimit; + c=ucase_toFullTitle(csm->csp, c, utf8_caseContextIterator, csc, &s, csm->locale, &csm->locCache); + destIndex=appendResult(dest, destIndex, destCapacity, c, s); + + /* lowercase [titleLimit..index[ */ + if(titleLimitoptions&U_TITLECASE_NO_LOWERCASE)==0) { + /* Normal operation: Lowercase the rest of the word. */ + destIndex+= + _caseMap( + csm, ucase_toFullLower, + dest+destIndex, destCapacity-destIndex, + src, csc, + titleLimit, index, + pErrorCode); + } else { + /* Optionally just copy the rest of the word unchanged. */ + length=index-titleLimit; + if((destIndex+length)<=destCapacity) { + uprv_memcpy(dest+destIndex, src+titleLimit, length); + } + destIndex+=length; + } + } + } + } + + prev=index; + } + + if(destIndex>destCapacity) { + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + utext_close(&utext); + return destIndex; +} + +#endif + +static int32_t +utf8_foldCase(const UCaseProps *csp, + uint8_t *dest, int32_t destCapacity, + const uint8_t *src, int32_t srcLength, + uint32_t options, + UErrorCode *pErrorCode) { + int32_t srcIndex, destIndex; + + const UChar *s; + UChar32 c, c2; + int32_t start; + + /* case mapping loop */ + srcIndex=destIndex=0; + while(srcIndexdestCapacity) { + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + return destIndex; +} + +/* + * Implement argument checking and buffer handling + * for string case mapping as a common function. + */ + +/* common internal function for public API functions */ + +static int32_t +caseMap(const UCaseMap *csm, + uint8_t *dest, int32_t destCapacity, + const uint8_t *src, int32_t srcLength, + int32_t toWhichCase, + UErrorCode *pErrorCode) { + int32_t destLength; + + /* check argument values */ + if(U_FAILURE(*pErrorCode)) { + return 0; + } + if( destCapacity<0 || + (dest==NULL && destCapacity>0) || + src==NULL || + srcLength<-1 + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* get the string length */ + if(srcLength==-1) { + srcLength=uprv_strlen((const char *)src); + } + + /* check for overlapping source and destination */ + if( dest!=NULL && + ((src>=dest && src<(dest+destCapacity)) || + (dest>=src && dest<(src+srcLength))) + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + destLength=0; + + if(toWhichCase==FOLD_CASE) { + destLength=utf8_foldCase(csm->csp, dest, destCapacity, src, srcLength, + csm->options, pErrorCode); + } else { + UCaseContext csc={ NULL }; + + csc.p=(void *)src; + csc.limit=srcLength; + + if(toWhichCase==TO_LOWER) { + destLength=_caseMap(csm, ucase_toFullLower, + dest, destCapacity, + src, &csc, + 0, srcLength, + pErrorCode); + } else if(toWhichCase==TO_UPPER) { + destLength=_caseMap(csm, ucase_toFullUpper, + dest, destCapacity, + src, &csc, + 0, srcLength, + pErrorCode); + } else /* if(toWhichCase==TO_TITLE) */ { +#if UCONFIG_NO_BREAK_ITERATION + *pErrorCode=U_UNSUPPORTED_ERROR; +#else + /* UCaseMap is actually non-const in toTitle() APIs. */ + destLength=_toTitle((UCaseMap *)csm, dest, destCapacity, + src, &csc, srcLength, + pErrorCode); +#endif + } + } + + return u_terminateChars((char *)dest, destCapacity, destLength, pErrorCode); +} + +/* public API functions */ + +U_CAPI int32_t U_EXPORT2 +ucasemap_utf8ToLower(const UCaseMap *csm, + char *dest, int32_t destCapacity, + const char *src, int32_t srcLength, + UErrorCode *pErrorCode) { + return caseMap(csm, + (uint8_t *)dest, destCapacity, + (const uint8_t *)src, srcLength, + TO_LOWER, pErrorCode); +} + +U_CAPI int32_t U_EXPORT2 +ucasemap_utf8ToUpper(const UCaseMap *csm, + char *dest, int32_t destCapacity, + const char *src, int32_t srcLength, + UErrorCode *pErrorCode) { + return caseMap(csm, + (uint8_t *)dest, destCapacity, + (const uint8_t *)src, srcLength, + TO_UPPER, pErrorCode); +} + +#if !UCONFIG_NO_BREAK_ITERATION + +U_CAPI int32_t U_EXPORT2 +ucasemap_utf8ToTitle(UCaseMap *csm, + char *dest, int32_t destCapacity, + const char *src, int32_t srcLength, + UErrorCode *pErrorCode) { + return caseMap(csm, + (uint8_t *)dest, destCapacity, + (const uint8_t *)src, srcLength, + TO_TITLE, pErrorCode); +} + +#endif + +U_CAPI int32_t U_EXPORT2 +ucasemap_utf8FoldCase(const UCaseMap *csm, + char *dest, int32_t destCapacity, + const char *src, int32_t srcLength, + UErrorCode *pErrorCode) { + return caseMap(csm, + (uint8_t *)dest, destCapacity, + (const uint8_t *)src, srcLength, + FOLD_CASE, pErrorCode); +} diff --git a/Build/source/libs/icu/icu-xetex/common/ucat.c b/Build/source/libs/icu/icu-xetex/common/ucat.c new file mode 100644 index 00000000000..5f6feb9d1be --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucat.c @@ -0,0 +1,76 @@ +/* +********************************************************************** +* Copyright (c) 2003, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Author: Alan Liu +* Created: March 19 2003 +* Since: ICU 2.6 +********************************************************************** +*/ +#include "unicode/ucat.h" +#include "unicode/ustring.h" +#include "cstring.h" +#include "uassert.h" + +/* Separator between set_num and msg_num */ +static const char SEPARATOR = '%'; + +/* Maximum length of a set_num/msg_num key, incl. terminating zero. + * Longest possible key is "-2147483648%-2147483648" */ +#define MAX_KEY_LEN (24) + +/** + * Fill in buffer with a set_num/msg_num key string, given the numeric + * values. Numeric values must be >= 0. Buffer must be of length + * MAX_KEY_LEN or more. + */ +static char* +_catkey(char* buffer, int32_t set_num, int32_t msg_num) { + int32_t i = 0; + i = T_CString_integerToString(buffer, set_num, 10); + buffer[i++] = SEPARATOR; + T_CString_integerToString(buffer+i, msg_num, 10); + return buffer; +} + +U_CAPI u_nl_catd U_EXPORT2 +u_catopen(const char* name, const char* locale, UErrorCode* ec) { + return (u_nl_catd) ures_open(name, locale, ec); +} + +U_CAPI void U_EXPORT2 +u_catclose(u_nl_catd catd) { + ures_close((UResourceBundle*) catd); /* may be NULL */ +} + +U_CAPI const UChar* U_EXPORT2 +u_catgets(u_nl_catd catd, int32_t set_num, int32_t msg_num, + const UChar* s, + int32_t* len, UErrorCode* ec) { + + char key[MAX_KEY_LEN]; + const UChar* result; + + if (ec == NULL || U_FAILURE(*ec)) { + goto ERROR; + } + + result = ures_getStringByKey((const UResourceBundle*) catd, + _catkey(key, set_num, msg_num), + len, ec); + if (U_FAILURE(*ec)) { + goto ERROR; + } + + return result; + + ERROR: + /* In case of any failure, return s */ + if (len != NULL) { + *len = u_strlen(s); + } + return s; +} + +/*eof*/ diff --git a/Build/source/libs/icu/icu-xetex/common/uchar.c b/Build/source/libs/icu/icu-xetex/common/uchar.c new file mode 100644 index 00000000000..e71d864bf2a --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uchar.c @@ -0,0 +1,979 @@ +/* +******************************************************************************** +* Copyright (C) 1996-2007, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************** +* +* File UCHAR.C +* +* Modification History: +* +* Date Name Description +* 04/02/97 aliu Creation. +* 4/15/99 Madhu Updated all the function definitions for C Implementation +* 5/20/99 Madhu Added the function u_getVersion() +* 8/19/1999 srl Upgraded scripts to Unicode3.0 +* 11/11/1999 weiv added u_isalnum(), cleaned comments +* 01/11/2000 helena Renamed u_getVersion to u_getUnicodeVersion. +* 06/20/2000 helena OS/400 port changes; mostly typecast. +****************************************************************************** +*/ + +#include "unicode/utypes.h" +#include "unicode/uchar.h" +#include "unicode/uscript.h" +#include "unicode/udata.h" +#include "umutex.h" +#include "cmemory.h" +#include "ucln_cmn.h" +#include "utrie.h" +#include "udataswp.h" +#include "unormimp.h" /* JAMO_L_BASE etc. */ +#include "uprops.h" + +#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) + +/* dynamically loaded Unicode character properties -------------------------- */ + +#define UCHAR_HARDCODE_DATA 1 + +#if UCHAR_HARDCODE_DATA + +/* uchar_props_data.c is machine-generated by genprops --csource */ +#include "uchar_props_data.c" + +#else + +/* + * loaded uprops.dat - + * for a description of the file format, see icu/source/tools/genprops/store.c + */ +static const char DATA_NAME[] = "uprops"; +static const char DATA_TYPE[] = "icu"; + +static UDataMemory *propsData=NULL; +static UErrorCode dataErrorCode=U_ZERO_ERROR; + +static uint8_t formatVersion[4]={ 0, 0, 0, 0 }; +static UVersionInfo dataVersion={ 0, 0, 0, 0 }; + +static UTrie propsTrie={ 0 }, propsVectorsTrie={ 0 }; +static const uint32_t *pData32=NULL, *propsVectors=NULL; +static int32_t countPropsVectors=0, propsVectorsColumns=0; + +static int8_t havePropsData=0; /* == 0 -> Data has not been loaded. + * < 0 -> Error occured attempting to load data. + * > 0 -> Data has been successfully loaded. + */ + +/* index values loaded from uprops.dat */ +static int32_t indexes[UPROPS_INDEX_COUNT]; + +static UBool U_CALLCONV +isAcceptable(void *context, + const char *type, const char *name, + const UDataInfo *pInfo) { + if( + pInfo->size>=20 && + pInfo->isBigEndian==U_IS_BIG_ENDIAN && + pInfo->charsetFamily==U_CHARSET_FAMILY && + pInfo->dataFormat[0]==0x55 && /* dataFormat="UPro" */ + pInfo->dataFormat[1]==0x50 && + pInfo->dataFormat[2]==0x72 && + pInfo->dataFormat[3]==0x6f && + pInfo->formatVersion[0]==4 && + pInfo->formatVersion[2]==UTRIE_SHIFT && + pInfo->formatVersion[3]==UTRIE_INDEX_SHIFT + ) { + uprv_memcpy(formatVersion, pInfo->formatVersion, 4); + uprv_memcpy(dataVersion, pInfo->dataVersion, 4); + return TRUE; + } else { + return FALSE; + } +} + +static UBool U_CALLCONV uchar_cleanup(void) +{ + if (propsData) { + udata_close(propsData); + propsData=NULL; + } + pData32=NULL; + propsVectors=NULL; + countPropsVectors=0; + uprv_memset(dataVersion, 0, U_MAX_VERSION_LENGTH); + dataErrorCode=U_ZERO_ERROR; + havePropsData=0; + + return TRUE; +} + +struct UCharProps { + UDataMemory *propsData; + UTrie propsTrie, propsVectorsTrie; + const uint32_t *pData32; +}; +typedef struct UCharProps UCharProps; + +/* open uprops.icu */ +static void +_openProps(UCharProps *ucp, UErrorCode *pErrorCode) { + const uint32_t *p; + int32_t length; + + ucp->propsData=udata_openChoice(NULL, DATA_TYPE, DATA_NAME, isAcceptable, NULL, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return; + } + + ucp->pData32=p=(const uint32_t *)udata_getMemory(ucp->propsData); + + /* unserialize the trie; it is directly after the int32_t indexes[UPROPS_INDEX_COUNT] */ + length=(int32_t)p[UPROPS_PROPS32_INDEX]*4; + length=utrie_unserialize(&ucp->propsTrie, (const uint8_t *)(p+UPROPS_INDEX_COUNT), length-64, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return; + } + + /* unserialize the properties vectors trie */ + length=(int32_t)(p[UPROPS_ADDITIONAL_VECTORS_INDEX]-p[UPROPS_ADDITIONAL_TRIE_INDEX])*4; + if(length>0) { + length=utrie_unserialize(&ucp->propsVectorsTrie, (const uint8_t *)(p+p[UPROPS_ADDITIONAL_TRIE_INDEX]), length, pErrorCode); + } + if(length<=0 || U_FAILURE(*pErrorCode)) { + /* + * length==0: + * Allow the properties vectors trie to be missing - + * also requires propsVectorsColumns=indexes[UPROPS_ADDITIONAL_VECTORS_COLUMNS_INDEX] + * to be zero so that this trie is never accessed. + */ + uprv_memset(&ucp->propsVectorsTrie, 0, sizeof(ucp->propsVectorsTrie)); + } +} + +#endif + +#if !UCHAR_HARDCODE_DATA +static int8_t +uprv_loadPropsData(UErrorCode *pErrorCode) { + /* load Unicode character properties data from file if necessary */ + + /* + * This lazy intialization with double-checked locking (without mutex protection for + * haveNormData==0) is transiently unsafe under certain circumstances. + * Check the readme and use u_init() if necessary. + */ + if(havePropsData==0) { + UCharProps ucp={ NULL }; + + if(U_FAILURE(*pErrorCode)) { + return havePropsData; + } + + /* open the data outside the mutex block */ + _openProps(&ucp, pErrorCode); + + if(U_SUCCESS(*pErrorCode)) { + /* in the mutex block, set the data for this process */ + umtx_lock(NULL); + if(propsData==NULL) { + propsData=ucp.propsData; + ucp.propsData=NULL; + pData32=ucp.pData32; + ucp.pData32=NULL; + uprv_memcpy(&propsTrie, &ucp.propsTrie, sizeof(propsTrie)); + uprv_memcpy(&propsVectorsTrie, &ucp.propsVectorsTrie, sizeof(propsVectorsTrie)); + } + + /* initialize some variables */ + uprv_memcpy(indexes, pData32, sizeof(indexes)); + + /* additional properties */ + if(indexes[UPROPS_ADDITIONAL_VECTORS_INDEX]!=0) { + propsVectors=pData32+indexes[UPROPS_ADDITIONAL_VECTORS_INDEX]; + countPropsVectors=indexes[UPROPS_RESERVED_INDEX]-indexes[UPROPS_ADDITIONAL_VECTORS_INDEX]; + propsVectorsColumns=indexes[UPROPS_ADDITIONAL_VECTORS_COLUMNS_INDEX]; + } + + havePropsData=1; + umtx_unlock(NULL); + } else { + dataErrorCode=*pErrorCode; + havePropsData=-1; + } + ucln_common_registerCleanup(UCLN_COMMON_UCHAR, uchar_cleanup); + + /* if a different thread set it first, then close the extra data */ + udata_close(ucp.propsData); /* NULL if it was set correctly */ + } + + return havePropsData; +} + +static int8_t +loadPropsData(void) { + UErrorCode errorCode = U_ZERO_ERROR; + int8_t retVal = uprv_loadPropsData(&errorCode); + return retVal; +} + +#endif + +/* constants and macros for access to the data ------------------------------ */ + +/* getting a uint32_t properties word from the data */ +#if UCHAR_HARDCODE_DATA + +#define GET_PROPS(c, result) UTRIE_GET16(&propsTrie, c, result); + +#else + +#define HAVE_DATA (havePropsData>0 || loadPropsData()>0) +#define GET_PROPS_UNSAFE(c, result) \ + UTRIE_GET16(&propsTrie, c, result); +#define GET_PROPS(c, result) \ + if(HAVE_DATA) { \ + GET_PROPS_UNSAFE(c, result); \ + } else { \ + (result)=0; \ + } + +#endif + +U_CFUNC UBool +uprv_haveProperties(UErrorCode *pErrorCode) { + if(U_FAILURE(*pErrorCode)) { + return FALSE; + } +#if !UCHAR_HARDCODE_DATA + if(havePropsData==0) { + uprv_loadPropsData(pErrorCode); + } + if(havePropsData<0) { + *pErrorCode=dataErrorCode; + return FALSE; + } +#endif + return TRUE; +} + +/* API functions ------------------------------------------------------------ */ + +/* Gets the Unicode character's general category.*/ +U_CAPI int8_t U_EXPORT2 +u_charType(UChar32 c) { + uint32_t props; + GET_PROPS(c, props); + return (int8_t)GET_CATEGORY(props); +} + +/* Enumerate all code points with their general categories. */ +struct _EnumTypeCallback { + UCharEnumTypeRange *enumRange; + const void *context; +}; + +static uint32_t U_CALLCONV +_enumTypeValue(const void *context, uint32_t value) { + return GET_CATEGORY(value); +} + +static UBool U_CALLCONV +_enumTypeRange(const void *context, UChar32 start, UChar32 limit, uint32_t value) { + /* just cast the value to UCharCategory */ + return ((struct _EnumTypeCallback *)context)-> + enumRange(((struct _EnumTypeCallback *)context)->context, + start, limit, (UCharCategory)value); +} + +U_CAPI void U_EXPORT2 +u_enumCharTypes(UCharEnumTypeRange *enumRange, const void *context) { + struct _EnumTypeCallback callback; + + if(enumRange==NULL +#if !UCHAR_HARDCODE_DATA + || !HAVE_DATA +#endif + ) { + return; + } + + callback.enumRange=enumRange; + callback.context=context; + utrie_enum(&propsTrie, _enumTypeValue, _enumTypeRange, &callback); +} + +/* Checks if ch is a lower case letter.*/ +U_CAPI UBool U_EXPORT2 +u_islower(UChar32 c) { + uint32_t props; + GET_PROPS(c, props); + return (UBool)(GET_CATEGORY(props)==U_LOWERCASE_LETTER); +} + +/* Checks if ch is an upper case letter.*/ +U_CAPI UBool U_EXPORT2 +u_isupper(UChar32 c) { + uint32_t props; + GET_PROPS(c, props); + return (UBool)(GET_CATEGORY(props)==U_UPPERCASE_LETTER); +} + +/* Checks if ch is a title case letter; usually upper case letters.*/ +U_CAPI UBool U_EXPORT2 +u_istitle(UChar32 c) { + uint32_t props; + GET_PROPS(c, props); + return (UBool)(GET_CATEGORY(props)==U_TITLECASE_LETTER); +} + +/* Checks if ch is a decimal digit. */ +U_CAPI UBool U_EXPORT2 +u_isdigit(UChar32 c) { + uint32_t props; + GET_PROPS(c, props); + return (UBool)(GET_CATEGORY(props)==U_DECIMAL_DIGIT_NUMBER); +} + +U_CAPI UBool U_EXPORT2 +u_isxdigit(UChar32 c) { + uint32_t props; + + /* check ASCII and Fullwidth ASCII a-fA-F */ + if( + (c<=0x66 && c>=0x41 && (c<=0x46 || c>=0x61)) || + (c>=0xff21 && c<=0xff46 && (c<=0xff26 || c>=0xff41)) + ) { + return TRUE; + } + + GET_PROPS(c, props); + return (UBool)(GET_CATEGORY(props)==U_DECIMAL_DIGIT_NUMBER); +} + +/* Checks if the Unicode character is a letter.*/ +U_CAPI UBool U_EXPORT2 +u_isalpha(UChar32 c) { + uint32_t props; + GET_PROPS(c, props); + return (UBool)((CAT_MASK(props)&U_GC_L_MASK)!=0); +} + +U_CAPI UBool U_EXPORT2 +u_isUAlphabetic(UChar32 c) { + return (u_getUnicodeProperties(c, 1)&U_MASK(UPROPS_ALPHABETIC))!=0; +} + +/* Checks if c is a letter or a decimal digit */ +U_CAPI UBool U_EXPORT2 +u_isalnum(UChar32 c) { + uint32_t props; + GET_PROPS(c, props); + return (UBool)((CAT_MASK(props)&(U_GC_L_MASK|U_GC_ND_MASK))!=0); +} + +/** + * Checks if c is alphabetic, or a decimal digit; implements UCHAR_POSIX_ALNUM. + * @internal + */ +U_CFUNC UBool +u_isalnumPOSIX(UChar32 c) { + return (UBool)(u_isUAlphabetic(c) || u_isdigit(c)); +} + +/* Checks if ch is a unicode character with assigned character type.*/ +U_CAPI UBool U_EXPORT2 +u_isdefined(UChar32 c) { + uint32_t props; + GET_PROPS(c, props); + return (UBool)(GET_CATEGORY(props)!=0); +} + +/* Checks if the Unicode character is a base form character that can take a diacritic.*/ +U_CAPI UBool U_EXPORT2 +u_isbase(UChar32 c) { + uint32_t props; + GET_PROPS(c, props); + return (UBool)((CAT_MASK(props)&(U_GC_L_MASK|U_GC_N_MASK|U_GC_MC_MASK|U_GC_ME_MASK))!=0); +} + +/* Checks if the Unicode character is a control character.*/ +U_CAPI UBool U_EXPORT2 +u_iscntrl(UChar32 c) { + uint32_t props; + GET_PROPS(c, props); + return (UBool)((CAT_MASK(props)&(U_GC_CC_MASK|U_GC_CF_MASK|U_GC_ZL_MASK|U_GC_ZP_MASK))!=0); +} + +U_CAPI UBool U_EXPORT2 +u_isISOControl(UChar32 c) { + return (uint32_t)c<=0x9f && (c<=0x1f || c>=0x7f); +} + +/* Some control characters that are used as space. */ +#define IS_THAT_CONTROL_SPACE(c) \ + (c<=0x9f && ((c>=TAB && c<=CR) || (c>=0x1c && c <=0x1f) || c==NL)) + +/* Checks if the Unicode character is a space character.*/ +U_CAPI UBool U_EXPORT2 +u_isspace(UChar32 c) { + uint32_t props; + GET_PROPS(c, props); + return (UBool)((CAT_MASK(props)&U_GC_Z_MASK)!=0 || IS_THAT_CONTROL_SPACE(c)); +} + +U_CAPI UBool U_EXPORT2 +u_isJavaSpaceChar(UChar32 c) { + uint32_t props; + GET_PROPS(c, props); + return (UBool)((CAT_MASK(props)&U_GC_Z_MASK)!=0); +} + +/* Checks if the Unicode character is a whitespace character.*/ +U_CAPI UBool U_EXPORT2 +u_isWhitespace(UChar32 c) { + uint32_t props; + GET_PROPS(c, props); + return (UBool)( + ((CAT_MASK(props)&U_GC_Z_MASK)!=0 && + c!=NBSP && c!=FIGURESP && c!=NNBSP) || /* exclude no-break spaces */ + IS_THAT_CONTROL_SPACE(c) + ); +} + +U_CAPI UBool U_EXPORT2 +u_isblank(UChar32 c) { + if((uint32_t)c<=0x9f) { + return c==9 || c==0x20; /* TAB or SPACE */ + } else { + /* Zs */ + uint32_t props; + GET_PROPS(c, props); + return (UBool)(GET_CATEGORY(props)==U_SPACE_SEPARATOR); + } +} + +U_CAPI UBool U_EXPORT2 +u_isUWhiteSpace(UChar32 c) { + return (u_getUnicodeProperties(c, 1)&U_MASK(UPROPS_WHITE_SPACE))!=0; +} + +/* Checks if the Unicode character is printable.*/ +U_CAPI UBool U_EXPORT2 +u_isprint(UChar32 c) { + uint32_t props; + GET_PROPS(c, props); + /* comparing ==0 returns FALSE for the categories mentioned */ + return (UBool)((CAT_MASK(props)&U_GC_C_MASK)==0); +} + +/** + * Checks if c is in \p{graph}\p{blank} - \p{cntrl}. + * Implements UCHAR_POSIX_PRINT. + * @internal + */ +U_CFUNC UBool +u_isprintPOSIX(UChar32 c) { + uint32_t props; + GET_PROPS(c, props); + /* + * The only cntrl character in graph+blank is TAB (in blank). + * Here we implement (blank-TAB)=Zs instead of calling u_isblank(). + */ + return (UBool)((GET_CATEGORY(props)==U_SPACE_SEPARATOR) || u_isgraphPOSIX(c)); +} + +U_CAPI UBool U_EXPORT2 +u_isgraph(UChar32 c) { + uint32_t props; + GET_PROPS(c, props); + /* comparing ==0 returns FALSE for the categories mentioned */ + return (UBool)((CAT_MASK(props)& + (U_GC_CC_MASK|U_GC_CF_MASK|U_GC_CS_MASK|U_GC_CN_MASK|U_GC_Z_MASK)) + ==0); +} + +/** + * Checks if c is in + * [^\p{space}\p{gc=Control}\p{gc=Surrogate}\p{gc=Unassigned}] + * with space=\p{Whitespace} and Control=Cc. + * Implements UCHAR_POSIX_GRAPH. + * @internal + */ +U_CFUNC UBool +u_isgraphPOSIX(UChar32 c) { + uint32_t props; + GET_PROPS(c, props); + /* \p{space}\p{gc=Control} == \p{gc=Z}\p{Control} */ + /* comparing ==0 returns FALSE for the categories mentioned */ + return (UBool)((CAT_MASK(props)& + (U_GC_CC_MASK|U_GC_CS_MASK|U_GC_CN_MASK|U_GC_Z_MASK)) + ==0); +} + +U_CAPI UBool U_EXPORT2 +u_ispunct(UChar32 c) { + uint32_t props; + GET_PROPS(c, props); + return (UBool)((CAT_MASK(props)&U_GC_P_MASK)!=0); +} + +/* Checks if the Unicode character can start a Unicode identifier.*/ +U_CAPI UBool U_EXPORT2 +u_isIDStart(UChar32 c) { + /* same as u_isalpha() */ + uint32_t props; + GET_PROPS(c, props); + return (UBool)((CAT_MASK(props)&(U_GC_L_MASK|U_GC_NL_MASK))!=0); +} + +/* Checks if the Unicode character can be a Unicode identifier part other than starting the + identifier.*/ +U_CAPI UBool U_EXPORT2 +u_isIDPart(UChar32 c) { + uint32_t props; + GET_PROPS(c, props); + return (UBool)( + (CAT_MASK(props)& + (U_GC_ND_MASK|U_GC_NL_MASK| + U_GC_L_MASK| + U_GC_PC_MASK|U_GC_MC_MASK|U_GC_MN_MASK) + )!=0 || + u_isIDIgnorable(c)); +} + +/*Checks if the Unicode character can be ignorable in a Java or Unicode identifier.*/ +U_CAPI UBool U_EXPORT2 +u_isIDIgnorable(UChar32 c) { + if(c<=0x9f) { + return u_isISOControl(c) && !IS_THAT_CONTROL_SPACE(c); + } else { + uint32_t props; + GET_PROPS(c, props); + return (UBool)(GET_CATEGORY(props)==U_FORMAT_CHAR); + } +} + +/*Checks if the Unicode character can start a Java identifier.*/ +U_CAPI UBool U_EXPORT2 +u_isJavaIDStart(UChar32 c) { + uint32_t props; + GET_PROPS(c, props); + return (UBool)((CAT_MASK(props)&(U_GC_L_MASK|U_GC_SC_MASK|U_GC_PC_MASK))!=0); +} + +/*Checks if the Unicode character can be a Java identifier part other than starting the + * identifier. + */ +U_CAPI UBool U_EXPORT2 +u_isJavaIDPart(UChar32 c) { + uint32_t props; + GET_PROPS(c, props); + return (UBool)( + (CAT_MASK(props)& + (U_GC_ND_MASK|U_GC_NL_MASK| + U_GC_L_MASK| + U_GC_SC_MASK|U_GC_PC_MASK| + U_GC_MC_MASK|U_GC_MN_MASK) + )!=0 || + u_isIDIgnorable(c)); +} + +U_CAPI int32_t U_EXPORT2 +u_charDigitValue(UChar32 c) { + uint32_t props; + GET_PROPS(c, props); + + if(GET_NUMERIC_TYPE(props)==1) { + return GET_NUMERIC_VALUE(props); + } else { + return -1; + } +} + +U_CAPI double U_EXPORT2 +u_getNumericValue(UChar32 c) { + uint32_t props, numericType, numericValue; + GET_PROPS(c, props); + numericType=GET_NUMERIC_TYPE(props); + + if(numericType==0 || numericType>=UPROPS_NT_COUNT) { + return U_NO_NUMERIC_VALUE; + } + + numericValue=GET_NUMERIC_VALUE(props); + + if(numericType>UPROPS_FRACTION_NUM_SHIFT; + denominator=(numericValue&UPROPS_FRACTION_DEN_MASK)+UPROPS_FRACTION_DEN_OFFSET; + + if(numerator==0) { + numerator=-1; + } + return (double)numerator/(double)denominator; + } else /* numericType==UPROPS_NT_LARGE */ { + /* large value with exponent */ + double numValue; + int32_t mant, exp; + + mant=(int32_t)numericValue>>UPROPS_LARGE_MANT_SHIFT; + exp=(int32_t)numericValue&UPROPS_LARGE_EXP_MASK; + if(mant==0) { + mant=1; + exp+=UPROPS_LARGE_EXP_OFFSET_EXTRA; + } else if(mant>9) { + return U_NO_NUMERIC_VALUE; /* reserved mantissa value */ + } else { + exp+=UPROPS_LARGE_EXP_OFFSET; + } + + numValue=mant; + + /* multiply by 10^exp without math.h */ + while(exp>=4) { + numValue*=10000.; + exp-=4; + } + switch(exp) { + case 3: + numValue*=1000.; + break; + case 2: + numValue*=100.; + break; + case 1: + numValue*=10.; + break; + case 0: + default: + break; + } + + return numValue; + } +} + +/* ICU 3.4: bidi/shaping properties moved to ubidi_props.c */ + +/* ICU 2.1: u_getCombiningClass() moved to unorm.cpp */ + +U_CAPI int32_t U_EXPORT2 +u_digit(UChar32 ch, int8_t radix) { + int8_t value; + if((uint8_t)(radix-2)<=(36-2)) { + value=(int8_t)u_charDigitValue(ch); + if(value<0) { + /* ch is not a decimal digit, try latin letters */ + if(ch>=0x61 && ch<=0x7A) { + value=(int8_t)(ch-0x57); /* ch - 'a' + 10 */ + } else if(ch>=0x41 && ch<=0x5A) { + value=(int8_t)(ch-0x37); /* ch - 'A' + 10 */ + } else if(ch>=0xFF41 && ch<=0xFF5A) { + value=(int8_t)(ch-0xFF37); /* fullwidth ASCII a-z */ + } else if(ch>=0xFF21 && ch<=0xFF3A) { + value=(int8_t)(ch-0xFF17); /* fullwidth ASCII A-Z */ + } + } + } else { + value=-1; /* invalid radix */ + } + return (int8_t)((value(36-2) || (uint32_t)digit>=(uint32_t)radix) { + return 0; + } else if(digit<10) { + return (UChar32)(0x30+digit); + } else { + return (UChar32)((0x61-10)+digit); + } +} + +/* miscellaneous, and support for uprops.c ---------------------------------- */ + +U_CAPI void U_EXPORT2 +u_getUnicodeVersion(UVersionInfo versionArray) { + if(versionArray!=NULL) { + uprv_memcpy(versionArray, dataVersion, U_MAX_VERSION_LENGTH); + } +} + +U_CFUNC uint32_t +u_getUnicodeProperties(UChar32 c, int32_t column) { + uint16_t vecIndex; + + if(column==-1) { + uint32_t props; + GET_PROPS(c, props); + return props; + } else if( +#if !UCHAR_HARDCODE_DATA + !HAVE_DATA || countPropsVectors==0 || +#endif + column<0 || column>=propsVectorsColumns + ) { + return 0; + } else { + UTRIE_GET16(&propsVectorsTrie, c, vecIndex); + return propsVectors[vecIndex+column]; + } +} + +U_CFUNC int32_t +uprv_getMaxValues(int32_t column) { +#if !UCHAR_HARDCODE_DATA + if(HAVE_DATA) { +#endif + switch(column) { + case 0: + return indexes[UPROPS_MAX_VALUES_INDEX]; + case 2: + return indexes[UPROPS_MAX_VALUES_2_INDEX]; + default: + return 0; + } +#if !UCHAR_HARDCODE_DATA + } else { + return 0; + } +#endif +} + +/* + * get Hangul Syllable Type + * implemented here so that uchar.c (uhst_addPropertyStarts()) + * does not depend on uprops.c (u_getIntPropertyValue(c, UCHAR_HANGUL_SYLLABLE_TYPE)) + */ +U_CFUNC UHangulSyllableType +uchar_getHST(UChar32 c) { + /* purely algorithmic; hardcode known characters, check for assigned new ones */ + if(c>UPROPS_AGE_SHIFT; + versionArray[0]=(uint8_t)(version>>4); + versionArray[1]=(uint8_t)(version&0xf); + versionArray[2]=versionArray[3]=0; + } +} + +U_CAPI UScriptCode U_EXPORT2 +uscript_getScript(UChar32 c, UErrorCode *pErrorCode) { + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return USCRIPT_INVALID_CODE; + } + if((uint32_t)c>0x10ffff) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return USCRIPT_INVALID_CODE; + } + + return (UScriptCode)(u_getUnicodeProperties(c, 0)&UPROPS_SCRIPT_MASK); +} + +U_CAPI UBlockCode U_EXPORT2 +ublock_getCode(UChar32 c) { + return (UBlockCode)((u_getUnicodeProperties(c, 0)&UPROPS_BLOCK_MASK)>>UPROPS_BLOCK_SHIFT); +} + +/* property starts for UnicodeSet ------------------------------------------- */ + +/* for Hangul_Syllable_Type */ +U_CFUNC void U_EXPORT2 +uhst_addPropertyStarts(const USetAdder *sa, UErrorCode *pErrorCode) { + UChar32 c; + int32_t value, value2; + + if(U_FAILURE(*pErrorCode)) { + return; + } + +#if !UCHAR_HARDCODE_DATA + if(!HAVE_DATA) { + *pErrorCode=dataErrorCode; + return; + } +#endif + + /* add code points with hardcoded properties, plus the ones following them */ + + /* + * Add Jamo type boundaries for UCHAR_HANGUL_SYLLABLE_TYPE. + * First, we add fixed boundaries for the blocks of Jamos. + * Then we check in loops to see where the current Unicode version + * actually stops assigning such Jamos. We start each loop + * at the end of the per-Jamo-block assignments in Unicode 4 or earlier. + * (These have not changed since Unicode 2.) + */ + sa->add(sa->set, 0x1100); + value=U_HST_LEADING_JAMO; + for(c=0x115a; c<=0x115f; ++c) { + value2=uchar_getHST(c); + if(value!=value2) { + value=value2; + sa->add(sa->set, c); + } + } + + sa->add(sa->set, 0x1160); + value=U_HST_VOWEL_JAMO; + for(c=0x11a3; c<=0x11a7; ++c) { + value2=uchar_getHST(c); + if(value!=value2) { + value=value2; + sa->add(sa->set, c); + } + } + + sa->add(sa->set, 0x11a8); + value=U_HST_TRAILING_JAMO; + for(c=0x11fa; c<=0x11ff; ++c) { + value2=uchar_getHST(c); + if(value!=value2) { + value=value2; + sa->add(sa->set, c); + } + } + + /* Add Hangul type boundaries for UCHAR_HANGUL_SYLLABLE_TYPE. */ + for(c=HANGUL_BASE; c<(HANGUL_BASE+HANGUL_COUNT); c+=JAMO_T_COUNT) { + sa->add(sa->set, c); + sa->add(sa->set, c+1); + } + sa->add(sa->set, c); +} + +static UBool U_CALLCONV +_enumPropertyStartsRange(const void *context, UChar32 start, UChar32 limit, uint32_t value) { + /* add the start code point to the USet */ + const USetAdder *sa=(const USetAdder *)context; + sa->add(sa->set, start); + return TRUE; +} + +#define USET_ADD_CP_AND_NEXT(sa, cp) sa->add(sa->set, cp); sa->add(sa->set, cp+1) + +U_CFUNC void U_EXPORT2 +uchar_addPropertyStarts(const USetAdder *sa, UErrorCode *pErrorCode) { + if(U_FAILURE(*pErrorCode)) { + return; + } + +#if !UCHAR_HARDCODE_DATA + if(!HAVE_DATA) { + *pErrorCode=dataErrorCode; + return; + } +#endif + + /* add the start code point of each same-value range of the main trie */ + utrie_enum(&propsTrie, NULL, _enumPropertyStartsRange, sa); + + /* add code points with hardcoded properties, plus the ones following them */ + + /* add for u_isblank() */ + USET_ADD_CP_AND_NEXT(sa, TAB); + + /* add for IS_THAT_CONTROL_SPACE() */ + sa->add(sa->set, CR+1); /* range TAB..CR */ + sa->add(sa->set, 0x1c); + sa->add(sa->set, 0x1f+1); + USET_ADD_CP_AND_NEXT(sa, NL); + + /* add for u_isIDIgnorable() what was not added above */ + sa->add(sa->set, DEL); /* range DEL..NBSP-1, NBSP added below */ + sa->add(sa->set, HAIRSP); + sa->add(sa->set, RLM+1); + sa->add(sa->set, INHSWAP); + sa->add(sa->set, NOMDIG+1); + USET_ADD_CP_AND_NEXT(sa, ZWNBSP); + + /* add no-break spaces for u_isWhitespace() what was not added above */ + USET_ADD_CP_AND_NEXT(sa, NBSP); + USET_ADD_CP_AND_NEXT(sa, FIGURESP); + USET_ADD_CP_AND_NEXT(sa, NNBSP); + + /* add for u_digit() */ + sa->add(sa->set, U_a); + sa->add(sa->set, U_z+1); + sa->add(sa->set, U_A); + sa->add(sa->set, U_Z+1); + sa->add(sa->set, U_FW_a); + sa->add(sa->set, U_FW_z+1); + sa->add(sa->set, U_FW_A); + sa->add(sa->set, U_FW_Z+1); + + /* add for u_isxdigit() */ + sa->add(sa->set, U_f+1); + sa->add(sa->set, U_F+1); + sa->add(sa->set, U_FW_f+1); + sa->add(sa->set, U_FW_F+1); + + /* add for UCHAR_DEFAULT_IGNORABLE_CODE_POINT what was not added above */ + sa->add(sa->set, WJ); /* range WJ..NOMDIG */ + sa->add(sa->set, 0xfff0); + sa->add(sa->set, 0xfffb+1); + sa->add(sa->set, 0xe0000); + sa->add(sa->set, 0xe0fff+1); + + /* add for UCHAR_GRAPHEME_BASE and others */ + USET_ADD_CP_AND_NEXT(sa, CGJ); +} + +U_CFUNC void U_EXPORT2 +upropsvec_addPropertyStarts(const USetAdder *sa, UErrorCode *pErrorCode) { + if(U_FAILURE(*pErrorCode)) { + return; + } + +#if !UCHAR_HARDCODE_DATA + if(!HAVE_DATA) { + *pErrorCode=dataErrorCode; + return; + } +#endif + + /* add the start code point of each same-value range of the properties vectors trie */ + if(propsVectorsColumns>0) { + /* if propsVectorsColumns==0 then the properties vectors trie may not be there at all */ + utrie_enum(&propsVectorsTrie, NULL, _enumPropertyStartsRange, sa); + } +} diff --git a/Build/source/libs/icu/icu-xetex/common/uchar_props_data.c b/Build/source/libs/icu/icu-xetex/common/uchar_props_data.c new file mode 100644 index 00000000000..aedc01dbdca --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uchar_props_data.c @@ -0,0 +1,2155 @@ +/* + * Copyright (C) 1999-2007, International Business Machines + * Corporation and others. All Rights Reserved. + * + * file name: uchar_props_data.c + * + * machine-generated on: 2006-06-13 + */ + +static const UVersionInfo formatVersion={4,0,5,2}; + +static const UVersionInfo dataVersion={5,0,0,0}; + +static const uint16_t propsTrie_index[13440]={ +0x2a8,0x2b0,0x2b8,0x2c0,0x2c8,0x2d0,0x2d8,0x2e0,0x2e8,0x2ea,0x2f0,0x2f3,0x2fb,0x303,0x30b,0x313, +0x2e8,0x319,0x321,0x325,0x328,0x32e,0x336,0x33e,0x346,0x346,0x346,0x34a,0x352,0x35a,0x35f,0x365, +0x36d,0x371,0x325,0x2e8,0x379,0x2e8,0x381,0x2e8,0x385,0x38a,0x38f,0x397,0x39d,0x3a2,0x3aa,0x3b0, +0x3b8,0x3c0,0x3c8,0x3d0,0x3d5,0x3d5,0x3d8,0x3e0,0x3e8,0x3ed,0x3f3,0x3f8,0x3d5,0x400,0x408,0x40e, +0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x416,0x418,0x420,0x428,0x430,0x436,0x43e,0x446, +0x44e,0x454,0x45c,0x464,0x46c,0x472,0x47a,0x482,0x430,0x48a,0x492,0x49a,0x4a2,0x4aa,0x4b2,0x4b9, +0x4c1,0x4c7,0x4cf,0x4d7,0x4df,0x4e5,0x4ed,0x4f5,0x4df,0x4fd,0x505,0x4d7,0x50d,0x514,0x51c,0x524, +0x52c,0x530,0x538,0x2a0,0x540,0x548,0x550,0x2a0,0x558,0x560,0x568,0x56e,0x576,0x57d,0x585,0x2a0, +0x3d5,0x58d,0x595,0x2a0,0x2a0,0x36d,0x59d,0x5a1,0x3d5,0x3d5,0x5a9,0x3d5,0x3d5,0x5b1,0x3d5,0x5b3, +0x3d5,0x3d5,0x5bb,0x3d5,0x5c3,0x5c7,0x5cf,0x3d5,0x5d5,0x3d5,0x5db,0x5e3,0x5eb,0x3d5,0x3d5,0x5f3, +0x52c,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x5fb,0x603,0x3d5,0x3d5,0x60b,0x613,0x61b,0x623,0x62b,0x3d5,0x633,0x63b,0x643, +0x64b,0x3d5,0x653,0x655,0x3d5,0x65d,0x2a0,0x2a0,0x665,0x66d,0x675,0x67a,0x3d5,0x682,0x68a,0x692, +0x69a,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x6a2,0x6a5,0x6ad,0x6b5,0x2a0,0x2a0,0x2a0,0x2a0, +0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x325,0x6bd,0x6c0,0x6c8,0x6cf,0x6c0,0x6d7,0x6da, +0x2e8,0x2e8,0x2e8,0x2e8,0x6e2,0x2e8,0x2e8,0x6ea,0x6f2,0x6fa,0x702,0x70a,0x712,0x716,0x71e,0x726, +0x72e,0x736,0x73e,0x746,0x74e,0x756,0x75c,0x764,0x76c,0x774,0x77c,0x784,0x78c,0x794,0x799,0x79f, +0x7a4,0x7a4,0x7a4,0x7a4,0x7a4,0x7a4,0x7a4,0x7a4,0x7ac,0x7b4,0x692,0x7b7,0x7bf,0x7c6,0x7cb,0x7d3, +0x692,0x7db,0x7e3,0x7eb,0x7ee,0x692,0x692,0x7f5,0x692,0x692,0x692,0x692,0x692,0x7fd,0x805,0x807, +0x692,0x692,0x692,0x80f,0x813,0x81b,0x2a0,0x2a0,0x823,0x829,0x82e,0x836,0x83e,0x844,0x84c,0x853, +0x692,0x692,0x692,0x692,0x692,0x692,0x692,0x692,0x7a4,0x7a4,0x7a4,0x7a4,0x85b,0x7a4,0x862,0x869, +0x7a4,0x7a4,0x7a4,0x7a4,0x7a4,0x7a4,0x7a4,0x7a4,0x871,0x879,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0, +0x36d,0x881,0x885,0x88d,0x2e8,0x2e8,0x2e8,0x895,0x325,0x89d,0x3d5,0x8a4,0x8ac,0x8b4,0x8b4,0x2a0, +0x8bc,0x2a0,0x2a0,0x2a0,0x8c4,0x692,0x692,0x8cb,0x692,0x692,0x692,0x692,0x692,0x692,0x8d3,0x8d9, +0x8e1,0x8e9,0x52c,0x3d5,0x8f1,0x8f9,0x3d5,0x8fb,0x52b,0x903,0x3d5,0x3d5,0x908,0x655,0x90e,0x661, +0x916,0x91e,0x925,0x692,0x91e,0x92d,0x692,0x916,0x692,0x692,0x692,0x692,0x692,0x692,0x692,0x692, +0xc64,0x3d5,0x3d5,0x3d5,0xbfc,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0xc6a,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0xc84,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0xbb3,0x692,0x692, +0xbdc,0x3d5,0xc99,0x3d5,0xc01,0xcde,0xcb1,0xc17,0xc62,0x3d5,0x3d5,0xccd,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0xce6,0x3d5,0xced,0xc07,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0xcb9,0x3d5,0x3d5,0x3d5,0xc1f,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0xc42,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0xbe0,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0xbe7,0x3d5,0x3d5,0x3d5,0xca1,0xbee,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0xcbd,0x3d5,0x3d5,0xc95,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0xc88,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0xca9,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0xcd2,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0xc49,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0xcda,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0xc0c,0x3d5,0x3d5,0x3d5,0xc11,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0xc79,0xc7d,0x3d5,0x3d5,0x3d5,0xbd4,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x654,0x2a0,0x2a0, +0x935,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x93b,0x692,0x7db,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0, +0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x943,0x94b,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0, +0x953,0x95b,0x3d5,0x963,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0, +0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0xbbb,0x2a0,0x2a0, +0xcf5,0xcf8,0xd00,0xd07,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0, +0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0xd0f,0x2a0,0xd17,0xd18,0xd17,0xd18, +0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3, +0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3, +0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb, +0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb, +0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb, +0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb, +0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb, +0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb, +0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb, +0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb, +0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb, +0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb, +0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb, +0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb, +0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x96b,0x3d5,0x971,0x3d5,0x3d5,0x5b3,0x2a0,0x979,0x981,0x989,0x3d5,0x3d5,0x3f7,0x991,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x996,0x661,0x3d5,0x99e,0x3d5,0x9a4,0x9a8, +0x9b0,0x9b8,0x9bf,0x9c7,0x3d5,0x3d5,0x3d5,0x9cd,0x9d5,0x2b8,0x9dd,0x9e5,0x9ea,0x9f2,0x9fa,0xa02, +0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3, +0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3,0xbc3, +0xa0a,0xa11,0xa19,0x2a0,0x3d5,0x3d5,0x3d5,0xa21,0xa29,0xa31,0xa39,0xa41,0xa48,0x2a0,0x2a0,0x2a0, +0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x9f2,0xa50,0xa58,0x2a0,0xa60,0x3d5,0xa68,0x2a0, +0x36d,0xa70,0xa74,0x3d5,0x417,0xa7c,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0, +0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0, +0xa84,0xa87,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0xa8f,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0, +0xa97,0xa9e,0xaa6,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0xaae,0x2a0,0x2a0,0x2a0,0x2a0, +0xab6,0xabe,0xac6,0xace,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0, +0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0, +0x692,0x692,0x692,0x692,0x692,0x692,0x692,0x8d3,0x692,0xad6,0x692,0xadd,0xae5,0xaeb,0xaef,0x2a0, +0x692,0x692,0xaf7,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x692,0x692,0xaff,0xb07,0x2a0,0x2a0,0x2a0,0x2a0, +0xb0f,0xb16,0xb1b,0xb21,0xb29,0xb31,0xb39,0xb13,0xb41,0xb49,0xb51,0xb56,0xb28,0xb0f,0xb16,0xb12, +0xb21,0xb5e,0xb10,0xb61,0xb13,0xb69,0xb71,0xb79,0xb80,0xb6c,0xb74,0xb7c,0xb83,0xb6f,0xb8b,0xb93, +0xc90,0x3d5,0x3d5,0xc50,0x3d5,0x3d5,0x3d5,0xc58,0x3d5,0xc72,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0xbf4,0x3d5,0x3d5,0xcc5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0xc25,0xc2d,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0xc11,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0xc34,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0xc3b,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0xc5d,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x8ac,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0, +0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5,0x3d5, +0x417,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0, +0xb9b,0xba3,0xba3,0xba3,0x2a0,0x2a0,0x2a0,0x2a0,0x346,0x346,0x346,0x346,0x346,0x346,0x346,0xbab, +0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0, +0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb, +0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb, +0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb, +0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcb,0xbcc, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf, +0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf, +0xc,0x17,0x17,0x17,0x19,0x17,0x17,0x17,0x14,0x15,0x17,0x18,0x17,0x13,0x17,0x17, +0x29,0x129,0x229,0x329,0x429,0x529,0x629,0x729,0x829,0x929,0x17,0x17,0x18,0x18,0x18,0x17, +0x17,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, +1,1,1,1,1,1,1,1,1,1,1,0x14,0x17,0x15,0x1a,0x16, +0x1a,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, +2,2,2,2,2,2,2,2,2,2,2,0x14,0x18,0x15,0x18,0xf, +0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf, +0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf, +0xc,0x17,0x19,0x19,0x19,0x19,0x1b,0x1b,0x1a,0x1b,2,0x1c,0x18,0x10,0x1b,0x1a, +0x1b,0x18,0x24b,0x34b,0x1a,2,0x1b,0x17,0x1a,0x14b,2,0x1d,0xa8b,0x88b,0x1a8b,0x17, +1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, +1,1,1,1,1,1,1,0x18,1,1,1,1,1,1,1,2, +2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, +2,2,2,2,2,2,2,0x18,2,2,2,2,2,2,2,2, +1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2, +1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2, +2,1,2,1,2,1,2,1,2,2,1,2,1,2,1,2, +1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2, +1,2,1,2,1,1,2,1,2,1,2,2,2,1,1,2, +1,2,1,1,2,1,1,1,2,2,1,1,1,1,2,1, +1,2,1,1,1,2,2,2,1,1,2,1,1,2,1,2, +1,2,1,1,2,1,2,2,1,2,1,1,2,1,1,1, +2,1,2,1,1,2,2,5,1,2,2,2,5,5,5,5, +1,3,2,1,3,2,1,3,2,1,2,1,2,1,2,1, +2,1,2,1,2,1,2,1,2,2,1,2,1,2,1,2, +1,2,1,2,1,2,1,2,1,2,1,2,2,1,3,2, +1,2,1,1,1,2,1,2,1,2,1,2,1,2,1,2, +1,2,1,2,1,2,1,2,2,2,2,2,2,2,1,1, +2,1,1,2,2,1,2,1,1,1,1,2,1,2,1,2, +1,2,1,2,2,2,2,2,2,2,2,2,2,2,2,2, +2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, +2,2,2,2,5,2,2,2,2,2,2,2,2,2,2,2, +2,2,2,2,2,2,2,2,4,4,4,4,4,4,4,4, +4,4,4,4,4,4,4,4,4,4,0x1a,0x1a,0x1a,0x1a,4,4, +4,4,4,4,4,4,4,4,4,4,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a, +0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,4,4,4,4,4,0x1a,0x1a,0x1a, +0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,4,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a, +0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,6,6,6,6,6,6,6,6, +6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6, +6,6,6,6,6,6,6,6,0,0,0,0,0x1a,0x1a,0,0, +0,0,4,2,2,2,0x17,0,0,0,0,0,0x1a,0x1a,1,0x17, +1,1,1,0,1,0,1,1,2,1,1,1,1,1,1,1, +1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1, +1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2, +2,2,2,2,2,2,2,2,2,2,2,0,2,2,1,1, +1,2,2,2,1,2,1,2,1,2,1,2,1,2,1,2, +1,2,1,2,2,2,2,2,1,2,0x18,1,2,1,1,2, +2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, +1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, +1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2, +2,2,2,2,1,2,0x1b,6,6,6,6,0,7,7,1,2, +1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2, +1,2,1,2,1,1,2,1,2,1,2,1,2,1,2,1, +2,1,2,2,1,2,1,2,1,2,1,2,1,2,1,2, +1,2,1,2,1,2,1,2,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1, +1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, +1,1,1,0,0,4,0x17,0x17,0x17,0x17,0x17,0x17,0,2,2,2, +2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, +2,2,2,2,2,2,2,2,2,2,2,2,0,0x17,0x13,0, +0,0,0,0,0,6,6,6,6,6,6,6,6,6,6,6, +6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6, +6,6,6,6,6,6,0x17,6,0x17,6,6,0x17,6,6,0x17,6, +0,0,0,0,0,0,0,0,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,0,0,0,0,0, +5,5,5,0x17,0x17,0,0,0,0,0,0,0,0,0,0,0, +0x10,0x10,0x10,0x10,0,0,0,0,0,0,0,0x19,0x17,0x17,0x1b,0x1b, +6,6,6,6,6,6,0,0,0,0,0,0x17,0,0,0x17,0x17, +0,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,0,0,0,0,0, +4,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6, +6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,0, +0x29,0x129,0x229,0x329,0x429,0x529,0x629,0x729,0x829,0x929,0x17,0x17,0x17,0x17,5,5, +6,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,0x17,5,6,6,6,6,6,6,6,0x10,7,6, +6,6,6,6,6,4,4,6,6,0x1b,6,6,6,6,5,5, +0x29,0x129,0x229,0x329,0x429,0x529,0x629,0x729,0x829,0x929,5,5,5,0x1b,0x1b,5, +0x17,0x17,0x17,0x17,0x17,0x17,0x17,0x17,0x17,0x17,0x17,0x17,0x17,0x17,0,0x10, +5,6,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6, +6,6,6,6,6,6,6,0,0,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6, +6,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0x29,0x129,0x229,0x329,0x429,0x529,0x629,0x729,0x829,0x929,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,6,6,6,6,6,6,6,6,6,4,4,0x1b,0x17, +0x17,0x17,4,0,0,0,0,0,0,6,6,8,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,0,0,6,5,8,8, +8,6,6,6,6,6,6,6,6,8,8,8,8,6,0,0, +5,6,6,6,6,0,0,0,5,5,5,5,5,5,5,5, +5,5,6,6,0x17,0x17,0x29,0x129,0x229,0x329,0x429,0x529,0x629,0x729,0x829,0x929, +0x17,0,0,0,0,0,0,0,0,0,0,5,5,5,5,5, +0,6,8,8,0,5,5,5,5,5,5,5,5,0,0,5, +5,0,0,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,0,5,5,5,5,5,5,5,0,5,0,0,0,5,5, +5,5,0,0,6,5,8,8,8,6,6,6,6,0,0,8, +8,0,0,8,8,6,5,0,0,0,0,0,0,0,0,8, +0,0,0,0,5,5,0,5,5,5,6,6,0,0,0x29,0x129, +0x229,0x329,0x429,0x529,0x629,0x729,0x829,0x929,5,5,0x19,0x19,0x16b,0x26b,0x36b,0x46b, +0xb,0x106b,0x1b,0,0,0,0,0,0,6,6,8,0,5,5,5, +5,5,5,0,0,0,0,5,5,0,0,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,0,5,5,5,5,5,5, +5,0,5,5,0,5,5,0,5,5,0,0,6,0,8,8, +8,6,6,0,0,0,0,6,6,0,0,6,6,6,0,0, +0,0,0,0,0,0,0,0,0,5,5,5,5,0,5,0, +0,0,0,0,0,0,0x29,0x129,0x229,0x329,0x429,0x529,0x629,0x729,0x829,0x929, +6,6,5,5,5,0,0,0,0,0,0,0,0,0,0,0, +0,6,6,8,0,5,5,5,5,5,5,5,5,5,0,5, +5,5,0,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,0,5,5,5,5,5,5,5,0,5,5,0,5,5,5, +5,5,0,0,6,5,8,8,8,6,6,6,6,6,0,6, +6,8,0,8,8,6,0,0,5,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,5,5,6,6,0,0,0x29,0x129, +0x229,0x329,0x429,0x529,0x629,0x729,0x829,0x929,0,0x19,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,5,5,5,5,5,5,5,5, +5,0,5,5,5,5,5,5,5,0,5,5,0,5,5,5, +5,5,0,0,6,5,8,6,8,6,6,6,0,0,0,8, +8,0,0,8,8,6,0,0,0,0,0,0,0,0,6,8, +0,0,0,0,5,5,0,5,5,5,0,0,0,0,0x29,0x129, +0x229,0x329,0x429,0x529,0x629,0x729,0x829,0x929,0x1b,5,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,6,5,0,5,5,5, +5,5,5,0,0,0,5,5,5,0,5,5,5,5,0,0, +0,5,5,0,5,0,5,5,0,0,0,5,5,0,0,0, +5,5,5,0,0,0,5,5,5,5,5,5,5,5,5,5, +5,5,0,0,0,0,8,8,6,8,8,0,0,0,8,8, +8,0,8,8,8,6,0,0,0,0,0,0,0,0,0,8, +0,0,0,0,0,0,0,0,0,0,0x29,0x129,0x229,0x329,0x429,0x529, +0x629,0x729,0x829,0x929,0xa6b,0x646b,0x11ab,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x19,0x1b,0, +0,0,0,0,0,8,8,8,0,5,5,5,5,5,5,5, +5,0,5,5,5,0,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,0,5,5,5,5,5,5,5,5,5,5, +0,5,5,5,5,5,0,0,0,0,6,6,6,8,8,8, +8,0,6,6,6,0,6,6,6,6,0,0,0,0,0,0, +0,6,6,0,0,0,0,0,0,0,0,0,5,5,0,0, +0,0,0x29,0x129,0x229,0x329,0x429,0x529,0x629,0x729,0x829,0x929,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,8, +0,5,5,5,5,5,5,5,5,0,5,5,5,0,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,0,5,5, +5,5,5,5,5,5,5,5,0,5,5,5,5,5,0,0, +6,5,8,6,8,8,8,8,8,0,6,8,8,0,8,8, +6,6,0,0,0,0,0,0,0,8,8,0,0,0,0,0, +0,0,5,0,5,5,6,6,0,0,0x29,0x129,0x229,0x329,0x429,0x529, +0x629,0x729,0x829,0x929,0,0x1b,0x1b,0,0,0,0,0,0,0,0,0, +0,0,0,0,5,5,5,5,5,5,5,5,5,0,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,0,0, +0,0,8,8,8,6,6,6,0,0,8,8,8,0,8,8, +8,6,0,0,0,0,0,0,0,0,0,8,0,0,0,0, +0,0,0,0,0,0,8,8,0,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,0,0,0,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,0,5,5,5,5,5,5,5,5,5,0,5,0,0, +5,5,5,5,5,5,5,0,0,0,6,0,0,0,0,8, +8,8,6,6,6,0,6,0,8,8,8,8,8,8,8,8, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,8,8,0x17,0,0,0,0,0,0,0,0,0,0,0, +0,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,6,5,5,6,6,6,6,6,6,6,0,0,0,0,0x19, +5,5,5,5,5,5,4,6,6,6,6,6,6,6,6,0x17, +0x29,0x129,0x229,0x329,0x429,0x529,0x629,0x729,0x829,0x929,0x17,0x17,0,0,0,0, +0,5,5,0,5,0,0,5,5,0,5,0,0,5,0,0, +0,0,0,0,5,5,5,5,0,5,5,5,5,5,5,5, +0,5,5,5,0,5,0,5,0,0,5,5,0,5,5,5, +5,6,5,5,6,6,6,6,6,6,0,6,6,5,0,0, +5,5,5,5,5,0,4,0,6,6,6,6,6,6,0,0, +0x29,0x129,0x229,0x329,0x429,0x529,0x629,0x729,0x829,0x929,0,0,5,5,0,0, +5,0x1b,0x1b,0x1b,0x17,0x17,0x17,0x17,0x17,0x17,0x17,0x17,0x17,0x17,0x17,0x17, +0x17,0x17,0x17,0x1b,0x1b,0x1b,0x1b,0x1b,6,6,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x29,0x129,0x229,0x329,0x429,0x529,0x629,0x729,0x829,0x929,0x88b,0x188b,0x288b,0x388b,0x488b,0x588b, +0x688b,0x788b,0x888b,0x8b,0x1b,6,0x1b,6,0x1b,6,0x14,0x15,0x14,0x15,8,8, +5,5,5,5,5,5,5,5,0,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,0,0,0,0,0,0,6,6,6,6,6,6,6, +6,6,6,6,6,6,6,8,6,6,6,6,6,0x17,6,6, +5,5,5,5,0,0,0,0,6,6,6,6,6,6,6,6, +0,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6, +6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6, +6,0,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,6,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0,0,0x1b,0x17,0x17,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,5,5,0,5,5,5,5,5,0,5,5,0, +8,6,6,6,6,8,6,0,0,0,6,6,8,6,0,0, +0,0,0,0,0x29,0x129,0x229,0x329,0x429,0x529,0x629,0x729,0x829,0x929,0x17,0x17, +0x17,0x17,0x17,0x17,5,5,5,5,5,5,8,8,6,6,0,0, +0,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0, +0,0,0,0,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,0x17, +4,0,0,0,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,0,0, +0,0,0,5,5,5,5,0,0,0,0,0,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,0,0,0,0,0,0,5,5,5,5, +5,5,5,5,5,0,5,5,5,5,0,0,5,5,5,5, +5,5,5,0,5,0,5,5,5,5,0,0,5,5,5,5, +5,5,5,5,5,0,5,5,5,5,0,0,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,0,5,5, +5,5,0,0,5,5,5,5,5,5,5,0,5,0,5,5, +5,5,0,0,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,0,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,0,5,5,5,5,0,0,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,0,0,0,0,6,0x1b,0x17,0x17,0x17, +0x17,0x17,0x17,0x17,0x17,0x14b,0x24b,0x34b,0x44b,0x54b,0x64b,0x74b,0x84b,0x94b,0xa6b,0x146b, +0x1e6b,0x286b,0x326b,0x3c6b,0x466b,0x506b,0x5a6b,0x646b,0x12ab,0,0,0,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0,0,0,0,0,0,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,0,0,0,0,0,0,0,0,0,0,0,5,5,5,5, +5,5,5,5,5,5,5,5,5,0x17,0x17,5,5,5,5,5, +5,5,5,0,0,0,0,0,0,0,0,0,0xc,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,0x14,0x15,0,0,0,5,5,5,5, +5,5,5,5,5,5,5,0x17,0x17,0x17,0x116a,0x126a,0x136a,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,5,5,5,5, +5,5,5,5,5,5,5,5,5,0,5,5,5,5,6,6, +6,0,0,0,0,0,0,0,0,0,0,0,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,6,6, +6,0x17,0x17,0,0,0,0,0,0,0,0,0,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,6,6, +0,0,0,0,0,0,0,0,0,0,0,0,5,5,5,5, +5,5,5,5,5,5,5,5,5,0,5,5,5,0,6,6, +0,0,0,0,0,0,0,0,0,0,0,0,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +0x10,0x10,8,6,6,6,6,6,6,6,8,8,8,8,8,8, +8,8,6,8,8,6,6,6,6,6,6,6,6,6,6,6, +0x17,0x17,0x17,4,0x17,0x17,0x17,0x19,5,6,0,0,0x29,0x129,0x229,0x329, +0x429,0x529,0x629,0x729,0x829,0x929,0,0,0,0,0,0,0x6b,0x16b,0x26b,0x36b, +0x46b,0x56b,0x66b,0x76b,0x86b,0x96b,0,0,0,0,0,0,0x17,0x17,0x17,0x17, +0x17,0x17,0x13,0x17,0x17,0x17,0x17,6,6,6,0xc,0,0x29,0x129,0x229,0x329, +0x429,0x529,0x629,0x729,0x829,0x929,0,0,0,0,0,0,5,5,5,4, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,0,0,0,0, +0,0,0,0,5,5,5,5,5,5,5,5,5,6,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,0,0,0,6,6,6,8,8,8,8,6,6,8,8,8, +0,0,0,0,8,8,6,8,8,8,8,8,8,6,6,6, +0,0,0,0,0x1b,0,0,0,0x17,0x17,0x29,0x129,0x229,0x329,0x429,0x529, +0x629,0x729,0x829,0x929,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,0,0,5,5,5,5,5,0,0,0, +0,0,0,0,0,0,0,0,5,5,5,5,5,5,5,5, +5,5,0,0,0,0,0,0,8,8,8,8,8,8,8,8, +8,8,8,8,8,8,8,8,8,5,5,5,5,5,5,5, +8,8,0,0,0,0,0,0,0x29,0x129,0x229,0x329,0x429,0x529,0x629,0x729, +0x829,0x929,0,0,0,0,0x17,0x17,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,6, +6,8,8,8,0,0,0x17,0x17,6,6,6,6,8,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,6,8,6,6,6,6,6,8, +6,8,8,8,8,8,6,8,8,5,5,5,5,5,5,5, +0,0,0,0,0x29,0x129,0x229,0x329,0x429,0x529,0x629,0x729,0x829,0x929,0x17,0x17, +0x17,0x17,0x17,0x17,0x17,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,6, +6,6,6,6,6,6,6,6,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0,0,0,2,2,2,2,2,2,2,2,2,2,2,2, +4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4, +4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4, +4,4,2,2,2,2,2,2,2,2,2,2,2,2,2,2, +2,2,2,2,2,2,2,2,4,2,2,2,2,2,2,2, +2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, +2,2,2,2,2,2,2,4,4,4,4,4,6,6,6,6, +6,6,6,6,6,6,6,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,6,6,1,2,1,2,1,2,1,2, +1,2,1,2,1,2,1,2,1,2,1,2,1,2,2,2, +2,2,2,2,0,0,0,0,1,2,1,2,1,2,1,2, +1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2, +1,2,0,0,0,0,0,0,2,2,2,2,2,2,2,2, +1,1,1,1,1,1,1,1,2,2,2,2,2,2,0,0, +1,1,1,1,1,1,0,0,2,2,2,2,2,2,2,2, +1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2, +1,1,1,1,1,1,1,1,2,2,2,2,2,2,0,0, +1,1,1,1,1,1,0,0,2,2,2,2,2,2,2,2, +0,1,0,1,0,1,0,1,2,2,2,2,2,2,2,2, +1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2, +2,2,2,2,2,2,0,0,2,2,2,2,2,2,2,2, +3,3,3,3,3,3,3,3,2,2,2,2,2,2,2,2, +3,3,3,3,3,3,3,3,2,2,2,2,2,0,2,2, +1,1,1,1,3,0x1a,2,0x1a,0x1a,0x1a,2,2,2,0,2,2, +1,1,1,1,3,0x1a,0x1a,0x1a,2,2,2,2,0,0,2,2, +1,1,1,1,0,0x1a,0x1a,0x1a,2,2,2,2,2,2,2,2, +1,1,1,1,1,0x1a,0x1a,0x1a,0,0,2,2,2,0,2,2, +1,1,1,1,3,0x1a,0x1a,0,0xc,0xc,0xc,0xc,0xc,0xc,0xc,0xc, +0xc,0xc,0xc,0x10,0x10,0x10,0x10,0x10,0x13,0x13,0x13,0x13,0x13,0x13,0x17,0x17, +0x1c,0x1d,0x14,0x1c,0x1c,0x1d,0x14,0x1c,0x17,0x17,0x17,0x17,0x17,0x17,0x17,0x17, +0xd,0xe,0x10,0x10,0x10,0x10,0x10,0xc,0x17,0x17,0x17,0x17,0x17,0x17,0x17,0x17, +0x17,0x1c,0x1d,0x17,0x17,0x17,0x17,0x16,0x16,0x17,0x17,0x17,0x18,0x14,0x15,0x17, +0x17,0x17,0x17,0x17,0x17,0x17,0x17,0x17,0x17,0x17,0x18,0x17,0x16,0x17,0x17,0x17, +0x17,0x17,0x17,0x17,0x17,0x17,0x17,0xc,0x10,0x10,0x10,0x10,0,0,0,0, +0,0,0x10,0x10,0x10,0x10,0x10,0x10,0x4b,2,0,0,0x44b,0x54b,0x64b,0x74b, +0x84b,0x94b,0x18,0x18,0x18,0x14,0x15,2,0x4b,0x14b,0x24b,0x34b,0x44b,0x54b,0x64b,0x74b, +0x84b,0x94b,0x18,0x18,0x18,0x14,0x15,0,4,4,4,4,4,0,0,0, +0,0,0,0,0,0,0,0,0x19,0x19,0x19,0x19,0x19,0x19,0x19,0x19, +0x19,0x19,0x19,0x19,0x19,0x19,0x19,0x19,0x19,0x19,0x19,0x19,0x19,0x19,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +6,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7, +7,6,7,7,7,6,6,6,6,6,6,6,6,6,6,6, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0x1b,0x1b,1,0x1b,0x1b,0x1b,0x1b,1,0x1b,0x1b,2,1,1,1,2,2, +1,1,1,2,0x1b,1,0x1b,0x1b,0x1b,1,1,1,1,1,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,1,0x1b,1,0x1b,1,0x1b,1,1,1,1,0x1b,2, +1,1,1,1,2,5,5,5,5,2,0x1b,0x1b,2,2,1,1, +0x18,0x18,0x18,0x18,0x18,1,2,2,2,2,0x1b,0x18,0x1b,0x1b,2,0, +0,0,0,0x98b,0x118b,0xb8b,0x138b,0x1b8b,0x238b,0xc8b,0x2c8b,0xe8b,0x1e8b,0x2e8b,0x3e8b,0x16b, +0x16a,0x26a,0x36a,0x46a,0x56a,0x66a,0x76a,0x86a,0x96a,0xa6a,0xb6a,0xc6a,0x326a,0x646a,0x50aa,0x11aa, +0x16a,0x26a,0x36a,0x46a,0x56a,0x66a,0x76a,0x86a,0x96a,0xa6a,0xb6a,0xc6a,0x326a,0x646a,0x50aa,0x11aa, +0x11aa,0x51aa,0x12aa,1,2,0,0,0,0,0,0,0,0,0,0,0, +0x18,0x18,0x18,0x18,0x18,0x1b,0x1b,0x1b,0x1b,0x1b,0x18,0x18,0x1b,0x1b,0x1b,0x1b, +0x18,0x1b,0x1b,0x18,0x1b,0x1b,0x18,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x18,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x18,0x18,0x1b,0x1b,0x18,0x1b,0x18,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18, +0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x18,0x18,0x18,0x18,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x18,0x18,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x14,0x15,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x18,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18, +0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x18,0x18,0x18,0x18,0x18,0x18,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0x14b,0x24b,0x34b,0x44b, +0x54b,0x64b,0x74b,0x84b,0x94b,0xa6b,0xb6b,0xc6b,0xd6b,0xe6b,0xf6b,0x106b,0x116b,0x126b,0x136b,0x146b, +0x14b,0x24b,0x34b,0x44b,0x54b,0x64b,0x74b,0x84b,0x94b,0xa6b,0xb6b,0xc6b,0xd6b,0xe6b,0xf6b,0x106b, +0x116b,0x126b,0x136b,0x146b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x4b,0xb6b, +0xc6b,0xd6b,0xe6b,0xf6b,0x106b,0x116b,0x126b,0x136b,0x146b,0x14b,0x24b,0x34b,0x44b,0x54b,0x64b,0x74b, +0x84b,0x94b,0xa6b,0x4b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x18,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x18,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x18,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0,0,0,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0x1b,0x1b,0x1b, +0x1b,0,0x1b,0x1b,0x1b,0x1b,0,0,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0,0x1b,0,0x1b,0x1b,0x1b,0x1b,0,0,0,0x1b,0, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0,0,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x14,0x15,0x14,0x15,0x14,0x15,0x14,0x15,0x14,0x15,0x14,0x15,0x14,0x15,0x14b,0x24b, +0x34b,0x44b,0x54b,0x64b,0x74b,0x84b,0x94b,0xa6b,0x14b,0x24b,0x34b,0x44b,0x54b,0x64b,0x74b,0x84b, +0x94b,0xa6b,0x14b,0x24b,0x34b,0x44b,0x54b,0x64b,0x74b,0x84b,0x94b,0xa6b,0x1b,0,0,0, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0, +0x18,0x18,0x18,0x18,0x18,0x14,0x15,0x18,0x18,0x18,0x18,0,0,0,0,0, +0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18, +0x18,0x18,0x14,0x15,0x14,0x15,0x14,0x15,0,0,0,0,0x18,0x18,0x18,0x18, +0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x14, +0x15,0x14,0x15,0x14,0x15,0x14,0x15,0x14,0x15,0x14,0x15,0x14,0x15,0x14,0x15,0x14, +0x15,0x14,0x15,0x14,0x15,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18, +0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18, +0x14,0x15,0x14,0x15,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18, +0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18, +0x14,0x15,0x18,0x18,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0, +0,0,0,0,0x1b,0x1b,0x1b,0x1b,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1, +1,1,1,0,2,2,2,2,2,2,2,2,2,2,2,2, +2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, +2,2,2,0,1,2,1,1,1,2,2,1,2,1,2,1, +2,0,0,0,0,0,0,0,2,1,2,2,0,0,0,0, +0,0,0,0,1,2,1,2,2,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0x17,0x17,0x17, +0x17,0x88b,0x17,0x17,2,2,2,2,2,2,0,0,0,0,0,0, +0,0,0,0,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,0,0,0,0,0,0,0,0,0,4, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,0,0,0,0,0,0,0,0,0, +5,5,5,5,5,5,5,0,5,5,5,5,5,5,5,0, +5,5,5,5,5,5,5,0,5,5,5,5,5,5,5,0, +0x17,0x17,0x1c,0x1d,0x1c,0x1d,0x17,0x17,0x17,0x1c,0x1d,0x17,0x1c,0x1d,0x17,0x17, +0x17,0x17,0x17,0x17,0x17,0x17,0x17,0x13,0,0,0,0,0x1c,0x1d,0,0, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0,0,0,0,0,0,0,0,0,0,0,0,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0,0,0,0,0xc,0x17,0x17,0x17,0x1b,4,5,0x6a,0x14,0x15,0x14,0x15, +0x14,0x15,0x14,0x15,0x14,0x15,0x1b,0x1b,0x14,0x15,0x14,0x15,0x14,0x15,0x14,0x15, +0x13,0x14,0x15,0x15,0x1b,0x16a,0x26a,0x36a,0x46a,0x56a,0x66a,0x76a,0x86a,0x96a,6,6, +6,6,6,6,0x13,4,4,4,4,4,0x1b,0x1b,0xa6a,0x146a,0x1e6a,4, +5,0x17,0x1b,0x1b,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,0,0,6,6,0x1a, +0x1a,4,4,5,0x13,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,0x17,4,4,4,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,0,0,0,0,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,0, +0x1b,0x1b,0x16b,0x26b,0x36b,0x46b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0,0x16b,0x26b,0x36b,0x46b,0x56b,0x66b,0x76b,0x86b, +0x96b,0xa6b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0,0,0,0,0,0,0,0, +0,0,0,0,0x1b,0x156b,0x166b,0x176b,0x186b,0x196b,0x1a6b,0x1b6b,0x1c6b,0x1d6b,0x1e6b,0x1f6b, +0x206b,0x216b,0x226b,0x236b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x246b,0x256b,0x266b,0x276b,0x286b,0x296b,0x2a6b,0x2b6b,0x2c6b,0x2d6b,0x2e6b, +0x2f6b,0x306b,0x316b,0x326b,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,4,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,0,0,0,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1a,0x1a,0x1a,0x1a, +0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a, +0x1a,0x1a,0x1a,4,4,4,4,0,0,0,0,0,0x1a,0x1a,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,5,5,8,5, +5,5,6,5,5,5,5,6,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,8, +8,6,6,8,0x1b,0x1b,0x1b,0x1b,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +0x17,0x17,0x17,0x17,0,0,0,0,0,0,0,0,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,0,0,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,0, +0,0,0,0,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,2,2,2,2,2,2,2,0,0,0,0,0, +0,0,0,0,0,0,0,2,2,2,2,2,0,0,0,0, +0,5,6,5,5,5,5,5,5,5,5,5,5,0x18,5,5, +5,5,5,5,5,5,5,5,5,5,5,0,5,5,5,5, +5,0,5,0,5,5,0,5,5,0,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,0x14,0x15,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,0,0,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +5,5,5,5,5,5,5,5,5,5,5,5,0x19,0x1b,0,0, +6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6, +0x17,0x17,0x17,0x17,0x17,0x17,0x17,0x14,0x15,0x17,0,0,0,0,0,0, +6,6,6,6,0,0,0,0,0,0,0,0,0,0,0,0, +0x17,0x13,0x13,0x16,0x16,0x14,0x15,0x14,0x15,0x14,0x15,0x14,0x15,0x14,0x15,0x14, +0x15,0x17,0x17,0x14,0x15,0x17,0x17,0x17,0x17,0x16,0x16,0x16,0x17,0x17,0x17,0, +0x17,0x17,0x17,0x17,0x13,0x14,0x15,0x14,0x15,0x14,0x15,0x17,0x17,0x17,0x18,0x13, +0x18,0x18,0x18,0,0x17,0x19,0x17,0x17,0,0,0,0,5,5,5,5, +5,0,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,0,0,0x10,0,0x17,0x17,0x17,0x19,0x17,0x17,0x17,0x14,0x15,0x17,0x18, +0x17,0x13,0x17,0x17,0x29,0x129,0x229,0x329,0x429,0x529,0x629,0x729,0x829,0x929,0x17,0x17, +0x18,0x18,0x18,0x17,0x1a,2,2,2,2,2,2,2,2,2,2,2, +2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,0x14, +0x18,0x15,0x18,0x14,0x15,0x17,0x14,0x15,0x17,0x17,5,5,5,5,5,5, +5,5,5,5,4,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,4,4,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,0,0,0,5,5,5,5,5,5, +0,0,5,5,5,5,5,5,0,0,5,5,5,5,5,5, +0,0,5,5,5,0,0,0,0x19,0x19,0x18,0x1a,0x1b,0x19,0x19,0, +0x1b,0x18,0x18,0x18,0x18,0x1b,0x1b,0,0,0,0,0,0,0,0,0, +0,0x10,0x10,0x10,0x1b,0x1b,0,0,5,5,5,5,5,5,5,5, +5,5,5,5,0,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,0,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,0, +5,5,0,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,0,0,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,0,0,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,0, +0,0,0,0,0x17,0x17,0x1b,0,0,0,0,0x16b,0x26b,0x36b,0x46b,0x56b, +0x66b,0x76b,0x86b,0x96b,0xa6b,0x146b,0x1e6b,0x286b,0x326b,0x3c6b,0x466b,0x506b,0x5a6b,0x646b,0xc86b,0x30ab, +0x40ab,0x50ab,0x60ab,0x70ab,0x80ab,0x90ab,0x11ab,0x21ab,0x31ab,0x41ab,0x51ab,0x61ab,0x71ab,0x81ab,0x91ab,0x12ab, +0x22ab,0x32ab,0x42ab,0x52ab,0x62ab,0x72ab,0x82ab,0x92ab,0,0,0,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0xa8a,0x88a,0x16a,0x56a,0x326a,0x50aa,0x51aa,0x52aa,0x56a,0xa6a,0x326a,0x646a, +0x50aa,0x11aa,0x51aa,0x56a,0xa6a,0x326a,0x646a,0x50aa,0x11aa,0x12aa,0x52aa,0xa6a,0x16a,0x16a,0x16a,0x26a, +0x26a,0x26a,0x26a,0x56a,0xa6a,0xa6a,0xa6a,0xa6a,0xa6a,0x1e6a,0x326a,0x326a,0x326a,0x326a,0x646a,0x30aa, +0x50aa,0x50aa,0x50aa,0x50aa,0x50aa,0x11aa,0x51aa,0x56a,0x326a,0x88b,0x88b,0x118b,0x1a8b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x6b,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0x16b,0x56b,0xa6b,0x326b,0,0,0,0,0,0,0,0,0,0,0,0, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,0x5a6a,5,5,5,5,5,5,5,5,0x90aa,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,0,0x17, +5,5,5,5,0,0,0,0,5,5,5,5,5,5,5,5, +0x17,0x16a,0x26a,0xa6a,0x146a,0x646a,0,0,0,0,0,0,0,0,0,0, +1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2, +2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +0x29,0x129,0x229,0x329,0x429,0x529,0x629,0x729,0x829,0x929,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +5,5,5,5,5,5,0,0,5,0,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,0,5,5,0,0,0,5,0,0,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,0x16b,0xa6b,0x146b,0x646b,0,0,0,0,0,0x17,5,6,6,6, +0,6,6,0,0,0,0,0,6,6,6,6,5,5,5,5, +0,5,5,5,0,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,0,0,0,0, +6,6,6,0,0,0,0,6,0x14b,0x24b,0x34b,0x44b,0xa6b,0x146b,0x646b,0x11ab, +0,0,0,0,0,0,0,0,0x17,0x17,0x17,0x17,0x17,0x17,0x17,0x17, +0x17,0,0,0,0,0,0,0,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0x26a,0x36a,0x46a,0x56a,0x66a,0x76a,0x86a,0x96a, +0x36a,0x46a,0x56a,0x66a,0x76a,0x86a,0x96a,0x46a,0x56a,0x66a,0x76a,0x86a,0x96a,0x16a,0x26a,0x36a, +0x46a,0x56a,0x66a,0x76a,0x86a,0x96a,0x16a,0x26a,0x36a,0x46a,0x56a,0x26a,0x36a,0x36a,0x46a,0x56a, +0x66a,0x76a,0x86a,0x96a,0x16a,0x26a,0x36a,0x36a,0x46a,0x56a,0xa,0xa,0x16a,0x26a,0x36a,0x36a, +0x46a,0x56a,0x36a,0x36a,0x46a,0x46a,0x46a,0x46a,0x66a,0x76a,0x76a,0x76a,0x86a,0x86a,0x96a,0x96a, +0x96a,0x96a,0x26a,0x36a,0x46a,0x56a,0x66a,0x16a,0x26a,0x36a,0x46a,0x46a,0x56a,0x56a,0xa,0xa, +0x16a,0x26a,0x98a,0x118a,0x2c8a,0x98a,0x118a,0xe8a,0xa8a,0xc8a,0xa8a,0,0,0,0,0, +0,0,0,0,0,0,0,0,0x17,0x17,0x17,0x17,0,0,0,0, +0,0,0,0,0,0,0,0,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0, +0,0,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,8,8,6,6,6,0x1b,0x1b, +0x1b,8,8,8,8,8,8,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,6, +6,6,6,6,6,6,6,0x1b,0x1b,6,6,6,6,6,6,6, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,6,6,6,6,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0,0,0x1b,0x1b,6,6, +6,0x1b,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b, +0x1b,0x1b,0x1b,0,0,0,0,0,0,0,0,0,0x16b,0x26b,0x36b,0x46b, +0x56b,0x66b,0x76b,0x86b,0x96b,0xa6b,0x146b,0x1e6b,0x286b,0x326b,0x3c6b,0x466b,0x506b,0x5a6b,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1, +1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, +1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2, +2,2,2,2,2,2,2,2,2,2,2,2,1,1,1,1, +1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2, +2,0,2,2,2,2,2,2,2,2,2,2,1,1,1,1, +1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, +1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2, +2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, +1,0,1,1,0,0,1,0,0,1,1,0,0,1,1,1, +1,0,1,1,1,1,1,1,1,1,2,2,2,2,0,2, +0,2,2,2,2,2,2,2,0,2,2,2,2,2,2,2, +2,2,2,2,1,1,1,1,1,1,1,1,1,1,1,1, +1,1,1,1,2,2,2,2,1,1,0,1,1,1,1,0, +0,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1, +1,0,2,2,2,2,2,2,2,2,2,2,2,2,2,2, +2,2,2,2,2,2,2,2,2,2,2,2,1,1,0,1, +1,1,1,0,1,1,1,1,1,0,1,0,0,0,1,1, +1,1,1,1,1,0,2,2,2,2,2,2,2,2,2,2, +2,2,2,2,1,1,1,1,1,1,1,1,1,1,1,1, +1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2, +2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, +2,2,2,2,1,1,1,1,1,1,1,1,1,1,1,1, +1,1,1,1,2,2,2,2,2,2,0,0,1,1,1,1, +1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, +1,1,1,1,1,0x18,2,2,2,2,2,2,2,2,2,2, +2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,0x18, +2,2,2,2,2,2,1,1,1,1,1,1,1,1,1,1, +1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0x18, +2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, +2,2,2,2,2,0x18,2,2,2,2,2,2,1,1,1,1, +1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,0x18, +2,2,2,2,2,2,1,2,0,0,0x29,0x129,0x229,0x329,0x429,0x529, +0x629,0x729,0x829,0x929,0x29,0x129,0x229,0x329,0x429,0x529,0x629,0x729,0x829,0x929,0x29,0x129, +0x229,0x329,0x429,0x529,0x629,0x729,0x829,0x929,0x29,0x129,0x229,0x329,0x429,0x529,0x629,0x729, +0x829,0x929,0x29,0x129,0x229,0x329,0x429,0x529,0x629,0x729,0x829,0x929,0,0x10,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0x10,0x10,0x10,0x10, +0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10, +0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,6,6,6,6, +6,6,6,6,6,6,6,6,6,6,6,6,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,0,0,0,0,0,0,0,0,0,0,5,5,5,5, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0x12,0x12,0x12,0x12, +0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, +0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x11,0x11,0x11,0x11, +0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11, +0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0,0, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,0x65,5,5,5,5,5,5,5,5,5, +0x165,5,5,0x765,5,5,5,0x12a5,5,0x365,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,0x165,5,5,5,5,5,5,5,0x165,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,0x165,5,5,5,5,5,5,5,5,5, +5,5,5,5,0x165,0x265,0x365,5,0x265,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,0x165,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,0x265,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +0x265,5,5,5,5,5,5,5,0x565,5,0x465,5,5,5,5,5, +5,5,5,5,5,0x265,5,0x865,5,0x665,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,0x265,5, +5,5,5,0x265,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,0x265,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,0x365,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,0x365,0x365,0x365, +0x365,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,0x665,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,0x365,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,0x365,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,0x365,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,0x365,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,0x465,5,5,5,5,5,5,0x465,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,0x465,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,0x465,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,0x465,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,0x565,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,0x565,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,0x565,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,0x665,5,5,5,5,5, +0x6465,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,0x665,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,0x765,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,0x765,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,0x765,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +0x865,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,0x965,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,0x965,0x1465,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,0x965,5,5,5,5,5, +5,5,5,5,0xa65,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,0x11a5,5,0xa65,5,0x11a5,0x1465,0x1e65,5,5,5,5,5,5, +0x2865,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,0xa65,5,5,5,5,0x1e65,5,5,5,5,5,5,5,5, +0x2865,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +0x2865,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,0x6465,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,0x6465,5,5,5,5,5,5,5,5,5, +5,5,5,5,0x12a5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,0x16a5,5,5,5,5,0x16a5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,0x1aa5,5,5,5,5,5, +5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +5,5,5,5,0x820,0x840,0x860,0,0,0,0,0,0x880,0x8a0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0x8c0,0x8e0,0,0,0,0,0,0,0,0,0,0, +0x900,0x920,0x940,0x920,0x920,0x920,0x920,0x920,0x960,0x920,0x980,0x920,0x920,0x920,0x9a0,0x920, +0x920,0x920,0x920,0x920,0x920,0x920,0x920,0x920,0x9c0,0x920,0x920,0x920,0x920,0x920,0x920,0x920, +0x920,0x920,0x920,0x920,0x920,0x9e0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0xa00,0,0xa20,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0xa40,0xa40,0xa40,0xa40, +0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40, +0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa60 +}; + +static const UTrie propsTrie={ + propsTrie_index, + NULL, + utrie_defaultGetFoldingOffset, + 2688, + 10752, + 0, + TRUE +}; + +static const uint16_t propsVectorsTrie_index[16924]={ +0x2a8,0x2b0,0x2b8,0x2c0,0x2c8,0x2d0,0x2d8,0x2e0,0xb1b,0xb22,0xb2a,0xb31,0xb39,0xb41,0xb49,0xb51, +0xb58,0xde3,0xb60,0xb64,0xb67,0xb6d,0xa52,0xb75,0xcdc,0xcdc,0xce4,0x340,0x348,0x350,0x358,0xb7d, +0xba9,0xca7,0xb85,0xb8d,0x360,0xb93,0xb9b,0xba1,0x860,0x368,0x36d,0x375,0x37c,0xcbc,0x384,0x38a, +0x392,0x39a,0x3a2,0xbb9,0xbc9,0xbcb,0xbb1,0xbc1,0x3aa,0xdeb,0x3b2,0x91a,0xdf3,0x3ba,0x1016,0x9cc, +0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x3c2,0x3c8,0x3d0,0x3d8,0x3e0,0x3e6,0x3ee,0x3f6, +0x3fe,0x404,0x40c,0x414,0x41c,0x422,0x42a,0x432,0x43a,0x440,0x448,0x450,0x458,0x460,0x468,0x46f, +0x477,0x47d,0x485,0x48d,0x495,0x49b,0x4a3,0x4ab,0x4b3,0x4b9,0x4c1,0x4c9,0x4d1,0x4d8,0x4e0,0x4e8, +0x4f0,0x4f4,0x4fc,0x503,0x50b,0x513,0x51b,0x523,0xdad,0xdb5,0x52b,0x533,0x53b,0x543,0x54b,0x550, +0xe70,0x558,0x560,0x567,0x567,0xbd3,0x56f,0x573,0xd74,0xd74,0x57b,0xd84,0xd85,0x583,0xd7c,0x585, +0xdfb,0xdfd,0x58d,0xdfd,0x595,0x59a,0x5a2,0xe02,0x5a8,0xdfd,0x5ae,0x5b6,0x959,0xe0a,0xe0a,0x5be, +0x5c6,0xe12,0xe12,0xe12,0xe12,0xe12,0xe12,0xe12,0xe12,0xe12,0xe12,0xe12,0xe12,0xe12,0xe12,0xe12, +0xe12,0xe12,0xe12,0x5cb,0x5d3,0xe1a,0xe1a,0x5db,0x868,0x870,0x878,0x880,0xe80,0xe78,0x5e3,0x5eb, +0x5f3,0xe22,0xe2a,0x5fb,0xe22,0x2e8,0x2a0,0x2a0,0x897,0x89f,0x8a7,0x8ac,0x100e,0x994,0x99c,0xf5e, +0x922,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x101e,0x1023,0x9d4,0x9dc,0x2a0,0x2a0,0x2a0,0x2a0, +0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0xf7e,0xf66,0xf76,0xf6e,0xfee,0xfe6,0x936,0x939, +0xbdb,0xbdb,0xbdb,0xbdb,0x603,0xbdb,0xbdb,0x60b,0x613,0xbe3,0x617,0x61f,0xbe3,0x627,0x62f,0x637, +0xa62,0xa5a,0xbeb,0x63f,0x647,0x64f,0x655,0x65d,0xa6a,0xa72,0x665,0xa7a,0x66d,0xbf3,0xa82,0xbfa, +0xa8a,0xa92,0xa9a,0xaa2,0xaaa,0xab1,0xc02,0xc0a,0xab9,0xc12,0xc15,0xc17,0xe32,0xf13,0xf19,0x675, +0xc1f,0x67d,0x685,0xac1,0xac6,0xac9,0xacf,0xa42,0xad7,0xad7,0xadc,0xa4a,0xae4,0xaec,0xaf4,0xafc, +0xb04,0xc27,0xb0b,0xb13,0x68d,0x695,0x69a,0x69a,0x6a2,0x6a8,0x6b0,0x6b8,0x6c0,0x6c6,0x888,0x88f, +0xe3a,0xe3a,0xe3a,0xe3a,0xe3a,0xe3a,0xe3a,0xe3a,0xf21,0xf21,0xf21,0xf21,0xf29,0xf30,0xf32,0xf39, +0xf41,0xf45,0xf45,0xf48,0xf45,0xf45,0xf4e,0xf45,0x8b4,0x8bc,0x8bd,0x8bd,0x8bd,0x8bd,0x8bd,0x8bd, +0xfd6,0x969,0x96d,0x9e4,0xfc6,0xfc6,0xfc6,0x941,0xfce,0x961,0xffe,0x9bc,0x949,0x951,0x951,0x2a0, +0x9ac,0x9b4,0x9b4,0x9b4,0x6ce,0xe42,0xe42,0x6d6,0xe4a,0xe4a,0xe4a,0xe4a,0xe4a,0xe4a,0x6de,0x2ec, +0xcd4,0xcec,0x6e6,0xcf4,0x6ee,0xcfc,0xd04,0xd0c,0x6f6,0x6fb,0xd14,0xd1b,0x700,0x708,0x92a,0x92e, +0x710,0xd2b,0x718,0xd23,0xd34,0xd38,0xd30,0x720,0xd56,0xd56,0xd40,0xd46,0xd56,0xd56,0xd57,0xd4e, +0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52, +0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52, +0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52, +0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52, +0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52, +0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52, +0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52, +0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52, +0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52, +0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52, +0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52, +0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52, +0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0xe52,0x728,0xf86,0xf86, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f, +0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0xd5f,0x730,0x737,0x737, +0xe5a,0xe60,0xe60,0xe60,0xe60,0xe60,0xe60,0xe60,0xe60,0xe60,0xe60,0xe60,0xe60,0xe60,0xe60,0xe60, +0xe60,0xe60,0xe60,0xe60,0xe60,0xe60,0xe60,0xe60,0xe60,0xe60,0xe60,0xe60,0xe60,0xe60,0xe60,0xe60, +0xe60,0xe60,0xe60,0xe60,0x73f,0xe68,0x2f4,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0, +0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x98c,0x9ec,0x9ed,0x9ed,0x9ed,0x9ed,0x9ed,0x9ed, +0xff6,0x2fc,0x102b,0x9f5,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0, +0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0, +0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6, +0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8, +0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda, +0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5, +0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7, +0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9, +0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb, +0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6, +0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8, +0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda, +0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5, +0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7, +0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9, +0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb, +0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6, +0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8, +0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda, +0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5, +0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7, +0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9, +0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb, +0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0xddb,0xdd5,0xdd6,0xdd7,0xdd8,0xdd9,0xdda,0x304,0x2a0,0x2a0, +0x1043,0x1046,0x104e,0x1054,0x105c,0x105d,0x2a0,0x1065,0x2a0,0x1065,0x2a0,0x1065,0x2a0,0x1065,0x2a0,0x1065, +0x2a0,0x1065,0x2a0,0x1065,0x2a0,0x1065,0x2a0,0x1065,0x2a0,0x1065,0x106d,0x1065,0x1075,0x1076,0x107e,0x107f, +0xdcd,0xdcd,0xdcd,0xdcd,0xdcd,0xdcd,0xdcd,0xdcd,0xdcd,0xdcd,0xdcd,0xdcd,0xdcd,0xdcd,0xdcd,0xdcd, +0xdcd,0xdcd,0xdcd,0xdcd,0xdcd,0xdcd,0xdcd,0xdcd,0xdcd,0xdcd,0xdcd,0xdcd,0xdcd,0xdcd,0xdcd,0xdcd, +0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a, +0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a, +0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a, +0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a, +0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a, +0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a, +0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a, +0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a, +0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a, +0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a, +0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a, +0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a, +0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xa3a,0xd67,0xd67,0xd67,0xd67,0xd67,0xd67,0xd67,0xd67, +0xd6c,0x747,0xf56,0x74d,0x1006,0x1006,0x751,0x758,0x760,0x768,0x770,0xc3f,0xc46,0x778,0x77d,0xc4e, +0xc84,0xc84,0xc2f,0xc37,0xc56,0xc7b,0xc7c,0xc8c,0xc5e,0xc63,0x785,0xc6b,0x78d,0xc73,0x795,0x799, +0x9c4,0x7a1,0x7a9,0x7b1,0xc94,0xc9a,0xc9f,0x7b9,0x7c9,0xcc4,0xccc,0xcaf,0xcb4,0x7d1,0x7d9,0x7c1, +0xdbd,0xdbd,0xdbd,0xdbd,0xdbd,0xdbd,0xdbd,0xdbd,0xdbd,0xdbd,0xdbd,0xdbd,0xdbd,0xdbd,0xdbd,0xdbd, +0xdbd,0xdbd,0xdbd,0xdbd,0xdbd,0xdbd,0xdbd,0xdbd,0xdbd,0xdbd,0xdbd,0xdbd,0xdc5,0xdc5,0xdc5,0xdc5, +0x8c5,0x8cc,0x8d4,0x8dc,0xf8e,0xf8e,0xf8e,0x8e4,0x8ec,0x8ef,0xfbe,0xfb6,0x30c,0x2a0,0x2a0,0x2a0, +0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x7e1,0x7e9,0x314,0x2a0,0x8f7,0xfde,0x9a4,0x2a0, +0xe95,0xe88,0xe8d,0xf96,0x8ff,0x31c,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0, +0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0, +0x907,0x90a,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x9fd,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0, +0x975,0x97c,0x984,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0, +0x1033,0x1033,0x1033,0x1033,0x1033,0x1033,0x1033,0x1033,0x1033,0x1033,0x1033,0x1033,0x1033,0x1033,0x1033,0x1033, +0x1033,0x1033,0x1033,0x1033,0x1033,0x1033,0x1033,0x1033,0x1033,0x1033,0x1033,0xa05,0xa09,0xa09,0xa09,0xa09, +0x103b,0x103b,0x103b,0xa11,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0, +0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0, +0xe9d,0xe9d,0xe9d,0xe9d,0xe9d,0xe9d,0xe9d,0x7f1,0xea5,0x7f9,0xea6,0xeae,0xeb6,0xebc,0x801,0x809, +0xfae,0xfae,0x324,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0xf9e,0xf9e,0x912,0xa19,0x2a0,0x2a0,0x2a0,0x2a0, +0xee5,0xee0,0x811,0xee3,0x819,0x821,0x829,0xee9,0x831,0x839,0x841,0xee2,0xeea,0xee5,0xee0,0xee8, +0xee3,0xeeb,0xee6,0xee1,0xee9,0x848,0xec4,0xecc,0xed3,0xeda,0xec7,0xecf,0xed6,0xedd,0x850,0xf0b, +0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0, +0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x2a0,0x328, +0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb, +0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb, +0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xefb, +0xefb,0xefb,0xefb,0xefb,0xefb,0xefb,0xa2a,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21, +0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21, +0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21, +0xf03,0xf03,0xf03,0xf03,0xf03,0xf03,0xf03,0xf03,0xf03,0xf03,0xf03,0xf03,0xf03,0xf03,0xf03,0xf03, +0xa32,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21, +0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21, +0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa21,0xa22, +0x858,0xef3,0xef3,0xef3,0x330,0x330,0x330,0x330,0xfa6,0xfa6,0xfa6,0xfa6,0xfa6,0xfa6,0xfa6,0x338, +0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330, +0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330, +0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330,0x330, +0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d, +0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d, +0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d, +0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd9d,0xd8d, +0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5, +0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5, +0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5, +0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xda5,0xd95, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0x495,0x495,0x495,0x495,0x495,0x495,0x495,0x495,0x495,0x426,0x621,0x450,0x450,0x54c,0x495,0x495, +0x495,0x495,0x495,0x495,0x495,0x495,0x495,0x495,0x495,0x495,0x495,0x495,0x495,0x495,0x495,0x495, +0x747,0x561,0x717,0x3ff,0x6fc,0x6d2,0x3ff,0x71a,0x69f,0x474,0x3ff,0x6ff,0x61b,0x573,0x61e,0x74a, +0x687,0x687,0x687,0x687,0x687,0x687,0x687,0x687,0x687,0x687,0x618,0x61b,0x402,0x402,0x402,0x561, +0x3ff,0x417,0x417,0x417,0x417,0x417,0x417,0x411,0x411,0x411,0x411,0x411,0x411,0x411,0x411,0x411, +0x411,0x411,0x411,0x411,0x411,0x411,0x411,0x411,0x411,0x411,0x411,0x69f,0x6fc,0x474,0x408,0x40b, +0x405,0x414,0x414,0x414,0x414,0x414,0x414,0x40e,0x40e,0x40e,0x40e,0x40e,0x40e,0x40e,0x40e,0x40e, +0x40e,0x40e,0x40e,0x40e,0x40e,0x40e,0x40e,0x40e,0x40e,0x40e,0x40e,0x69f,0x447,0x474,0x402,0x495, +0x498,0x498,0x498,0x498,0x498,0x750,0x498,0x498,0x498,0x498,0x498,0x498,0x498,0x498,0x498,0x498, +0x498,0x498,0x498,0x498,0x498,0x498,0x498,0x498,0x498,0x498,0x498,0x498,0x498,0x498,0x498,0x498, +0x567,0x17d,0x6d5,0x702,0x6ed,0x702,0x41a,0x17d,0x183,0x1f2,0x189,0x708,0x41d,0x43b,0x3a8,0x420, +0x6c0,0x6f0,0x17a,0x17a,0x44d,0x1f5,0x17d,0x186,0x183,0x17a,0x189,0x708,0x177,0x177,0x177,0x17d, +0x1fb,0x1fb,0x1fb,0x1fb,0x1fb,0x1fb,0x3b1,0x1fb,0x1fb,0x1fb,0x1fb,0x1fb,0x1fb,0x1fb,0x1fb,0x1fb, +0x3b1,0x1fb,0x1fb,0x1fb,0x1fb,0x1fb,0x1fb,0x180,0x3b1,0x1fb,0x1fb,0x1fb,0x1fb,0x1fb,0x3b1,0x3ab, +0x3ae,0x3ae,0x1f8,0x1f8,0x1f8,0x1f8,0x3ab,0x1f8,0x3ae,0x3ae,0x3ae,0x1f8,0x3ae,0x3ae,0x1f8,0x1f8, +0x3ab,0x1f8,0x3ae,0x3ae,0x1f8,0x1f8,0x1f8,0x180,0x3ab,0x3ae,0x3ae,0x1f8,0x3ae,0x1f8,0x3ab,0x1f8, +0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x8b8,0x5d,0x5d,0x5d,0x5d,0x5d,0x5d, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0x8d9,0x8d9,0x8dc,0x8dc,0x8d9,0x8d9,0x8d9,0x8d9,0x8d9,0x8d9,0x8d9,0x8d9,0x8d,0x8d,0x8d,0x8d, +0x8ee,0xa41,0x8ee,0x8ee,0x8ee,0xa41,0x8ee,0xa8,0xa8,0xa8,0xa8,0xa8,0xa8,0xa8,0xa8,0xa8, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0xbf1,0xbf1,0xbf1,0xc4b,0xc4b,0xc42,0xc42,0xc4b,0xbee,0xbee,0xbee,0xbee,0x138,0x138,0x138,0x138, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0x7c5,0x7c5,0x7c5,0x7c5,0xab,0xab,0xab,0xab,0xab,0xab,0xab,0xab,0xab,0xab,0xab,0xab, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0xbac,0xbac,0xbac,0xbac,0xbac,0xbac,0xbac,0xbac,0xbac,0xbac,0xbac,0x10b,0x10b,0x10b,0x10b,0x10b, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0x945,0x945,0x945,0x945,0x945,0x945,0x945,0x945,0x945,0x945,0x945,0xc9,0xc9,0xc9,0xc9,0xc9, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0xb4f,0xb4f,0xb4f,0xb4f,0xb4f,0xb4f,0xb4f,0xb4f,0xb4f,0xb4f,0xff,0xff,0xff,0xff,0xff,0xff, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0xba9,0xba9,0xc2a,0xc2a,0xc2a,0xba9,0x108,0x108,0x108,0x108,0x108,0x108,0x108,0x108,0x108,0x108, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0x765,0x765, +3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, +3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, +0xb34,0xb34,0xb34,0xb34,0xb34,0xb34,0xb34,0xb34,0xb34,0xb34,0xb34,0xb34,0xb34,0xb34,0xb34,0xb34, +3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, +0x570,0x570,0x8cd,0xa2c,0xa2c,0xa2c,0xa2c,0xa2c,0xa2c,0xa2c,0xa2c,0xa2c,0xa2c,0xa2c,0xa2c,0xa2c, +6,6,6,6,0x243,0x240,6,6,6,6,0x249,0xc93,0xc93,0xc93,0x60f,6, +6,6,6,6,0x246,0x243,0x258,0x237,0x258,0x258,0x258,6,0x258,6,0x258,0x258, +0x24f,0x3cf,0x3cf,0x3cf,0x3cf,0x3cf,0x3cf,0x3cf,0x3cf,0x3cf,0x3cf,0x3cf,0x3cf,0x3cf,0x3cf,0x3cf, +0x3cf,0x3cf,6,0x3cf,0x3cf,0x3cf,0x3cf,0x3cf,0x3cf,0x3cf,0x258,0x258,0x24f,0x24f,0x24f,0x24f, +0x24f,0x3cc,0x3cc,0x3cc,0x3cc,0x3cc,0x3cc,0x3cc,0x3cc,0x3cc,0x3cc,0x3cc,0x3cc,0x3cc,0x3cc,0x3cc, +0x3cc,0x3cc,0x24c,0x3cc,0x3cc,0x3cc,0x3cc,0x3cc,0x3cc,0x3cc,0x24f,0x24f,0x24f,0x24f,0x24f,6, +0x25b,0x25b,0x25e,0x258,0x258,0x25b,0x252,0x7e3,0x98d,0x98a,0x255,0x7e3,0x255,0x7e3,0x255,0x7e3, +0x26a,0x264,0x261,0x49b,0x49b,0x49b,0x49b,9,0x894,0x894,0x993,0x990,0x7ec,0x7e6,0x7ec,0x7e6, +0x26a,0x264,0x26a,0x264,0x26a,0x264,0x26a,0x264,0x26a,0x264,0x26a,0x264,0x26a,0x264,0x26a,0x264, +0xd8,0xd8,0xd8,0xd8,0xd8,0xd8,0xd8,0xd8,0xd8,0xd8,0xd8,0xd8,0xd8,0xd8,0xd8,0xd8, +0xc,0x27c,0x27c,0x27c,0x27c,0x27c,0x27c,0x27c,0x27c,0x27c,0x27c,0x27c,0x27c,0x27c,0x27c,0x27c, +0x27c,0x27c,0x27c,0x27c,0x27c,0x27c,0x27c,0x27c,0x27c,0x27c,0x27c,0xc,0xc,0x27f,0x270,0x270, +0x273,0x270,0x273,0x270,0xc,0x276,0x276,0x276,0x276,0x276,0x276,0x276,0x276,0x276,0x276,0x276, +0x276,0x276,0x276,0x276,0x276,0x276,0x276,0x276,0x276,0x276,0x276,0x276,0x276,0x276,0x276,0x276, +0x276,0x276,0x276,0x276,0x276,0x276,0x276,0x279,0xc,0x612,0x86a,0xc,0xc,0xc,0xc,0xc, +0xf,0x78c,0x78c,0x78c,0x78c,0x78c,0x78c,0x78c,0x78c,0x78c,0x78c,0x78c,0x78c,0x78c,0x78c,0x78c, +0x282,0x49e,0x49e,0x28b,0x78f,0xc1e,0xc51,0xc1e,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf, +0x28e,0x28e,0x28e,0x28e,0x28e,0x28e,0x28e,0x28e,0x28e,0x28e,0x28e,0x28e,0x28e,0x28e,0x28e,0x28e, +0x28e,0x28e,0x28e,0xf,0xf,0xf,0xf,0xf,0x28e,0x28e,0x28e,0x288,0x285,0xf,0xf,0xf, +0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xf,0xa89,0xa89,0xa89,0xa89,0x12,0x12,0x12,0x12, +0x12,0x12,0x12,0xc7b,0x54f,0xb49,0xa8c,0xa8c,0xb13,0xb13,0xb13,0xb13,0xb13,0xb13,0x12,0x12, +0x12,0x12,0x12,0x54f,0x12,0x12,0xc54,0x552,0x12,0x29a,0x29d,0x29d,0x29d,0x29d,0x29d,0x29a, +0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a, +0x29a,0x29a,0x29a,0x12,0x12,0x12,0x12,0x12,0x294,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a, +0x29a,0x29a,0x29a,0x4a4,0x4a4,0x4a4,0x4a4,0x4a4,0x4a4,0x4a4,0x4a4,0x897,0x897,0x897,0xb13,0xb19, +0xb16,0xc21,0xc21,0xc21,0xc21,0xc21,0xc21,0x12,0x801,0x801,0x801,0x7fe,0x7fe,0x7fe,0x7fe,0x7fe, +0x7fe,0x7fe,0x7fe,0x7fb,0x7fe,0x7fb,0x15,0x7f8,0x804,0x89a,0x804,0x804,0x804,0x804,0x804,0x804, +0x804,0x804,0x804,0x804,0x804,0x804,0x804,0x804,0x89d,0x89d,0x89d,0x89d,0x89d,0x89d,0x89d,0x89d, +0x89d,0x89d,0x89d,0x15,0x15,0xa92,0xa92,0xa92,0xbb2,0xbb2,0xbb2,0xbb2,0xbb2,0xbb2,0xbb2,0xbb2, +0xbb2,0xbb2,0xbb2,0xbb2,0xbb2,0xbb2,0xbb2,0xbb2,0x807,0x807,0x807,0x807,0x807,0x807,0x8a3,0x8a3, +0x8a3,0x8a3,0x8a3,0x8a3,0x8a3,0x8a3,0x8a3,0x8a3,0x8a3,0x999,0x18,0x18,0x18,0x18,0x18,0x18, +0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x1b,0x4b0,0x4b0,0x4b9,0xa95,0x2a9,0x2a9,0x2a9, +0x2a9,0x2a9,0x2a9,0x2a9,0x2a9,0x2a9,0x2a9,0x2a9,0x2a9,0x2a9,0x2a9,0x2a9,0x2a9,0x2a9,0x2a9,0x2a9, +0x2a9,0x2a9,0x2a9,0x2a9,0x2a9,0x2a9,0x2a9,0x2a9,0x2a9,0x2ac,0x2a9,0x2a9,0x2a9,0x2a9,0x2a9,0x2a9, +0x2a9,0x2ac,0x2a9,0x2a9,0x2ac,0x2a9,0x2a9,0x2a9,0x2a9,0x2a9,0x1b,0x1b,0x4b3,0x2a9,0x4b9,0x4b9, +0x4b9,0x4b0,0x4b0,0x4b0,0x4b0,0x4b0,0x4b0,0x4b0,0x4b0,0x4b9,0x4b9,0x4b9,0x4b9,0x4b6,0x1b,0x1b, +0x2a9,0x4b3,0x4b3,0x4b3,0x4b3,0x1b,0x1b,0x1b,0x2ac,0x2ac,0x2ac,0x2ac,0x2ac,0x2ac,0x2ac,0x2ac, +0x2a9,0x2a9,0x4b0,0x4b0,0x42c,0x42c,0x666,0x666,0x666,0x666,0x666,0x666,0x666,0x666,0x666,0x666, +0x2a6,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0x1b,0xc9c,0xc9c,0xb73,0xc9c,0xc9c, +0x1e,0x4bc,0x4c5,0x4c5,0x1e,0x2b2,0x2b2,0x2b2,0x2b2,0x2b2,0x2b2,0x2b2,0x2b2,0x1e,0x1e,0x2b2, +0x2b2,0x1e,0x1e,0x2b2,0x2b2,0x2b2,0x2b2,0x2b2,0x2b2,0x2b2,0x2b2,0x2b2,0x2b2,0x2b2,0x2b2,0x2b2, +0x2b2,0x1e,0x2b2,0x2b2,0x2b2,0x2b2,0x2b2,0x2b2,0x2b2,0x1e,0x2b2,0x1e,0x1e,0x1e,0x2b2,0x2b2, +0x2b2,0x2b2,0x1e,0x1e,0x4bf,0xa98,0x4bc,0x4c5,0x4c5,0x4bc,0x4bc,0x4bc,0x4bc,0x1e,0x1e,0x4c5, +0x4c5,0x1e,0x1e,0x4c8,0x4c8,0x4c2,0xb76,0x1e,0x1e,0x1e,0x1e,0x1e,0x1e,0x1e,0x1e,0x4bc, +0x1e,0x1e,0x1e,0x1e,0x2b5,0x2b5,0x1e,0x2b5,0x2b2,0x2b2,0x4bc,0x4bc,0x1e,0x1e,0x669,0x669, +0x669,0x669,0x669,0x669,0x669,0x669,0x669,0x669,0x2b2,0x2b2,0x6db,0x6db,0x2af,0x2af,0x2af,0x2af, +0x2af,0x2af,0x2af,0x1e,0x1e,0x1e,0x1e,0x1e,0x21,0xb1c,0x4cb,0xb1f,0x21,0x2b8,0x2b8,0x2b8, +0x2b8,0x2b8,0x2b8,0x21,0x21,0x21,0x21,0x2b8,0x2b8,0x21,0x21,0x2b8,0x2b8,0x2b8,0x2b8,0x2b8, +0x2b8,0x2b8,0x2b8,0x2b8,0x2b8,0x2b8,0x2b8,0x2b8,0x2b8,0x21,0x2b8,0x2b8,0x2b8,0x2b8,0x2b8,0x2b8, +0x2b8,0x21,0x2b8,0x2bb,0x21,0x2b8,0x2bb,0x21,0x2b8,0x2b8,0x21,0x21,0x4ce,0x21,0x4d4,0x4d4, +0x4d4,0x4cb,0x4cb,0x21,0x21,0x21,0x21,0x4cb,0x4cb,0x21,0x21,0x4cb,0x4cb,0x4d1,0x21,0x21, +0x21,0x21,0x21,0x21,0x21,0x21,0x21,0x21,0x21,0x2bb,0x2bb,0x2bb,0x2b8,0x21,0x2bb,0x21, +0x21,0x21,0x21,0x21,0x21,0x21,0x66c,0x66c,0x66c,0x66c,0x66c,0x66c,0x66c,0x66c,0x66c,0x66c, +0x4cb,0x4cb,0x2b8,0x2b8,0x2b8,0x21,0x21,0x21,0x21,0x21,0x21,0x21,0x21,0x21,0x21,0x21, +0x24,0x4d7,0x4d7,0x4e0,0x24,0x2be,0x2be,0x2be,0x2be,0x2be,0x2be,0x2be,0xa9b,0x2be,0x24,0x2be, +0x2be,0x2be,0x24,0x2be,0x2be,0x2be,0x2be,0x2be,0x2be,0x2be,0x2be,0x2be,0x2be,0x2be,0x2be,0x2be, +0x2be,0x24,0x2be,0x2be,0x2be,0x2be,0x2be,0x2be,0x2be,0x24,0x2be,0x2be,0x24,0x2be,0x2be,0x2be, +0x2be,0x2be,0x24,0x24,0x4da,0x2be,0x4e0,0x4e0,0x4e0,0x4d7,0x4d7,0x4d7,0x4d7,0x4d7,0x24,0x4d7, +0x4d7,0x4e0,0x24,0x4e0,0x4e0,0x4dd,0x24,0x24,0x2be,0x24,0x24,0x24,0x24,0x24,0x24,0x24, +0x24,0x24,0x24,0x24,0x24,0x24,0x24,0x24,0x2be,0xa9b,0xb22,0xb22,0x24,0x24,0x66f,0x66f, +0x66f,0x66f,0x66f,0x66f,0x66f,0x66f,0x66f,0x66f,0x24,0xb55,0x24,0x24,0x24,0x24,0x24,0x24, +0x24,0x24,0x24,0x24,0x24,0x24,0x24,0x24,0x27,0x4e3,0x4ec,0x4ec,0x27,0x2c4,0x2c4,0x2c4, +0x2c4,0x2c4,0x2c4,0x2c4,0x2c4,0x27,0x27,0x2c4,0x2c4,0x27,0x27,0x2c4,0x2c4,0x2c4,0x2c4,0x2c4, +0x2c4,0x2c4,0x2c4,0x2c4,0x2c4,0x2c4,0x2c4,0x2c4,0x2c4,0x27,0x2c4,0x2c4,0x2c4,0x2c4,0x2c4,0x2c4, +0x2c4,0x27,0x2c4,0x2c4,0x27,0xa9e,0x2c4,0x2c4,0x2c4,0x2c4,0x27,0x27,0x4e6,0x2c4,0x4e3,0x4e3, +0x4ec,0x4e3,0x4e3,0x4e3,0x27,0x27,0x27,0x4ec,0x4ef,0x27,0x27,0x4ef,0x4ef,0x4e9,0x27,0x27, +0x27,0x27,0x27,0x27,0x27,0x27,0x4e3,0x4e3,0x27,0x27,0x27,0x27,0x2c7,0x2c7,0x27,0x2c4, +0x2c4,0x2c4,0x27,0x27,0x27,0x27,0x672,0x672,0x672,0x672,0x672,0x672,0x672,0x672,0x672,0x672, +0x2c1,0xa9e,0x27,0x27,0x27,0x27,0x27,0x27,0x27,0x27,0x27,0x27,0x27,0x27,0x27,0x27, +0x2a,0x2a,0x4f2,0x2cd,0x2a,0x2cd,0x2cd,0x2cd,0x2cd,0x2cd,0x2cd,0x2a,0x2a,0x2a,0x2cd,0x2cd, +0x2cd,0x2a,0x2cd,0x2cd,0x2d0,0x2cd,0x2a,0x2a,0x2a,0x2cd,0x2cd,0x2a,0x2cd,0x2a,0x2cd,0x2cd, +0x2a,0x2a,0x2a,0x2cd,0x2cd,0x2a,0x2a,0x2a,0x2cd,0x2cd,0x2cd,0x2a,0x2a,0x2a,0x2cd,0x2cd, +0x2cd,0x2cd,0x2cd,0x2cd,0x2cd,0x2cd,0xb79,0x2cd,0x2cd,0x2cd,0x2a,0x2a,0x2a,0x2a,0x4f2,0x4f8, +0x4f2,0x4f8,0x4f8,0x2a,0x2a,0x2a,0x4f8,0x4f8,0x4f8,0x2a,0x4fb,0x4fb,0x4fb,0x4f5,0x2a,0x2a, +0x2a,0x2a,0x2a,0x2a,0x2a,0x2a,0x2a,0x4f2,0x2a,0x2a,0x2a,0x2a,0x2a,0x2a,0x2a,0x2a, +0x2a,0x2a,0xc6f,0x675,0x675,0x675,0x675,0x675,0x675,0x675,0x675,0x675,0x2ca,0x2ca,0x2ca,0xaa1, +0xaa1,0xaa1,0xaa1,0xaa1,0xaa1,0xb58,0xaa1,0x2a,0x2a,0x2a,0x2a,0x2a,0x2d,0x507,0x507,0x507, +0x2d,0x2d3,0x2d3,0x2d3,0x2d3,0x2d3,0x2d3,0x2d3,0x2d3,0x2d,0x2d3,0x2d3,0x2d3,0x2d,0x2d3,0x2d3, +0x2d3,0x2d3,0x2d3,0x2d3,0x2d3,0x2d3,0x2d3,0x2d3,0x2d3,0x2d3,0x2d3,0x2d3,0x2d3,0x2d,0x2d3,0x2d3, +0x2d3,0x2d3,0x2d3,0x2d3,0x2d3,0x2d3,0x2d3,0x2d3,0x2d,0x2d3,0x2d3,0x2d3,0x2d3,0x2d3,0x2d,0x2d, +0x2d,0x2d,0x4fe,0x4fe,0x4fe,0x507,0x507,0x507,0x507,0x2d,0x4fe,0x4fe,0x501,0x2d,0x4fe,0x4fe, +0x4fe,0x504,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x4fe,0x4fe,0x2d,0x2d,0x2d,0x2d,0x2d, +0x2d,0x2d,0x2d,0x2d,0x2d3,0x2d3,0x2d,0x2d,0x2d,0x2d,0x678,0x678,0x678,0x678,0x678,0x678, +0x678,0x678,0x678,0x678,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, +0x2d,0x2d,0x2d,0x2d,0x30,0x30,0x510,0x510,0x30,0x2d6,0x2d6,0x2d6,0x2d6,0x2d6,0x2d6,0x2d6, +0x2d6,0x30,0x2d6,0x2d6,0x2d6,0x30,0x2d6,0x2d6,0x2d6,0x2d6,0x2d6,0x2d6,0x2d6,0x2d6,0x2d6,0x2d6, +0x2d6,0x2d6,0x2d6,0x2d6,0x2d6,0x30,0x2d6,0x2d6,0x2d6,0x2d6,0x2d6,0x2d6,0x2d6,0x2d6,0x2d6,0x2d6, +0x30,0x2d6,0x2d6,0x2d6,0x2d6,0x2d6,0x30,0x30,0xb25,0xaa4,0x510,0x50a,0x513,0x510,0x50a,0x510, +0x510,0x30,0x50a,0x513,0x513,0x30,0x513,0x513,0x50a,0x50d,0x30,0x30,0x30,0x30,0x30,0x30, +0x30,0x50a,0x50a,0x30,0x30,0x30,0x30,0x30,0x30,0x30,0x2d6,0x30,0x2d6,0x2d6,0xd0e,0xd0e, +0x30,0x30,0x67b,0x67b,0x67b,0x67b,0x67b,0x67b,0x67b,0x67b,0x67b,0x67b,0x30,0xc9f,0xc9f,0x30, +0x30,0x30,0x30,0x30,0x30,0x30,0x30,0x30,0x30,0x30,0x30,0x30,0x33,0x33,0x51c,0x51c, +0x33,0x2d9,0x2d9,0x2d9,0x2d9,0x2d9,0x2d9,0x2d9,0x2d9,0x33,0x2d9,0x2d9,0x2d9,0x33,0x2d9,0x2d9, +0x2d9,0x2d9,0x2d9,0x2d9,0x2d9,0x2d9,0x2d9,0x2d9,0x2d9,0x2d9,0x2d9,0x2d9,0x2d9,0x33,0x2d9,0x2d9, +0x2d9,0x2d9,0x2d9,0x2d9,0x2d9,0x2d9,0x2d9,0x2d9,0x2d9,0x2d9,0x2d9,0x2d9,0x2d9,0x2d9,0x33,0x33, +0x33,0x33,0x516,0x51c,0x51c,0x516,0x516,0x516,0x33,0x33,0x51c,0x51c,0x51c,0x33,0x51f,0x51f, +0x51f,0x519,0x33,0x33,0x33,0x33,0x33,0x33,0x33,0x33,0x33,0x516,0x33,0x33,0x33,0x33, +0x33,0x33,0x33,0x33,0x2d9,0x2d9,0x33,0x33,0x33,0x33,0x67e,0x67e,0x67e,0x67e,0x67e,0x67e, +0x67e,0x67e,0x67e,0x67e,0x33,0x33,0x33,0x33,0x33,0x33,0x33,0x33,0x33,0x33,0x33,0x33, +0x33,0x33,0x33,0x33,0x36,0x36,0x8ac,0x8ac,0x36,0x80d,0x80d,0x80d,0x80d,0x80d,0x80d,0x80d, +0x80d,0x80d,0x80d,0x80d,0x80d,0x80d,0x80d,0x80d,0x80d,0x80d,0x80d,0x36,0x36,0x36,0x80d,0x80d, +0x80d,0x80d,0x80d,0x80d,0x80d,0x80d,0x80d,0x80d,0x80d,0x80d,0x80d,0x80d,0x80d,0x80d,0x80d,0x80d, +0x80d,0x80d,0x36,0x80d,0x80d,0x80d,0x80d,0x80d,0x80d,0x80d,0x80d,0x80d,0x36,0x80d,0x36,0x36, +0x80d,0x80d,0x80d,0x80d,0x80d,0x80d,0x80d,0x36,0x36,0x36,0x8a9,0x36,0x36,0x36,0x36,0x8a6, +0x8ac,0x8ac,0x8a6,0x8a6,0x8a6,0x36,0x8a6,0x36,0x8ac,0x8ac,0x8af,0x8ac,0x8af,0x8af,0x8af,0x8a6, +0x36,0x36,0x36,0x36,0x36,0x36,0x36,0x36,0x36,0x36,0x36,0x36,0x36,0x36,0x36,0x36, +0x36,0x36,0x8ac,0x8ac,0x80a,0x36,0x36,0x36,0x36,0x36,0x36,0x36,0x36,0x36,0x36,0x36, +0x39,0x729,0x729,0x729,0x729,0x729,0x729,0x729,0x729,0x729,0x729,0x729,0x729,0x729,0x729,0x729, +0x729,0x729,0x729,0x729,0x729,0x729,0x729,0x729,0x729,0x729,0x729,0x729,0x729,0x729,0x729,0x729, +0x729,0x71d,0x729,0x726,0x71d,0x71d,0x71d,0x71d,0x71d,0x71d,0x723,0x39,0x39,0x39,0x39,0x6de, +0x72f,0x72f,0x72f,0x72f,0x72f,0x729,0x72c,0x720,0x720,0x720,0x720,0x720,0x720,0x71d,0x720,0x2dc, +0x681,0x681,0x681,0x681,0x681,0x681,0x681,0x681,0x681,0x681,0x42f,0x42f,0x39,0x39,0x39,0x39, +0x39,0x39,0x39,0x39,0x39,0x39,0x39,0x39,0x39,0x39,0x39,0x39,0x39,0x39,0x39,0x39, +0x39,0x39,0x39,0x39,0x39,0x39,0x39,0x39,0x39,0x39,0x39,0x39,0x3c,0x73b,0x73b,0x3c, +0x73b,0x3c,0x3c,0x73b,0x73b,0x3c,0x73b,0x3c,0x3c,0x73b,0x3c,0x3c,0x3c,0x3c,0x3c,0x3c, +0x73b,0x73b,0x73b,0x73b,0x3c,0x73b,0x73b,0x73b,0x73b,0x73b,0x73b,0x73b,0x3c,0x73b,0x73b,0x73b, +0x3c,0x73b,0x3c,0x73b,0x3c,0x3c,0x73b,0x73b,0x3c,0x73b,0x73b,0x73b,0x73b,0x732,0x73b,0x738, +0x732,0x732,0x732,0x732,0x732,0x732,0x3c,0x732,0x732,0x73b,0x3c,0x3c,0x744,0x744,0x744,0x744, +0x744,0x3c,0x741,0x3c,0x735,0x735,0x735,0x735,0x735,0x732,0x3c,0x3c,0x684,0x684,0x684,0x684, +0x684,0x684,0x684,0x684,0x684,0x684,0x3c,0x3c,0x73e,0x73e,0x3c,0x3c,0x3c,0x3c,0x3c,0x3c, +0x3c,0x3c,0x3c,0x3c,0x3c,0x3c,0x3c,0x3c,0x3c,0x3c,0x3c,0x3c,0x3c,0x3c,0x3c,0x3c, +0x3c,0x3c,0x3c,0x3c,0x3c,0x3c,0x3c,0x3c,0x3c,0x3c,0x3c,0x3c,0x777,0x777,0x777,0x77a, +0x777,0x777,0x777,0x777,0x3f,0x777,0x777,0x777,0x777,0x77a,0x777,0x777,0x777,0x777,0x77a,0x777, +0x777,0x777,0x777,0x77a,0x777,0x777,0x777,0x777,0x77a,0x777,0x777,0x777,0x777,0x777,0x777,0x777, +0x777,0x777,0x777,0x777,0x777,0x77a,0x813,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x792,0x792,0x795, +0x792,0x795,0x795,0x798,0x795,0x798,0x792,0x792,0x792,0x792,0x792,0x783,0x792,0x795,0x79b,0x79b, +0x79e,0x780,0x79b,0x79b,0x777,0x777,0x777,0x777,0x3f,0x3f,0x3f,0x3f,0x792,0x792,0x792,0x795, +0x792,0x792,0x8b2,0x792,0x3f,0x792,0x792,0x792,0x792,0x795,0x792,0x792,0x792,0x792,0x795,0x792, +0x792,0x792,0x792,0x795,0x792,0x792,0x792,0x792,0x795,0x792,0x8b2,0x8b2,0x8b2,0x792,0x792,0x792, +0x792,0x792,0x792,0x792,0x8b2,0x795,0x8b2,0x8b2,0x8b2,0x3f,0x86d,0x86d,0x810,0x810,0x810,0x810, +0x810,0x810,0x8b5,0x810,0x810,0x810,0x810,0x810,0x810,0x3f,0x3f,0x810,0xc0f,0xc0f,0x3f,0x3f, +0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f, +0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f, +0x918,0x918,0x42,0x918,0x918,0x918,0x91b,0x918,0x42,0x918,0x918,0x42,0x915,0x90c,0x90c,0x90c, +0x90c,0x915,0x90c,0x42,0x42,0x42,0x90c,0x90f,0x915,0x912,0x42,0x42,0x42,0x42,0x42,0x42, +0x8fa,0x8fa,0x8fa,0x8fa,0x8fa,0x8fa,0x8fa,0x8fa,0x8fa,0x8fa,0x870,0x870,0x816,0x816,0x816,0x816, +0x918,0x918,0x918,0x918,0x918,0x918,0x915,0x915,0x90c,0x90c,0x42,0x42,0x42,0x42,0x42,0x42, +0x42,0x42,0x42,0x42,0x42,0x42,0x42,0x42,0x42,0x42,0x42,0x42,0x42,0x42,0x42,0x42, +0x42,0x42,0x42,0x42,0x42,0x42,0x42,0x42,0x42,0x42,0x42,0x42,0x2e5,0x2e5,0x2e5,0x2e5, +0x2e5,0x2e5,0x45,0x45,0x45,0x45,0x45,0x45,0x45,0x45,0x45,0x45,0x2e2,0x2e2,0x2e2,0x2e2, +0x2e2,0x2e2,0x2e2,0x2e2,0x2e2,0x2e2,0x2e2,0x2e2,0x2e2,0x2e2,0x2e2,0x2e2,0x2e2,0x2e2,0x2e2,0x2e2, +0x2e2,0x2e2,0x2e2,0x99c,0x99c,0xb7c,0xb7c,0x2df,0xb7f,0x45,0x45,0x45,0x756,0x756,0x756,0x756, +0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756, +0x756,0x756,0x756,0x756,0x756,0x756,0x48,0x48,0x48,0x48,0x48,0x759,0x75f,0x75f,0x75f,0x48, +0x48,0x48,0x48,0x48,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c, +0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x48,0x48, +0x48,0x48,0x48,0x48,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0xb85,0x825,0x4b,0x825,0x825, +0x825,0x825,0x4b,0x4b,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x4b,0x825,0x4b,0x825,0x825, +0x825,0x825,0x4b,0x4b,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0xb85,0x825,0x4b,0x825,0x825, +0x825,0x825,0x4b,0x4b,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825, +0x825,0x825,0x825,0x825,0x825,0x825,0x825,0xb85,0x825,0x4b,0x825,0x825,0x825,0x825,0x4b,0x4b, +0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x4b,0x825,0x4b,0x825,0x825,0x825,0x825,0x4b,0x4b, +0x825,0x825,0x825,0x825,0x825,0x825,0x825,0xb85,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x4b, +0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0xb85, +0x825,0x4b,0x825,0x825,0x825,0x825,0x4b,0x4b,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0xb85, +0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825, +0x825,0x825,0x825,0x4b,0x4b,0x4b,0x4b,0xc24,0xb82,0x873,0x81f,0x81c,0x81c,0x81c,0x81c,0x81f, +0x81f,0x822,0x822,0x822,0x822,0x822,0x822,0x822,0x822,0x822,0x819,0x819,0x819,0x819,0x819,0x819, +0x819,0x819,0x819,0x819,0x819,0x4b,0x4b,0x4b,0x828,0x828,0x828,0x828,0x828,0x828,0x828,0x828, +0x828,0x828,0x828,0x828,0x828,0x828,0x828,0x828,0x828,0x828,0x828,0x828,0x828,0x4e,0x4e,0x4e, +0x4e,0x4e,0x4e,0x4e,0x4e,0x4e,0x4e,0x4e,0x51,0x831,0x831,0x831,0x831,0x831,0x831,0x831, +0x831,0x831,0x831,0x831,0x831,0x831,0x831,0x831,0x831,0x831,0x831,0x831,0x831,0x831,0x831,0x831, +0x831,0x831,0x831,0x831,0x831,0x831,0x831,0x831,0x831,0x82b,0x82e,0x831,0x831,0x831,0x831,0x831, +0x831,0x831,0x831,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x876,0x834,0x834,0x834, +0x834,0x834,0x834,0x834,0x834,0x834,0x834,0x834,0x834,0x834,0x834,0x834,0x834,0x834,0x834,0x834, +0x834,0x834,0x834,0x834,0x834,0x834,0x834,0x903,0x891,0x54,0x54,0x54,0x837,0x837,0x837,0x837, +0x837,0x837,0x837,0x837,0x837,0x837,0x837,0x879,0x879,0x879,0x837,0x837,0x837,0x57,0x57,0x57, +0x57,0x57,0x57,0x57,0x57,0x57,0x57,0x57,0x57,0x57,0x57,0x57,0x92d,0x92d,0x92d,0x92d, +0x92d,0x92d,0x921,0x92d,0x92d,0x924,0x924,0x924,0x924,0x924,0x924,0x924,0x924,0x924,0x927,0x92a, +0x87f,0x87f,0x8f1,0x930,0x87c,0x83a,0x87f,0x906,0x930,0xb5b,0x5a,0x5a,0x8fd,0x8fd,0x8fd,0x8fd, +0x8fd,0x8fd,0x8fd,0x8fd,0x8fd,0x8fd,0x5a,0x5a,0x5a,0x5a,0x5a,0x5a,0xaa7,0xaa7,0xaa7,0xaa7, +0xaa7,0xaa7,0xaa7,0xaa7,0xaa7,0xaa7,0x5a,0x5a,0x5a,0x5a,0x5a,0x5a,0x83d,0x83d,0x882,0x885, +0x888,0x882,0x88e,0x83d,0x888,0x88b,0x83d,0x8bb,0x8bb,0x8bb,0x8c7,0x5d,0x900,0x900,0x900,0x900, +0x900,0x900,0x900,0x900,0x900,0x900,0x5d,0x5d,0x5d,0x5d,0x5d,0x5d,0x840,0x840,0x840,0x840, +0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840, +0x840,0x840,0x840,0x840,0x5d,0x5d,0x5d,0x5d,0x5d,0x5d,0x5d,0x5d,0x2ee,0x2e8,0x2ee,0x2e8, +0x2ee,0x2e8,0x2ee,0x2e8,0x2ee,0x2e8,0x2ee,0x2e8,0x2ee,0x2e8,0x2ee,0x2e8,0x2ee,0x2e8,0x2ee,0x2e8, +0x2ee,0x2e8,0x2e8,0x2e8,0x2e8,0x2e8,0x2eb,0x77d,0x60,0x60,0x60,0x60,0x2ee,0x2e8,0x2ee,0x2e8, +0x2ee,0x2e8,0x2ee,0x2e8,0x2ee,0x2e8,0x2ee,0x2e8,0x2ee,0x2e8,0x2ee,0x2e8,0x2ee,0x2e8,0x2ee,0x2e8, +0x2ee,0x2e8,0x2ee,0x2e8,0x2ee,0x2e8,0x60,0x60,0x60,0x60,0x60,0x60,0x2f7,0x2f7,0x2f7,0x2f7, +0x2f7,0x2f7,0x2f7,0x2f7,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x2f7,0x2f7,0x2f7,0x2f7, +0x2f7,0x2f7,0x63,0x63,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x63,0x63,0x2f7,0x2f7,0x2f7,0x2f7, +0x2f7,0x2f7,0x2f7,0x2f7,0x63,0x2fa,0x63,0x2fa,0x63,0x2fa,0x63,0x2fa,0x2f7,0x2f7,0x2f7,0x2f7, +0x2f7,0x2f7,0x2f7,0x2f7,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x2f7,0x2f7,0x2f7,0x2f7, +0x2f7,0x2f7,0x2f7,0x2f7,0x2f7,0x2f7,0x2f7,0x2f7,0x2f7,0x2f7,0x63,0x63,0x2f7,0x2f7,0x2f7,0x2f7, +0x2f7,0x2f7,0x2f7,0x2f7,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x2f7,0x2f7,0x2f7,0x2f7, +0x2f7,0x63,0x2f7,0x2f7,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x2f4,0x2f7,0x2f4,0x2f4,0x2f1,0x2f7,0x2f7, +0x2f7,0x63,0x2f7,0x2f7,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x2f1,0x2f1,0x2f1,0x2f7,0x2f7,0x2f7,0x2f7, +0x63,0x63,0x2f7,0x2f7,0x2fa,0x2fa,0x2fa,0x2fa,0x63,0x2f1,0x2f1,0x2f1,0x2f7,0x2f7,0x2f7,0x2f7, +0x2f7,0x2f7,0x2f7,0x2f7,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x2f1,0x2f1,0x2f1,0x63,0x63,0x2f7,0x2f7, +0x2f7,0x63,0x2f7,0x2f7,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x2f1,0x2f4,0x63,0xa71,0x99f,0x99f,0x99f, +0x66,0x66,0x66,0x66,0x66,0x66,0x528,0x528,0x528,0x528,0x528,0x528,0x30c,0x9ab,0x69,0x69, +0x1a1,0x30c,0x30c,0x30c,0x30c,0x30c,0x312,0x318,0x312,0x693,0x45c,0x1a4,0x309,0x19e,0x19e,0x19e, +0x19e,0x309,0x309,0x309,0x309,0x309,0x30f,0x315,0x30f,0x690,0x459,0x69,0xb8b,0xb8b,0xb8b,0xb8b, +0xb8b,0x69,0x69,0x69,0x69,0x69,0x69,0x69,0x69,0x69,0x69,0x69,0x6e1,0x6e1,0x6e1,0x6e1, +0x6e1,0x6e1,0x6e1,0x6bd,0x6e4,0x6f6,0x6e1,0x7b6,0x7cb,0x909,0x909,0x909,0xa6e,0xa6e,0xc7e,0xc7e, +0xc7e,0xc7e,0x6c,0x6c,0x6c,0x6c,0x6c,0x6c,0x6c,0x6c,0x6c,0x6c,0x6c,0x6c,0x6c,0x6c, +0x6c,0x6c,0x6c,0x6c,0x531,0x531,0x531,0x531,0x531,0x531,0x531,0x531,0x531,0x531,0x531,0x531, +0x531,0x52e,0x52e,0x52e,0x52e,0x531,0x8be,0x8be,0xa11,0xa17,0xa17,0xa14,0xa14,0xa14,0xa14,0xc27, +0xd11,0xd11,0xd11,0xd11,0x6f,0x6f,0x6f,0x6f,0x6f,0x6f,0x6f,0x6f,0x6f,0x6f,0x6f,0x6f, +0x6f,0x6f,0x6f,0x6f,0x9b4,0x9b1,0x9b1,0x9b1,0x9b1,0x9ba,0x9b7,0x9b7,0x9b7,0x9b7,0x9ae,0x9b1, +0xb8e,0xca2,0xca5,0x72,0x75,0x75,0x75,0x3e7,0x1b3,0x16b,0x33c,0x33c,0x33c,0x33c,0x33c,0x1b3, +0x3e7,0x3e7,0x1b3,0x33c,0x342,0x342,0x342,0x84f,0xca8,0x75,0x75,0x75,0x75,0x75,0x75,0x75, +0x75,0x75,0x75,0x75,0x1bf,0x1bf,0x1bf,0x1bf,0x1bf,0x1bc,0x1bc,0x1bc,0x1bc,0x1bc,0x351,0x351, +0x348,0x348,0x348,0x348,0xcae,0xcae,0xcab,0xcab,0xcab,0xcab,0xcab,0xcab,0x78,0x78,0x78,0x78, +0x78,0x78,0x78,0x78,0x78,0x78,0x78,0x78,0x78,0x78,0x78,0x78,0x78,0x78,0x78,0x78, +0x78,0x78,0x78,0x78,0x363,0x363,0x363,0x363,0x363,0x858,0x858,0x7b,0x7b,0x7b,0x7b,0x7b, +0x7b,0x7b,0x7b,0x7b,0x7b,0x7b,0x7b,0x7b,0x7b,0x7b,0x7b,0x7b,0x7b,0x7b,0x7b,0x7b, +0x7b,0x7b,0x7b,0x7b,0x366,0x366,0x366,0x366,0x366,0x366,0x366,0x366,0x366,0x366,0x366,0x7e, +0x7e,0x7e,0x7e,0x7e,0x7e,0x7e,0x7e,0x7e,0x7e,0x7e,0x7e,0x7e,0x7e,0x7e,0x7e,0x7e, +0x7e,0x7e,0x7e,0x7e,0x9cf,0x9cf,0x9cf,0x9cf,0x9cf,0x9cf,0x9cf,0x9cf,0x9cf,0x9cf,0xab9,0xab9, +0xab9,0xab9,0xab9,0xab9,0xab9,0xab9,0xb97,0xb97,0xb97,0xb97,0xb97,0xb97,0xb97,0xb97,0xb97,0xb97, +0xb97,0x81,0x81,0x81,0xab9,0xab9,0xb97,0xb97,0xb97,0xb97,0xb97,0xb97,0xb97,0xb97,0xb97,0xb97, +0xb97,0xb97,0xb97,0xb97,0xb97,0xb97,0xcb1,0x81,0x81,0x81,0x81,0x81,0x81,0x81,0x81,0x81, +0x81,0x81,0x81,0x81,0x81,0x81,0x81,0x81,0x81,0x81,0x81,0x81,0x81,0x81,0x81,0x81, +0x81,0x81,0x81,0x81,0x81,0x81,0x81,0x81,0x84,0x375,0x375,0x375,0x375,0x84,0x375,0x375, +0x375,0x375,0x84,0x84,0x375,0x375,0x375,0x375,0x375,0x375,0x375,0x375,0x375,0x375,0x375,0x375, +0x375,0x375,0x375,0x375,0x375,0x375,0x375,0x375,0x84,0x375,0x375,0x375,0x375,0x375,0x375,0x375, +0x375,0x375,0x375,0x375,0x375,0x375,0x375,0x375,0x375,0x375,0x375,0x375,0x375,0x3ed,0x375,0x375, +0x375,0x375,0x375,0x375,0x375,0x375,0x375,0x375,0x375,0x375,0x375,0x375,0x84,0x375,0x84,0x375, +0x375,0x375,0x375,0x84,0x84,0x84,0x375,0x84,0x375,0x375,0x375,0x70e,0x70e,0x70e,0x70e,0x84, +0x84,0x375,0x55b,0x55b,0x375,0x375,0x375,0x375,0xa5c,0xa02,0xa5c,0xa02,0xa5c,0xa02,0xa5c,0xa02, +0xa5c,0xa02,0xa5c,0xa02,0xa5c,0xa02,0x1ec,0x1ec,0x1ec,0x1ec,0x1ec,0x1ec,0x1ec,0x1ec,0x1ec,0x1ec, +0x174,0x174,0x174,0x174,0x174,0x174,0x174,0x174,0x174,0x174,0x174,0x174,0x174,0x174,0x174,0x174, +0x174,0x174,0x174,0x174,0x375,0x84,0x84,0x84,0x375,0x375,0x375,0x375,0x375,0x375,0x375,0x375, +0x375,0x375,0x375,0x375,0x375,0x375,0x375,0x375,0x84,0x375,0x375,0x375,0x375,0x375,0x375,0x375, +0x375,0x375,0x375,0x375,0x375,0x375,0x375,0x84,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0, +0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0, +0x8d0,0x8d0,0x87,0x8d0,0x8d0,0x8d0,0x8d0,0x8d3,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0, +0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d3,0x87,0x87,0x87,0x87, +0x87,0x87,0x87,0x87,0x87,0x87,0x87,0x87,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6, +0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8a,0x8a, +0x8a,0x8a,0x8a,0x8a,0x8a,0x8a,0x8a,0x8a,0x90,0x642,0x5b5,0x642,0x5b5,0x642,0x5b5,0x642, +0x5b5,0x642,0x5b5,0x5b5,0x5b8,0x5b5,0x5b8,0x5b5,0x5b8,0x5b5,0x5b8,0x5b5,0x5b8,0x5b5,0x5b8,0x5b5, +0x5b8,0x5b5,0x5b8,0x5b5,0x5b8,0x5b5,0x5b8,0x5b5,0x5b5,0x5b5,0x5b5,0x642,0x5b5,0x642,0x5b5,0x642, +0x5b5,0x5b5,0x5b5,0x5b5,0x5b5,0x5b5,0x642,0x5b5,0x5b5,0x5b5,0x5b5,0x5b5,0x5b8,0xa53,0xa53,0x90, +0x90,0x549,0x549,0x63f,0x63f,0x645,0x648,0xa38,0x93,0x93,0x93,0x93,0x93,0x5c1,0x5c1,0x5c1, +0x5c1,0x5c1,0x5c1,0x5c1,0x5c1,0x5c1,0x5c1,0x5c1,0x5c1,0x5c1,0x5c1,0x5c1,0x5c1,0x5c1,0x5c1,0x5c1, +0x5c1,0x5c1,0x5c1,0x5c1,0x5c1,0x5c1,0x5c1,0x5c1,0x5c1,0x93,0x93,0x93,0x96,0x5c4,0x5c4,0x5c4, +0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x96, +0x5ca,0x5ca,0x5cd,0x5cd,0x5cd,0x5cd,0x5cd,0x5cd,0x5cd,0x5cd,0x5cd,0x5cd,0x5cd,0x5cd,0x5cd,0x5cd, +0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5, +0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x99,0x99,0x99,0x99,0x99,0x99,0x99,0x99, +0x5d9,0x5d9,0x5d9,0x5d9,0x5d9,0x5d9,0x5d9,0x5d9,0x5d9,0x5d9,0x5d9,0x5d9,0x5d9,0x5d9,0x5d9,0x5d9, +0x5d9,0x5d9,0x5d9,0x5d9,0x5d9,0x5d9,0x5d9,0x5d9,0x5d9,0x5d9,0x5d9,0x5d9,0x5d9,0xb40,0xb40,0x9c, +0x5d3,0x5d3,0x5d3,0x5d3,0x9c,0x9c,0x9c,0x9c,0x9c,0x9c,0x9c,0x9c,0x9c,0x9c,0x9c,0x9c, +0xb3d,0xa3e,0xa3e,0xa3e,0xa3e,0xa3e,0xa3e,0xa3e,0xa3e,0xa3e,0xa3e,0xa3e,0xa3e,0xa3e,0xa3e,0xa3e, +0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6, +0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x9c, +0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8, +0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x9f,0x9f,0x9f,0x9f,0x9f,0x9f,0x9f,0x9f,0x9f,0x9f, +0x5e5,0x5e5,0x5e5,0x5e5,0x5e5,0x5e5,0xc60,0xc60,0xc60,0xc60,0xc60,0xc60,0xc60,0xc60,0xc60,0xc60, +0xc60,0xc60,0xc60,0xc60,0xc60,0xc60,0xc60,0xc60,0xc60,0xc60,0xc60,0xc60,0xa2,0xa2,0xa2,0xa2, +0xa2,0xa2,0xa2,0xa2,0xa2,0xa2,0xa2,0xa2,0xa2,0xa2,0xa2,0xa2,0xa2,0xa2,0xa2,0xa2, +0xa2,0xa2,0xa2,0xa2,0xa2,0xa2,0xa2,0xa2,0xa2,0xa2,0xa2,0xa2,0x8eb,0x8eb,0x8eb,0x8eb, +0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0xa5,0xa5,0xa5,0x8ee,0x8ee,0x8ee,0x8ee, +0x8ee,0x8ee,0x8ee,0x8ee,0x8ee,0x8ee,0x8ee,0x8ee,0x8ee,0x8ee,0x8ee,0x8ee,0x5e8,0x5eb,0x5e8,0x5eb, +0x5eb,0x5e8,0x5e8,0x5eb,0x5eb,0x5eb,0x5e8,0x5e8,0x5e8,0x5e8,0xae,0xae,0xa44,0xa44,0xa44,0xa44, +0xa44,0xa44,0xa44,0xa44,0xa44,0xa44,0xa44,0xa44,0xa44,0xa44,0xa44,0xa44,0xa44,0xa44,0xa44,0xae, +0xae,0xae,0xae,0xae,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63, +0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xae,0xae, +0xae,0xae,0xae,0xae,0xae,0xae,0xae,0xae,0xae,0xae,0xae,0xae,0xae,0xae,0xae,0xae, +0xae,0xae,0xae,0xae,0xae,0xae,0xae,0xae,0xae,0xae,0xae,0xae,0xae,0xae,0xae,0xae, +0x387,0x387,0x387,0x387,0x387,0x387,0x387,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1, +0xb1,0xb1,0xb1,0x378,0x378,0x378,0x378,0x378,0xb1,0xb1,0xb1,0xb1,0xb1,0x864,0x534,0x37e, +0x384,0x384,0x384,0x384,0x384,0x384,0x384,0x384,0x384,0x37b,0x37e,0x37e,0x37e,0x37e,0x37e,0x37e, +0x37e,0x37e,0x37e,0x37e,0x37e,0x37e,0x37e,0xb1,0x37e,0x37e,0x37e,0x37e,0x37e,0xb1,0x37e,0xb1, +0x37e,0x37e,0xb1,0x37e,0x37e,0xb1,0x37e,0x37e,0x37e,0x37e,0x37e,0x37e,0x37e,0x37e,0x37e,0x381, +0x393,0x38d,0x393,0x38d,0x390,0x396,0x393,0x38d,0x390,0x396,0x393,0x38d,0x390,0x396,0x393,0x38d, +0x393,0x38d,0x390,0x396,0x393,0x38d,0x393,0x38d,0x390,0x396,0x393,0x38d,0x390,0x396,0x393,0x38d, +0x393,0x38d,0xb4,0xb4,0xb4,0xb4,0xb4,0xb4,0xb4,0xb4,0xb4,0xb4,0xb4,0xb4,0xb4,0xb4, +0xb4,0xb4,0xb4,0xb4,0xb4,0xb4,0xb4,0x393,0x38d,0x390,0x396,0x393,0x38d,0x393,0x38d,0x393, +0x38d,0x393,0x393,0x38d,0xb4,0xb4,0xb4,0xb4,0xb4,0xb4,0xb4,0xb4,0xb4,0xb4,0xb4,0xb4, +0xb4,0xb4,0xb4,0xb4,0x390,0x38d,0x390,0x390,0x390,0x390,0x390,0x390,0x38d,0x390,0x38d,0x38d, +0x390,0x390,0x38d,0x38d,0x38d,0x38d,0x38d,0x390,0x38d,0x38d,0x390,0x38d,0x390,0x390,0x390,0x38d, +0x390,0x390,0x390,0x390,0xb4,0xb4,0x390,0x390,0x390,0x390,0x38d,0x38d,0x390,0x38d,0x38d,0x38d, +0x38d,0x390,0x38d,0x38d,0x38d,0x38d,0x38d,0x390,0x390,0x390,0x38d,0x38d,0xb4,0xb4,0xb4,0xb4, +0xb4,0xb4,0xb4,0xb4,0x936,0x936,0x936,0x936,0x936,0x936,0x936,0x936,0x936,0x936,0x936,0x936, +0x936,0x936,0x936,0x936,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x38a,0x38a, +0xa6b,0xabc,0xb4,0xb4,0x537,0x537,0x537,0x537,0xb7,0xb7,0xb7,0xb7,0xb7,0xb7,0xb7,0xb7, +0xb7,0xb7,0xb7,0xb7,0x5f1,0x5f4,0x5f4,0x5fa,0x5fa,0x6ab,0x486,0x6ab,0x486,0x6ab,0x486,0x6ab, +0x486,0x6ab,0x486,0x6ab,0x486,0x6ae,0x489,0x6ae,0x489,0xa47,0xa47,0xb52,0xb10,0x5ee,0x5ee,0x5ee, +0x5ee,0x5f7,0x5f7,0x5f7,0x48f,0x603,0x492,0xba,0x65a,0x65a,0x564,0x564,0x600,0x6b1,0x48c,0x6b1, +0x48c,0x6b1,0x48c,0x5fd,0x5fd,0x606,0x606,0x609,0x606,0x606,0x606,0xba,0x606,0x705,0x6d8,0x5fd, +0xba,0xba,0xba,0xba,0x399,0x3a5,0x399,0x9d2,0x399,0xbd,0x399,0x3a5,0x399,0x3a5,0x399,0x3a5, +0x399,0x3a5,0x399,0x3a5,0x3a5,0x3a2,0x39c,0x39f,0x3a5,0x3a2,0x39c,0x39f,0x3a5,0x3a2,0x39c,0x39f, +0x3a5,0x3a2,0x39c,0x3a2,0x39c,0x3a2,0x39c,0x39f,0x3a5,0x3a2,0x39c,0x3a2,0x39c,0x3a2,0x39c,0x3a2, +0x39c,0xbd,0xbd,0x753,0x6cf,0x6f9,0x585,0x588,0x57c,0x6f9,0x6f9,0xc3,0x3f0,0x3f3,0x3f3,0x3f3, +0x3f3,0x3f0,0x3f0,0xc3,0xc0,0xc0,0xc0,0xc0,0xc0,0xc0,0xc0,0xc0,0xc0,0x8c1,0x8c1,0x8c1, +0x7c8,0x1ef,0x165,0x165,0xc3,0x55e,0x582,0x57c,0x6f9,0x6cf,0x57c,0x582,0x69c,0x46b,0x57c,0x585, +0x46e,0x57f,0x471,0x57c,0x591,0x591,0x591,0x591,0x591,0x591,0x591,0x591,0x591,0x591,0x636,0x636, +0x585,0x585,0x585,0x55e,0x3f9,0x3f6,0x3f6,0x3f6,0x3f6,0x3f6,0x3f6,0x3f6,0x3f6,0x3f6,0x3f6,0x3f6, +0x3f6,0x3f6,0x3f6,0x3f6,0x3f6,0x3f6,0x3f6,0x3f6,0x3f6,0x3f6,0x3f6,0x3f6,0x3f6,0x3f6,0x3f6,0x3f6, +0x3f6,0x3f6,0x3f6,0xc3,0xc3,0xc3,0x3f6,0x3f6,0x3f6,0x3f6,0x3f6,0x3f6,0xc3,0xc3,0x3f6,0x3f6, +0x3f6,0x3f6,0x3f6,0x3f6,0xc3,0xc3,0x3f6,0x3f6,0x3f6,0x3f6,0x3f6,0x3f6,0xc3,0xc3,0x3f6,0x3f6, +0x3f6,0xc3,0xc3,0xc3,0x942,0x942,0x942,0x942,0x942,0x942,0x942,0x942,0x942,0x942,0x942,0x942, +0x942,0x942,0x942,0x942,0x942,0x942,0x942,0x942,0x942,0x942,0x942,0x942,0x942,0x942,0x942,0x942, +0x942,0x942,0x942,0xc6,0x93f,0x93f,0x93f,0x93f,0xc6,0xc6,0xc6,0xc6,0xc6,0xc6,0xc6,0xc6, +0xc6,0xc6,0xc6,0xc6,0x945,0x945,0x945,0x945,0x945,0x945,0x945,0x945,0x945,0x945,0x945,0x945, +0x945,0x945,0x945,0x945,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e, +0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0xcc,0xcc,0xcc,0xcc,0xcc,0xcc, +0xcc,0xcc,0xcc,0xcc,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0xcf,0xcf,0xcf,0x951,0x951, +0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951, +0x951,0x951,0x951,0x951,0x954,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951, +0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951, +0x951,0x951,0xcf,0xcf,0xcf,0xcf,0xcf,0xcf,0xcf,0xcf,0xcf,0xcf,0xcf,0xcf,0xcf,0xcf, +0xcf,0xcf,0xcf,0xcf,0xcf,0xcf,0xcf,0xcf,0xcf,0xcf,0xcf,0xcf,0xcf,0xcf,0xcf,0xcf, +0xcf,0xcf,0xcf,0xcf,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d, +0x95d,0x95d,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0xd2,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a, +0x95a,0x95a,0x95a,0x95a,0x95d,0x95d,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a, +0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a, +0x95d,0xd2,0x95d,0x95d,0xd2,0xd2,0x95d,0xd2,0xd2,0x95d,0x95d,0xd2,0xd2,0x95d,0x95d,0x95d, +0x95d,0xd2,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95a,0x95a,0x95a,0x95a,0xd2,0x95a, +0xd2,0x95a,0x95a,0x95a,0x95a,0xac5,0x95a,0x95a,0xd2,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a, +0x95a,0x95a,0x95a,0x95a,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d, +0x95d,0x95d,0x95d,0x95d,0x95a,0x95a,0x95a,0x95a,0x95d,0x95d,0xd2,0x95d,0x95d,0x95d,0x95d,0xd2, +0xd2,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0xd2,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d, +0x95d,0xd2,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a, +0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95d,0x95d,0xd2,0x95d, +0x95d,0x95d,0x95d,0xd2,0x95d,0x95d,0x95d,0x95d,0x95d,0xd2,0x95d,0xd2,0xd2,0xd2,0x95d,0x95d, +0x95d,0x95d,0x95d,0x95d,0x95d,0xd2,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a, +0x95a,0x95a,0x95a,0x95a,0xb9a,0xb9a,0xd2,0xd2,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d, +0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d, +0x95a,0x95a,0x95a,0x957,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0xcb7,0xcb4,0xd2,0xd2,0x97b,0x97b, +0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b, +0xd5,0x972,0xd5,0xd5,0xd5,0xd5,0xd5,0xd5,0xd5,0xd5,0xd5,0xd5,0xd5,0xd5,0xd5,0xd5, +0xd5,0xd5,0xd5,0xd5,0xd5,0xd5,0xd5,0xd5,0xd5,0xd5,0xd5,0xd5,0xd5,0xd5,0xd5,0xd5, +0x9d8,0x9d5,0x9d8,0x9d5,0x9d8,0x9d5,0x9d8,0x9d5,0x9d8,0x9d5,0x9d8,0x9d5,0x9d8,0x9d5,0x9d8,0x9d5, +0xcbd,0xcba,0xcbd,0xcba,0xd8,0xd8,0xd8,0xd8,0xd8,0xd8,0xd8,0xd8,0xd8,0xd8,0xd8,0xd8, +0x9db,0x9db,0x9db,0x9db,0x9db,0x9db,0x9db,0x9db,0x9db,0x9db,0x9db,0x9db,0x9db,0xdb,0x9db,0x9db, +0x9db,0x9db,0xa1a,0xa1a,0xa1d,0xdb,0xdb,0xdb,0xdb,0xdb,0xdb,0xdb,0xdb,0xdb,0xdb,0xdb, +0x9de,0x9de,0x9de,0x9de,0x9de,0x9de,0x9de,0x9de,0x9de,0x9de,0x9de,0x9de,0x9de,0x9de,0x9de,0x9de, +0x9de,0x9de,0xa20,0xa20,0xa23,0x9ff,0x9ff,0xde,0xde,0xde,0xde,0xde,0xde,0xde,0xde,0xde, +0x9e1,0x9e1,0x9e1,0x9e1,0x9e1,0x9e1,0x9e1,0x9e1,0x9e1,0x9e1,0x9e1,0x9e1,0x9e1,0x9e1,0x9e1,0x9e1, +0x9e1,0x9e1,0xa26,0xa26,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1, +0x9e4,0x9e4,0x9e4,0x9e4,0x9e4,0x9e4,0x9e4,0x9e4,0x9e4,0x9e4,0x9e4,0x9e4,0x9e4,0xe4,0x9e4,0x9e4, +0x9e4,0xe4,0xa29,0xa29,0xe4,0xe4,0xe4,0xe4,0xe4,0xe4,0xe4,0xe4,0xe4,0xe4,0xe4,0xe4, +0xb9d,0xb9d,0xb9d,0xb9d,0xb9d,0xc75,0xc12,0xcc0,0xcc0,0xcc0,0xcc0,0xe7,0xe7,0xe7,0xe7,0xe7, +0x9e7,0x9e7,0x9e7,0x9e7,0x9e7,0x9e7,0x9e7,0x9e7,0x9e7,0x9e7,0x9e7,0x9e7,0x9e7,0x9e7,0x9e7,0x9e7, +0x9e7,0x9e7,0xa65,0xa0b,0xa65,0xa0b,0xa65,0xa0b,0xe7,0xe7,0xe7,0xe7,0x9ea,0x9ea,0x9ea,0x9ea, +0x9ea,0x9ea,0x9ea,0x9ea,0x9ea,0x9ea,0x9ea,0x9ea,0x9ea,0x9ea,0x9ea,0x9ea,0xacb,0xacb,0xacb,0xacb, +0xacb,0xacb,0xacb,0xacb,0xacb,0xacb,0xacb,0xacb,0xacb,0xacb,0xacb,0xacb,0xacb,0xacb,0xacb,0xacb, +0xacb,0xacb,0xacb,0xacb,0xacb,0xacb,0xacb,0xacb,0xacb,0xea,0xea,0xea,0xb28,0xb28,0xb28,0xb2e, +0xb2e,0xb2e,0xb2e,0xb28,0xb28,0xb2e,0xb2e,0xb2e,0xea,0xea,0xea,0xea,0xb2e,0xb2e,0xb28,0xb2e, +0xb2e,0xb2e,0xb2e,0xb2e,0xb2e,0xb2b,0xb2b,0xb2b,0xea,0xea,0xea,0xea,0xac8,0xea,0xea,0xea, +0xb37,0xb37,0xb4c,0xb4c,0xb4c,0xb4c,0xb4c,0xb4c,0xb4c,0xb4c,0xb4c,0xb4c,0xb5e,0xb5e,0xb5e,0xb5e, +0xb5e,0xb5e,0xb5e,0xb5e,0xb5e,0xb5e,0xb5e,0xb5e,0xb5e,0xb5e,0xb5e,0xb5e,0xb5e,0xb5e,0xed,0xed, +0xb5e,0xb5e,0xb5e,0xb5e,0xb5e,0xed,0xed,0xed,0xed,0xed,0xed,0xed,0xed,0xed,0xed,0xed, +0xae9,0xae9,0xae9,0xae9,0xae9,0xae9,0xae9,0xae9,0xae9,0xae9,0xae9,0xae9,0xae9,0xae9,0xba6,0xba6, +0xba6,0xba6,0xba6,0xba6,0xcc3,0xcc3,0xcc3,0xcc3,0xcc3,0xcc3,0xcc3,0xf0,0xf0,0xf0,0xf0,0xf0, +0xcc3,0xcc3,0xcc3,0xcc3,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0, +0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0, +0xf0,0xf0,0xf0,0xf0,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef, +0xf3,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef, +0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xf3,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef, +0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xf3,0xaef,0xaef,0xf3,0xaef, +0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xf3,0xf3, +0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xaef,0xf3,0xf3, +0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3, +0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3, +0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2, +0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xf6,0xf6,0xf6,0xf6,0xf6, +0xb0a,0xb0a,0xb0a,0xf9,0xf9,0xf9,0xf9,0xaf5,0xaf5,0xaf5,0xaf5,0xaf5,0xaf5,0xaf5,0xaf5,0xaf5, +0xaf5,0xaf5,0xaf5,0xaf5,0xaf5,0xaf5,0xaf5,0xaf5,0xaf5,0xaf5,0xaf5,0xaf5,0xaf5,0xaf5,0xaf5,0xaf5, +0xf9,0xf9,0xf9,0xaf5,0xaf5,0xaf5,0xaf5,0xaf5,0xaf5,0xaf5,0xaf5,0xaf5,0xaf8,0xaf8,0xaf8,0xaf8, +0xaf8,0xaf8,0xaf8,0xaf8,0xaf8,0xaf8,0xaf8,0xaf8,0xaf8,0xaf8,0xaf8,0xaf8,0xaf8,0xaf8,0xaf8,0xaf8, +0xaf8,0xaf8,0xaf8,0xaf8,0xaf8,0xaf8,0xaf8,0xaf8,0xaf8,0xaf8,0xfc,0xb0d,0xafe,0xafe,0xafe,0xafe, +0xafe,0xafe,0xafe,0xafe,0xafe,0xafe,0xafe,0xafe,0xafe,0xafe,0xafe,0xafe,0xafe,0xafe,0xafe,0xafe, +0xafe,0xafe,0xafe,0xafe,0xafe,0xafe,0xafe,0xafe,0xafe,0xafe,0xff,0xff,0xb01,0xb01,0xb01,0xb01, +0xb01,0xb01,0x102,0x102,0xb01,0x102,0xb01,0xb01,0xb01,0xb01,0xb01,0xb01,0xb01,0xb01,0xb01,0xb01, +0xb01,0xb01,0xb01,0xb01,0xb01,0xb01,0xb01,0xb01,0xb01,0xb01,0xb01,0xb01,0xb01,0xb01,0x102,0xb01, +0xb01,0x102,0x102,0x102,0xb01,0x102,0x102,0xb01,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04, +0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0x105, +0x105,0x105,0x105,0x105,0x105,0x105,0x105,0x105,0xbb2,0xbb2,0xbb2,0xbb2,0xbb2,0xbb2,0xbb2,0xbb2, +0xbb2,0xbb2,0xbb2,0xbb2,0xbb2,0xbb2,0x10e,0x10e,0x10e,0x10e,0x10e,0x10e,0x10e,0x10e,0x10e,0x10e, +0x10e,0x10e,0x10e,0x10e,0x10e,0x10e,0x10e,0x10e,0xbb8,0xbb8,0xbb8,0xbb8,0xbb8,0xbb8,0xbb8,0xbb8, +0xbb8,0xbb8,0xbb8,0xbb8,0xbb8,0xbb8,0xbb8,0xbb8,0xbb8,0xbb8,0xbb8,0xbb8,0xbb8,0xbb8,0xbb8,0xc2d, +0xc2d,0xc30,0xc30,0xc30,0x111,0x111,0xbfd,0xbb5,0xc66,0xc66,0xc66,0xc66,0xc66,0xc66,0xc66,0xc66, +0xc66,0xc66,0xc66,0xc66,0xc66,0xc66,0xc66,0xc66,0x114,0x114,0x114,0x114,0x114,0x114,0x114,0x114, +0x114,0x114,0x114,0x114,0x114,0x114,0x114,0x114,0xa59,0xa59,0xa59,0xa59,0xa59,0xa59,0xa59,0xa59, +0xa59,0xa59,0xa59,0xa59,0xa59,0xa59,0xa59,0xa59,0xc33,0xc33,0xc33,0xc33,0xd14,0xd14,0xd14,0xd14, +0xd14,0xd14,0xd14,0x117,0x117,0x117,0x117,0x117,0x117,0x117,0x117,0x117,0x117,0x117,0x117,0x117, +0x117,0x117,0x117,0x117,0x117,0x117,0x117,0x117,0x117,0x117,0x117,0x117,0x117,0x117,0x117,0x117, +0x117,0x117,0xd14,0xd14,0xbc1,0xbbe,0xbc1,0xbbe,0xbbe,0xbbb,0xbbb,0xbbb,0xbbb,0xbbb,0xbbb,0x11a, +0x11a,0x11a,0x11a,0x11a,0x11a,0x11a,0x11a,0x11a,0x11a,0x11a,0x11a,0x11a,0x11a,0xc00,0xc00,0xc00, +0xc00,0xbbb,0xc00,0xc00,0xbc4,0xbc4,0xbc4,0xbc4,0xbc4,0xbc4,0xbc4,0xbc4,0xbc4,0xbc4,0xbc4,0xbc4, +0xbc4,0xbc4,0xbc4,0xbc4,0xbc4,0xbc4,0xbc4,0xbc4,0xbc4,0xbc4,0xbc4,0x11d,0x11d,0x11d,0x11d,0x11d, +0x11d,0x11d,0x11d,0x11d,0xbc4,0xbc4,0xbc4,0xbc4,0xbc4,0xbc4,0xbc4,0x11d,0xbc4,0xbc4,0xbc4,0xbc4, +0xbc4,0xbc4,0xbc4,0x11d,0xbc4,0xbc4,0xbc4,0xbc4,0xbc4,0xbc4,0xbc4,0x11d,0xbc4,0xbc4,0xbc4,0xbc4, +0xbc4,0xbc4,0xbc4,0x11d,0xbca,0xbca,0xbca,0xbca,0xbca,0xbca,0xbca,0xbca,0xbca,0xbca,0xbca,0xbca, +0xbca,0xbca,0xbca,0xbca,0xbc7,0xbc7,0xbc7,0xbc7,0xbc7,0xbc7,0xbc7,0xbc7,0xbc7,0xbc7,0x120,0x120, +0x120,0x120,0x120,0x120,0xbcd,0xbcd,0xbcd,0xbcd,0xbcd,0xbcd,0x123,0x123,0x123,0x123,0x123,0x123, +0x123,0x123,0x123,0x123,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4, +0xbf4,0xbf4,0xbf4,0xbf4,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3, +0xbd3,0xbd3,0xbd3,0x126,0xbd0,0xbd0,0xbd0,0xbd0,0xbd0,0xbd0,0xbd0,0xbd0,0xbd0,0xbd0,0xbd0,0xbd0, +0xbd0,0xbd0,0xbd0,0xbd0,0xbd0,0xbd0,0xbd0,0xbd0,0xbd0,0xbd0,0xbd0,0xbd0,0xbd0,0xbd0,0xbd0,0xbd0, +0xbd0,0xbd0,0xbd0,0x126,0xbd9,0xc39,0xc39,0xc39,0x129,0xc39,0xc39,0x129,0x129,0x129,0x129,0x129, +0xc39,0xc39,0xc39,0xc39,0xbd9,0xbd9,0xbd9,0xbd9,0x129,0xbd9,0xbd9,0xbd9,0x129,0xbd9,0xbd9,0xbd9, +0xbd9,0xbd9,0xbd9,0xbd9,0xbd9,0xbd9,0xbd9,0xbd9,0xbd9,0xbd9,0xbd9,0xbd9,0xbd9,0xbd9,0xbd9,0xbd9, +0xbd9,0xbd9,0xbd9,0xbd9,0x129,0x129,0x129,0x129,0xc36,0xc36,0xc36,0x129,0x129,0x129,0x129,0xc3c, +0xbd6,0xbd6,0xbd6,0xbd6,0xbd6,0xbd6,0xbd6,0xbd6,0x129,0x129,0x129,0x129,0x129,0x129,0x129,0x129, +0xc03,0xc03,0xc03,0xc03,0xc03,0xc03,0xc03,0xc03,0xbd6,0x129,0x129,0x129,0x129,0x129,0x129,0x129, +0xbdc,0xbdc,0xbdc,0xbdc,0xbdc,0xbdc,0xbdc,0xbdc,0xbdc,0xbdc,0xbdc,0xbdc,0xbdc,0xbdc,0xbdc,0xbdc, +0xbdc,0xbdc,0xbdc,0xbdc,0xbdc,0xbdc,0xbdc,0xcc6,0xcc6,0xcc6,0xcc6,0x12c,0x12c,0x12c,0x12c,0x12c, +0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0x12f,0x12f,0x12f,0x12f,0x12f,0x12f, +0xc87,0xc87,0xc87,0xc87,0xc87,0xc87,0xc87,0xc87,0xc87,0xc87,0xc87,0xc87,0xc87,0xc87,0xc87,0xc87, +0xc87,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xc87,0xc87,0x12f,0x12f,0x12f,0x12f,0x12f,0x12f, +0xc72,0xc72,0xc72,0xc72,0xc72,0xc72,0xc72,0xc72,0xc72,0xc72,0x12f,0x12f,0x12f,0x12f,0xc84,0xc84, +0xbdf,0xbdf,0xbdf,0xbdf,0x132,0x132,0x132,0x132,0xbdf,0xbdf,0xbdf,0xbdf,0xbdf,0xbdf,0xbdf,0xbdf, +0xc06,0xbdf,0xbdf,0xbdf,0xbdf,0xbdf,0x132,0x132,0x132,0x132,0x132,0x132,0x132,0x132,0x132,0x132, +0xc81,0xc81,0xc81,0xc81,0xc81,0xc81,0xc81,0xc81,0xc81,0xc81,0xc81,0xc81,0xc81,0xc81,0xc09,0xc09, +0xc09,0xc09,0xc09,0xc09,0xc09,0xc09,0xbeb,0xc0c,0x135,0x135,0x135,0x135,0xc81,0xc81,0x135,0x135, +0x135,0x135,0x135,0x135,0x135,0x135,0x135,0x135,0x135,0x135,0x135,0x135,0x135,0x135,0x135,0x135, +0x135,0x135,0x135,0x135,0x135,0x135,0x135,0x135,0x135,0x135,0x135,0x135,0x135,0x135,0x135,0x135, +0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0x13b,0x13b,0x13b,0x13b,0x13b,0x13b,0x13b,0x13b,0x13b,0xbf7, +0x13b,0x13b,0x13b,0x13b,0x13b,0x13b,0x13b,0x13b,0x13b,0x13b,0x13b,0x13b,0x13b,0x13b,0x13b,0x13b, +0xa2f,0xa2f,0xa2f,0xa2f,0xa2f,0xa2f,0xa2f,0xa2f,0xa2f,0xa2f,0xa2f,0xa2f,0xa2f,0xa2f,0xa2f,0xa2f, +0xc6c,0xc15,0xc15,0xc6c,0xc6c,0xc57,0xc57,0xc78,0xc18,0xc69,0x13e,0x13e,0x13e,0x13e,0x13e,0x13e, +0xccc,0xccc,0xccc,0xccc,0xccc,0xccc,0xccc,0xccc,0xccc,0xccc,0xccc,0xd17,0xd17,0xd17,0xd17,0xd17, +0xd17,0xd17,0xd17,0xd17,0xccf,0xccf,0xcc9,0xcc9,0xd2f,0xd29,0xcd2,0x141,0x141,0x141,0x141,0x141, +0xd23,0xd23,0xd1a,0xd23,0xd26,0xcd8,0xcd8,0xcd8,0xcd8,0xcd8,0xcd8,0xcd8,0x144,0x144,0x144,0x144, +0xd35,0xd35,0xd35,0xd35,0xd35,0xd35,0xd35,0xd35,0xd35,0xd35,0xcff,0xcff,0xcf9,0xcfc,0xcff,0xcff, +0xcf9,0xcd5,0xcd5,0xcd5,0xcd5,0xcd5,0xcd5,0xcd5,0xcd5,0xcd5,0xcd5,0xd1d,0xd1d,0xd1d,0xd1d,0xd1d, +0xd1d,0xd1d,0xd1d,0xd1d,0xcd5,0xcd5,0xcd5,0xcd5,0xcd5,0xcd5,0xcd5,0xcd5,0xcd5,0x144,0x144,0x144, +0xce1,0xcde,0xce1,0xce1,0xce1,0xcde,0xcde,0xce1,0xcde,0xce1,0xcde,0xce1,0xcde,0x147,0x147,0x147, +0x147,0x147,0x147,0x147,0xcde,0xce1,0xcde,0xcde,0x147,0x147,0x147,0x147,0x147,0x147,0x147,0x147, +0xce4,0xce4,0x14a,0x14a,0x14a,0x14a,0x14a,0x14a,0x14a,0x14a,0x14a,0x14a,0x14a,0x14a,0x14a,0x14a, +0x14a,0x14a,0x14a,0x14a,0x14a,0x14a,0x14a,0x14a,0x14a,0x14a,0x14a,0x14a,0x14a,0x14a,0x14a,0x14a, +0x14a,0x14a,0x14a,0x14a,0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xce7, +0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xd08,0xd08,0xd2c,0xd2c,0x14d,0x14d,0x14d,0x14d, +0x14d,0x14d,0x14d,0x14d,0xced,0xced,0xced,0xced,0xced,0xced,0xced,0xced,0xced,0xced,0xced,0xced, +0xced,0xced,0xced,0xced,0xced,0xced,0xced,0xced,0xced,0xced,0xcea,0xcea,0xcea,0xcea,0x150,0x150, +0x150,0x150,0x150,0xd02,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0, +0xcf0,0xcf0,0xcf0,0x153,0x153,0x153,0x153,0x153,0x153,0x153,0x153,0x153,0x153,0x153,0x153,0x153, +0x153,0x153,0x153,0x153,0x153,0x153,0x153,0x153,0x153,0x153,0x153,0x153,0x153,0x153,0x153,0x153, +0x153,0x153,0x153,0x153,0xcf3,0xcf3,0xcf3,0x156,0x156,0x156,0x156,0x156,0x156,0x156,0x156,0x156, +0x156,0x156,0x156,0x156,0xd05,0xd05,0xd05,0xd05,0x156,0x156,0x156,0x156,0x156,0x156,0x156,0x156, +0x156,0x156,0x156,0x156,0xcf6,0xcf6,0xcf6,0xcf6,0xcf6,0xcf6,0xcf6,0xcf6,0xcf6,0xcf6,0xcf6,0xcf6, +0xcf6,0xcf6,0xcf6,0xcf6,0xcf6,0xcf6,0x159,0x159,0x159,0x159,0x159,0x159,0x159,0x159,0x159,0x159, +0x159,0x159,0x159,0x159,0x15c,0x15c,0x15c,0x15c,0x15c,0x15c,0x15c,0x15c,0x15c,0x15c,0x15c,0x15c, +0x15c,0x15c,0x15c,0x15c,0x15c,0x15c,0x15c,0x15c,0x15c,0x15c,0x15c,0x15c,0x15c,0x15c,0x15c,0x15c, +0x15c,0x15c,0x15c,0x15c,0x15c,0x15c,0x765,0x765,0x975,0x975,0x975,0x975,0x975,0x975,0x975,0x975, +0x975,0x975,0x975,0x975,0x975,0x975,0x975,0x975,0x975,0x975,0x975,0x975,0x975,0x975,0x975,0x15f, +0x15f,0x15f,0x15f,0x15f,0x15f,0x15f,0x15f,0x15f,0x978,0x978,0x978,0x978,0x978,0x978,0x978,0x978, +0x978,0x978,0x978,0x978,0x978,0x978,0x978,0x978,0x978,0x978,0x978,0x978,0x978,0x978,0x978,0x978, +0x978,0x978,0x978,0x978,0x978,0x978,0x162,0x162,0x168,0x168,0x168,0x168,0x168,0x168,0x168,0x168, +0x168,0x168,0x168,0x168,0x168,0x168,0x168,0x168,0x168,0x168,0x168,0x168,0x168,0x168,0x168,0x168, +0x168,0x168,0x168,0x168,0x168,0x168,0x168,0x168,0x1d4,0x1d4,0x1d4,0x1d4,0x1d4,0x1d4,0x1d4,0x1d4, +0x1d4,0x1d4,0x16e,0x981,0x981,0x981,0x981,0x981,0x981,0x981,0x981,0x981,0x981,0x981,0x981,0x981, +0x981,0x981,0x981,0x981,0x981,0x981,0x981,0xb07,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da, +0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x171,0x369,0x369,0x369, +0x369,0x369,0x369,0x369,0x369,0x369,0x369,0x369,0x22b,0x22b,0x225,0x225,0x3c9,0x225,0x22e,0x18f, +0x44a,0x18f,0x18f,0x18f,0x44a,0x18f,0x22e,0x22e,0x192,0x231,0x225,0x225,0x225,0x225,0x225,0x225, +0x18c,0x18c,0x18c,0x18c,0x228,0x18c,0x225,0x867,0x195,0x195,0x3de,0x2fd,0x60c,0x60c,0x60c,0x43e, +0x453,0x453,0x522,0x522,0x522,0x522,0x522,0x8ca,0x6c3,0x6b4,0x6c6,0x6c9,0x6ba,0x6c3,0x6b7,0x6b7, +0x2fd,0x70b,0x70b,0x195,0x627,0x624,0x3e1,0x303,0x432,0x432,0x435,0x435,0x435,0x435,0x435,0x56a, +0x435,0x435,0x435,0x74d,0x52b,0x52b,0x525,0x525,0x444,0x56d,0x438,0x441,0x423,0x198,0x19b,0x300, +0x711,0x714,0x68d,0x70b,0x711,0x711,0x68d,0x70b,0x31e,0x31e,0x333,0x6cc,0x31b,0x1a7,0x31e,0x32a, +0x31b,0x6cc,0x32d,0x333,0x333,0x333,0x32d,0x32d,0x333,0x333,0x333,0x1ad,0x31b,0x333,0x6f3,0x31b, +0x327,0x333,0x333,0x333,0x333,0x333,0x31b,0x31b,0x321,0x1a7,0x1aa,0x31b,0x333,0x31b,0x3e4,0x31b, +0x333,0x324,0x339,0x1b0,0x333,0x333,0x327,0x32d,0x333,0x333,0x336,0x333,0x32d,0x330,0x330,0x330, +0x330,0x84c,0x849,0xab0,0xb91,0x9b7,0x9ba,0x9ba,0x1b9,0x1b9,0x1b9,0x1b9,0x1b9,0x1b9,0x1b9,0x1b9, +0x1b9,0x1b9,0x1b9,0x1b9,0x345,0x345,0x345,0x345,0x1b6,0x1b6,0x1b6,0x1b6,0x1b6,0x1b6,0x1b6,0x1b6, +0x1b6,0x1b6,0x33f,0x33f,0x33f,0x33f,0x33f,0x33f,0x348,0x348,0x348,0x348,0x348,0x348,0x348,0x348, +0x348,0x348,0x348,0x348,0x348,0x34b,0x351,0x351,0x348,0x348,0x1bf,0x348,0x1bf,0x348,0x348,0x348, +0x348,0x348,0x348,0x348,0x348,0x348,0x348,0x348,0x1c2,0x354,0x1c2,0x1c2,0x357,0x354,0x354,0x1c2, +0x1c2,0x357,0x354,0x1c2,0x357,0x354,0x354,0x1c2,0x354,0x1c2,0x6ea,0x6e7,0x354,0x1c2,0x354,0x354, +0x354,0x354,0x1c2,0x354,0x354,0x1c2,0x1c2,0x1c2,0x1c2,0x354,0x354,0x1c2,0x357,0x1c2,0x357,0x1c2, +0x1c2,0x1c2,0x1c2,0x1c2,0x1c8,0x35a,0x1c2,0x35a,0x35a,0x354,0x354,0x354,0x1c2,0x1c2,0x1c2,0x1c2, +0x354,0x354,0x354,0x354,0x1c2,0x1c2,0x354,0x354,0x354,0x357,0x354,0x354,0x357,0x354,0x354,0x357, +0x1c2,0x357,0x354,0x354,0x1c2,0x354,0x354,0x354,0x354,0x354,0x1c2,0x354,0x354,0x354,0x354,0x354, +0x354,0x354,0x354,0x354,0x354,0x354,0x354,0x354,0x1c5,0x1c2,0x357,0x354,0x1c2,0x1c2,0x1c2,0x1c2, +0x354,0x354,0x1c2,0x1c2,0x354,0x357,0x1c5,0x1c5,0x357,0x357,0x354,0x354,0x357,0x357,0x354,0x354, +0x357,0x357,0x354,0x354,0x354,0x354,0x354,0x354,0x357,0x357,0x1c2,0x1c2,0x357,0x357,0x1c2,0x1c2, +0x357,0x357,0x354,0x354,0x354,0x354,0x354,0x354,0x354,0x354,0x354,0x354,0x354,0x1c2,0x354,0x354, +0x354,0x1c2,0x354,0x354,0x354,0x354,0x354,0x354,0x354,0x1c2,0x354,0x354,0x354,0x354,0x354,0x354, +0x357,0x357,0x357,0x357,0x354,0x354,0x354,0x354,0x354,0x354,0x354,0x354,0x354,0x354,0x354,0x354, +0x354,0x354,0x354,0x1c2,0x35d,0x855,0x35d,0x35d,0x35d,0x35d,0x35d,0x35d,0x360,0x360,0x360,0x360, +0x35d,0x35d,0x35d,0x35d,0x35d,0x35d,0x1cb,0x35d,0x35d,0x35d,0x35d,0x35d,0x35d,0x35d,0x35d,0x35d, +0x35d,0x35d,0x35d,0x35d,0x1d1,0x1d1,0x1d1,0x1d1,0x1d1,0x1d1,0x1d1,0x1d1,0x1d1,0x1d1,0x1d1,0x1d1, +0x1d1,0x1d1,0x1d1,0x1d1,0x1d1,0x1d1,0x1d1,0x1d1,0x1ce,0x1ce,0x1ce,0x1ce,0x1ce,0x1ce,0x1ce,0x1ce, +0x1ce,0x1ce,0x1ce,0x1ce,0x1ce,0x1ce,0x1ce,0x1ce,0x1ce,0x1ce,0x1ce,0x1ce,0x1ce,0x1ce,0x1ce,0x1ce, +0x1ce,0x1ce,0x1ce,0x1ce,0x1ce,0x1ce,0x1ce,0x1ce,0x1ce,0x1ce,0x1d7,0x1d7,0x1d7,0x1d7,0x1d7,0x1d7, +0x1d7,0x1d7,0x1d7,0x1d7,0x1d7,0x1d7,0x1d7,0x1d7,0x1d7,0x1d7,0x1d7,0x1d7,0x1d4,0x1d4,0x1d4,0x1d4, +0x1d4,0x1d4,0x1d4,0x1d4,0x1d4,0x1d4,0x1d4,0x1d4,0x1d4,0x1d4,0x1d4,0x1d4,0x1da,0x1da,0x1da,0x1da, +0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da, +0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x369,0x369,0x369,0x369, +0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da,0x1da, +0x1dd,0x1dd,0x1dd,0x1dd,0x1dd,0x1dd,0x1dd,0x1dd,0x1dd,0x1dd,0x1dd,0x1dd,0x1dd,0x1dd,0x1dd,0x1dd, +0x36c,0x36c,0x1dd,0x1dd,0x1dd,0x1dd,0x9c9,0x9c9,0x9c9,0x9c9,0x9c9,0x9c9,0x9c9,0x9c9,0x9c9,0x9c9, +0x1e0,0x1e0,0x36f,0x1e0,0x1e0,0x1e0,0x1e0,0x1e0,0x1e0,0x1e0,0x36f,0x36f,0x36f,0x36f,0x36f,0x36f, +0x36f,0x36f,0x1e0,0x1e0,0x36f,0x36f,0x1e0,0x1e3,0x36f,0x36f,0x36f,0x36f,0x1e0,0x1e0,0x36f,0x36f, +0x1e0,0x1e3,0x36f,0x36f,0x36f,0x36f,0x1e0,0x1e0,0x1e0,0x36f,0x36f,0x1e0,0x36f,0x36f,0x1e0,0x1e0, +0x1e0,0x1e0,0x36f,0x36f,0x36f,0x36f,0x36f,0x36f,0x36f,0x36f,0x36f,0x36f,0x36f,0x36f,0x36f,0x36f, +0x36f,0x36f,0x1e0,0x1e0,0x1e0,0x1e0,0x36f,0x36f,0x36f,0x36f,0x36f,0x36f,0x36f,0x36f,0x36f,0x1e0, +0x85b,0x85b,0x85b,0x85b,0x85b,0x85b,0x85b,0x85b,0x9cc,0x9cc,0x9cc,0x9cc,0x9cc,0x9cc,0x9cc,0x9cc, +0x372,0x372,0x372,0x372,0x372,0x1e6,0x1e6,0x372,0x372,0x1e6,0x372,0x372,0x372,0x372,0x1e6,0x1e6, +0x372,0x372,0x372,0x372,0xa74,0xa74,0x97e,0x97e,0xb97,0x85e,0x372,0x372,0x1e6,0x372,0x1e6,0x372, +0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372, +0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x1e6,0x1e6,0x372,0x1e6, +0x1e6,0x1e6,0x372,0x1e6,0x1e6,0x1e6,0x1e6,0x372,0x1e6,0x1e6,0x372,0x1e9,0x85e,0x85e,0x9cf,0x9cf, +0x9cf,0x9cf,0x9cf,0x9cf,0x9cf,0x9cf,0x9cf,0x9cf,0x9cf,0x9cf,0xb97,0xb97,0x207,0x3b7,0x207,0x1fe, +0x207,0x1fe,0x207,0x1fe,0x207,0x1fe,0x207,0x1fe,0x207,0x1fe,0x207,0x1fe,0x204,0x3b4,0x207,0x3b7, +0x207,0x1fe,0x207,0x1fe,0x207,0x1fe,0x207,0x3b7,0x207,0x1fe,0x207,0x1fe,0x207,0x1fe,0x3bd,0x3b4, +0x207,0x1fe,0x207,0x3b7,0x207,0x1fe,0x207,0x1fe,0x207,0x3b4,0x3c0,0x3ba,0x207,0x1fe,0x207,0x1fe, +0x3b4,0x207,0x1fe,0x207,0x1fe,0x207,0x1fe,0x3c0,0x3ba,0x3bd,0x3b4,0x207,0x3b7,0x207,0x1fe,0x207, +0x3b7,0x3ba,0x3bd,0x3b4,0x207,0x3b7,0x207,0x1fe,0x207,0x1fe,0x3bd,0x3b4,0x207,0x1fe,0x207,0x1fe, +0x207,0x1fe,0x207,0x1fe,0x207,0x1fe,0x207,0x1fe,0x207,0x1fe,0x3bd,0x3b4,0x207,0x1fe,0x207,0x3b7, +0x207,0x1fe,0x207,0x1fe,0x207,0x1fe,0x207,0x1fe,0x207,0x1fe,0x207,0x1fe,0x207,0x207,0x1fe,0x207, +0x1fe,0x207,0x1fe,0x201,0x20a,0x216,0x216,0x20a,0x216,0x20a,0x216,0x216,0x20a,0x216,0x216,0x216, +0x20a,0x20a,0x216,0x216,0x216,0x216,0x20a,0x216,0x216,0x20a,0x216,0x216,0x216,0x20a,0x20a,0x20a, +0x216,0x216,0x20a,0x216,0x219,0x20d,0x216,0x20a,0x216,0x20a,0x216,0x216,0x20a,0x216,0x20a,0x20a, +0x216,0x20a,0x216,0x219,0x20d,0x216,0x216,0x216,0x20a,0x216,0x20a,0x216,0x216,0x20a,0x20a,0x213, +0x216,0x20a,0x20a,0x20a,0x213,0x213,0x213,0x213,0x21c,0x21c,0x210,0x21c,0x21c,0x210,0x21c,0x21c, +0x210,0x219,0x3c3,0x219,0x3c3,0x219,0x3c3,0x219,0x3c3,0x219,0x3c3,0x219,0x3c3,0x219,0x3c3,0x219, +0x3c3,0x20a,0x219,0x20d,0x219,0x20d,0x219,0x20d,0x216,0x20a,0x219,0x20d,0x219,0x20d,0x219,0x20d, +0x219,0x20d,0x219,0x20d,0x20d,0x21c,0x21c,0x210,0x219,0x20d,0x7d4,0x7d4,0x7d7,0x7d1,0x219,0x20d, +0x219,0x20d,0x219,0x20d,0x219,0x20d,0x219,0x20d,0x219,0x20d,0x219,0x20d,0x219,0x20d,0x219,0x20d, +0x219,0x20d,0x219,0x20d,0x219,0x20d,0x219,0x20d,0x7d7,0x7d1,0x7d7,0x7d1,0x7d4,0x7ce,0x7d7,0x7d1, +0xb61,0xb64,0xc8d,0xc90,0xc90,0xc90,0xc90,0xc8d,0xc90,0xc8d,0xc90,0xc8d,0xc90,0xc8d,0xc90,0xc8d, +0x21f,0x3c6,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f, +0x21f,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f, +0x222,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f,0x21f,0x7da,0x7da,0x7da, +0x7da,0x7da,0xa7a,0xa7a,0x234,0x234,0x234,0x234,0x234,0x234,0x234,0x234,0x234,0x22e,0x22e,0x22e, +0x22e,0x22e,0x22e,0x22e,0x234,0x234,0x234,0x234,0x234,0x225,0x225,0x225,0x225,0x225,0x7dd,0x7dd, +0x7dd,0x7dd,0x7e0,0xa7d,0xa7d,0xa7d,0xa7d,0xa7d,0xa7d,0xa7d,0xa7d,0xa7d,0xa7d,0xa7d,0xa7d,0xa7d, +0xa7d,0xa7d,0xa7d,0xa7d,0x255,0x7e3,0x23d,0x23a,0x23d,0x23a,0x23d,0x23a,0x23d,0x23a,0x23d,0x23a, +0x23d,0x23a,0x23d,0x23a,0x25b,0x25b,0x252,0x24c,0x93c,0x939,0x987,0xa83,0xa80,0xa86,0xa83,0xa80, +0xb67,0xb6a,0xb6a,0xb6a,0x3d2,0x3d2,0x3d2,0x3d2,0x3d2,0x3d2,0x3d2,0x3d2,0x3d2,0x3d2,0x3d2,0x3d2, +0x3d2,0x3d2,0x3d2,0x3d2,0x7e9,0x3d5,0x264,0x267,0x264,0x264,0x264,0x267,0x264,0x264,0x264,0x264, +0x267,0x7e9,0x267,0x264,0x26a,0x264,0x26a,0x264,0x26a,0x264,0x26a,0x264,0x26a,0x264,0x26a,0x264, +0x26a,0x264,0x26a,0x264,0x26a,0x264,0x26a,0x264,0x26a,0x264,0x26d,0x267,0x26a,0x264,0x26a,0x264, +0x26a,0x264,0x26a,0x264,0x26a,0x264,0x26a,0x264,0x26a,0x264,0x26a,0x264,0x26a,0x264,0x26a,0x264, +0x26a,0x264,0x26a,0x264,0x26a,0x264,0x26a,0x264,0x26a,0x264,0x26a,0x264,0x26a,0x26d,0x267,0x26a, +0x264,0x993,0x990,0x26a,0x264,0x993,0x990,0x26a,0x264,0x993,0x990,0xc96,0x26d,0x267,0x26d,0x267, +0x26a,0x264,0x26d,0x267,0x26a,0x264,0x26d,0x267,0x26d,0x267,0x26d,0x267,0x26a,0x264,0x26d,0x267, +0x7ef,0x7e9,0x26d,0x267,0x26d,0x267,0x26d,0x267,0x26d,0x267,0xb70,0xb6d,0x26d,0x267,0xc99,0xc96, +0xc99,0xc96,0xc99,0xc96,0x7ef,0x3db,0x26a,0x26d,0x26a,0x26a,0x26a,0x26d,0x26a,0x26a,0x26a,0x26a, +0x26d,0x7ef,0x26d,0x26a,0x3d8,0x3d8,0x3d8,0x3d8,0x3d8,0x3d8,0x3d8,0x3d8,0x3d8,0x3db,0x3d8,0x3d8, +0x3d8,0x3d8,0x3d8,0x3d8,0x29d,0x29a,0x29d,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a, +0x29a,0x29a,0x29a,0x7f5,0x29a,0x29a,0x29a,0x29d,0x558,0x29a,0x4aa,0x4aa,0x4aa,0x4aa,0x4aa,0x4aa, +0x4aa,0x291,0x4a7,0x4ad,0x65d,0x65d,0x65d,0x65d,0x65d,0x65d,0x65d,0x65d,0x65d,0x65d,0x555,0x660, +0x660,0x297,0x996,0x996,0x4a1,0x29a,0x29a,0x29a,0x29a,0x2a0,0x2a0,0x2a0,0x2a0,0x29a,0x29a,0x29a, +0x29a,0x29a,0x29a,0x29a,0x4ad,0x4aa,0x4aa,0x4aa,0x4aa,0x2a3,0x2a3,0x4aa,0x4aa,0x297,0x4ad,0x4ad, +0x4ad,0x4aa,0xa8f,0xa8f,0x663,0x663,0x663,0x663,0x663,0x663,0x663,0x663,0x663,0x663,0x7f5,0x7f5, +0x7f5,0x7f2,0x7f2,0xa8f,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a, +0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a,0x29a, +0x29a,0x29a,0x29a,0x29a,0x7f5,0x7f5,0x29a,0x29a,0x29a,0x29a,0x29a,0x7f5,0x2e5,0x2e5,0x2e5,0x2e5, +0x2e5,0x2e5,0x2e5,0x2e5,0x2e5,0x2e5,0x2e5,0x2e5,0x2e5,0x2e5,0x2e5,0x2e5,0x2e5,0x2e5,0x2e5,0x2e5, +0x2e5,0x2e5,0x2e5,0x2e5,0x2e5,0x2e5,0x2e5,0x2e5,0x2e5,0x2e5,0x2e5,0x2e5,0x2ee,0x2e8,0x2ee,0x2e8, +0x2ee,0x2e8,0x2ee,0x2e8,0x2ee,0x2e8,0x2ee,0x2e8,0x2ee,0x2e8,0x2ee,0x2e8,0x2ee,0x2e8,0x2ee,0x2e8, +0x2ee,0x2e8,0x2ee,0x2e8,0x2ee,0x2e8,0x2ee,0x2e8,0x2ee,0x2e8,0x2ee,0x2e8,0x2f7,0x2f7,0x2f7,0x2f7, +0x2f7,0x2f7,0x2f7,0x2f7,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x2f7,0x2f7,0x2f7,0x2f7, +0x2f7,0x2f7,0x2f7,0x2f7,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x2fa,0x306,0x2fd,0x2fd,0x2fd, +0x615,0x68a,0x456,0xa4a,0x8f4,0x8f4,0x846,0x846,0x846,0x846,0x9a2,0x9a2,0x9a2,0x9a2,0x9a8,0xaaa, +0xaad,0xb88,0xbfa,0x9a5,0xbfa,0xbfa,0xbfa,0xbfa,0xb88,0xbfa,0xbfa,0x9fc,0x34e,0x348,0x348,0x34e, +0x348,0x348,0x34e,0x348,0x348,0x348,0x348,0x348,0x348,0x348,0x351,0x348,0x348,0x348,0x348,0x348, +0x348,0x348,0x348,0x348,0x3ea,0x3ea,0x348,0x348,0x348,0x348,0x348,0x348,0x348,0x348,0x348,0x3ea, +0x348,0x348,0x348,0x852,0x852,0x852,0x852,0x852,0x852,0x852,0x852,0x852,0x9bd,0x9bd,0x9bd,0x9bd, +0x9bd,0x9bd,0x9bd,0x9bd,0x9bd,0x9bd,0x9bd,0x9bd,0x354,0x354,0x354,0x354,0x354,0x354,0x354,0x354, +0x354,0x354,0x354,0x354,0x354,0x354,0x354,0x354,0x354,0x354,0x354,0x354,0x354,0x354,0x354,0x354, +0x354,0x354,0x354,0x354,0x354,0x354,0x354,0x354,0x357,0x357,0x357,0x357,0x354,0x354,0x354,0x354, +0x354,0x354,0x357,0x357,0x357,0x357,0x354,0x354,0x354,0x354,0x9c0,0x9c0,0x9c0,0x9c0,0x9c0,0x9c0, +0x9c0,0x9c0,0x9c0,0x9c0,0x9c0,0x9c0,0x9c0,0x9c0,0x360,0x360,0x35d,0x35d,0x35d,0x35d,0x35d,0x35d, +0x35d,0x6a2,0x477,0x35d,0x35d,0x35d,0x35d,0x35d,0x35d,0x35d,0x35d,0x35d,0x35d,0x35d,0x35d,0x35d, +0x35d,0x35d,0x35d,0x35d,0x35d,0x35d,0x35d,0x35d,0x35d,0x35d,0x35d,0x35d,0x35d,0x35d,0x35d,0x35d, +0x35d,0x35d,0x35d,0x35d,0x35d,0x35d,0x35d,0x855,0x9c6,0x855,0x855,0x855,0x363,0x363,0x363,0x363, +0x363,0x363,0x363,0x363,0x363,0x363,0x363,0x363,0x363,0x363,0x363,0x363,0x363,0x363,0x363,0x363, +0x363,0x363,0x363,0x363,0x363,0x363,0x363,0x363,0x363,0x363,0x363,0x363,0x372,0x372,0x372,0x372, +0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372, +0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x372,0x393,0x393,0x393,0x393, +0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393, +0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a, +0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d, +0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x390,0x396,0x393,0x38d, +0x390,0x396,0x393,0x38d,0x390,0x396,0x393,0x38d,0x390,0x396,0x393,0x38d,0x390,0x396,0x393,0x38d, +0x390,0x396,0x393,0x38d,0x390,0x396,0x393,0x38d,0x390,0x396,0x393,0x38d,0x393,0x38d,0x393,0x38d, +0x393,0x38d,0x393,0x38d,0x393,0x38d,0x393,0x38d,0x390,0x396,0x393,0x38d,0x390,0x396,0x393,0x38d, +0x390,0x396,0x393,0x38d,0x390,0x396,0x393,0x38d,0x393,0x38d,0x393,0x38d,0x393,0x38d,0x390,0x396, +0x390,0x396,0x393,0x38d,0x393,0x38d,0x393,0x38d,0x393,0x38d,0x393,0x38d,0x393,0x38d,0x393,0x38d, +0x390,0x393,0x38d,0x390,0x393,0x38d,0x390,0x396,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d, +0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x390, +0x390,0x390,0x390,0x390,0x390,0x390,0x390,0x390,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393, +0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d, +0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x390,0x390,0x390,0x390,0x390,0x390,0x390, +0x396,0x396,0x396,0x396,0x396,0x396,0x396,0x396,0x38d,0x393,0x696,0x45f,0x390,0x390,0x38d,0x390, +0x38d,0x390,0x38d,0x38d,0x390,0x38d,0x38d,0x390,0x38d,0x390,0x38d,0x38d,0x390,0x38d,0x390,0x390, +0x38d,0x38d,0x38d,0x390,0x38d,0x38d,0x38d,0x38d,0x38d,0x390,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d, +0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d,0x38d, +0x390,0x390,0x38d,0x38d,0x390,0x38d,0x390,0x38d,0x38d,0x38d,0x38d,0x38d,0x390,0x390,0x390,0x390, +0x390,0x390,0x390,0x390,0x390,0x390,0x390,0x390,0x390,0x390,0x390,0x390,0x390,0x390,0x390,0x390, +0x390,0x390,0x390,0x390,0x390,0x390,0x390,0x390,0x390,0x390,0x390,0x390,0x390,0x390,0x390,0x396, +0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393, +0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393, +0x396,0x396,0x396,0x396,0x396,0x396,0x396,0x396,0x396,0x396,0x396,0x396,0x396,0x396,0x396,0x396, +0x396,0x396,0x396,0x396,0x396,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393,0x393, +0x3a2,0x3a2,0x39c,0x3a2,0x39c,0x3a2,0x39c,0x3a2,0x39c,0x3a2,0x39c,0x39f,0x3a5,0x3a2,0x39c,0x3a2, +0x39c,0x39f,0x3a5,0x3a2,0x39c,0x3a2,0x39c,0x39f,0x3a5,0x3a2,0x39c,0x39f,0x3a5,0x3a2,0x39c,0x39f, +0x3a5,0x3a2,0x39c,0x3a2,0x39c,0x3a2,0x39c,0x3a2,0x39c,0x3a2,0x39c,0x39f,0x3a5,0x3a2,0x39c,0x39f, +0x3a5,0x3a2,0x39c,0x39f,0x3a5,0x3a2,0x39c,0x39f,0x3a5,0x3a2,0x39c,0x39f,0x3a5,0x3a2,0x39c,0x39f, +0x3a5,0x3a2,0x39c,0x39f,0x3a5,0x3a2,0x39c,0x39f,0x3a5,0x3a2,0x39c,0x39f,0x3d8,0x3d8,0x3d8,0x3d8, +0x3d8,0x3d8,0x3d8,0x3d8,0x3d8,0x3d8,0x3d8,0x3d8,0x3d8,0x3d8,0x3d8,0x3d8,0x3d2,0x3d2,0x3d2,0x3d2, +0x3d2,0x3d2,0x3d2,0x3d2,0x3d2,0x3d5,0x3d2,0x3d2,0x3d2,0x3d2,0x3d2,0x3d2,0xa08,0x468,0x699,0x462, +0x465,0x62a,0x3fc,0x633,0x633,0x633,0x633,0x633,0x633,0x633,0x633,0x633,0x630,0x3fc,0x3fc,0x3fc, +0x3fc,0x3fc,0x3fc,0x3fc,0x3fc,0x3fc,0x3fc,0x3fc,0x3fc,0x3fc,0x3fc,0x3fc,0x3fc,0x3fc,0x3fc,0x3fc, +0x3fc,0x3fc,0x3fc,0x3fc,0x3fc,0x3fc,0x3fc,0x3fc,0x3fc,0x3fc,0x3fc,0x3fc,0x3fc,0x3fc,0x62d,0x62d, +0x78c,0x78c,0xc1b,0x78c,0x78c,0x78c,0x78c,0x78c,0x78c,0x78c,0x78c,0x78c,0x78c,0x78c,0x78c,0x78c, +0x49e,0x49e,0x49e,0x49e,0x49e,0x49e,0x49e,0x49e,0x49e,0x49e,0xd0b,0x49e,0x49e,0x49e,0x429,0x49e, +0x57c,0x59d,0x59d,0x59d,0x59d,0x59d,0x59d,0x597,0x597,0x597,0x597,0x597,0x597,0x597,0x597,0x597, +0x597,0x597,0x597,0x597,0x597,0x597,0x597,0x597,0x597,0x597,0x597,0x69c,0x585,0x46b,0x58b,0x58e, +0x588,0x59a,0x59a,0x59a,0x59a,0x59a,0x59a,0x594,0x594,0x594,0x594,0x594,0x594,0x594,0x594,0x594, +0x594,0x594,0x594,0x594,0x594,0x594,0x594,0x594,0x594,0x594,0x594,0x69c,0x585,0x46b,0x585,0xa62, +0x579,0x480,0x483,0x5a6,0x5a0,0x63c,0x5ac,0x5b2,0x6a5,0x47a,0x6a5,0x47a,0x6a8,0x47d,0x6a8,0x47d, +0x6a5,0x47a,0x5a6,0x5a6,0x6a5,0x47a,0x6a5,0x47a,0x6a5,0x47a,0x6a5,0x47a,0x639,0x6a8,0x47d,0x47d, +0x53a,0x53a,0x53a,0x53a,0x53a,0x53a,0x53a,0x53a,0x53a,0x53a,0x53a,0x53a,0x53a,0x53a,0x53a,0x53a, +0x53a,0x53a,0x53a,0x53a,0x53a,0x53a,0x53a,0x53a,0x53a,0x53a,0x53a,0x53a,0x53a,0x53a,0x53a,0x53a, +0x543,0x543,0x53a,0x53d,0x53d,0x540,0x8c4,0x8c4,0x8c4,0x8c4,0x8c4,0x8c4,0x8c4,0x8c4,0x8c4,0xa32, +0xb31,0xb31,0xb31,0xb31,0xb31,0xb31,0xb31,0xb31,0xc4e,0xc4e,0xc4e,0xc4e,0xc5a,0xb3a,0xb3a,0xb3a, +0x5a6,0x5b2,0x5b2,0x5b2,0x5b2,0x5b2,0x5b2,0x5b2,0x5b2,0x5b2,0x546,0x546,0x546,0x546,0x546,0x546, +0x5a9,0x5af,0x5af,0x5af,0x5af,0x5af,0x5a3,0x5a0,0x8e2,0x8e2,0x8e2,0xa50,0xa4d,0xa35,0x8df,0x576, +0x5b8,0x5b5,0x5b8,0x642,0x5b5,0x5b8,0x5b5,0x5b8,0x5b5,0x5b8,0x5b5,0x5b5,0x5b5,0x5b5,0x5b5,0x5b5, +0x5b8,0x5b8,0x5b5,0x5b8,0x5b8,0x5b5,0x5b8,0x5b8,0x5b5,0x5b8,0x5b8,0x5b5,0x5b8,0x5b8,0x5b5,0x5b5, +0xa56,0x651,0x5bb,0x651,0x5bb,0x651,0x5bb,0x651,0x5bb,0x651,0x5bb,0x5bb,0x5be,0x5bb,0x5be,0x5bb, +0x5be,0x5bb,0x5be,0x5bb,0x5be,0x5bb,0x5be,0x5bb,0x5be,0x5bb,0x5be,0x5bb,0x5be,0x5bb,0x5be,0x5bb, +0x5be,0x5bb,0x5be,0x651,0x5bb,0x5be,0x5bb,0x5be,0x5bb,0x5be,0x5bb,0x5bb,0x5bb,0x5bb,0x5bb,0x5bb, +0x5be,0x5be,0x5bb,0x5be,0x5be,0x5bb,0x5be,0x5be,0x5bb,0x5be,0x5be,0x5bb,0x5be,0x5be,0x5bb,0x5bb, +0x5bb,0x5bb,0x5bb,0x651,0x5bb,0x651,0x5bb,0x651,0x5bb,0x5bb,0x5bb,0x5bb,0x5bb,0x5bb,0x651,0x5bb, +0x5bb,0x5bb,0x5bb,0x5bb,0x5be,0x651,0x651,0x5be,0x5be,0x5be,0x5be,0x64b,0x64e,0x654,0x657,0xa3b, +0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4, +0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4, +0x5c7,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4, +0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5c4,0x5dc,0x5dc,0x5dc,0x5dc, +0x5dc,0x5dc,0x5dc,0x5dc,0x5dc,0x5dc,0x5dc,0x5dc,0x5dc,0x5dc,0x5dc,0x5dc,0x5dc,0x5dc,0x5dc,0x5dc, +0x5dc,0x5dc,0x5dc,0x5dc,0x5dc,0x5dc,0x5dc,0x5dc,0xb43,0xb43,0xc5d,0x5d0,0x5d3,0x5d3,0x5d3,0x5d3, +0x5d3,0x5d3,0x5d3,0x5d3,0x5d3,0x5d3,0x5d3,0x5d3,0x5d3,0x5d3,0x5d3,0x5d3,0x5d3,0x5d3,0x5d3,0x5d3, +0x5d3,0x5d3,0x5d3,0x5d3,0x5d3,0x5d3,0x5d3,0x5d3,0x5d3,0x5d3,0x5d3,0x5d3,0xb3d,0xb3d,0xb3d,0xb3d, +0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6, +0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6,0x5d6, +0x5d6,0xa3e,0xa3e,0xa3e,0xa3e,0xa3e,0xa3e,0xa3e,0xa3e,0xa3e,0xa3e,0xa3e,0xa3e,0xa3e,0xa3e,0xa3e, +0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2, +0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5df,0x5df,0x5df,0x5df,0x5df,0x5df,0x5df,0x5df, +0x5df,0x5df,0x5df,0x5df,0x5df,0x5df,0x5df,0x5df,0x5df,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0xb46, +0xb46,0xb46,0xb46,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5df,0x5df,0x5df,0x5df,0x5df,0x5df,0x5df,0x5df, +0x5df,0x5df,0x5df,0x5df,0x5df,0x5df,0x5df,0x5df,0x5df,0x5df,0x5df,0x5df,0x5df,0x5df,0x5df,0x5df, +0x5df,0x5df,0x5df,0x5df,0x5df,0x5df,0x5df,0xb46,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2, +0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2, +0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0x5e2,0xb46,0xb46,0x5e5,0x5e5,0x5e5,0x5e5, +0x5e5,0x5e5,0x5e5,0x5e5,0x5e5,0x5e5,0x5e5,0x5e5,0x5e5,0x5e5,0x5e5,0x5e5,0x5e5,0x5e5,0x5e5,0x5e5, +0x5e5,0x5e5,0x5e5,0x5e5,0x5e5,0x5e5,0x5e5,0x5e5,0x5e5,0x5e5,0x5e5,0x5e5,0x5e8,0x5e8,0x5e8,0x5e8, +0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8, +0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5eb,0x5eb, +0x5e8,0x5eb,0x5e8,0x5eb,0x5eb,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5e8,0x5eb, +0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756, +0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756,0x756, +0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c, +0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c,0x75c, +0x762,0x75f,0x75f,0x75f,0x75f,0x75f,0x75f,0x75f,0x75f,0x75f,0x75f,0x75f,0x75f,0x75f,0x75f,0x75f, +0x75f,0x75f,0x75f,0x75f,0x75f,0x75f,0x75f,0x75f,0x75f,0x75f,0x75f,0x75f,0x75f,0x75f,0x75f,0x75f, +0x75f,0x75f,0x75f,0x75f,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e, +0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e, +0x76e,0x76e,0x768,0x768,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771, +0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771, +0x771,0x771,0x76b,0x76b,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e, +0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e,0x76e, +0x76e,0x76e,0x76e,0x76e,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771, +0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771,0x771, +0x771,0x771,0x771,0x771,0x777,0x786,0x786,0x786,0x786,0x774,0x786,0x786,0x7ad,0x786,0x786,0x780, +0x7aa,0x7a7,0x7a7,0x7a7,0x7a7,0x7a7,0x7ad,0x774,0x7a4,0x774,0x774,0x774,0x79b,0x79b,0x774,0x774, +0x774,0x774,0x774,0x774,0x7b0,0x7b0,0x7b0,0x7b0,0x7b0,0x7b0,0x7b0,0x7b0,0x7b0,0x7b0,0x774,0x774, +0x774,0x774,0x774,0x774,0x774,0x774,0x774,0x774,0x780,0x79b,0x774,0x79b,0x774,0x79b,0x7b3,0x789, +0x7b3,0x789,0x7a1,0x7a1,0x7b9,0x7b9,0x7b9,0x7b9,0x7b9,0x7b9,0x7b9,0x7b9,0x7b9,0x7b9,0x7b9,0x7b9, +0x7b9,0x7b9,0x7b9,0x7b9,0x7b9,0x7b9,0x7b9,0x7b9,0x7b9,0x7b9,0x7b9,0x7b9,0x7b9,0x7b9,0x7b9,0x7b9, +0x7b9,0x7b9,0x7b9,0x7b9,0x7bc,0x7bc,0x7bc,0x7bc,0x7bc,0x7bc,0x7bc,0x7bc,0x7bc,0x7bc,0x7bc,0x7bc, +0x7bc,0x7bc,0x7bc,0x7bc,0x7bc,0x7bc,0x7bc,0x7bc,0x7bc,0x7bc,0x7bc,0x7bc,0x7bc,0x7bc,0x7bc,0x7bc, +0x7bc,0x7bc,0x7bc,0x7bc,0x7bf,0x7bf,0x7bf,0x7bf,0x7bf,0x7bf,0x7bf,0x7bf,0x7bf,0x7bf,0x7bf,0x7bf, +0x7bf,0x7bf,0x7bf,0x7bf,0x7bf,0x7bf,0x7bf,0x7bf,0x7bf,0x7bf,0x7bf,0x7bf,0x7bf,0x7bf,0x7bf,0x7bf, +0x7bf,0x7bf,0x7bf,0x7bf,0x7c2,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5, +0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5, +0x7c2,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5, +0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x7c5,0x984,0xa77,0x7d4,0x7ce, +0x7d4,0x7ce,0x7d7,0x7d1,0x7d7,0x7d1,0x7d7,0x7d1,0x7d7,0x7d1,0x7d7,0x7d1,0x7d7,0x7d1,0x7d7,0x7d1, +0xa77,0xa77,0xa77,0xb61,0xb61,0xb61,0xb64,0xb64,0xb61,0xb64,0xb64,0xb61,0x804,0x804,0x804,0x804, +0x804,0x804,0x804,0x804,0x804,0x804,0x804,0x804,0x804,0xa92,0xa92,0xa92,0x8a0,0x8a0,0x8a0,0x8a0, +0x8a0,0x8a0,0x8a0,0x8a0,0x8a0,0x8a0,0x8a0,0x8a0,0x8a0,0x8a0,0x8a0,0x8a0,0x807,0x807,0x807,0x807, +0x807,0x807,0x807,0x807,0x807,0x807,0x807,0x807,0x807,0x807,0x807,0x807,0x807,0x807,0x807,0x807, +0x807,0x807,0x807,0x807,0x807,0x807,0x807,0x807,0x807,0x807,0x807,0x807,0x825,0x825,0x825,0x825, +0x825,0x825,0x825,0xb85,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825, +0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825, +0x825,0x825,0x825,0x825,0x825,0x825,0x825,0xb85,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825, +0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x825,0x828,0x828,0x828,0x828,0x828,0x828,0x828,0x828, +0x828,0x828,0x828,0x828,0x828,0x828,0x828,0x828,0x828,0x828,0x828,0x828,0x828,0x828,0x828,0x828, +0x828,0x828,0x828,0x828,0x828,0x828,0x828,0x828,0x831,0x831,0x831,0x831,0x831,0x831,0x831,0x831, +0x831,0x831,0x831,0x831,0x831,0x831,0x831,0x831,0x831,0x831,0x831,0x831,0x831,0x831,0x831,0x831, +0x831,0x831,0x831,0x831,0x831,0x831,0x831,0x831,0x837,0x837,0x837,0x837,0x837,0x837,0x837,0x837, +0x837,0x837,0x837,0x837,0x837,0x837,0x837,0x837,0x837,0x837,0x837,0x837,0x837,0x837,0x837,0x837, +0x837,0x837,0x837,0x837,0x837,0x837,0x837,0x837,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840, +0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840, +0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x843,0x840,0x840,0x840,0x840, +0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840, +0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x840,0x855,0x855,0x855,0x855,0x855,0x855,0x855,0x855, +0x855,0x855,0x855,0x855,0x855,0x855,0x855,0x855,0x855,0x855,0x855,0x855,0x855,0x855,0x855,0x855, +0x855,0x855,0x855,0x9c6,0x9c6,0x9c6,0x9c6,0x9c6,0x861,0x861,0x861,0x861,0x861,0x861,0x861,0x861, +0x861,0x861,0x861,0x861,0x861,0x861,0x861,0x861,0x861,0x861,0x861,0x861,0x861,0x861,0x861,0x861, +0x861,0x861,0x861,0x861,0x861,0x861,0x861,0x861,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0, +0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0, +0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d0,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6, +0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6, +0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8d6,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8, +0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8, +0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8e8,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb, +0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8f7,0x8eb,0x8eb, +0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb, +0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb,0x8eb, +0x8ee,0x8ee,0xa41,0xa41,0x8ee,0x8ee,0x8ee,0x8ee,0x8ee,0x8ee,0x8ee,0x8ee,0x8ee,0x8ee,0x8ee,0x8ee, +0x8ee,0x8ee,0x8ee,0x8ee,0xa41,0x8ee,0x8ee,0x8ee,0x8ee,0x8ee,0x8ee,0x8ee,0x8ee,0x8ee,0x8ee,0x8ee, +0x918,0x918,0x918,0x918,0x918,0x918,0x918,0x918,0x918,0x918,0x918,0x918,0x918,0x918,0x918,0x918, +0x918,0x918,0x918,0x918,0x918,0x918,0x918,0x918,0x918,0x918,0x918,0x918,0x918,0x918,0x918,0x918, +0x930,0x930,0x930,0x933,0x930,0x930,0x930,0x930,0x930,0x930,0x930,0x930,0x930,0x930,0x930,0x930, +0x930,0x930,0x930,0x930,0x91e,0x91e,0x92d,0x921,0x921,0x921,0x921,0x921,0x921,0x921,0x92d,0x92d, +0x930,0x930,0x930,0x930,0x930,0x930,0x930,0x930,0x930,0x930,0x930,0x930,0x930,0x930,0x930,0x930, +0x930,0x930,0x930,0x930,0x930,0x930,0x930,0x930,0x930,0x930,0x930,0x930,0x930,0x930,0x930,0x930, +0x94b,0x94b,0x94b,0x94b,0x94b,0x94b,0xac2,0xac2,0x948,0x948,0x948,0x948,0x948,0x948,0x948,0x948, +0x948,0x948,0x948,0x948,0x948,0x948,0x948,0x948,0x948,0x948,0x948,0x948,0x948,0x948,0x948,0x948, +0x948,0x948,0xabf,0xabf,0xafb,0xafb,0xafb,0xafb,0xafb,0xafb,0xafb,0xafb,0xafb,0xafb,0xafb,0xafb, +0xafb,0xafb,0xafb,0xafb,0x94b,0x94b,0x94b,0x94b,0x94b,0x94b,0x94b,0x94b,0x94b,0x94b,0x94b,0x94b, +0x94b,0x94b,0x94b,0x94b,0x94b,0x94b,0x94b,0x94b,0x94b,0x94b,0x94b,0x94b,0x94b,0x94b,0x94b,0x94b, +0x94b,0x94b,0x94b,0x94b,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e, +0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e,0x94e, +0x94e,0x94e,0x94e,0x94e,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951, +0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951, +0x951,0x951,0x951,0x951,0x951,0x951,0x954,0x954,0x954,0x954,0x954,0x954,0x954,0x963,0x969,0x96f, +0x96f,0x96f,0x951,0x951,0x951,0x96c,0x966,0x966,0x966,0x966,0x966,0x960,0x960,0x960,0x960,0x960, +0x960,0x960,0x960,0x96f,0x96f,0x96f,0x96f,0x96f,0x96f,0x96f,0x96f,0x951,0x951,0x96f,0x96f,0x96f, +0x96f,0x96f,0x96f,0x96f,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951, +0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x96f,0x96f,0x96f,0x96f,0x951,0x951, +0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x951,0x954,0x954,0x954,0x954,0x954, +0x95d,0x957,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a, +0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x957,0x95a,0x95a,0x95a,0x95a, +0x95a,0x95a,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d, +0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x957,0x95a,0x95a,0x95a,0x95a, +0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a, +0x95a,0x957,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d, +0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x957,0x95a,0x95a, +0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a, +0x95a,0x95a,0x95a,0x95a,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d, +0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95d,0x95a,0x95a, +0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a, +0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95a,0x95d,0x95d,0x95d,0x95d,0x972,0x972,0x972,0x972, +0x972,0x972,0x972,0x972,0x972,0x972,0x972,0x972,0x972,0x972,0x972,0x972,0x972,0x972,0x972,0x972, +0x972,0x972,0x972,0x972,0x972,0x972,0x972,0x972,0x972,0x972,0x972,0x972,0x975,0x975,0x975,0x975, +0x975,0x975,0x975,0x975,0x975,0x975,0x975,0x975,0x975,0x975,0x975,0x975,0x975,0x975,0x975,0x975, +0x975,0x975,0x975,0x975,0x975,0x975,0x975,0x975,0x975,0x975,0x975,0x975,0x978,0x978,0x978,0x978, +0x978,0x978,0x978,0x978,0x978,0x978,0x978,0x978,0x978,0x978,0x978,0x978,0x978,0x978,0x978,0x978, +0x978,0x978,0x978,0x978,0x978,0x978,0x978,0x978,0x978,0x978,0x978,0x978,0x97b,0x97b,0x97b,0x97b, +0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b, +0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x97b,0x9c6,0x9c6,0x9c6,0x9c6, +0x9c6,0x9c6,0x9c6,0x9c6,0x9c6,0x9c6,0x9c6,0x9c6,0x9c6,0x9c6,0x9c6,0x9c6,0x9c6,0x9c6,0x9c6,0x9c6, +0x9c3,0x9c3,0x9c3,0x9c6,0x9c3,0x9c3,0x9c3,0x9c3,0x9c3,0x9c3,0x9c3,0x9c3,0x9c3,0x9c3,0x9c3,0x9c3, +0x9c3,0x9c3,0x9c3,0xab3,0xab6,0xb94,0xb94,0xb94,0xb94,0xb94,0xb94,0xb94,0xb94,0xb94,0xb94,0xb94, +0xcae,0xcae,0xcae,0xcae,0x9ed,0x9ed,0x9ed,0x9ed,0x9ed,0x9ed,0x9ed,0x9ed,0x9ed,0x9ed,0x9ed,0x9ed, +0x9ed,0x9ed,0x9ed,0x9ed,0x9ed,0x9ed,0x9ed,0x9ed,0x9ed,0x9ed,0x9ed,0x9ed,0x9ed,0x9ed,0x9ed,0x9ed, +0x9ed,0x9ed,0x9ed,0x9ed,0x9f0,0x9f0,0x9f0,0xa5f,0xa05,0xa68,0xa0e,0xa5f,0xa05,0xa5f,0xa05,0xa5f, +0xa05,0xa5f,0xa05,0xa5f,0xa05,0xa5f,0xa05,0xa5f,0xa05,0xa5f,0xa05,0xa5f,0xa05,0x9f0,0x9f0,0x9f0, +0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0, +0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0, +0xa5f,0xa05,0xa5f,0xa05,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0, +0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0,0x9f0, +0xa5f,0xa05,0x9f0,0x9f0,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3, +0x9f9,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3, +0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3, +0x9f3,0x9f3,0x9f3,0x9f3,0x9f9,0x9f9,0x9f9,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3, +0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3,0x9f3, +0x9f3,0x9f3,0x9f3,0x9f3,0x9f6,0x9f3,0x9f3,0x9f3,0xa44,0xa44,0xa44,0xa44,0xa44,0xa44,0xa44,0xa44, +0xa44,0xa44,0xa44,0xa44,0xa44,0xa44,0xa44,0xa44,0xa44,0xa44,0xa44,0xa44,0xa44,0xa44,0xa44,0xa44, +0xa44,0xa44,0xa44,0xa44,0xa44,0xa44,0xa44,0xa44,0xace,0xace,0xace,0xace,0xace,0xace,0xace,0xace, +0xace,0xace,0xace,0xace,0xace,0xace,0xace,0xace,0xace,0xace,0xace,0xace,0xace,0xace,0xace,0xace, +0xace,0xace,0xace,0xace,0xace,0xace,0xace,0xace,0xadd,0xadd,0xadd,0xadd,0xadd,0xadd,0xad4,0xad4, +0xad4,0xad4,0xad4,0xad1,0xae6,0xae6,0xae6,0xae0,0xae6,0xae6,0xae6,0xae6,0xae6,0xae6,0xae6,0xae6, +0xae6,0xae6,0xae6,0xae0,0xae6,0xae6,0xae6,0xae6,0xada,0xada,0xae3,0xae3,0xae3,0xae3,0xad7,0xad7, +0xad7,0xad7,0xad7,0xadd,0xba3,0xba3,0xba3,0xba3,0xba3,0xba3,0xba3,0xba3,0xba3,0xba3,0xba3,0xba3, +0xba0,0xba3,0xba3,0xba3,0xba3,0xba3,0xba3,0xba3,0xae6,0xae6,0xae6,0xae6,0xae6,0xae6,0xae6,0xae6, +0xae6,0xae6,0xae6,0xae6,0xae6,0xae6,0xae0,0xae6,0xae6,0xae6,0xae6,0xae6,0xae6,0xae6,0xae6,0xae6, +0xae6,0xae6,0xae6,0xae6,0xae6,0xada,0xada,0xada,0xadd,0xadd,0xadd,0xadd,0xadd,0xadd,0xadd,0xadd, +0xadd,0xadd,0xadd,0xadd,0xadd,0xadd,0xadd,0xadd,0xadd,0xadd,0xadd,0xadd,0xadd,0xadd,0xadd,0xadd, +0xadd,0xadd,0xadd,0xadd,0xadd,0xadd,0xadd,0xadd,0xaec,0xaec,0xaec,0xaec,0xaec,0xaec,0xaec,0xaec, +0xaec,0xaec,0xaec,0xaec,0xaec,0xaec,0xaec,0xaec,0xaec,0xaec,0xaec,0xaec,0xaec,0xaec,0xaec,0xaec, +0xaec,0xaec,0xaec,0xaec,0xaec,0xaec,0xaec,0xaec,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2, +0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2, +0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xaf2,0xafb,0xafb,0xafb,0xafb,0xafb,0xafb,0xafb,0xafb, +0xafb,0xafb,0xafb,0xafb,0xafb,0xafb,0xafb,0xafb,0xafb,0xafb,0xafb,0xafb,0xafb,0xafb,0xafb,0xafb, +0xafb,0xafb,0xafb,0xafb,0xafb,0xafb,0xafb,0xafb,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04, +0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04, +0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb04,0xb34,0xb34,0xb34,0xb34,0xb34,0xb34,0xb34,0xb34, +0xb34,0xb34,0xb34,0xb34,0xb34,0xb34,0xb34,0xb34,0xb34,0xb34,0xb34,0xb34,0xb34,0xb34,0xb34,0xb34, +0xb34,0xb34,0xb34,0xb34,0xb34,0xb34,0xb34,0xb34,0xba9,0xba9,0xba9,0xba9,0xba9,0xba9,0xba9,0xba9, +0xba9,0xba9,0xba9,0xba9,0xba9,0xba9,0xba9,0xba9,0xba9,0xba9,0xba9,0xba9,0xba9,0xba9,0xba9,0xba9, +0xba9,0xba9,0xba9,0xba9,0xba9,0xba9,0xba9,0xba9,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf, +0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbac,0xbac,0xbac, +0xbac,0xbac,0xbac,0xbac,0xbac,0xbac,0xbac,0xbac,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf, +0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf, +0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbaf,0xbc1,0xbbe,0xbc1,0xbbe,0xbc1,0xbbe,0xbc1,0xbbe, +0xbc1,0xbbe,0xbc1,0xbbe,0xbc1,0xbbe,0xbc1,0xbbe,0xbc1,0xbbe,0xbc1,0xbbe,0xbc1,0xbbe,0xbc1,0xbbe, +0xbc1,0xbbe,0xbc1,0xbbe,0xbc1,0xbbe,0xbc1,0xbbe,0xbcd,0xbcd,0xbcd,0xbcd,0xbcd,0xbcd,0xbcd,0xbcd, +0xbcd,0xbcd,0xbcd,0xbcd,0xbcd,0xbcd,0xbcd,0xbcd,0xbcd,0xbcd,0xbcd,0xbcd,0xbcd,0xbcd,0xbcd,0xbcd, +0xbcd,0xbcd,0xbcd,0xbcd,0xbcd,0xbcd,0xbcd,0xbcd,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3, +0xbd3,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3, +0xbd3,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3,0xbd3,0xbdf,0xbdf,0xbdf,0xbdf,0xbdf,0xbdf,0xbdf,0xbdf, +0xbdf,0xbdf,0xbdf,0xbdf,0xbdf,0xbdf,0xbdf,0xbdf,0xbdf,0xbdf,0xbdf,0xbdf,0xbdf,0xbdf,0xbdf,0xbdf, +0xbdf,0xbdf,0xbdf,0xbdf,0xbdf,0xbdf,0xbdf,0xbdf,0xbe8,0xbe8,0xbe8,0xbe8,0xbe8,0xbe8,0xbe8,0xbe8, +0xbe8,0xbe8,0xbe8,0xbe8,0xbe8,0xbe8,0xbe8,0xbe8,0xbe8,0xbe8,0xbe8,0xbe8,0xbe8,0xbe8,0xbe8,0xbe8, +0xbe8,0xbe8,0xbe8,0xbe8,0xbe8,0xbe8,0xbe8,0xbe2,0xbe5,0xbe5,0xbe5,0xbe5,0xbe5,0xbe5,0xbe5,0xbe5, +0xbe5,0xbe5,0xbe5,0xbe5,0xbe5,0xbe5,0xbe5,0xbe5,0xbe5,0xbe5,0xbe5,0xbe5,0xbe5,0xbe5,0xbe5,0xbe5, +0xbe5,0xbe5,0xbe5,0xbe8,0xbe8,0xbe8,0xbe8,0xbe8,0xbf1,0xbf1,0xc48,0xbf1,0xbf1,0xbf1,0xc45,0xbf1, +0xbf1,0xbf1,0xbf1,0xc3f,0xbf1,0xbf1,0xbf1,0xbf1,0xbf1,0xbf1,0xbf1,0xbf1,0xbf1,0xbf1,0xbf1,0xbf1, +0xbf1,0xbf1,0xbf1,0xbf1,0xbf1,0xbf1,0xbf1,0xbf1,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4, +0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4, +0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xbf4,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63, +0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63, +0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc63,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a, +0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a, +0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xc8a,0xd32,0xd32,0xd32,0xd32,0xd32,0xd32,0xd32,0xd32, +0xd32,0xd32,0xccc,0xccc,0xccc,0xccc,0xccc,0xccc,0xccc,0xccc,0xccc,0xccc,0xccc,0xccc,0xccc,0xccc, +0xccc,0xccc,0xccc,0xccc,0xccc,0xccc,0xccc,0xccc,0xd1a,0xd1a,0xd1a,0xd1a,0xd20,0xcd8,0xcdb,0xcd8, +0xcdb,0xcd8,0xcdb,0xcd8,0xcdb,0xcd8,0xcdb,0xcd8,0xcd8,0xcd8,0xcdb,0xcd8,0xcd8,0xcd8,0xcd8,0xcd8, +0xcd8,0xcd8,0xcd8,0xcd8,0xcd8,0xcd8,0xcd8,0xcd8,0xcd8,0xcd8,0xcd8,0xcd8,0xcd8,0xcd8,0xcd8,0xcd8, +0xd1d,0xd20,0xd1a,0xd1a,0xd1a,0xd1a,0xd1a,0xd23,0xd1a,0xd23,0xd20,0xd20,0xce7,0xce7,0xce7,0xce7, +0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xce7, +0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xce7,0xcf0,0xcf0,0xcf0,0xcf0, +0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0, +0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf0,0xcf3,0xcf3,0xcf3,0xcf3, +0xcf3,0xcf3,0xcf3,0xcf3,0xcf3,0xcf3,0xcf3,0xcf3,0xcf3,0xcf3,0xcf3,0xcf3,0xcf3,0xcf3,0xcf3,0xcf3, +0xcf3,0xcf3,0xcf3,0xcf3,0xcf3,0xcf3,0xcf3,0xcf3,0xcf3,0xcf3,0xcf3,0xcf3,0x820,0x840,0x860,0, +0,0,0,0,0x880,0x8a0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0x8c0,0x8e0,0,0, +0,0,0,0,0,0,0,0x900,0x920,0x920,0x920,0x920,0x920,0x920,0x920,0x920, +0x920,0x920,0x920,0x920,0x920,0x920,0x920,0x920,0x920,0x920,0x920,0x920,0x920,0x920,0x920,0x920, +0x920,0x920,0x920,0x920,0x920,0x920,0x920,0x920,0x920,0x940,0x960,0x960,0x960,0x960,0x960,0x960, +0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x980,0x9a0, +0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x960, +0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x960,0x960, +0x960,0x960,0x960,0x9a0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0x900,0x9c0,0x9e0,0x9e0,0x9e0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0xa00,0xa00,0xa00,0xa00,0xa00,0xa00,0xa00,0xa00,0xa00,0xa00,0xa00,0xa00, +0xa00,0xa00,0xa00,0xa00,0xa00,0xa00,0xa00,0xa00,0xa00,0xa00,0xa00,0xa00,0xa00,0xa00,0xa00,0xa00, +0xa00,0xa00,0xa00,0xa00,0xa00,0xa00,0xa00,0xa20,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40, +0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40, +0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa40,0xa60 +}; + +static const UTrie propsVectorsTrie={ + propsVectorsTrie_index, + NULL, + utrie_defaultGetFoldingOffset, + 2688, + 14236, + 0, + TRUE +}; + +static const uint32_t propsVectors[3384]={ +0x67,0,0,0x67,0x800000,0,0x467,0,0,0x4e7,0,0,0x567,0,0,0x5e7, +0,0,0x667,0,0,0x6e7,0,0,0x767,0,0,0x7e7,0,0,0x867,0, +0,0x8e7,0,0,0x967,0,0,0x9e7,0,0,0xa67,0,0,0xae7,0,0, +0xb67,0,0,0xbe7,0,0,0xc67,0,0,0xce7,0,0,0xd67,0,0,0xde7, +0,0,0xe67,0,0,0xee7,0,0,0xf67,0,0,0xfe7,0,0,0x1067,0, +0,0x10e7,0,0,0x1167,0,0,0x11e7,0,0,0x1267,0,0,0x12e7,0,0, +0x1367,0,0,0x13e7,0,0,0x1467,0x800000,0,0x14e7,0,0,0x1567,0,0,0x15e7, +0,0,0x1667,0,0,0x16e7,0,0,0x1867,0,0x4000000,0x18e7,0,0x4000000,0x1967,0, +0x4000000,0x1be7,0,0x4000000,0x1c67,0,0x4000000,0x1d67,0,0,0x1de7,0,0,0x1e67,0,0, +0x1f67,0,0,0x2067,0,0,0x20e7,0,0,0x21e7,0,0,0x2267,0,0,0x2367, +0,0,0x23e7,0,0,0x2467,0,0,0x24e7,0,0,0x2567,0,0,0x27e7,0, +0,0x2867,0,0,0x28e7,0,0,0x2967,0,0,0x2a67,0,0,0x2ae7,0,0, +0x2b67,0x800000,0,0x2be7,0,0,0x2c67,0,0,0x2ce7,0,0,0x2de7,0,0,0x2e67, +0,0,0x2ee7,0,0,0x3067,0x800000,0,0x30e7,0,0,0x3167,0,0,0x31e7,0, +0,0x3267,0,0,0x32e7,0,0,0x3367,0,0x4000000,0x37e7,0,0,0x3867,0,0, +0x39e7,0,0x4000000,0x3ae7,0,0,0x3b67,0,0,0x3be7,0,0,0x3c67,0,0,0x3d67, +0,0,0x3de7,0,0,0x3e67,0,0,0x3f67,0,0,0x3fe7,0,0,0x4067,0, +0,0x40e7,0,0,0x4167,0,0,0x41e7,0,0,0x4267,0,0,0x42e7,0,0, +0x4367,0,0,0x43e7,0,0,0x4467,0,0,0x44e7,0,0,0x4567,0,0,0x45e7, +0,0,0x4667,0,0,0x4767,0,0x4000000,0x47e7,0,0,0x4867,0,0,0x48e7,0, +0,0x4967,0,0,0x49e7,0,0,0x4a67,0,0,0x4ae7,0,0,0x4b67,0,0, +0x4be7,0,0,0x4c67,0,0,0x4ce7,0,0,0x4d67,0,0,0x28067,0,0,0x2af67, +0,0,0x2afe7,0,0,0x11002b67,0x810000,0,0x1100a767,0,0,0x11041680,0x80000000,6,0x11041980,0x80000000, +3,0x11041a00,0x80000000,0x4000000,0x11041c00,0x80000000,0,0x11048100,0x80000000,6,0x11048100,0x80000000,0xf,0x11048100,0x80000000,0x4000000, +0x11048100,0x80000080,0x4000000,0x11048100,0x80001000,2,0x11048100,0x90003000,0x1000,0x11048119,0xf8000400,0x1040f,0x11048300,0x80001000,2,0x11048300, +0xf8001400,0x18400,0x11048300,0xf8003400,0x18400,0x11049400,0x80000000,0x4000000,0x11049400,0x80000008,0x4000000,0x11049400,0x80000080,0x4000000,0x11049480,0x80000000, +0xe,0x11049480,0x80000000,0xf,0x11049499,0xf8000400,0x1040f,0x11049600,0x80000000,2,0x11049600,0x80000000,0xf,0x11049600,0xf8000480,0x10405, +0x11049619,0xf8000400,0x28401,0x11049680,0x80000000,6,0x11049680,0xf8000400,0x10402,0x11049680,0xf8000400,0x28402,0x11049700,0x80000000,0x4000000,0x11049700, +0x80000080,0x4000000,0x11049780,0x80000080,0x4000000,0x11049780,0x80000080,0x4000001,0x11049780,0x80000080,0x4000002,0x11049800,0x80000000,0x4000000,0x11049980,0x80000000, +2,0x11049980,0x80000000,3,0x11049980,0x80000400,0x10403,0x11049980,0x80000400,0x28403,0x11049a00,0x80000000,0x4000000,0x11049a80,0x80000000,0x4000000, +0x11049b00,0x80000000,0x4000000,0x11049b00,0x80000080,0x4000000,0x11049b80,0x80000000,0x4000000,0x11049b80,0x80000080,0x4000000,0x11049c00,0x80000000,0,0x1104ab00, +0x80000000,0,0x11080100,0x80000000,0x4000000,0x11080100,0xf8000400,0x10402,0x11080119,0xf8000400,0x10401,0x11080119,0xf8000400,0x28401,0x11080199,0xf8000400, +0x10401,0x11080199,0xf8000400,0x10402,0x11080199,0xf8000400,0x28400,0x11080199,0xf8000400,0x28401,0x11080219,0xf8000400,0x10400,0x11080219,0xf8000400,0x10401, +0x11080219,0xf8000400,0x10402,0x11080219,0xf8000400,0x18400,0x11080219,0xf8000400,0x28400,0x11080219,0xf8000400,0x28401,0x11080219,0xf8000400,0x28402,0x11080299, +0xf8000400,0x10400,0x11080299,0xf8000400,0x18400,0x11080300,0x80001000,0,0x11080300,0x80001000,2,0x11080300,0xf8001400,0x10400,0x11080300,0xf8001400, +0x18400,0x11080300,0xf8003400,0x18400,0x11080319,0xf8001400,0x1040f,0x11080400,0x80000040,1,0x11080407,0xf8000400,0x10400,0x11080407,0xf8000400,0x28400, +0x1108040e,0x80001000,0,0x1108040e,0x80001000,1,0x1108040e,0x80001000,2,0x1108040e,0xe0001400,0x10402,0x1108040e,0xf8000400,0x10400,0x1108040e, +0xf8000400,0x10401,0x1108040e,0xf8000400,0x10402,0x1108040e,0xf8000400,0x28400,0x1108040e,0xf8000400,0x28401,0x1108040e,0xf8000480,0x10402,0x1108040e,0xf8000480, +0x28402,0x11080488,0x80000000,0,0x11080488,0xf8000400,0x10400,0x11080488,0xf8000400,0x10401,0x11080488,0xf8000400,0x28400,0x11080488,0xf8000400,0x28401, +0x11080503,0x80000000,0,0x11080503,0x80000000,0x1024000,0x11080503,0xf8000400,0x10400,0x11080503,0xf8000400,0x10402,0x11080503,0xf8000400,0x28400,0x11080503, +0xf8001400,0x18400,0x11080593,0x80000000,0,0x11080593,0x80000000,0x1000,0x11080593,0x80000000,0x18400,0x11080593,0x80000040,0,0x11080593,0xf8000400, +0x18400,0x11080600,0x800000,0xc820,0x11080600,0xf8002400,0x18400,0x11080602,0x80000000,0,0x11080602,0xf8000400,0x18400,0x11080602,0xf8000400,0x18401, +0x11080602,0xf8000400,0x18402,0x11080602,0xf8001400,0x18400,0x11080780,0x80000000,0,0x1108078a,0xf8000400,0x18400,0x1108078a,0xf8000400,0x18401,0x11080804, +0x80000000,0,0x11080804,0xf8000400,0x18400,0x11080804,0xf8000400,0x18401,0x11080890,0xf8000400,0x18400,0x11080890,0xf8000400,0x18401,0x1108090f,0xf8000400, +0x18400,0x1108099f,0x80000000,0,0x1108099f,0xf8000400,0x18400,0x1108099f,0xf8000400,0x18401,0x11080a23,0x80000000,0,0x11080a23,0xf8000400,0x18400, +0x11080a23,0xf8000400,0x18401,0x11080aa4,0xf8000400,0x18400,0x11080b15,0xf8000400,0x18400,0x11080b9a,0xf8000400,0x18400,0x11080ca6,0x80000000,0,0x11080e80, +0x80000000,0,0x11080e8c,0xf8000400,0x18400,0x11080e8c,0xf8000400,0x28400,0x11081319,0xf8000400,0x10401,0x11081319,0xf8000400,0x10402,0x11081319,0xf8000400, +0x28401,0x1108138e,0x80001000,1,0x1108138e,0x80001000,2,0x1108138e,0xf8000400,0x10401,0x1108138e,0xf8000400,0x28401,0x11081400,0x80000000,0x4000000, +0x11081400,0x80000000,0x4000002,0x11081400,0xd0000000,0x1c00,0x11081400,0xd0000080,0x1c00,0x11081480,0x80000000,0xe,0x11081480,0x80000000,0xf,0x11081480, +0x80000080,0xe,0x11081480,0x80000080,0xf,0x11081480,0x80000088,0xe,0x11081480,0x80000088,0xf,0x11081600,0x80000000,0,0x11081600,0x80000000, +2,0x11081600,0x80000000,0xf,0x11081600,0x80000080,0,0x11081600,0xf8000000,0,0x11081600,0xf8000400,0x28402,0x11081600,0xf8000480,0x10405, +0x11081600,0xf8000480,0x18402,0x11081600,0xf8000480,0x28405,0x11081619,0xf8000400,0x28400,0x11081619,0xf8000400,0x28401,0x11081680,0x80000000,6,0x11081680, +0xf8000400,0x10402,0x11081680,0xf8000400,0x18400,0x11081680,0xf8000400,0x28402,0x11081700,0x80000000,0x4000000,0x11081700,0x80000000,0x4000001,0x11081700,0x80000080, +0x4000000,0x11081700,0x80000080,0x4000001,0x11081780,0x80000080,0x4000000,0x11081780,0x80000080,0x4000001,0x11081780,0x80000080,0x4000002,0x11081800,0x80000000,0x4000000, +0x11081800,0x80000080,0x4000000,0x11081880,0x80000000,0x4000000,0x11081900,0x80000000,0x4000000,0x11081a00,0x80000000,0x4000000,0x11081a80,0x80000000,0x4000000,0x11081b00, +0x80000000,0x4000000,0x11081b80,0x80000000,0x4000000,0x11081c00,0x80000000,0x4000000,0x11082803,0xf8000400,0x10402,0x11082813,0x80000080,5,0x11082813,0xf8000400, +0x18401,0x11082813,0xf8000400,0x18402,0x11082813,0xf8000400,0x18405,0x11082819,0xf8000400,0x10402,0x11082882,0xe0000400,0x18408,0x11082882,0xf8000400,0x18404, +0x11082882,0xf8000400,0x18407,0x11082882,0xf8000400,0x18408,0x11082882,0xf8000400,0x18409,0x11082a82,0xe0000400,0x18408,0x11082a82,0xf8000400,0x18404,0x11082a82, +0xf8000400,0x18407,0x11082a82,0xf8000400,0x18408,0x11082a82,0xf8000400,0x18409,0x11088100,0x80000000,0x4000000,0x11088119,0xf8000400,0x10400,0x11088119,0xf8000400, +0x10401,0x11088119,0xf8000400,0x28400,0x11088199,0xf8000400,0x10400,0x11088199,0xf8000400,0x10401,0x11088199,0xf8000400,0x10402,0x11088199,0xf8000400,0x28400, +0x11088199,0xf8000400,0x28402,0x11088219,0xf8000400,0x10401,0x11088299,0xf8000400,0x10400,0x11088300,0x80001000,0,0x1108840e,0xf8000400,0x10400,0x1108840e, +0xf8000400,0x28400,0x11088488,0xf8000400,0x10400,0x11088488,0xf8000400,0x10401,0x11088488,0xf8000400,0x28400,0x11088488,0xf8000400,0x28401,0x11089400,0x80000000, +0x4000000,0x11089400,0x80000000,0x4000002,0x1108960e,0xf8000400,0x28401,0x11089680,0x80000000,6,0x11089700,0x80000000,0x4000000,0x11089c00,0x80000000,0x4000000, +0x11092b80,0x80000000,0xa,0x11092b80,0x80000080,0xa,0x11092b92,0xf8000400,0x1840a,0x11092b92,0xf8800400,0x1840a,0x11092b96,0xf8000400,0x18c0a,0x110a0080, +0x80000000,0x4000000,0x110a0080,0x80000080,0x4000000,0x110a0080,0x80001000,0x4000000,0x110a0080,0x80001080,0x4000000,0x110a0080,0xd0000000,0x1c00,0x110a0099,0xf8000400, +0x10400,0x110a0099,0xf8000400,0x28400,0x110a0099,0xf8000700,0x10400,0x110a0099,0xf8000700,0x28400,0x110a0100,0x80000000,0x4000000,0x110a0100,0x80000080,0x4000000, +0x110a0100,0x80001000,2,0x110c9400,0x80000008,0x4000000,0x11100080,1,0x8020020,0x11100593,0x80000008,0,0x11100780,0x80000040,0x1024000,0x11100ca6, +0x80000040,0,0x11101400,0x80000001,0x20001,0x11101400,0x80000001,0x20002,0x11101400,0x80000008,0x4000000,0x11108100,0x800010,0xc820,0x11109400,0x80000000, +0x4001000,0x11109400,0x80000008,0x4000000,0x11109400,0x80000018,0x4000000,0x11120080,0x80000080,0x4000000,0x11140300,0xf8001400,0x18400,0x11148100,0x80001000,2, +0x11180080,1,0x8020020,0x11181400,1,0x801c020,0x11201400,0x80000000,0x4008000,0x11201480,0x80000080,0x800e,0x11201480,0x80000080,0x800f,0x11202880, +0x80000000,0x4008000,0x11212b80,0x80000020,0x800a,0x11212b80,0x80000040,0xa,0x11212b80,0x80000040,0x102400a,0x1121ab80,0x80000000,0x8011,0x1121ab80,0x80000040, +0x11,0x1121ab80,0x80000040,0x1024011,0x11220080,0x80000000,0x4008000,0x11229800,0x80000000,0x4008001,0x11229e80,0x80000000,0x4008000,0x11229e80,0x80000020,0x4008000, +0x11229e80,0x80000040,0x4000000,0x11229e80,0x80000040,0x5024000,0x1122a980,0x80000000,0x8010,0x1122a980,0x80000020,0x8010,0x1122aa00,0x80000000,0x800c,0x1122aa00, +0x80000040,0xc,0x1122aa00,0x80000040,0x102400c,0x11240080,0x800000,0x20,0x11240100,0x800000,0x20,0x11240488,0x50021000,0x60,0x11240593,0x50021400, +0x60,0x11240601,0x50020400,0x60,0x11240601,0x50021400,0x60,0x11240602,0x20000,0x60,0x11240602,0x50020400,0x60,0x11240602,0x50021000,0x60, +0x1124078a,0x50020400,0x60,0x1124078a,0x50021000,0x60,0x1124078a,0x50061000,0x60,0x1124078a,0xd0000400,0x18400,0x11240804,0x50020400,0x60,0x11240804, +0x50021000,0x60,0x11240804,0x50061000,0x60,0x11240804,0xd0000400,0x18400,0x11240804,0xd0000400,0x18401,0x11240890,0x50020400,0x60,0x11240890,0x50021000, +0x60,0x11240890,0x50061000,0x60,0x11240890,0xd0000400,0x18400,0x1124090f,0x50020400,0x60,0x1124090f,0x50021000,0x60,0x1124090f,0x50061000,0x60, +0x1124090f,0xd0000400,0x18400,0x1124099f,0x50020400,0x60,0x1124099f,0x50021000,0x60,0x1124099f,0x50061000,0x60,0x1124099f,0xd0000400,0x18400,0x1124099f, +0xd0000400,0x18401,0x11240a23,0x50020400,0x60,0x11240a23,0x50061000,0x60,0x11240a23,0xd0000400,0x18400,0x11240a23,0xd0000400,0x18401,0x11240aa4,0x50020400, +0x60,0x11240aa4,0x50020400,0x61,0x11240aa4,0x50061000,0x60,0x11240aa4,0xd0000400,0x18400,0x11240b15,0x50020400,0x60,0x11240b15,0x50061000,0x60, +0x11240b15,0xd0000400,0x18400,0x11240b15,0xd0000400,0x18401,0x11240b9a,0x50020400,0x60,0x11240b9a,0x50061000,0x60,0x11240b9a,0xd0000400,0x18400,0x11240b9a, +0xd0000400,0x18401,0x11241400,0x800000,0xc820,0x11241400,0x800000,0x800c820,0x11241400,0x1800000,0xc820,0x11241401,0x820000,0x60,0x11241581,0x20000, +0x60,0x11241581,0x50020080,0x60,0x11242813,0x50021400,0x60,0x11242901,0x50021000,0x60,0x11248381,0x50021000,0x60,0x11248381,0x50021000,0x61, +0x11248381,0x50021400,0x60,0x11248381,0x51021000,0x61,0x11269e81,0x50021000,0x60,0x11269f01,0x50021000,0x60,0x11280080,1,0x801c040,0x112c0600, +0x80000040,0,0x112c0600,0x80000040,0x1024000,0x112c0602,0x80000000,0,0x112c0602,0x80000040,0x1024000,0x112c1c00,0x80000000,0x4000000,0x112dab80,0x80000040, +0x1024011,0x112e0080,0x80000040,0x5024000,0x112eaa00,0x80000040,0x102400c,0x11300100,0x80000001,0x1800b,0x11301400,0x80000001,0x2000b,0x11301400,0x80000018,0x400000b, +0x11308381,0x50021000,0x60,0x11360080,0x80000018,0x4000000,0x11381e80,0x80000000,0,0x11399e80,0x80000001,0x20011,0x1139ab80,0x80000000,0x11,0x1139ab80, +0x80000018,0x11,0x1139ab80,0x80000020,0x11,0x1139ab80,0x80000080,0x11,0x1139ab80,0x80001000,0x11,0x1139ab80,0x80001080,0x11,0x1139ab80,0xd0000000, +0x1c11,0x1139ab80,0xd0000100,0x11,0x1139ab99,0xf8000400,0x10411,0x1139ab99,0xf8000400,0x28411,0x1139ab99,0xf8000500,0x10411,0x1139ab99,0xf8000500,0x28411, +0x113a9e80,0x80000000,0,0x113a9e80,0x80000000,2,0x113a9e80,0x80000000,0x4000000,0x113a9e80,0x80000008,0x4000000,0x113a9e80,0xf8000c00,0x18000,0x113a9e80, +0xf8002400,0x18c00,0x113a9e91,0xf8000c00,0x18000,0x113a9f14,0xf8000400,0x18000,0x113a9f14,0xf8000400,0x18001,0x113a9f96,0xf8000400,0x18c00,0x113a9f96,0xf8000400, +0x18c01,0x113aa005,0xf8000400,0x18400,0x113aa092,0xf8000400,0x18402,0x113aa092,0xf8800400,0x18402,0x113aa100,0x80000000,0,0x113aa100,0x80000000,0xf, +0x113aa200,0x80000000,0,0x113aa200,0x80000000,2,0x113aa200,0x80000000,3,0x113aa212,0x80000000,2,0x113aa212,0x80000000,3,0x113aa280, +0x80000000,2,0x113aa280,0x80000000,0xd,0x113aa391,0xf8400c00,0x18000,0x113aa791,0xf8000c00,0x18001,0x113aa791,0xf8400c00,0x18000,0x113aa980,0x80000000, +2,0x113aa980,0x80000000,0x10,0x113aa980,0x80000008,0x10,0x113aa980,0xd0000000,0x1c02,0x113aa980,0xd0000000,0x1c10,0x113aaa00,0x80000000,0xc, +0x113aaa00,0x80000008,0xc,0x113aaa00,0x80000040,0xc,0x113aaa00,0x80000080,0xc,0x113aaa00,0x80000098,0xc,0x113c9400,0x80000000,0x4000002,0x11400400, +0x80000040,0x1401,0x11400500,0x80000040,0x1025400,0x11401400,0x80000080,0x4001400,0x11420080,0x80000040,0x4001000,0x11420080,0x80000040,0x4001400,0x11420080,0x80000040, +0x5005400,0x11440080,1,0x801c0a0,0x11481400,0x80000040,0x5024000,0x11481400,0x80000040,0x5024002,0x11492b80,0x80000010,0xa,0x11492b80,0xf0001400,0x18c0a, +0x11492b80,0xf8003400,0x18c0a,0x11492b96,0xf8000400,0x18c0a,0x1149ab80,0x80000040,0x11,0x114a9e80,0x80000008,0x4000000,0x114a9e91,0xf8002400,0x18400,0x114a9f00, +0xe0001000,0xc02,0x114a9f14,0xf8000400,0x18000,0x114a9f14,0xf8002400,0x18000,0x114a9f14,0xf8002400,0x18001,0x114a9f80,0x80000010,0,0x114a9f80,0xf8003400, +0x18c00,0x114a9f96,0xf8000400,0x18c00,0x114a9f96,0xf8002400,0x18c00,0x114a9f96,0xf8002400,0x18c01,0x114aaa00,0x80000040,0xc,0x114c0600,0xd0000000,0x15800, +0x114c0602,0x80000000,0x15800,0x114c0602,0xd0000000,0x15800,0x114c078a,0xd0000000,0x15800,0x114c0804,0xd0000000,0x15800,0x114c0890,0xd0000000,0x15800,0x114c090f, +0xd0000000,0x15800,0x114c099f,0xd0000000,0x15800,0x114c0a23,0xd0000000,0x15800,0x114c0aa4,0xd0000000,0x15800,0x114c0b15,0xd0000000,0x15800,0x114c0b9a,0xd0000000, +0x15800,0x114c0ca6,0xd0000000,0x15800,0x114c0d18,0xd0000000,0x15800,0x114e0080,0xd0000300,0x15800,0x11501400,0x80000000,0x4008000,0x11501400,0x80000020,0x4008000, +0x11501480,0x80000080,0x800e,0x11501480,0x80000080,0x800f,0x11502880,0x80000000,0x4008000,0x11512b80,0x80000020,0x800a,0x1151ab80,0x80000000,0x8011,0x11520080, +0x80000000,0x4008000,0x11529800,0x80000000,0x4008001,0x11529e80,0x80000000,0x4008000,0x11529e80,0x80000020,0x4008000,0x1152a980,0x80000000,0x8010,0x1152a980,0x80000020, +0x8010,0x1152aa00,0x80000000,0x800c,0x11541400,0x80000000,0x4000000,0x11541400,0x80000000,0x4000002,0x11541400,0x80000080,0x4000002,0x11541500,0x80000000,0, +0x11548100,0x80000000,0x4000000,0x11549400,0x80000000,0x4000000,0x11549400,0x80000080,0x4000000,0x11549400,0x80000080,0x4000002,0x11549600,0x80000000,2,0x1155ab80, +0x80000000,0x11,0x11560080,0x80000000,0x4000000,0x11560100,0x80000000,0x4000000,0x1156aa00,0x80000000,0xc,0x11580804,0x80000000,0,0x11580c80,0x80000000, +0,0x11581500,0x80000000,0,0x11581500,0x80000000,2,0x11581780,0x80000080,0x4000000,0x11581780,0x80000088,0x4000000,0x11588100,0x80000000,0x4000000, +0x11588100,0x80000080,0x4000000,0x11589600,0x80000000,2,0x11591500,0x80000000,0,0x1159ab80,0x80000000,0x11,0x115a0080,0x80000000,0x4000000,0x115a0080, +0x80000080,0x4000000,0x115a0100,0x80000000,0x4000000,0x115aaa00,0x80000000,0xc,0x115c0100,0x80000020,0x4008000,0x115c1400,0x80000020,0x4008000,0x115c1c00,0x80000000, +0x4008000,0x115c9400,0x80000020,0x4008000,0x115c9400,0x80000020,0x4009000,0x115e0080,0x80000020,0x4008000,0x115e0080,0x80000020,0x4009000,0x11600ca6,0x50020400,0x60, +0x11600ca6,0x50021000,0x60,0x11600ca6,0x50060400,0x60,0x11600ca6,0xf0000400,0x18002,0x11600ca6,0xf8000400,0x18000,0x11600ca6,0xf8002400,0x18000,0x11600ca6, +0xfc000400,0x18000,0x11600d18,0x50020400,0x60,0x11600d18,0x50021000,0x60,0x11600d18,0xf0000400,0x18002,0x11600d18,0xf8000400,0x18000,0x11600d18,0xf8000400, +0x18002,0x11600d18,0xf8002400,0x18000,0x11600d18,0xfc000400,0x18000,0x116a0080,0x80000001,0x8020000,0x116e0080,0x80000000,0x4000000,0x11701400,0x800000,0xc820, +0x11740100,1,0x801c020,0x11782a80,0x800000,0xc820,0x11868f12,0xf8000400,0x18480,0x11868f12,0xf8800400,0x18480,0x11880f12,0xf8000400,0x18500,0x118c0f12, +0xf8000400,0x18520,0x118c0f12,0xf8800400,0x18520,0x20000067,0x810000,0,0x200036e7,0x810000,0,0x20003767,0x810000,0,0x2000b6e7,0, +0,0x2000b767,0,0,0x20080da7,0x80000000,0,0x20080da7,0xf8000400,0x18400,0x20080da7,0xf8000400,0x18401,0x20081319,0xf8000400,0x10401, +0x20100da7,0x80000000,0,0x20100da7,0xd0000400,0x18400,0x20140da7,0x80000000,0,0x20200da7,0x80000000,0x8000,0x20240593,0x50021000,0x60,0x20240593, +0x50021400,0x60,0x20240da7,0x50020400,0x60,0x20240da7,0x50020400,0x61,0x20240da7,0x50020400,0x62,0x20240da7,0x50021000,0x60,0x20240da7,0x50061000, +0x60,0x20240da7,0xd0001000,0,0x202c0da7,0x80000000,0,0x202c0da7,0x80000040,0,0x20300da7,0x80000000,0xb,0x20300da7,0x80000040,0, +0x204c0da7,0xd0000000,0x15800,0x20500da7,0x80000000,0x8000,0x20581500,0x80000000,0,0x206425e7,0x800000,0,0x20642667,0x800000,0,0x206426e7, +0x800000,0,0x207ea512,0xf8000400,0x184c1,0x2082a512,0xf8000400,0x184e1,0x211c2b00,0x80000000,0,0x21589500,0x80000000,0,0x30080219,0xf8000400, +0x10400,0x30080219,0xf8000400,0x10401,0x30080219,0xf8000400,0x28400,0x30080219,0xf8000400,0x28401,0x30080299,0xf8000400,0x10400,0x30080300,0x80001000,0, +0x30080300,0xf8001400,0x18400,0x3008040e,0xf8000400,0x10400,0x30080488,0xf8000400,0x10400,0x30080488,0xf8000400,0x10401,0x30080488,0xf8000400,0x28400,0x30080488, +0xf8000400,0x28401,0x30080602,0x80000000,0,0x30080602,0xf8000400,0x18400,0x300806a2,0x800000,0xc820,0x300806a2,0x80000000,0,0x300806a2,0x80000040, +0,0x300806a2,0x80000040,0x1024000,0x300806a2,0xf8000400,0x18400,0x30080725,0xf8000400,0x18400,0x30080c21,0x80000000,0,0x30080c21,0xf8000400,0x18400, +0x30080da7,0x80000000,0,0x30080da7,0xf8000400,0x18400,0x30080e1c,0x80000000,0,0x30080f8b,0x80000000,0,0x30080f8b,0x80000040,0,0x30080f8b, +0x80000040,0x1024000,0x30080f8b,0xd0000000,0,0x30080f8b,0xf8000400,0x18400,0x30081006,0xf8000400,0x18400,0x300810a8,0x80000040,0,0x300810a8,0x80000040, +0x1024000,0x300810a8,0xf8000400,0x18400,0x3008111d,0xf8000400,0x18400,0x300811a0,0xf8000400,0x18400,0x30081217,0x80000000,0,0x3008129b,0x80000000,0, +0x3008129b,0xf8000400,0x18400,0x3008129b,0xf8002400,0x18400,0x30081400,0x80000000,0x4000000,0x30081600,0x80000000,0,0x30081600,0xf8000400,0x10405,0x30081680, +0xf8000400,0x28400,0x30081700,0x80000000,0x4000000,0x30081800,0x80000000,0x4000000,0x30081880,0x80000000,0x4000000,0x30081b00,0x80000000,0x4000000,0x30081b80,0x80000000, +0x4000000,0x30081cae,0x80000000,0x4000000,0x30082813,0xf8000400,0x18401,0x30088300,0x80001000,0,0x30100503,0x80000018,0,0x30100da7,0x80000000,0, +0x30100e1c,0x80000040,0x1024000,0x30100f8b,0x80000040,0,0x3010111d,0x80000001,0x20000,0x30101180,0x80000040,0,0x30101217,0x80000000,0,0x30101217, +0x80000040,0,0x30101280,0x80000040,0,0x30101280,0x80000040,0x1024000,0x3010129b,0x80000040,0,0x3010129b,0x80000040,0x1024000,0x3014129b,0x80000018, +0,0x3020111d,0x80000000,0x8000,0x30240488,0x20000,0x60,0x30240601,0x50020400,0x60,0x302406a2,0x50020400,0x60,0x302406a2,0x50021000,0x60, +0x302406a2,0x50021400,0x60,0x30240725,0x50021400,0x60,0x30240c21,0x50020400,0x60,0x30240c21,0x50061000,0x60,0x30240c21,0xd0000400,0x18400,0x30240c21, +0xd0000400,0x18401,0x30240da7,0x50020400,0x60,0x30240da7,0x50021000,0x60,0x3024129b,0x50020400,0x60,0x3024129b,0x50820000,0x2000060,0x30241581,0x20000, +0x60,0x30242b00,0,0xc820,0x30248381,0x50021000,0x60,0x3030129b,0x80000001,0x20000,0x30301400,0x80000001,0x2000b,0x30308381,0x50021000,0x60, +0x303a9d11,0x80200000,0,0x303a9d11,0x80200000,2,0x303a9d91,0x80200000,2,0x303a9e00,0x80080000,0,0x303a9e00,0x80100000,0,0x303a9e80, +0x80000000,0,0x303a9e91,0xf8000c00,0x18002,0x303aa185,0xf8000400,0x18400,0x303aa311,0xf8400c00,0x18000,0x303aa429,0xf8000400,0x18400,0x303aa4a9,0x80000000, +0,0x30481217,0x80000040,0,0x30481400,0x80000040,0x5024002,0x304aa429,0xf8002400,0x18400,0x304c0e1c,0xd0000000,0x15800,0x304c1217,0xd0000000,0x15800, +0x304c129b,0xd0000000,0x15800,0x3050111d,0x80000000,0x8000,0x30581217,0x80000000,0,0x30581500,0x80000000,0,0x30600e1c,0x50020400,0x60,0x30600e1c, +0x50021000,0x60,0x30600e1c,0x50061000,0x60,0x30600e1c,0xd0000400,0x18000,0x30600e1c,0xf8000400,0x18000,0x30600e1c,0xf8000400,0x18001,0x30601217,0x800000, +0xc820,0x30601217,0x50020400,0x60,0x30601217,0x50021000,0x60,0x30601217,0x50061000,0x60,0x30601217,0x51021000,0x60,0x30601217,0xd0000400,0x18000, +0x30601217,0xf8000400,0x18000,0x30601217,0xf9000400,0x18000,0x310028e7,0x810000,0,0x3108040e,0xf8000480,0x10402,0x3108040e,0xf8000480,0x28402,0x31082c1e, +0x80000000,0,0x31082c1e,0xf8000400,0x18400,0x31082c8d,0xf8000400,0x18400,0x31082d09,0xf8000400,0x10400,0x31082d09,0xf8000400,0x28400,0x31082d80,0x80000000, +0,0x31082e00,0x80000000,0,0x31082e00,0x80000000,1,0x31082e80,0x80000080,5,0x31082e80,0xf8000480,0x10405,0x31082e80,0xf8000480,0x28405, +0x31242e00,0x800000,0xc820,0x31242e00,0x50020000,0x60,0x31242e00,0x50021000,0x60,0x31242e00,0xd0000000,0,0x31242e00,0xd0001000,0,0x31242e01, +0x50021000,0x60,0x31243000,0x800000,0xc820,0x313aaf11,0xf8400c00,0x18000,0x313aaf91,0xf8000c00,0x18001,0x314c2e80,0xd0000080,0x15805,0x32041b80,0x80000000, +0x4000000,0x32049980,0x80000000,0,0x32080219,0xf8000400,0x28400,0x3208040e,0x80000080,0,0x3208040e,0xf8000400,0x10400,0x3208040e,0xf8000400,0x28400, +0x32080488,0xf8000400,0x10400,0x32080488,0xf8000400,0x28400,0x32080602,0xf8000400,0x18400,0x32080725,0xf8000400,0x18400,0x32080e8c,0xf8000400,0x18400,0x32081400, +0x800080,0xc820,0x32081400,0x80000000,0x4000000,0x32081400,0x80000000,0x4000002,0x32081400,0x80000080,0x4000000,0x32081499,0xf8000400,0x1040f,0x32081600,0x80000000, +0,0x32081600,0x80000080,0,0x32081600,0x80000080,5,0x32081600,0xf8000480,0x10405,0x32081600,0xf8000480,0x28405,0x32081700,0x80000080,0x4000000, +0x32081780,0x80000080,0x4000000,0x32081800,0x80000000,0x4000000,0x32081800,0x80000080,0x4000000,0x32081a80,0x80000000,0x4000000,0x32081b00,0x80000080,0x4000000,0x32081b80, +0x80000000,0x4000000,0x32082a82,0xf8000400,0x18400,0x32083088,0xf8000400,0x10400,0x32083088,0xf8000400,0x28400,0x3208312a,0xf8000400,0x18400,0x320831ab,0xf8000400, +0x18400,0x3208322c,0xf8000400,0x18400,0x320832ad,0xf8000400,0x18400,0x32083300,0x80000080,0x4000000,0x32083380,0x80000080,0x4000000,0x32083400,0x80000080,0x4000000, +0x32083480,0x80000080,0x4000000,0x32083500,0x80000080,0x4000000,0x32083500,0x80000080,0x4000001,0x32083500,0x80000080,0x4000002,0x32101400,0x80000001,0x20002,0x32103180, +0x80000000,0,0x32201c00,0x80000000,0x4008000,0x32203480,0x80000080,0x4008000,0x3221ab80,0x80000000,0x8011,0x32223300,0x80000080,0x4008000,0x32223480,0x80000080, +0x4008000,0x32241581,0x20000,0x60,0x32241581,0x50020000,0x60,0x32241581,0x50020080,0x60,0x3224312a,0x50020400,0x60,0x3224312a,0x50060000,0x60, +0x322431ab,0x50020400,0x60,0x322431ab,0x50060000,0x60,0x3224322c,0x50020400,0x60,0x322432ad,0x50020400,0x60,0x32248381,0x50020000,0x60,0x3224b601, +0x50820000,0x2000060,0x32308381,0x50820000,0x60,0x323a9e80,0x80000000,0,0x323a9f14,0xf8000400,0x18010,0x323a9f96,0xf8000400,0x18c10,0x323aa200,0x80000000, +3,0x323aa4a9,0x80000000,0,0x323aa791,0xf8000400,0x18401,0x323aa980,0x80000000,0x4000000,0x32481400,0x80000040,0x5024002,0x324a9e80,0xf8000400,0x18400, +0x324a9e91,0xf8000400,0x18400,0x324a9f14,0xf8000400,0x18000,0x324a9f80,0x80000008,0xc00,0x324ab596,0xf8000400,0x18c00,0x32501c00,0x80000000,0x4008000,0x32503480, +0x80000080,0x4008000,0x3251ab80,0x80000000,0x8011,0x32523300,0x80000080,0x4008000,0x32523480,0x80000080,0x4008000,0x32542882,0x80000000,8,0x32581500,0x80000000, +0,0x32781400,0x800000,0xc820,0x40049b80,0x80000000,0x4000000,0x40080219,0xf8000400,0x10400,0x40080299,0xf8000400,0x10400,0x40080300,0x80001000,0, +0x4008040e,0xf8000400,0x10400,0x4008040e,0xf8000400,0x28400,0x4008040e,0xf8000400,0x28402,0x40080600,0x800000,0xc820,0x40080602,0x80000000,0,0x40080602, +0xf8000400,0x18400,0x400806a2,0xf8000400,0x18400,0x4008078a,0xf8000400,0x18400,0x40080804,0xf8000400,0x18400,0x4008090f,0xf8000400,0x18400,0x4008099f,0xf8000400, +0x18400,0x40080a23,0x80000000,0,0x40080b15,0xf8000400,0x18400,0x40081217,0x80000000,0,0x40081400,0x80000008,0x4000000,0x40081400,0xd0000000,0x1c00, +0x40081600,0x80000000,2,0x40081800,0x80000000,0x4000000,0x40081800,0x80000080,0x4000000,0x40081b80,0x80000000,0x4000000,0x40082880,0x80000000,0,0x40082d09, +0xf8000400,0x10400,0x40082d09,0xf8000400,0x28400,0x40082e80,0xf8000480,0x10405,0x400837b0,0x80000000,0,0x400837b0,0xf8000400,0x18400,0x40083897,0x80000000, +0,0x40083908,0xf8000400,0x10400,0x4008390e,0xf8000400,0x10400,0x4008390e,0xf8001400,0x1040e,0x4008390e,0xf8001400,0x1040f,0x40083919,0xf8000400,0x10400, +0x40083919,0xf8001400,0x10400,0x40083919,0xf8001400,0x1040e,0x40083919,0xf8001400,0x1040f,0x40083980,0x80000000,0x4000000,0x40083a00,0x80000000,0,0x40083ab1, +0xf8000400,0x18400,0x40083b31,0xf8000400,0x18400,0x40083b80,0x80000000,0,0x40083c35,0xf8000400,0x18400,0x40083cb3,0xf8000400,0x18400,0x40083d32,0xf8000400, +0x18400,0x40083daf,0xf8000400,0x18400,0x40083e00,0x80000000,0,0x40089980,0x80000000,0,0x40103b80,0x80000000,0,0x40103c35,0x80000040,0, +0x4022a980,0x80000000,0x8010,0x40240602,0x50020400,0x60,0x40240602,0x50021000,0x60,0x40240602,0x50021400,0x60,0x40240890,0x50020400,0x60,0x40240890, +0xd0000400,0x18400,0x4024090f,0x50020400,0x60,0x40240b15,0x50021000,0x60,0x402437b0,0x50020400,0x60,0x402437b0,0x50021000,0x60,0x402437b0,0xd0000400, +0x18400,0x40248381,0x50021000,0x60,0x4024be81,0x50820000,0x2000060,0x402c37b0,0x80000040,0x1024000,0x40308381,0x50021000,0x60,0x403aa200,0x80000000,0xd, +0x403aa212,0x80000000,2,0x403aa212,0x80000000,3,0x403aa280,0x80000000,0xd,0x40400602,0x80000000,0x1400,0x404c37b0,0xd0000000,0x15800,0x404c3d32, +0xd0000000,0x15800,0x4052a980,0x80000000,0x8010,0x4058090f,0x80000000,0,0x40580a23,0x80000000,0,0x40601217,0x50021000,0x60,0x40603834,0xf8000400, +0x18000,0x41080219,0xf8000400,0x10400,0x41080219,0xf8000400,0x28400,0x4108040e,0xf8000400,0x10400,0x4108040e,0xf8000400,0x28400,0x41080488,0xf8000400,0x10400, +0x41080488,0xf8000400,0x28400,0x4108078a,0xf8000400,0x18400,0x41080804,0xf8000400,0x18400,0x41080a23,0xf8000400,0x18400,0x41080e8c,0xf8000400,0x18400,0x41080e8c, +0xf8000400,0x1840f,0x41080f8b,0x80000000,0,0x41080f8b,0xf8000400,0x18400,0x41081400,0x80000000,0x4000000,0x41081499,0xf8000400,0x1040e,0x41081600,0x80000000, +0,0x41081600,0xf8000480,0x10405,0x41081800,0x80000000,0x4000000,0x41081b80,0x80000000,0x4000000,0x41082e80,0xf8000480,0x10405,0x41083300,0x80000080,0x4000000, +0x41083908,0xf8000400,0x1040f,0x41083919,0xf8000400,0x10400,0x41083980,0x80000000,0x4000000,0x41083f0e,0x80000000,0,0x41083f8e,0x80000000,0,0x41083f8e, +0xf8000400,0x18400,0x41084002,0xf8000400,0x18400,0x410840b7,0x80000000,0,0x410840b7,0xf8000400,0x18400,0x41084207,0x80000000,0,0x41084207,0xf8000400, +0x10400,0x41084207,0xf8000400,0x28400,0x4108428b,0xf8000400,0x18400,0x4108430b,0x80000000,0,0x4108430b,0xf8000400,0x18400,0x4108438c,0xf8000400,0x10400, +0x41084438,0xf8000400,0x10400,0x41084438,0xf8000400,0x28400,0x410844b9,0x80000000,0,0x410844b9,0xf8000400,0x18400,0x41084500,0x80000000,0,0x4108463d, +0xf8000400,0x18400,0x4108468e,0xf8000400,0x1040f,0x41084699,0xf8000400,0x10400,0x41084699,0xf8000400,0x1040f,0x41084700,0x80000000,0x4000000,0x410847ba,0x80000000, +0,0x410847ba,0xf8000400,0x18400,0x4108483c,0xf8000400,0x18400,0x4108483c,0xf8000400,0x1840f,0x41101400,0x80000000,0x4000000,0x411040b7,0x80000000,0, +0x41104207,0x80000000,0,0x411044b9,0x80000000,0,0x4110463d,0x80000040,0,0x41104700,0x80000000,0x4000000,0x41104700,0x80000018,0x4000000,0x41140da7, +0x80000000,0,0x41203300,0x80000080,0x4008000,0x4122c880,0x80000000,0x10,0x4122c880,0x80000000,0x8010,0x41240593,0x50020000,0x60,0x41240593,0x50020400, +0x60,0x41240602,0x50020400,0x60,0x41240f8b,0x50020400,0x60,0x41241581,0x50020000,0x60,0x41243f0e,0x50020000,0x60,0x412440b7,0x50020400,0x60, +0x412440b7,0xd0000400,0x18400,0x41244181,0x50020000,0x60,0x412444b9,0x50020000,0x60,0x412444b9,0x50020400,0x60,0x412444b9,0x50060000,0x60,0x412447ba, +0x50020000,0x60,0x412447ba,0x50020400,0x60,0x412447ba,0x50060000,0x60,0x412447ba,0xd0000000,0,0x412447ba,0xd0000400,0x18400,0x41248381,0x50020000, +0x60,0x412c0593,0x80000000,0,0x412c0602,0x80000000,0,0x412ec880,0x80000000,0x10,0x41308381,0x50020000,0x60,0x413aa200,0x80000000,3, +0x413aa391,0xf8400c00,0x18000,0x413aa791,0xf8000c00,0x18001,0x413ac100,0x80000000,0,0x413ec880,0x80000000,0x10,0x4142c880,0x80000000,0x1410,0x414c0a23, +0xd0000000,0x15800,0x414c45bb,0xd0000000,0x15800,0x41503300,0x80000080,0x4008000,0x4152c880,0x80000000,0x8010,0x41540602,0x80000000,0,0x41581500,0x80000000, +0,0x415c4700,0x80000000,0x4008000,0x416045bb,0x80000000,0,0x416045bb,0xd0000400,0x18000,0x416045bb,0xf8000400,0x18000,0x50080219,0xf8000400,0x10400, +0x50080219,0xf8000400,0x28400,0x5008040e,0xf8000400,0x10400,0x50080488,0xf8000400,0x10400,0x50080488,0xf8000400,0x28400,0x5008078a,0xf8000400,0x18400,0x50080b15, +0x80000000,0,0x50081600,0x80000000,0,0x50081619,0xf8000400,0x10400,0x50081699,0xf8000400,0x10400,0x50081800,0x80000000,0x4000000,0x50081800,0x80000080, +0x4000000,0x50081b80,0x80000000,0x4000000,0x50082e80,0xf8000480,0x10405,0x50082e80,0xf8000480,0x28405,0x50083088,0xf8000400,0x10400,0x50083088,0xf8000400,0x28400, +0x50083300,0x80000080,0x4000000,0x50083980,0x80000000,0x4000000,0x50084500,0xf8001400,0x18400,0x50084957,0x80000000,0,0x50084957,0xf8000400,0x18400,0x50084957, +0xf8001400,0x18400,0x50084957,0xf8002400,0x18400,0x500849be,0x80000000,0,0x500849be,0xf8000400,0x18400,0x500849be,0xf8000400,0x18401,0x50084a19,0xf8000400, +0x10400,0x50084a19,0xf8000400,0x28400,0x50084a80,0x80001000,0,0x50084b5a,0xf8000400,0x18400,0x50084bdb,0x80000000,0,0x50084bdb,0xf8000400,0x18400, +0x50084c65,0xf8000400,0x18400,0x50084ce5,0xf8000400,0x18400,0x50084d00,0x80000000,0,0x501049be,0x80000000,0,0x501049be,0x80000040,0,0x501049be, +0x80000040,0x1024000,0x50104bdb,0x80000040,0,0x50104ce5,0x80000040,0,0x50144b5a,0x80000000,0,0x50240593,0x50021400,0x60,0x50240b15,0x50020400, +0x60,0x50241581,0x50020080,0x60,0x50244181,0x50021000,0x60,0x50244957,0x50021000,0x60,0x502449be,0x50020400,0x60,0x502449be,0x50021000,0x60, +0x502449be,0xd0000400,0x18400,0x502449be,0xd0000400,0x18401,0x502449be,0xd0041000,0,0x502c4957,0x80000040,0x1024000,0x502c4b5a,0x80000040,0x1024000,0x50404957, +0x80000040,0x1400,0x504c4957,0xd0000000,0x15800,0x504c49be,0xd0000000,0x15800}; + +static const int32_t countPropsVectors=3384; +static const int32_t propsVectorsColumns=3; +static const int32_t indexes[UPROPS_INDEX_COUNT]={0x1a54,0x1a54,0x1a54,0x1a54,0x3b66,3,0x489e,0,0,0,0x8ecd73,0x29d31,0,0,0,0}; + diff --git a/Build/source/libs/icu/icu-xetex/common/uchriter.cpp b/Build/source/libs/icu/icu-xetex/common/uchriter.cpp new file mode 100644 index 00000000000..ae73695a810 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uchriter.cpp @@ -0,0 +1,363 @@ +/* +****************************************************************************** +* Copyright (C) 1998-2004, International Business Machines Corporation and * +* others. All Rights Reserved. * +****************************************************************************** +*/ + +#include "unicode/uchriter.h" +#include "unicode/ustring.h" +#include "uhash.h" + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(UCharCharacterIterator) + +UCharCharacterIterator::UCharCharacterIterator() + : CharacterIterator(), + text(0) +{ + // never default construct! +} + +UCharCharacterIterator::UCharCharacterIterator(const UChar* textPtr, + int32_t length) + : CharacterIterator(textPtr != 0 ? (length>=0 ? length : u_strlen(textPtr)) : 0), + text(textPtr) +{ +} + +UCharCharacterIterator::UCharCharacterIterator(const UChar* textPtr, + int32_t length, + int32_t position) + : CharacterIterator(textPtr != 0 ? (length>=0 ? length : u_strlen(textPtr)) : 0, position), + text(textPtr) +{ +} + +UCharCharacterIterator::UCharCharacterIterator(const UChar* textPtr, + int32_t length, + int32_t textBegin, + int32_t textEnd, + int32_t position) + : CharacterIterator(textPtr != 0 ? (length>=0 ? length : u_strlen(textPtr)) : 0, textBegin, textEnd, position), + text(textPtr) +{ +} + +UCharCharacterIterator::UCharCharacterIterator(const UCharCharacterIterator& that) +: CharacterIterator(that), + text(that.text) +{ +} + +UCharCharacterIterator& +UCharCharacterIterator::operator=(const UCharCharacterIterator& that) { + CharacterIterator::operator=(that); + text = that.text; + return *this; +} + +UCharCharacterIterator::~UCharCharacterIterator() { +} + +UBool +UCharCharacterIterator::operator==(const ForwardCharacterIterator& that) const { + if (this == &that) { + return TRUE; + } + + if (getDynamicClassID() != that.getDynamicClassID()) { + return FALSE; + } + + UCharCharacterIterator& realThat = (UCharCharacterIterator&)that; + + return text == realThat.text + && textLength == realThat.textLength + && pos == realThat.pos + && begin == realThat.begin + && end == realThat.end; +} + +int32_t +UCharCharacterIterator::hashCode() const { + return uhash_hashUCharsN(text, textLength) ^ pos ^ begin ^ end; +} + +CharacterIterator* +UCharCharacterIterator::clone() const { + return new UCharCharacterIterator(*this); +} + +UChar +UCharCharacterIterator::first() { + pos = begin; + if(pos < end) { + return text[pos]; + } else { + return DONE; + } +} + +UChar +UCharCharacterIterator::firstPostInc() { + pos = begin; + if(pos < end) { + return text[pos++]; + } else { + return DONE; + } +} + +UChar +UCharCharacterIterator::last() { + pos = end; + if(pos > begin) { + return text[--pos]; + } else { + return DONE; + } +} + +UChar +UCharCharacterIterator::setIndex(int32_t position) { + if(position < begin) { + pos = begin; + } else if(position > end) { + pos = end; + } else { + pos = position; + } + if(pos < end) { + return text[pos]; + } else { + return DONE; + } +} + +UChar +UCharCharacterIterator::current() const { + if (pos >= begin && pos < end) { + return text[pos]; + } else { + return DONE; + } +} + +UChar +UCharCharacterIterator::next() { + if (pos + 1 < end) { + return text[++pos]; + } else { + /* make current() return DONE */ + pos = end; + return DONE; + } +} + +UChar +UCharCharacterIterator::nextPostInc() { + if (pos < end) { + return text[pos++]; + } else { + return DONE; + } +} + +UBool +UCharCharacterIterator::hasNext() { + return (UBool)(pos < end ? TRUE : FALSE); +} + +UChar +UCharCharacterIterator::previous() { + if (pos > begin) { + return text[--pos]; + } else { + return DONE; + } +} + +UBool +UCharCharacterIterator::hasPrevious() { + return (UBool)(pos > begin ? TRUE : FALSE); +} + +UChar32 +UCharCharacterIterator::first32() { + pos = begin; + if(pos < end) { + int32_t i = pos; + UChar32 c; + UTF_NEXT_CHAR(text, i, end, c); + return c; + } else { + return DONE; + } +} + +UChar32 +UCharCharacterIterator::first32PostInc() { + pos = begin; + if(pos < end) { + UChar32 c; + UTF_NEXT_CHAR(text, pos, end, c); + return c; + } else { + return DONE; + } +} + +UChar32 +UCharCharacterIterator::last32() { + pos = end; + if(pos > begin) { + UChar32 c; + UTF_PREV_CHAR(text, begin, pos, c); + return c; + } else { + return DONE; + } +} + +UChar32 +UCharCharacterIterator::setIndex32(int32_t position) { + if(position < begin) { + position = begin; + } else if(position > end) { + position = end; + } + if(position < end) { + UTF_SET_CHAR_START(text, begin, position); + int32_t i = this->pos = position; + UChar32 c; + UTF_NEXT_CHAR(text, i, end, c); + return c; + } else { + this->pos = position; + return DONE; + } +} + +UChar32 +UCharCharacterIterator::current32() const { + if (pos >= begin && pos < end) { + UChar32 c; + UTF_GET_CHAR(text, begin, pos, end, c); + return c; + } else { + return DONE; + } +} + +UChar32 +UCharCharacterIterator::next32() { + if (pos < end) { + UTF_FWD_1(text, pos, end); + if(pos < end) { + int32_t i = pos; + UChar32 c; + UTF_NEXT_CHAR(text, i, end, c); + return c; + } + } + /* make current() return DONE */ + pos = end; + return DONE; +} + +UChar32 +UCharCharacterIterator::next32PostInc() { + if (pos < end) { + UChar32 c; + UTF_NEXT_CHAR(text, pos, end, c); + return c; + } else { + return DONE; + } +} + +UChar32 +UCharCharacterIterator::previous32() { + if (pos > begin) { + UChar32 c; + UTF_PREV_CHAR(text, begin, pos, c); + return c; + } else { + return DONE; + } +} + +int32_t +UCharCharacterIterator::move(int32_t delta, CharacterIterator::EOrigin origin) { + switch(origin) { + case kStart: + pos = begin + delta; + break; + case kCurrent: + pos += delta; + break; + case kEnd: + pos = end + delta; + break; + default: + break; + } + + if(pos < begin) { + pos = begin; + } else if(pos > end) { + pos = end; + } + + return pos; +} + +int32_t +UCharCharacterIterator::move32(int32_t delta, CharacterIterator::EOrigin origin) { + // this implementation relies on the "safe" version of the UTF macros + // (or the trustworthiness of the caller) + switch(origin) { + case kStart: + pos = begin; + if(delta > 0) { + UTF_FWD_N(text, pos, end, delta); + } + break; + case kCurrent: + if(delta > 0) { + UTF_FWD_N(text, pos, end, delta); + } else { + UTF_BACK_N(text, begin, pos, -delta); + } + break; + case kEnd: + pos = end; + if(delta < 0) { + UTF_BACK_N(text, begin, pos, -delta); + } + break; + default: + break; + } + + return pos; +} + +void UCharCharacterIterator::setText(const UChar* newText, + int32_t newTextLength) { + text = newText; + if(newText == 0 || newTextLength < 0) { + newTextLength = 0; + } + end = textLength = newTextLength; + pos = begin = 0; +} + +void +UCharCharacterIterator::getText(UnicodeString& result) { + result = UnicodeString(text, textLength); +} + +U_NAMESPACE_END diff --git a/Build/source/libs/icu/icu-xetex/common/ucln.h b/Build/source/libs/icu/icu-xetex/common/ucln.h new file mode 100644 index 00000000000..a4573e93bb2 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucln.h @@ -0,0 +1,68 @@ +/* +****************************************************************************** +* * +* Copyright (C) 2001-2007, International Business Machines * +* Corporation and others. All Rights Reserved. * +* * +****************************************************************************** +* file name: ucln_cmn.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2001July05 +* created by: George Rhoten +*/ + +#ifndef __UCLN_H__ +#define __UCLN_H__ + +#include "unicode/utypes.h" + +/** These are the functions used to register a library's memory cleanup + * functions. Each library should define a single library register function + * to call this API. In the i18n library, it is ucln_i18n_registerCleanup(). + * + * None of the cleanup functions should use a mutex to clean up an API's + * allocated memory because a cleanup function is not meant to be thread safe, + * and plenty of data cannot be reference counted in order to make sure that + * no one else needs the allocated data. + * + * In order to make a cleanup function get called when u_cleanup is called, + * You should add your function to the library specific cleanup function. + * If the cleanup function is not in the common library, the code that + * allocates the memory should call the library specific cleanup function. + * For instance, in the i18n library, any memory allocated statically must + * call ucln_i18n_registerCleanup() from the ucln_in.h header. These library + * cleanup functions are needed in order to prevent a circular dependency + * between the common library and any other library. + * + * The order of the cleanup is very important. In general, an API that + * depends on a second API should be cleaned up before the second API. + * For instance, the default converter in ustring depends upon the converter + * API. So the default converter should be closed before the converter API + * has its cache flushed. This will prevent any memory leaks due to + * reference counting. + * + * Please see common/ucln_cmn.{h,c} and i18n/ucln_in.{h,c} for examples. + */ + +typedef enum ECleanupLibraryType { + UCLN_START = -1, + UCLN_CUSTOM, /* Custom is for anyone else. */ + UCLN_CTESTFW, + UCLN_LAYOUTEX, + UCLN_LAYOUT, + UCLN_IO, + UCLN_I18N, + UCLN_COMMON /* This must be the last one to cleanup. */ +} ECleanupLibraryType; + +U_CDECL_BEGIN +typedef UBool U_CALLCONV cleanupFunc(void); +U_CDECL_END + +U_CAPI void U_EXPORT2 ucln_registerCleanup(ECleanupLibraryType type, + cleanupFunc *func); + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/ucln_cmn.c b/Build/source/libs/icu/icu-xetex/common/ucln_cmn.c new file mode 100644 index 00000000000..452e628bb38 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucln_cmn.c @@ -0,0 +1,74 @@ +/* +****************************************************************************** +* * +* Copyright (C) 2001-2006, International Business Machines * +* Corporation and others. All Rights Reserved. * +* * +****************************************************************************** +* file name: ucln_cmn.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2001July05 +* created by: George Rhoten +*/ + +#include "unicode/utypes.h" +#include "unicode/uclean.h" +#include "utracimp.h" +#include "ustr_imp.h" +#include "unormimp.h" +#include "ucln_cmn.h" +#include "umutex.h" +#include "ucln.h" +#include "cmemory.h" +#include "uassert.h" + +static cleanupFunc *gCommonCleanupFunctions[UCLN_COMMON_COUNT]; +static cleanupFunc *gLibCleanupFunctions[UCLN_COMMON]; + +U_CFUNC void +ucln_common_registerCleanup(ECleanupCommonType type, + cleanupFunc *func) +{ + U_ASSERT(UCLN_COMMON_START < type && type < UCLN_COMMON_COUNT); + if (UCLN_COMMON_START < type && type < UCLN_COMMON_COUNT) + { + gCommonCleanupFunctions[type] = func; + } +} + +U_CAPI void U_EXPORT2 +ucln_registerCleanup(ECleanupLibraryType type, + cleanupFunc *func) +{ + U_ASSERT(UCLN_START < type && type < UCLN_COMMON); + if (UCLN_START < type && type < UCLN_COMMON) + { + gLibCleanupFunctions[type] = func; + } +} + +U_CFUNC UBool ucln_lib_cleanup(void) { + ECleanupLibraryType libType = UCLN_START; + ECleanupCommonType commonFunc = UCLN_COMMON_START; + + for (libType++; libType +#endif + +U_CFUNC uint16_t +udata_getHeaderSize(const DataHeader *udh) { + if(udh==NULL) { + return 0; + } else if(udh->info.isBigEndian==U_IS_BIG_ENDIAN) { + /* same endianness */ + return udh->dataHeader.headerSize; + } else { + /* opposite endianness */ + uint16_t x=udh->dataHeader.headerSize; + return (uint16_t)((x<<8)|(x>>8)); + } +} + +U_CFUNC uint16_t +udata_getInfoSize(const UDataInfo *info) { + if(info==NULL) { + return 0; + } else if(info->isBigEndian==U_IS_BIG_ENDIAN) { + /* same endianness */ + return info->size; + } else { + /* opposite endianness */ + uint16_t x=info->size; + return (uint16_t)((x<<8)|(x>>8)); + } +} + +/*-----------------------------------------------------------------------------* + * * + * Pointer TOCs. TODO: This form of table-of-contents should be removed * + * because DLLs must be relocated on loading to correct the * + * pointer values and this operation makes shared memory * + * mapping of the data much less likely to work. * + * * + *-----------------------------------------------------------------------------*/ +typedef struct { + const char *entryName; + const DataHeader *pHeader; +} PointerTOCEntry; + + +typedef struct { + uint32_t count; + uint32_t reserved; + PointerTOCEntry entry[2]; /* Actual size is from count. */ +} PointerTOC; + + +/* definition of OffsetTOC struct types moved to ucmndata.h */ + +/*-----------------------------------------------------------------------------* + * * + * entry point lookup implementations * + * * + *-----------------------------------------------------------------------------*/ +static uint32_t offsetTOCEntryCount(const UDataMemory *pData) { + int32_t retVal=0; + const UDataOffsetTOC *toc = (UDataOffsetTOC *)pData->toc; + if (toc != NULL) { + retVal = toc->count; + } + return retVal; +} + + +static const DataHeader * +offsetTOCLookupFn(const UDataMemory *pData, + const char *tocEntryName, + int32_t *pLength, + UErrorCode *pErrorCode) { + const UDataOffsetTOC *toc = (UDataOffsetTOC *)pData->toc; + if(toc!=NULL) { + const char *base=(const char *)toc; + uint32_t start, limit, number, lastNumber; + int32_t strResult; + const UDataOffsetTOCEntry *entry; + + /* perform a binary search for the data in the common data's table of contents */ +#if defined (UDATA_DEBUG_DUMP) + /* list the contents of the TOC each time .. not recommended */ + for(start=0;startcount;start++) { + fprintf(stderr, "\tx%d: %s\n", start, &base[toc->entry[start].nameOffset]); + } +#endif + + start=0; + limit=toc->count; /* number of names in this table of contents */ + lastNumber=limit; + entry=toc->entry; + for (;;) { + number = (start+limit)/2; + if (lastNumber == number) { /* Have we moved? */ + break; /* We haven't moved, and it wasn't found; */ + /* or the empty stub common data library was used during build. */ + } + lastNumber = number; + strResult = uprv_strcmp(tocEntryName, base+entry[number].nameOffset); + if(strResult<0) { + limit=number; + } else if (strResult>0) { + start=number; + } + else { + /* found it */ +#ifdef UDATA_DEBUG + fprintf(stderr, "%s: Found.\n", tocEntryName); +#endif + entry += number; /* Alias the entry to the current entry. */ + if((number+1) < toc->count) { + *pLength = (int32_t)(entry[1].dataOffset - entry->dataOffset); + } else { + *pLength = -1; + } + return (const DataHeader *)(base+entry->dataOffset); + } + } +#ifdef UDATA_DEBUG + fprintf(stderr, "%s: Not found.\n", tocEntryName); +#endif + return NULL; + } else { +#ifdef UDATA_DEBUG + fprintf(stderr, "returning header\n"); +#endif + + return pData->pHeader; + } +} + + +static uint32_t pointerTOCEntryCount(const UDataMemory *pData) { + const PointerTOC *toc = (PointerTOC *)pData->toc; + return (uint32_t)((toc != NULL) ? (toc->count) : 0); +} + + +static const DataHeader *pointerTOCLookupFn(const UDataMemory *pData, + const char *name, + int32_t *pLength, + UErrorCode *pErrorCode) { + if(pData->toc!=NULL) { + const PointerTOC *toc = (PointerTOC *)pData->toc; + uint32_t start, limit, number, lastNumber; + int32_t strResult; + +#if defined (UDATA_DEBUG_DUMP) + /* list the contents of the TOC each time .. not recommended */ + for(start=0;startcount;start++) { + fprintf(stderr, "\tx%d: %s\n", start, toc->entry[start].entryName); + } +#endif + + /* perform a binary search for the data in the common data's table of contents */ + start=0; + limit=toc->count; + lastNumber=limit; + + for (;;) { + number = (start+limit)/2; + if (lastNumber == number) { /* Have we moved? */ + break; /* We haven't moved, and it wasn't found, */ + /* or the empty stub common data library was used during build. */ + } + lastNumber = number; + strResult = uprv_strcmp(name, toc->entry[number].entryName); + if(strResult<0) { + limit=number; + } else if (strResult>0) { + start=number; + } + else { + /* found it */ +#ifdef UDATA_DEBUG + fprintf(STDErr, "%s: Found.\n", toc->entry[number].entryName); +#endif + *pLength=-1; + return UDataMemory_normalizeDataPointer(toc->entry[number].pHeader); + } + } +#ifdef UDATA_DEBUG + fprintf(stderr, "%s: Not found.\n", name); +#endif + return NULL; + } else { + return pData->pHeader; + } +} + +static const commonDataFuncs CmnDFuncs = {offsetTOCLookupFn, offsetTOCEntryCount}; +static const commonDataFuncs ToCPFuncs = {pointerTOCLookupFn, pointerTOCEntryCount}; + + + +/*----------------------------------------------------------------------* + * * + * checkCommonData Validate the format of a common data file. * + * Fill in the virtual function ptr based on TOC type * + * If the data is invalid, close the UDataMemory * + * and set the appropriate error code. * + * * + *----------------------------------------------------------------------*/ +void udata_checkCommonData(UDataMemory *udm, UErrorCode *err) { + if (U_FAILURE(*err)) { + return; + } + + if(!(udm->pHeader->dataHeader.magic1==0xda && + udm->pHeader->dataHeader.magic2==0x27 && + udm->pHeader->info.isBigEndian==U_IS_BIG_ENDIAN && + udm->pHeader->info.charsetFamily==U_CHARSET_FAMILY) + ) { + /* header not valid */ + *err=U_INVALID_FORMAT_ERROR; + } + else if (udm->pHeader->info.dataFormat[0]==0x43 && + udm->pHeader->info.dataFormat[1]==0x6d && + udm->pHeader->info.dataFormat[2]==0x6e && + udm->pHeader->info.dataFormat[3]==0x44 && + udm->pHeader->info.formatVersion[0]==1 + ) { + /* dataFormat="CmnD" */ + udm->vFuncs = &CmnDFuncs; + udm->toc=(const char *)udm->pHeader+udata_getHeaderSize(udm->pHeader); + } + else if(udm->pHeader->info.dataFormat[0]==0x54 && + udm->pHeader->info.dataFormat[1]==0x6f && + udm->pHeader->info.dataFormat[2]==0x43 && + udm->pHeader->info.dataFormat[3]==0x50 && + udm->pHeader->info.formatVersion[0]==1 + ) { + /* dataFormat="ToCP" */ + udm->vFuncs = &ToCPFuncs; + udm->toc=(const char *)udm->pHeader+udata_getHeaderSize(udm->pHeader); + } + else { + /* dataFormat not recognized */ + *err=U_INVALID_FORMAT_ERROR; + } + + if (U_FAILURE(*err)) { + /* If the data is no good and we memory-mapped it ourselves, + * close the memory mapping so it doesn't leak. Note that this has + * no effect on non-memory mapped data, other than clearing fields in udm. + */ + udata_close(udm); + } +} + +/* + * TODO: Add a udata_swapPackageHeader() function that swaps an ICU .dat package + * header but not its sub-items. + * This function will be needed for automatic runtime swapping. + * Sub-items should not be swapped to limit the swapping to the parts of the + * package that are actually used. + * + * Since lengths of items are implicit in the order and offsets of their + * ToC entries, and since offsets are relative to the start of the ToC, + * a swapped version may need to generate a different data structure + * with pointers to the original data items and with their lengths + * (-1 for the last one if it is not known), and maybe even pointers to the + * swapped versions of the items. + * These pointers to swapped versions would establish a cache; + * instead, each open data item could simply own the storage for its swapped + * data. This fits better with the current design. + * + * markus 2003sep18 Jitterbug 2235 + */ diff --git a/Build/source/libs/icu/icu-xetex/common/ucmndata.h b/Build/source/libs/icu/icu-xetex/common/ucmndata.h new file mode 100644 index 00000000000..9634ed840f6 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucmndata.h @@ -0,0 +1,109 @@ +/* +****************************************************************************** +* +* Copyright (C) 1999-2003, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************/ + + +/*---------------------------------------------------------------------------------- + * + * UCommonData An abstract interface for dealing with ICU Common Data Files. + * ICU Common Data Files are a grouping of a number of individual + * data items (resources, converters, tables, anything) into a + * single file or dll. The combined format includes a table of + * contents for locating the individual items by name. + * + * Two formats for the table of contents are supported, which is + * why there is an abstract inteface involved. + * + * These functions are part of the ICU internal implementation, and + * are not inteded to be used directly by applications. + */ + +#ifndef __UCMNDATA_H__ +#define __UCMNDATA_H__ + +#include "unicode/udata.h" +#include "umapfile.h" + + +#define COMMON_DATA_NAME U_ICUDATA_NAME + +typedef struct { + uint16_t headerSize; + uint8_t magic1; + uint8_t magic2; +} MappedData; + + +typedef struct { + MappedData dataHeader; + UDataInfo info; +} DataHeader; + +typedef struct { + uint32_t nameOffset; + uint32_t dataOffset; +} UDataOffsetTOCEntry; + +typedef struct { + uint32_t count; + UDataOffsetTOCEntry entry[2]; /* Actual size of array is from count. */ +} UDataOffsetTOC; + +/** + * Get the header size from a const DataHeader *udh. + * Handles opposite-endian data. + * + * @internal + */ +U_CFUNC uint16_t +udata_getHeaderSize(const DataHeader *udh); + +/** + * Get the UDataInfo.size from a const UDataInfo *info. + * Handles opposite-endian data. + * + * @internal + */ +U_CFUNC uint16_t +udata_getInfoSize(const UDataInfo *info); + +/* + * "Virtual" functions for data lookup. + * To call one, given a UDataMemory *p, the code looks like this: + * p->vFuncs.Lookup(p, tocEntryName, pErrorCode); + * (I sure do wish this was written in C++, not C) + */ + +typedef const DataHeader * +(* LookupFn)(const UDataMemory *pData, + const char *tocEntryName, + int32_t *pLength, + UErrorCode *pErrorCode); + +typedef uint32_t +(* NumEntriesFn)(const UDataMemory *pData); + +typedef struct { + LookupFn Lookup; + NumEntriesFn NumEntries; +} commonDataFuncs; + + +/* + * Functions to check whether a UDataMemory refers to memory containing + * a recognizable header and table of contents a Common Data Format + * + * If a valid header and TOC are found, + * set the CommonDataFuncs function dispatch vector in the UDataMemory + * to point to the right functions for the TOC type. + * otherwise + * set an errorcode. + */ +void udata_checkCommonData(UDataMemory *pData, UErrorCode *pErrorCode); + + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/ucnv.c b/Build/source/libs/icu/icu-xetex/common/ucnv.c new file mode 100644 index 00000000000..9ef346fb612 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnv.c @@ -0,0 +1,2888 @@ +/* +****************************************************************************** +* +* Copyright (C) 1998-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* +* ucnv.c: +* Implements APIs for the ICU's codeset conversion library; +* mostly calls through internal functions; +* created by Bertrand A. Damiba +* +* Modification History: +* +* Date Name Description +* 04/04/99 helena Fixed internal header inclusion. +* 05/09/00 helena Added implementation to handle fallback mappings. +* 06/20/2000 helena OS/400 port changes; mostly typecast. +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "unicode/ustring.h" +#include "unicode/ucnv.h" +#include "unicode/ucnv_err.h" +#include "unicode/uset.h" +#include "putilimp.h" +#include "cmemory.h" +#include "cstring.h" +#include "uassert.h" +#include "utracimp.h" +#include "ustr_imp.h" +#include "ucnv_imp.h" +#include "ucnv_cnv.h" +#include "ucnv_bld.h" + +/* size of intermediate and preflighting buffers in ucnv_convert() */ +#define CHUNK_SIZE 1024 + +typedef struct UAmbiguousConverter { + const char *name; + const UChar variant5c; +} UAmbiguousConverter; + +static const UAmbiguousConverter ambiguousConverters[]={ + { "ibm-897_P100-1995", 0xa5 }, + { "ibm-942_P120-1999", 0xa5 }, + { "ibm-943_P130-1999", 0xa5 }, + { "ibm-946_P100-1995", 0xa5 }, + { "ibm-33722_P120-1999", 0xa5 }, + /*{ "ibm-54191_P100-2006", 0xa5 },*/ + /*{ "ibm-62383_P100-2007", 0xa5 },*/ + /*{ "ibm-891_P100-1995", 0x20a9 },*/ + { "ibm-944_P100-1995", 0x20a9 }, + { "ibm-949_P110-1999", 0x20a9 }, + { "ibm-1363_P110-1997", 0x20a9 }, + { "ISO_2022,locale=ko,version=0", 0x20a9 } +}; + +/*Calls through createConverter */ +U_CAPI UConverter* U_EXPORT2 +ucnv_open (const char *name, + UErrorCode * err) +{ + UConverter *r; + + if (err == NULL || U_FAILURE (*err)) { + return NULL; + } + + r = ucnv_createConverter(NULL, name, err); + return r; +} + +U_CAPI UConverter* U_EXPORT2 +ucnv_openPackage (const char *packageName, const char *converterName, UErrorCode * err) +{ + return ucnv_createConverterFromPackage(packageName, converterName, err); +} + +/*Extracts the UChar* to a char* and calls through createConverter */ +U_CAPI UConverter* U_EXPORT2 +ucnv_openU (const UChar * name, + UErrorCode * err) +{ + char asciiName[UCNV_MAX_CONVERTER_NAME_LENGTH]; + + if (err == NULL || U_FAILURE(*err)) + return NULL; + if (name == NULL) + return ucnv_open (NULL, err); + if (u_strlen(name) >= UCNV_MAX_CONVERTER_NAME_LENGTH) + { + *err = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + return ucnv_open(u_austrcpy(asciiName, name), err); +} + +/* Copy the string that is represented by the UConverterPlatform enum + * @param platformString An output buffer + * @param platform An enum representing a platform + * @return the length of the copied string. + */ +static int32_t +ucnv_copyPlatformString(char *platformString, UConverterPlatform pltfrm) +{ + switch (pltfrm) + { + case UCNV_IBM: + uprv_strcpy(platformString, "ibm-"); + return 4; + case UCNV_UNKNOWN: + break; + } + + /* default to empty string */ + *platformString = 0; + return 0; +} + +/*Assumes a $platform-#codepage.$CONVERTER_FILE_EXTENSION scheme and calls + *through createConverter*/ +U_CAPI UConverter* U_EXPORT2 +ucnv_openCCSID (int32_t codepage, + UConverterPlatform platform, + UErrorCode * err) +{ + char myName[UCNV_MAX_CONVERTER_NAME_LENGTH]; + int32_t myNameLen; + + if (err == NULL || U_FAILURE (*err)) + return NULL; + + /* ucnv_copyPlatformString could return "ibm-" or "cp" */ + myNameLen = ucnv_copyPlatformString(myName, platform); + T_CString_integerToString(myName + myNameLen, codepage, 10); + + return ucnv_createConverter(NULL, myName, err); +} + +/* Creating a temporary stack-based object that can be used in one thread, +and created from a converter that is shared across threads. +*/ + +U_CAPI UConverter* U_EXPORT2 +ucnv_safeClone(const UConverter* cnv, void *stackBuffer, int32_t *pBufferSize, UErrorCode *status) +{ + UConverter *localConverter, *allocatedConverter; + int32_t bufferSizeNeeded; + char *stackBufferChars = (char *)stackBuffer; + UErrorCode cbErr; + UConverterToUnicodeArgs toUArgs = { + sizeof(UConverterToUnicodeArgs), + TRUE, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + }; + UConverterFromUnicodeArgs fromUArgs = { + sizeof(UConverterFromUnicodeArgs), + TRUE, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + }; + + UTRACE_ENTRY_OC(UTRACE_UCNV_CLONE); + + if (status == NULL || U_FAILURE(*status)){ + UTRACE_EXIT_STATUS(status? *status: U_ILLEGAL_ARGUMENT_ERROR); + return 0; + } + + if (!pBufferSize || !cnv){ + *status = U_ILLEGAL_ARGUMENT_ERROR; + UTRACE_EXIT_STATUS(*status); + return 0; + } + + UTRACE_DATA3(UTRACE_OPEN_CLOSE, "clone converter %s at %p into stackBuffer %p", + ucnv_getName(cnv, status), cnv, stackBuffer); + + if (cnv->sharedData->impl->safeClone != NULL) { + /* call the custom safeClone function for sizing */ + bufferSizeNeeded = 0; + cnv->sharedData->impl->safeClone(cnv, NULL, &bufferSizeNeeded, status); + } + else + { + /* inherent sizing */ + bufferSizeNeeded = sizeof(UConverter); + } + + if (*pBufferSize <= 0){ /* 'preflighting' request - set needed size into *pBufferSize */ + *pBufferSize = bufferSizeNeeded; + UTRACE_EXIT_VALUE(bufferSizeNeeded); + return 0; + } + + + /* Pointers on 64-bit platforms need to be aligned + * on a 64-bit boundary in memory. + */ + if (U_ALIGNMENT_OFFSET(stackBuffer) != 0) { + int32_t offsetUp = (int32_t)U_ALIGNMENT_OFFSET_UP(stackBufferChars); + if(*pBufferSize > offsetUp) { + *pBufferSize -= offsetUp; + stackBufferChars += offsetUp; + } else { + /* prevent using the stack buffer but keep the size > 0 so that we do not just preflight */ + *pBufferSize = 1; + } + } + + stackBuffer = (void *)stackBufferChars; + + /* Now, see if we must allocate any memory */ + if (*pBufferSize < bufferSizeNeeded || stackBuffer == NULL) + { + /* allocate one here...*/ + localConverter = allocatedConverter = (UConverter *) uprv_malloc (bufferSizeNeeded); + + if(localConverter == NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + UTRACE_EXIT_STATUS(*status); + return NULL; + } + + if (U_SUCCESS(*status)) { + *status = U_SAFECLONE_ALLOCATED_WARNING; + } + + /* record the fact that memory was allocated */ + *pBufferSize = bufferSizeNeeded; + } else { + /* just use the stack buffer */ + localConverter = (UConverter*) stackBuffer; + allocatedConverter = NULL; + } + + uprv_memset(localConverter, 0, bufferSizeNeeded); + + /* Copy initial state */ + uprv_memcpy(localConverter, cnv, sizeof(UConverter)); + localConverter->isCopyLocal = localConverter->isExtraLocal = FALSE; + + /* copy the substitution string */ + if (cnv->subChars == (uint8_t *)cnv->subUChars) { + localConverter->subChars = (uint8_t *)localConverter->subUChars; + } else { + localConverter->subChars = (uint8_t *)uprv_malloc(UCNV_ERROR_BUFFER_LENGTH * U_SIZEOF_UCHAR); + if (localConverter->subChars == NULL) { + uprv_free(allocatedConverter); + UTRACE_EXIT_STATUS(*status); + return NULL; + } + uprv_memcpy(localConverter->subChars, cnv->subChars, UCNV_ERROR_BUFFER_LENGTH * U_SIZEOF_UCHAR); + } + + /* now either call the safeclone fcn or not */ + if (cnv->sharedData->impl->safeClone != NULL) { + /* call the custom safeClone function */ + localConverter = cnv->sharedData->impl->safeClone(cnv, localConverter, pBufferSize, status); + } + + if(localConverter==NULL || U_FAILURE(*status)) { + if (allocatedConverter != NULL && allocatedConverter->subChars != (uint8_t *)allocatedConverter->subUChars) { + uprv_free(allocatedConverter->subChars); + } + uprv_free(allocatedConverter); + UTRACE_EXIT_STATUS(*status); + return NULL; + } + + /* increment refcount of shared data if needed */ + /* + Checking whether it's an algorithic converter is okay + in multithreaded applications because the value never changes. + Don't check referenceCounter for any other value. + */ + if (cnv->sharedData->referenceCounter != ~0) { + ucnv_incrementRefCount(cnv->sharedData); + } + + if(localConverter == (UConverter*)stackBuffer) { + /* we're using user provided data - set to not destroy */ + localConverter->isCopyLocal = TRUE; + } + + /* allow callback functions to handle any memory allocation */ + toUArgs.converter = fromUArgs.converter = localConverter; + cbErr = U_ZERO_ERROR; + cnv->fromCharErrorBehaviour(cnv->toUContext, &toUArgs, NULL, 0, UCNV_CLONE, &cbErr); + cbErr = U_ZERO_ERROR; + cnv->fromUCharErrorBehaviour(cnv->fromUContext, &fromUArgs, NULL, 0, 0, UCNV_CLONE, &cbErr); + + UTRACE_EXIT_PTR_STATUS(localConverter, *status); + return localConverter; +} + + + +/*Decreases the reference counter in the shared immutable section of the object + *and frees the mutable part*/ + +U_CAPI void U_EXPORT2 +ucnv_close (UConverter * converter) +{ + UErrorCode errorCode = U_ZERO_ERROR; + + UTRACE_ENTRY_OC(UTRACE_UCNV_CLOSE); + + if (converter == NULL) + { + UTRACE_EXIT(); + return; + } + + UTRACE_DATA3(UTRACE_OPEN_CLOSE, "close converter %s at %p, isCopyLocal=%b", + ucnv_getName(converter, &errorCode), converter, converter->isCopyLocal); + + /* In order to speed up the close, only call the callbacks when they have been changed. + This performance check will only work when the callbacks are set within a shared library + or from user code that statically links this code. */ + /* first, notify the callback functions that the converter is closed */ + if (converter->fromCharErrorBehaviour != UCNV_TO_U_DEFAULT_CALLBACK) { + UConverterToUnicodeArgs toUArgs = { + sizeof(UConverterToUnicodeArgs), + TRUE, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + }; + + toUArgs.converter = converter; + errorCode = U_ZERO_ERROR; + converter->fromCharErrorBehaviour(converter->toUContext, &toUArgs, NULL, 0, UCNV_CLOSE, &errorCode); + } + if (converter->fromUCharErrorBehaviour != UCNV_FROM_U_DEFAULT_CALLBACK) { + UConverterFromUnicodeArgs fromUArgs = { + sizeof(UConverterFromUnicodeArgs), + TRUE, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + }; + fromUArgs.converter = converter; + errorCode = U_ZERO_ERROR; + converter->fromUCharErrorBehaviour(converter->fromUContext, &fromUArgs, NULL, 0, 0, UCNV_CLOSE, &errorCode); + } + + if (converter->sharedData->impl->close != NULL) { + converter->sharedData->impl->close(converter); + } + + if (converter->subChars != (uint8_t *)converter->subUChars) { + uprv_free(converter->subChars); + } + + /* + Checking whether it's an algorithic converter is okay + in multithreaded applications because the value never changes. + Don't check referenceCounter for any other value. + */ + if (converter->sharedData->referenceCounter != ~0) { + ucnv_unloadSharedDataIfReady(converter->sharedData); + } + + if(!converter->isCopyLocal){ + uprv_free(converter); + } + + UTRACE_EXIT(); +} + +/*returns a single Name from the list, will return NULL if out of bounds + */ +U_CAPI const char* U_EXPORT2 +ucnv_getAvailableName (int32_t n) +{ + if (0 <= n && n <= 0xffff) { + UErrorCode err = U_ZERO_ERROR; + const char *name = ucnv_bld_getAvailableConverter((uint16_t)n, &err); + if (U_SUCCESS(err)) { + return name; + } + } + return NULL; +} + +U_CAPI int32_t U_EXPORT2 +ucnv_countAvailable () +{ + UErrorCode err = U_ZERO_ERROR; + return ucnv_bld_countAvailableConverters(&err); +} + +U_CAPI void U_EXPORT2 +ucnv_getSubstChars (const UConverter * converter, + char *mySubChar, + int8_t * len, + UErrorCode * err) +{ + if (U_FAILURE (*err)) + return; + + if (converter->subCharLen <= 0) { + /* Unicode string or empty string from ucnv_setSubstString(). */ + *len = 0; + return; + } + + if (*len < converter->subCharLen) /*not enough space in subChars */ + { + *err = U_INDEX_OUTOFBOUNDS_ERROR; + return; + } + + uprv_memcpy (mySubChar, converter->subChars, converter->subCharLen); /*fills in the subchars */ + *len = converter->subCharLen; /*store # of bytes copied to buffer */ +} + +U_CAPI void U_EXPORT2 +ucnv_setSubstChars (UConverter * converter, + const char *mySubChar, + int8_t len, + UErrorCode * err) +{ + if (U_FAILURE (*err)) + return; + + /*Makes sure that the subChar is within the codepages char length boundaries */ + if ((len > converter->sharedData->staticData->maxBytesPerChar) + || (len < converter->sharedData->staticData->minBytesPerChar)) + { + *err = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + uprv_memcpy (converter->subChars, mySubChar, len); /*copies the subchars */ + converter->subCharLen = len; /*sets the new len */ + + /* + * There is currently (2001Feb) no separate API to set/get subChar1. + * In order to always have subChar written after it is explicitly set, + * we set subChar1 to 0. + */ + converter->subChar1 = 0; + + return; +} + +U_CAPI void U_EXPORT2 +ucnv_setSubstString(UConverter *cnv, + const UChar *s, + int32_t length, + UErrorCode *err) { + UAlignedMemory cloneBuffer[U_CNV_SAFECLONE_BUFFERSIZE / sizeof(UAlignedMemory) + 1]; + char chars[UCNV_ERROR_BUFFER_LENGTH]; + + UConverter *clone; + uint8_t *subChars; + int32_t cloneSize, length8; + + /* Let the following functions check all arguments. */ + cloneSize = sizeof(cloneBuffer); + clone = ucnv_safeClone(cnv, cloneBuffer, &cloneSize, err); + ucnv_setFromUCallBack(clone, UCNV_FROM_U_CALLBACK_STOP, NULL, NULL, NULL, err); + length8 = ucnv_fromUChars(clone, chars, (int32_t)sizeof(chars), s, length, err); + ucnv_close(clone); + if (U_FAILURE(*err)) { + return; + } + + if (cnv->sharedData->impl->writeSub == NULL +#if !UCONFIG_NO_LEGACY_CONVERSION + || (cnv->sharedData->staticData->conversionType == UCNV_MBCS && + ucnv_MBCSGetType(cnv) != UCNV_EBCDIC_STATEFUL) +#endif + ) { + /* The converter is not stateful. Store the charset bytes as a fixed string. */ + subChars = (uint8_t *)chars; + } else { + /* + * The converter has a non-default writeSub() function, indicating + * that it is stateful. + * Store the Unicode string for on-the-fly conversion for correct + * state handling. + */ + if (length > UCNV_ERROR_BUFFER_LENGTH) { + /* + * Should not occur. The converter should output at least one byte + * per UChar, which means that ucnv_fromUChars() should catch all + * overflows. + */ + *err = U_BUFFER_OVERFLOW_ERROR; + return; + } + subChars = (uint8_t *)s; + if (length < 0) { + length = u_strlen(s); + } + length8 = length * U_SIZEOF_UCHAR; + } + + /* + * For storing the substitution string, select either the small buffer inside + * UConverter or allocate a subChars buffer. + */ + if (length8 > UCNV_MAX_SUBCHAR_LEN) { + /* Use a separate buffer for the string. Outside UConverter to not make it too large. */ + if (cnv->subChars == (uint8_t *)cnv->subUChars) { + /* Allocate a new buffer for the string. */ + cnv->subChars = (uint8_t *)uprv_malloc(UCNV_ERROR_BUFFER_LENGTH * U_SIZEOF_UCHAR); + if (cnv->subChars == NULL) { + cnv->subChars = (uint8_t *)cnv->subUChars; + *err = U_MEMORY_ALLOCATION_ERROR; + return; + } + uprv_memset(cnv->subChars, 0, UCNV_ERROR_BUFFER_LENGTH * U_SIZEOF_UCHAR); + } + } + + /* Copy the substitution string into the UConverter or its subChars buffer. */ + if (length8 == 0) { + cnv->subCharLen = 0; + } else { + uprv_memcpy(cnv->subChars, subChars, length8); + if (subChars == (uint8_t *)chars) { + cnv->subCharLen = (int8_t)length8; + } else /* subChars == s */ { + cnv->subCharLen = (int8_t)-length; + } + } + + /* See comment in ucnv_setSubstChars(). */ + cnv->subChar1 = 0; +} + +/*resets the internal states of a converter + *goal : have the same behaviour than a freshly created converter + */ +static void _reset(UConverter *converter, UConverterResetChoice choice, + UBool callCallback) { + if(converter == NULL) { + return; + } + + if(callCallback) { + /* first, notify the callback functions that the converter is reset */ + UConverterToUnicodeArgs toUArgs = { + sizeof(UConverterToUnicodeArgs), + TRUE, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + }; + UConverterFromUnicodeArgs fromUArgs = { + sizeof(UConverterFromUnicodeArgs), + TRUE, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + }; + UErrorCode errorCode; + + toUArgs.converter = fromUArgs.converter = converter; + if(choice<=UCNV_RESET_TO_UNICODE) { + errorCode = U_ZERO_ERROR; + converter->fromCharErrorBehaviour(converter->toUContext, &toUArgs, NULL, 0, UCNV_RESET, &errorCode); + } + if(choice!=UCNV_RESET_TO_UNICODE) { + errorCode = U_ZERO_ERROR; + converter->fromUCharErrorBehaviour(converter->fromUContext, &fromUArgs, NULL, 0, 0, UCNV_RESET, &errorCode); + } + } + + /* now reset the converter itself */ + if(choice<=UCNV_RESET_TO_UNICODE) { + converter->toUnicodeStatus = converter->sharedData->toUnicodeStatus; + converter->mode = 0; + converter->toULength = 0; + converter->invalidCharLength = converter->UCharErrorBufferLength = 0; + converter->preToULength = 0; + } + if(choice!=UCNV_RESET_TO_UNICODE) { + converter->fromUnicodeStatus = 0; + converter->fromUChar32 = 0; + converter->invalidUCharLength = converter->charErrorBufferLength = 0; + converter->preFromUFirstCP = U_SENTINEL; + converter->preFromULength = 0; + } + + if (converter->sharedData->impl->reset != NULL) { + /* call the custom reset function */ + converter->sharedData->impl->reset(converter, choice); + } +} + +U_CAPI void U_EXPORT2 +ucnv_reset(UConverter *converter) +{ + _reset(converter, UCNV_RESET_BOTH, TRUE); +} + +U_CAPI void U_EXPORT2 +ucnv_resetToUnicode(UConverter *converter) +{ + _reset(converter, UCNV_RESET_TO_UNICODE, TRUE); +} + +U_CAPI void U_EXPORT2 +ucnv_resetFromUnicode(UConverter *converter) +{ + _reset(converter, UCNV_RESET_FROM_UNICODE, TRUE); +} + +U_CAPI int8_t U_EXPORT2 +ucnv_getMaxCharSize (const UConverter * converter) +{ + return converter->maxBytesPerUChar; +} + + +U_CAPI int8_t U_EXPORT2 +ucnv_getMinCharSize (const UConverter * converter) +{ + return converter->sharedData->staticData->minBytesPerChar; +} + +U_CAPI const char* U_EXPORT2 +ucnv_getName (const UConverter * converter, UErrorCode * err) + +{ + if (U_FAILURE (*err)) + return NULL; + if(converter->sharedData->impl->getName){ + const char* temp= converter->sharedData->impl->getName(converter); + if(temp) + return temp; + } + return converter->sharedData->staticData->name; +} + +U_CAPI int32_t U_EXPORT2 +ucnv_getCCSID(const UConverter * converter, + UErrorCode * err) +{ + int32_t ccsid; + if (U_FAILURE (*err)) + return -1; + + ccsid = converter->sharedData->staticData->codepage; + if (ccsid == 0) { + /* Rare case. This is for cases like gb18030, + which doesn't have an IBM cannonical name, but does have an IBM alias. */ + const char *standardName = ucnv_getStandardName(ucnv_getName(converter, err), "IBM", err); + if (U_SUCCESS(*err) && standardName) { + const char *ccsidStr = uprv_strchr(standardName, '-'); + if (ccsidStr) { + ccsid = (int32_t)atol(ccsidStr+1); /* +1 to skip '-' */ + } + } + } + return ccsid; +} + + +U_CAPI UConverterPlatform U_EXPORT2 +ucnv_getPlatform (const UConverter * converter, + UErrorCode * err) +{ + if (U_FAILURE (*err)) + return UCNV_UNKNOWN; + + return (UConverterPlatform)converter->sharedData->staticData->platform; +} + +U_CAPI void U_EXPORT2 + ucnv_getToUCallBack (const UConverter * converter, + UConverterToUCallback *action, + const void **context) +{ + *action = converter->fromCharErrorBehaviour; + *context = converter->toUContext; +} + +U_CAPI void U_EXPORT2 + ucnv_getFromUCallBack (const UConverter * converter, + UConverterFromUCallback *action, + const void **context) +{ + *action = converter->fromUCharErrorBehaviour; + *context = converter->fromUContext; +} + +U_CAPI void U_EXPORT2 +ucnv_setToUCallBack (UConverter * converter, + UConverterToUCallback newAction, + const void* newContext, + UConverterToUCallback *oldAction, + const void** oldContext, + UErrorCode * err) +{ + if (U_FAILURE (*err)) + return; + if (oldAction) *oldAction = converter->fromCharErrorBehaviour; + converter->fromCharErrorBehaviour = newAction; + if (oldContext) *oldContext = converter->toUContext; + converter->toUContext = newContext; +} + +U_CAPI void U_EXPORT2 +ucnv_setFromUCallBack (UConverter * converter, + UConverterFromUCallback newAction, + const void* newContext, + UConverterFromUCallback *oldAction, + const void** oldContext, + UErrorCode * err) +{ + if (U_FAILURE (*err)) + return; + if (oldAction) *oldAction = converter->fromUCharErrorBehaviour; + converter->fromUCharErrorBehaviour = newAction; + if (oldContext) *oldContext = converter->fromUContext; + converter->fromUContext = newContext; +} + +static void +_updateOffsets(int32_t *offsets, int32_t length, + int32_t sourceIndex, int32_t errorInputLength) { + int32_t *limit; + int32_t delta, offset; + + if(sourceIndex>=0) { + /* + * adjust each offset by adding the previous sourceIndex + * minus the length of the input sequence that caused an + * error, if any + */ + delta=sourceIndex-errorInputLength; + } else { + /* + * set each offset to -1 because this conversion function + * does not handle offsets + */ + delta=-1; + } + + limit=offsets+length; + if(delta==0) { + /* most common case, nothing to do */ + } else if(delta>0) { + /* add the delta to each offset (but not if the offset is <0) */ + while(offsets=0) { + *offsets=offset+delta; + } + ++offsets; + } + } else /* delta<0 */ { + /* + * set each offset to -1 because this conversion function + * does not handle offsets + * or the error input sequence started in a previous buffer + */ + while(offsetsconverter; + s=pArgs->source; + t=pArgs->target; + offsets=pArgs->offsets; + + /* get the converter implementation function */ + sourceIndex=0; + if(offsets==NULL) { + fromUnicode=cnv->sharedData->impl->fromUnicode; + } else { + fromUnicode=cnv->sharedData->impl->fromUnicodeWithOffsets; + if(fromUnicode==NULL) { + /* there is no WithOffsets implementation */ + fromUnicode=cnv->sharedData->impl->fromUnicode; + /* we will write -1 for each offset */ + sourceIndex=-1; + } + } + + if(cnv->preFromULength>=0) { + /* normal mode */ + realSource=NULL; + + /* avoid compiler warnings - not otherwise necessary, and the values do not matter */ + realSourceLimit=NULL; + realFlush=FALSE; + realSourceIndex=0; + } else { + /* + * Previous m:n conversion stored source units from a partial match + * and failed to consume all of them. + * We need to "replay" them from a temporary buffer and convert them first. + */ + realSource=pArgs->source; + realSourceLimit=pArgs->sourceLimit; + realFlush=pArgs->flush; + realSourceIndex=sourceIndex; + + uprv_memcpy(replay, cnv->preFromU, -cnv->preFromULength*U_SIZEOF_UCHAR); + pArgs->source=replay; + pArgs->sourceLimit=replay-cnv->preFromULength; + pArgs->flush=FALSE; + sourceIndex=-1; + + cnv->preFromULength=0; + } + + /* + * loop for conversion and error handling + * + * loop { + * convert + * loop { + * update offsets + * handle end of input + * handle errors/call callback + * } + * } + */ + for(;;) { + if(U_SUCCESS(*err)) { + /* convert */ + fromUnicode(pArgs, err); + + /* + * set a flag for whether the converter + * successfully processed the end of the input + * + * need not check cnv->preFromULength==0 because a replay (<0) will cause + * sflush && pArgs->source==pArgs->sourceLimit && + cnv->fromUChar32==0); + } else { + /* handle error from ucnv_convertEx() */ + converterSawEndOfInput=FALSE; + } + + /* no callback called yet for this iteration */ + calledCallback=FALSE; + + /* no sourceIndex adjustment for conversion, only for callback output */ + errorInputLength=0; + + /* + * loop for offsets and error handling + * + * iterates at most 3 times: + * 1. to clean up after the conversion function + * 2. after the callback + * 3. after the callback again if there was truncated input + */ + for(;;) { + /* update offsets if we write any */ + if(offsets!=NULL) { + int32_t length=(int32_t)(pArgs->target-t); + if(length>0) { + _updateOffsets(offsets, length, sourceIndex, errorInputLength); + + /* + * if a converter handles offsets and updates the offsets + * pointer at the end, then pArgs->offset should not change + * here; + * however, some converters do not handle offsets at all + * (sourceIndex<0) or may not update the offsets pointer + */ + pArgs->offsets=offsets+=length; + } + + if(sourceIndex>=0) { + sourceIndex+=(int32_t)(pArgs->source-s); + } + } + + if(cnv->preFromULength<0) { + /* + * switch the source to new replay units (cannot occur while replaying) + * after offset handling and before end-of-input and callback handling + */ + if(realSource==NULL) { + realSource=pArgs->source; + realSourceLimit=pArgs->sourceLimit; + realFlush=pArgs->flush; + realSourceIndex=sourceIndex; + + uprv_memcpy(replay, cnv->preFromU, -cnv->preFromULength*U_SIZEOF_UCHAR); + pArgs->source=replay; + pArgs->sourceLimit=replay-cnv->preFromULength; + pArgs->flush=FALSE; + if((sourceIndex+=cnv->preFromULength)<0) { + sourceIndex=-1; + } + + cnv->preFromULength=0; + } else { + /* see implementation note before _fromUnicodeWithCallback() */ + U_ASSERT(realSource==NULL); + *err=U_INTERNAL_PROGRAM_ERROR; + } + } + + /* update pointers */ + s=pArgs->source; + t=pArgs->target; + + if(U_SUCCESS(*err)) { + if(ssourceLimit) { + /* + * continue with the conversion loop while there is still input left + * (continue converting by breaking out of only the inner loop) + */ + break; + } else if(realSource!=NULL) { + /* switch back from replaying to the real source and continue */ + pArgs->source=realSource; + pArgs->sourceLimit=realSourceLimit; + pArgs->flush=realFlush; + sourceIndex=realSourceIndex; + + realSource=NULL; + break; + } else if(pArgs->flush && cnv->fromUChar32!=0) { + /* + * the entire input stream is consumed + * and there is a partial, truncated input sequence left + */ + + /* inject an error and continue with callback handling */ + *err=U_TRUNCATED_CHAR_FOUND; + calledCallback=FALSE; /* new error condition */ + } else { + /* input consumed */ + if(pArgs->flush) { + /* + * return to the conversion loop once more if the flush + * flag is set and the conversion function has not + * successfully processed the end of the input yet + * + * (continue converting by breaking out of only the inner loop) + */ + if(!converterSawEndOfInput) { + break; + } + + /* reset the converter without calling the callback function */ + _reset(cnv, UCNV_RESET_FROM_UNICODE, FALSE); + } + + /* done successfully */ + return; + } + } + + /* U_FAILURE(*err) */ + { + UErrorCode e; + + if( calledCallback || + (e=*err)==U_BUFFER_OVERFLOW_ERROR || + (e!=U_INVALID_CHAR_FOUND && + e!=U_ILLEGAL_CHAR_FOUND && + e!=U_TRUNCATED_CHAR_FOUND) + ) { + /* + * the callback did not or cannot resolve the error: + * set output pointers and return + * + * the check for buffer overflow is redundant but it is + * a high-runner case and hopefully documents the intent + * well + * + * if we were replaying, then the replay buffer must be + * copied back into the UConverter + * and the real arguments must be restored + */ + if(realSource!=NULL) { + int32_t length; + + U_ASSERT(cnv->preFromULength==0); + + length=(int32_t)(pArgs->sourceLimit-pArgs->source); + if(length>0) { + uprv_memcpy(cnv->preFromU, pArgs->source, length*U_SIZEOF_UCHAR); + cnv->preFromULength=(int8_t)-length; + } + + pArgs->source=realSource; + pArgs->sourceLimit=realSourceLimit; + pArgs->flush=realFlush; + } + + return; + } + } + + /* callback handling */ + { + UChar32 codePoint; + + /* get and write the code point */ + codePoint=cnv->fromUChar32; + errorInputLength=0; + U16_APPEND_UNSAFE(cnv->invalidUCharBuffer, errorInputLength, codePoint); + cnv->invalidUCharLength=(int8_t)errorInputLength; + + /* set the converter state to deal with the next character */ + cnv->fromUChar32=0; + + /* call the callback function */ + cnv->fromUCharErrorBehaviour(cnv->fromUContext, pArgs, + cnv->invalidUCharBuffer, errorInputLength, codePoint, + *err==U_INVALID_CHAR_FOUND ? UCNV_UNASSIGNED : UCNV_ILLEGAL, + err); + } + + /* + * loop back to the offset handling + * + * this flag will indicate after offset handling + * that a callback was called; + * if the callback did not resolve the error, then we return + */ + calledCallback=TRUE; + } + } +} + +/* + * Output the fromUnicode overflow buffer. + * Call this function if(cnv->charErrorBufferLength>0). + * @return TRUE if overflow + */ +static UBool +ucnv_outputOverflowFromUnicode(UConverter *cnv, + char **target, const char *targetLimit, + int32_t **pOffsets, + UErrorCode *err) { + int32_t *offsets; + char *overflow, *t; + int32_t i, length; + + t=*target; + if(pOffsets!=NULL) { + offsets=*pOffsets; + } else { + offsets=NULL; + } + + overflow=(char *)cnv->charErrorBuffer; + length=cnv->charErrorBufferLength; + i=0; + while(icharErrorBufferLength=(int8_t)j; + *target=t; + if(offsets!=NULL) { + *pOffsets=offsets; + } + *err=U_BUFFER_OVERFLOW_ERROR; + return TRUE; + } + + /* copy the overflow contents to the target */ + *t++=overflow[i++]; + if(offsets!=NULL) { + *offsets++=-1; /* no source index available for old output */ + } + } + + /* the overflow buffer is completely copied to the target */ + cnv->charErrorBufferLength=0; + *target=t; + if(offsets!=NULL) { + *pOffsets=offsets; + } + return FALSE; +} + +U_CAPI void U_EXPORT2 +ucnv_fromUnicode(UConverter *cnv, + char **target, const char *targetLimit, + const UChar **source, const UChar *sourceLimit, + int32_t *offsets, + UBool flush, + UErrorCode *err) { + UConverterFromUnicodeArgs args; + const UChar *s; + char *t; + + /* check parameters */ + if(err==NULL || U_FAILURE(*err)) { + return; + } + + if(cnv==NULL || target==NULL || source==NULL) { + *err=U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + s=*source; + t=*target; + + if ((const void *)U_MAX_PTR(sourceLimit) == (const void *)sourceLimit) { + /* + Prevent code from going into an infinite loop in case we do hit this + limit. The limit pointer is expected to be on a UChar * boundary. + This also prevents the next argument check from failing. + */ + sourceLimit = (const UChar *)(((const char *)sourceLimit) - 1); + } + + /* + * All these conditions should never happen. + * + * 1) Make sure that the limits are >= to the address source or target + * + * 2) Make sure that the buffer sizes do not exceed the number range for + * int32_t because some functions use the size (in units or bytes) + * rather than comparing pointers, and because offsets are int32_t values. + * + * size_t is guaranteed to be unsigned and large enough for the job. + * + * Return with an error instead of adjusting the limits because we would + * not be able to maintain the semantics that either the source must be + * consumed or the target filled (unless an error occurs). + * An adjustment would be targetLimit=t+0x7fffffff; for example. + * + * 3) Make sure that the user didn't incorrectly cast a UChar * pointer + * to a char * pointer and provide an incomplete UChar code unit. + */ + if (sourceLimit(size_t)0x3fffffff && sourceLimit>s) || + ((size_t)(targetLimit-t)>(size_t)0x7fffffff && targetLimit>t) || + (((const char *)sourceLimit-(const char *)s) & 1) != 0) + { + *err=U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + /* output the target overflow buffer */ + if( cnv->charErrorBufferLength>0 && + ucnv_outputOverflowFromUnicode(cnv, target, targetLimit, &offsets, err) + ) { + /* U_BUFFER_OVERFLOW_ERROR */ + return; + } + /* *target may have moved, therefore stop using t */ + + if(!flush && s==sourceLimit && cnv->preFromULength>=0) { + /* the overflow buffer is emptied and there is no new input: we are done */ + return; + } + + /* + * Do not simply return with a buffer overflow error if + * !flush && t==targetLimit + * because it is possible that the source will not generate any output. + * For example, the skip callback may be called; + * it does not output anything. + */ + + /* prepare the converter arguments */ + args.converter=cnv; + args.flush=flush; + args.offsets=offsets; + args.source=s; + args.sourceLimit=sourceLimit; + args.target=*target; + args.targetLimit=targetLimit; + args.size=sizeof(args); + + _fromUnicodeWithCallback(&args, err); + + *source=args.source; + *target=args.target; +} + +/* ucnv_toUnicode() --------------------------------------------------------- */ + +static void +_toUnicodeWithCallback(UConverterToUnicodeArgs *pArgs, UErrorCode *err) { + UConverterToUnicode toUnicode; + UConverter *cnv; + const char *s; + UChar *t; + int32_t *offsets; + int32_t sourceIndex; + int32_t errorInputLength; + UBool converterSawEndOfInput, calledCallback; + + /* variables for m:n conversion */ + char replay[UCNV_EXT_MAX_BYTES]; + const char *realSource, *realSourceLimit; + int32_t realSourceIndex; + UBool realFlush; + + cnv=pArgs->converter; + s=pArgs->source; + t=pArgs->target; + offsets=pArgs->offsets; + + /* get the converter implementation function */ + sourceIndex=0; + if(offsets==NULL) { + toUnicode=cnv->sharedData->impl->toUnicode; + } else { + toUnicode=cnv->sharedData->impl->toUnicodeWithOffsets; + if(toUnicode==NULL) { + /* there is no WithOffsets implementation */ + toUnicode=cnv->sharedData->impl->toUnicode; + /* we will write -1 for each offset */ + sourceIndex=-1; + } + } + + if(cnv->preToULength>=0) { + /* normal mode */ + realSource=NULL; + + /* avoid compiler warnings - not otherwise necessary, and the values do not matter */ + realSourceLimit=NULL; + realFlush=FALSE; + realSourceIndex=0; + } else { + /* + * Previous m:n conversion stored source units from a partial match + * and failed to consume all of them. + * We need to "replay" them from a temporary buffer and convert them first. + */ + realSource=pArgs->source; + realSourceLimit=pArgs->sourceLimit; + realFlush=pArgs->flush; + realSourceIndex=sourceIndex; + + uprv_memcpy(replay, cnv->preToU, -cnv->preToULength); + pArgs->source=replay; + pArgs->sourceLimit=replay-cnv->preToULength; + pArgs->flush=FALSE; + sourceIndex=-1; + + cnv->preToULength=0; + } + + /* + * loop for conversion and error handling + * + * loop { + * convert + * loop { + * update offsets + * handle end of input + * handle errors/call callback + * } + * } + */ + for(;;) { + if(U_SUCCESS(*err)) { + /* convert */ + toUnicode(pArgs, err); + + /* + * set a flag for whether the converter + * successfully processed the end of the input + * + * need not check cnv->preToULength==0 because a replay (<0) will cause + * sflush && pArgs->source==pArgs->sourceLimit && + cnv->toULength==0); + } else { + /* handle error from getNextUChar() or ucnv_convertEx() */ + converterSawEndOfInput=FALSE; + } + + /* no callback called yet for this iteration */ + calledCallback=FALSE; + + /* no sourceIndex adjustment for conversion, only for callback output */ + errorInputLength=0; + + /* + * loop for offsets and error handling + * + * iterates at most 3 times: + * 1. to clean up after the conversion function + * 2. after the callback + * 3. after the callback again if there was truncated input + */ + for(;;) { + /* update offsets if we write any */ + if(offsets!=NULL) { + int32_t length=(int32_t)(pArgs->target-t); + if(length>0) { + _updateOffsets(offsets, length, sourceIndex, errorInputLength); + + /* + * if a converter handles offsets and updates the offsets + * pointer at the end, then pArgs->offset should not change + * here; + * however, some converters do not handle offsets at all + * (sourceIndex<0) or may not update the offsets pointer + */ + pArgs->offsets=offsets+=length; + } + + if(sourceIndex>=0) { + sourceIndex+=(int32_t)(pArgs->source-s); + } + } + + if(cnv->preToULength<0) { + /* + * switch the source to new replay units (cannot occur while replaying) + * after offset handling and before end-of-input and callback handling + */ + if(realSource==NULL) { + realSource=pArgs->source; + realSourceLimit=pArgs->sourceLimit; + realFlush=pArgs->flush; + realSourceIndex=sourceIndex; + + uprv_memcpy(replay, cnv->preToU, -cnv->preToULength); + pArgs->source=replay; + pArgs->sourceLimit=replay-cnv->preToULength; + pArgs->flush=FALSE; + if((sourceIndex+=cnv->preToULength)<0) { + sourceIndex=-1; + } + + cnv->preToULength=0; + } else { + /* see implementation note before _fromUnicodeWithCallback() */ + U_ASSERT(realSource==NULL); + *err=U_INTERNAL_PROGRAM_ERROR; + } + } + + /* update pointers */ + s=pArgs->source; + t=pArgs->target; + + if(U_SUCCESS(*err)) { + if(ssourceLimit) { + /* + * continue with the conversion loop while there is still input left + * (continue converting by breaking out of only the inner loop) + */ + break; + } else if(realSource!=NULL) { + /* switch back from replaying to the real source and continue */ + pArgs->source=realSource; + pArgs->sourceLimit=realSourceLimit; + pArgs->flush=realFlush; + sourceIndex=realSourceIndex; + + realSource=NULL; + break; + } else if(pArgs->flush && cnv->toULength>0) { + /* + * the entire input stream is consumed + * and there is a partial, truncated input sequence left + */ + + /* inject an error and continue with callback handling */ + *err=U_TRUNCATED_CHAR_FOUND; + calledCallback=FALSE; /* new error condition */ + } else { + /* input consumed */ + if(pArgs->flush) { + /* + * return to the conversion loop once more if the flush + * flag is set and the conversion function has not + * successfully processed the end of the input yet + * + * (continue converting by breaking out of only the inner loop) + */ + if(!converterSawEndOfInput) { + break; + } + + /* reset the converter without calling the callback function */ + _reset(cnv, UCNV_RESET_TO_UNICODE, FALSE); + } + + /* done successfully */ + return; + } + } + + /* U_FAILURE(*err) */ + { + UErrorCode e; + + if( calledCallback || + (e=*err)==U_BUFFER_OVERFLOW_ERROR || + (e!=U_INVALID_CHAR_FOUND && + e!=U_ILLEGAL_CHAR_FOUND && + e!=U_TRUNCATED_CHAR_FOUND && + e!=U_ILLEGAL_ESCAPE_SEQUENCE && + e!=U_UNSUPPORTED_ESCAPE_SEQUENCE) + ) { + /* + * the callback did not or cannot resolve the error: + * set output pointers and return + * + * the check for buffer overflow is redundant but it is + * a high-runner case and hopefully documents the intent + * well + * + * if we were replaying, then the replay buffer must be + * copied back into the UConverter + * and the real arguments must be restored + */ + if(realSource!=NULL) { + int32_t length; + + U_ASSERT(cnv->preToULength==0); + + length=(int32_t)(pArgs->sourceLimit-pArgs->source); + if(length>0) { + uprv_memcpy(cnv->preToU, pArgs->source, length); + cnv->preToULength=(int8_t)-length; + } + + pArgs->source=realSource; + pArgs->sourceLimit=realSourceLimit; + pArgs->flush=realFlush; + } + + return; + } + } + + /* copy toUBytes[] to invalidCharBuffer[] */ + errorInputLength=cnv->invalidCharLength=cnv->toULength; + if(errorInputLength>0) { + uprv_memcpy(cnv->invalidCharBuffer, cnv->toUBytes, errorInputLength); + } + + /* set the converter state to deal with the next character */ + cnv->toULength=0; + + /* call the callback function */ + cnv->fromCharErrorBehaviour(cnv->toUContext, pArgs, + cnv->invalidCharBuffer, errorInputLength, + (*err==U_INVALID_CHAR_FOUND || *err==U_UNSUPPORTED_ESCAPE_SEQUENCE) ? + UCNV_UNASSIGNED : UCNV_ILLEGAL, + err); + + /* + * loop back to the offset handling + * + * this flag will indicate after offset handling + * that a callback was called; + * if the callback did not resolve the error, then we return + */ + calledCallback=TRUE; + } + } +} + +/* + * Output the toUnicode overflow buffer. + * Call this function if(cnv->UCharErrorBufferLength>0). + * @return TRUE if overflow + */ +static UBool +ucnv_outputOverflowToUnicode(UConverter *cnv, + UChar **target, const UChar *targetLimit, + int32_t **pOffsets, + UErrorCode *err) { + int32_t *offsets; + UChar *overflow, *t; + int32_t i, length; + + t=*target; + if(pOffsets!=NULL) { + offsets=*pOffsets; + } else { + offsets=NULL; + } + + overflow=cnv->UCharErrorBuffer; + length=cnv->UCharErrorBufferLength; + i=0; + while(iUCharErrorBufferLength=(int8_t)j; + *target=t; + if(offsets!=NULL) { + *pOffsets=offsets; + } + *err=U_BUFFER_OVERFLOW_ERROR; + return TRUE; + } + + /* copy the overflow contents to the target */ + *t++=overflow[i++]; + if(offsets!=NULL) { + *offsets++=-1; /* no source index available for old output */ + } + } + + /* the overflow buffer is completely copied to the target */ + cnv->UCharErrorBufferLength=0; + *target=t; + if(offsets!=NULL) { + *pOffsets=offsets; + } + return FALSE; +} + +U_CAPI void U_EXPORT2 +ucnv_toUnicode(UConverter *cnv, + UChar **target, const UChar *targetLimit, + const char **source, const char *sourceLimit, + int32_t *offsets, + UBool flush, + UErrorCode *err) { + UConverterToUnicodeArgs args; + const char *s; + UChar *t; + + /* check parameters */ + if(err==NULL || U_FAILURE(*err)) { + return; + } + + if(cnv==NULL || target==NULL || source==NULL) { + *err=U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + s=*source; + t=*target; + + if ((const void *)U_MAX_PTR(targetLimit) == (const void *)targetLimit) { + /* + Prevent code from going into an infinite loop in case we do hit this + limit. The limit pointer is expected to be on a UChar * boundary. + This also prevents the next argument check from failing. + */ + targetLimit = (const UChar *)(((const char *)targetLimit) - 1); + } + + /* + * All these conditions should never happen. + * + * 1) Make sure that the limits are >= to the address source or target + * + * 2) Make sure that the buffer sizes do not exceed the number range for + * int32_t because some functions use the size (in units or bytes) + * rather than comparing pointers, and because offsets are int32_t values. + * + * size_t is guaranteed to be unsigned and large enough for the job. + * + * Return with an error instead of adjusting the limits because we would + * not be able to maintain the semantics that either the source must be + * consumed or the target filled (unless an error occurs). + * An adjustment would be sourceLimit=t+0x7fffffff; for example. + * + * 3) Make sure that the user didn't incorrectly cast a UChar * pointer + * to a char * pointer and provide an incomplete UChar code unit. + */ + if (sourceLimit(size_t)0x7fffffff && sourceLimit>s) || + ((size_t)(targetLimit-t)>(size_t)0x3fffffff && targetLimit>t) || + (((const char *)targetLimit-(const char *)t) & 1) != 0 + ) { + *err=U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + /* output the target overflow buffer */ + if( cnv->UCharErrorBufferLength>0 && + ucnv_outputOverflowToUnicode(cnv, target, targetLimit, &offsets, err) + ) { + /* U_BUFFER_OVERFLOW_ERROR */ + return; + } + /* *target may have moved, therefore stop using t */ + + if(!flush && s==sourceLimit && cnv->preToULength>=0) { + /* the overflow buffer is emptied and there is no new input: we are done */ + return; + } + + /* + * Do not simply return with a buffer overflow error if + * !flush && t==targetLimit + * because it is possible that the source will not generate any output. + * For example, the skip callback may be called; + * it does not output anything. + */ + + /* prepare the converter arguments */ + args.converter=cnv; + args.flush=flush; + args.offsets=offsets; + args.source=s; + args.sourceLimit=sourceLimit; + args.target=*target; + args.targetLimit=targetLimit; + args.size=sizeof(args); + + _toUnicodeWithCallback(&args, err); + + *source=args.source; + *target=args.target; +} + +/* ucnv_to/fromUChars() ----------------------------------------------------- */ + +U_CAPI int32_t U_EXPORT2 +ucnv_fromUChars(UConverter *cnv, + char *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + UErrorCode *pErrorCode) { + const UChar *srcLimit; + char *originalDest, *destLimit; + int32_t destLength; + + /* check arguments */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + if( cnv==NULL || + destCapacity<0 || (destCapacity>0 && dest==NULL) || + srcLength<-1 || (srcLength!=0 && src==NULL) + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* initialize */ + ucnv_resetFromUnicode(cnv); + originalDest=dest; + if(srcLength==-1) { + srcLength=u_strlen(src); + } + if(srcLength>0) { + srcLimit=src+srcLength; + destLimit=dest+destCapacity; + + /* pin the destination limit to U_MAX_PTR; NULL check is for OS/400 */ + if(destLimit0 && dest==NULL) || + srcLength<-1 || (srcLength!=0 && src==NULL)) + { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* initialize */ + ucnv_resetToUnicode(cnv); + originalDest=dest; + if(srcLength==-1) { + srcLength=(int32_t)uprv_strlen(src); + } + if(srcLength>0) { + srcLimit=src+srcLength; + destLimit=dest+destCapacity; + + /* pin the destination limit to U_MAX_PTR; NULL check is for OS/400 */ + if(destLimit(size_t)0x7fffffff && sourceLimit>s)) { + *err=U_ILLEGAL_ARGUMENT_ERROR; + return 0xffff; + } + + c=U_SENTINEL; + + /* flush the target overflow buffer */ + if(cnv->UCharErrorBufferLength>0) { + UChar *overflow; + + overflow=cnv->UCharErrorBuffer; + i=0; + length=cnv->UCharErrorBufferLength; + U16_NEXT(overflow, i, length, c); + + /* move the remaining overflow contents up to the beginning */ + if((cnv->UCharErrorBufferLength=(int8_t)(length-i))>0) { + uprv_memmove(cnv->UCharErrorBuffer, cnv->UCharErrorBuffer+i, + cnv->UCharErrorBufferLength*U_SIZEOF_UCHAR); + } + + if(!U16_IS_LEAD(c) || itoULength==0 && cnv->sharedData->impl->getNextUChar!=NULL) { + c=cnv->sharedData->impl->getNextUChar(&args, err); + *source=s=args.source; + if(*err==U_INDEX_OUTOFBOUNDS_ERROR) { + /* reset the converter without calling the callback function */ + _reset(cnv, UCNV_RESET_TO_UNICODE, FALSE); + return 0xffff; /* no output */ + } else if(U_SUCCESS(*err) && c>=0) { + return c; + /* + * else fall through to use _toUnicode() because + * UCNV_GET_NEXT_UCHAR_USE_TO_U: the native function did not want to handle it after all + * U_FAILURE: call _toUnicode() for callback handling (do not output c) + */ + } + } + + /* convert to one UChar in buffer[0], or handle getNextUChar() errors */ + _toUnicodeWithCallback(&args, err); + + if(*err==U_BUFFER_OVERFLOW_ERROR) { + *err=U_ZERO_ERROR; + } + + i=0; + length=(int32_t)(args.target-buffer); + } else { + /* write the lead surrogate from the overflow buffer */ + buffer[0]=(UChar)c; + args.target=buffer+1; + i=0; + length=1; + } + + /* buffer contents starts at i and ends before length */ + + if(U_FAILURE(*err)) { + c=0xffff; /* no output */ + } else if(length==0) { + /* no input or only state changes */ + *err=U_INDEX_OUTOFBOUNDS_ERROR; + /* no need to reset explicitly because _toUnicodeWithCallback() did it */ + c=0xffff; /* no output */ + } else { + c=buffer[0]; + i=1; + if(!U16_IS_LEAD(c)) { + /* consume c=buffer[0], done */ + } else { + /* got a lead surrogate, see if a trail surrogate follows */ + UChar c2; + + if(cnv->UCharErrorBufferLength>0) { + /* got overflow output from the conversion */ + if(U16_IS_TRAIL(c2=cnv->UCharErrorBuffer[0])) { + /* got a trail surrogate, too */ + c=U16_GET_SUPPLEMENTARY(c, c2); + + /* move the remaining overflow contents up to the beginning */ + if((--cnv->UCharErrorBufferLength)>0) { + uprv_memmove(cnv->UCharErrorBuffer, cnv->UCharErrorBuffer+1, + cnv->UCharErrorBufferLength*U_SIZEOF_UCHAR); + } + } else { + /* c is an unpaired lead surrogate, just return it */ + } + } else if(args.sourceUCharErrorBufferLength)>0) { + uprv_memmove(cnv->UCharErrorBuffer+delta, cnv->UCharErrorBuffer, + length*U_SIZEOF_UCHAR); + } + cnv->UCharErrorBufferLength=(int8_t)(length+delta); + + cnv->UCharErrorBuffer[0]=buffer[i++]; + if(delta>1) { + cnv->UCharErrorBuffer[1]=buffer[i]; + } + } + + *source=args.source; + return c; +} + +/* ucnv_convert() and siblings ---------------------------------------------- */ + +U_CAPI void U_EXPORT2 +ucnv_convertEx(UConverter *targetCnv, UConverter *sourceCnv, + char **target, const char *targetLimit, + const char **source, const char *sourceLimit, + UChar *pivotStart, UChar **pivotSource, + UChar **pivotTarget, const UChar *pivotLimit, + UBool reset, UBool flush, + UErrorCode *pErrorCode) { + UChar pivotBuffer[CHUNK_SIZE]; + const UChar *myPivotSource; + UChar *myPivotTarget; + const char *s; + char *t; + + UConverterToUnicodeArgs toUArgs; + UConverterFromUnicodeArgs fromUArgs; + UConverterConvert convert; + + /* error checking */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return; + } + + if( targetCnv==NULL || sourceCnv==NULL || + source==NULL || *source==NULL || + target==NULL || *target==NULL || targetLimit==NULL + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + s=*source; + t=*target; + if((sourceLimit!=NULL && sourceLimit(size_t)0x7fffffff && sourceLimit>s)) || + ((size_t)(targetLimit-t)>(size_t)0x7fffffff && targetLimit>t) + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + if(pivotStart==NULL) { + if(!flush) { + /* streaming conversion requires an explicit pivot buffer */ + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + /* use the stack pivot buffer */ + myPivotSource=myPivotTarget=pivotStart=pivotBuffer; + pivotSource=(UChar **)&myPivotSource; + pivotTarget=&myPivotTarget; + pivotLimit=pivotBuffer+CHUNK_SIZE; + } else if( pivotStart>=pivotLimit || + pivotSource==NULL || *pivotSource==NULL || + pivotTarget==NULL || *pivotTarget==NULL || + pivotLimit==NULL + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + if(sourceLimit==NULL) { + /* get limit of single-byte-NUL-terminated source string */ + sourceLimit=uprv_strchr(*source, 0); + } + + if(reset) { + ucnv_resetToUnicode(sourceCnv); + ucnv_resetFromUnicode(targetCnv); + *pivotSource=*pivotTarget=pivotStart; + } else if(targetCnv->charErrorBufferLength>0) { + /* output the targetCnv overflow buffer */ + if(ucnv_outputOverflowFromUnicode(targetCnv, target, targetLimit, NULL, pErrorCode)) { + /* U_BUFFER_OVERFLOW_ERROR */ + return; + } + /* *target has moved, therefore stop using t */ + + if( !flush && + targetCnv->preFromULength>=0 && *pivotSource==*pivotTarget && + sourceCnv->UCharErrorBufferLength==0 && sourceCnv->preToULength>=0 && s==sourceLimit + ) { + /* the fromUnicode overflow buffer is emptied and there is no new input: we are done */ + return; + } + } + + /* Is direct-UTF-8 conversion available? */ + if( sourceCnv->sharedData->staticData->conversionType==UCNV_UTF8 && + targetCnv->sharedData->impl->fromUTF8!=NULL + ) { + convert=targetCnv->sharedData->impl->fromUTF8; + } else if( targetCnv->sharedData->staticData->conversionType==UCNV_UTF8 && + sourceCnv->sharedData->impl->toUTF8!=NULL + ) { + convert=sourceCnv->sharedData->impl->toUTF8; + } else { + convert=NULL; + } + + /* + * If direct-UTF-8 conversion is available, then we use a smaller + * pivot buffer for error handling and partial matches + * so that we quickly return to direct conversion. + * + * 32 is large enough for UCNV_EXT_MAX_UCHARS and UCNV_ERROR_BUFFER_LENGTH. + * + * We could reduce the pivot buffer size further, at the cost of + * buffer overflows from callbacks. + * The pivot buffer should not be smaller than the maximum number of + * fromUnicode extension table input UChars + * (for m:n conversion, see + * targetCnv->sharedData->mbcs.extIndexes[UCNV_EXT_COUNT_UCHARS]) + * or 2 for surrogate pairs. + * + * Too small a buffer can cause thrashing between pivoting and direct + * conversion, with function call overhead outweighing the benefits + * of direct conversion. + */ + if(convert!=NULL && (pivotLimit-pivotStart)>32) { + pivotLimit=pivotStart+32; + } + + /* prepare the converter arguments */ + fromUArgs.converter=targetCnv; + fromUArgs.flush=FALSE; + fromUArgs.offsets=NULL; + fromUArgs.target=*target; + fromUArgs.targetLimit=targetLimit; + fromUArgs.size=sizeof(fromUArgs); + + toUArgs.converter=sourceCnv; + toUArgs.flush=flush; + toUArgs.offsets=NULL; + toUArgs.source=s; + toUArgs.sourceLimit=sourceLimit; + toUArgs.targetLimit=pivotLimit; + toUArgs.size=sizeof(toUArgs); + + /* + * TODO: Consider separating this function into two functions, + * extracting exactly the conversion loop, + * for readability and to reduce the set of visible variables. + * + * Otherwise stop using s and t from here on. + */ + s=t=NULL; + + /* + * conversion loop + * + * The sequence of steps in the loop may appear backward, + * but the principle is simple: + * In the chain of + * source - sourceCnv overflow - pivot - targetCnv overflow - target + * empty out later buffers before refilling them from earlier ones. + * + * The targetCnv overflow buffer is flushed out only once before the loop. + */ + for(;;) { + /* + * if(pivot not empty or error or replay or flush fromUnicode) { + * fromUnicode(pivot -> target); + * } + * + * For pivoting conversion; and for direct conversion for + * error callback handling and flushing the replay buffer. + */ + if( *pivotSource<*pivotTarget || + U_FAILURE(*pErrorCode) || + targetCnv->preFromULength<0 || + fromUArgs.flush + ) { + fromUArgs.source=*pivotSource; + fromUArgs.sourceLimit=*pivotTarget; + _fromUnicodeWithCallback(&fromUArgs, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + /* target overflow, or conversion error */ + *pivotSource=(UChar *)fromUArgs.source; + break; + } + + /* + * _fromUnicodeWithCallback() must have consumed the pivot contents + * (*pivotSource==*pivotTarget) since it returned with U_SUCCESS() + */ + } + + /* The pivot buffer is empty; reset it so we start at pivotStart. */ + *pivotSource=*pivotTarget=pivotStart; + + /* + * if(sourceCnv overflow buffer not empty) { + * move(sourceCnv overflow buffer -> pivot); + * continue; + * } + */ + /* output the sourceCnv overflow buffer */ + if(sourceCnv->UCharErrorBufferLength>0) { + if(ucnv_outputOverflowToUnicode(sourceCnv, pivotTarget, pivotLimit, NULL, pErrorCode)) { + /* U_BUFFER_OVERFLOW_ERROR */ + *pErrorCode=U_ZERO_ERROR; + } + continue; + } + + /* + * check for end of input and break if done + * + * Checking both flush and fromUArgs.flush ensures that the converters + * have been called with the flush flag set if the ucnv_convertEx() + * caller set it. + */ + if( toUArgs.source==sourceLimit && + sourceCnv->preToULength>=0 && sourceCnv->toULength==0 && + (!flush || fromUArgs.flush) + ) { + /* done successfully */ + break; + } + + /* + * use direct conversion if available + * but not if continuing a partial match + * or flushing the toUnicode replay buffer + */ + if(convert!=NULL && targetCnv->preFromUFirstCP<0 && sourceCnv->preToULength==0) { + if(*pErrorCode==U_USING_DEFAULT_WARNING) { + /* remove a warning that may be set by this function */ + *pErrorCode=U_ZERO_ERROR; + } + convert(&fromUArgs, &toUArgs, pErrorCode); + if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) { + break; + } else if(U_FAILURE(*pErrorCode)) { + if(sourceCnv->toULength>0) { + /* + * Fall through to calling _toUnicodeWithCallback() + * for callback handling. + * + * The pivot buffer will be reset with + * *pivotSource=*pivotTarget=pivotStart; + * which indicates a toUnicode error to the caller + * (*pivotSource==pivotStart shows no pivot UChars consumed). + */ + } else { + /* + * Indicate a fromUnicode error to the caller + * (*pivotSource>pivotStart shows some pivot UChars consumed). + */ + *pivotSource=*pivotTarget=pivotStart+1; + /* + * Loop around to calling _fromUnicodeWithCallbacks() + * for callback handling. + */ + continue; + } + } else if(*pErrorCode==U_USING_DEFAULT_WARNING) { + /* + * No error, but the implementation requested to temporarily + * fall back to pivoting. + */ + *pErrorCode=U_ZERO_ERROR; + /* + * The following else branches are almost identical to the end-of-input + * handling in _toUnicodeWithCallback(). + * Avoid calling it just for the end of input. + */ + } else if(flush && sourceCnv->toULength>0) { /* flush==toUArgs.flush */ + /* + * the entire input stream is consumed + * and there is a partial, truncated input sequence left + */ + + /* inject an error and continue with callback handling */ + *pErrorCode=U_TRUNCATED_CHAR_FOUND; + } else { + /* input consumed */ + if(flush) { + /* reset the converters without calling the callback functions */ + _reset(sourceCnv, UCNV_RESET_TO_UNICODE, FALSE); + _reset(targetCnv, UCNV_RESET_FROM_UNICODE, FALSE); + } + + /* done successfully */ + break; + } + } + + /* + * toUnicode(source -> pivot); + * + * For pivoting conversion; and for direct conversion for + * error callback handling, continuing partial matches + * and flushing the replay buffer. + * + * The pivot buffer is empty and reset. + */ + toUArgs.target=pivotStart; /* ==*pivotTarget */ + /* toUArgs.targetLimit=pivotLimit; already set before the loop */ + _toUnicodeWithCallback(&toUArgs, pErrorCode); + *pivotTarget=toUArgs.target; + if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) { + /* pivot overflow: continue with the conversion loop */ + *pErrorCode=U_ZERO_ERROR; + } else if(U_FAILURE(*pErrorCode) || (!flush && *pivotTarget==pivotStart)) { + /* conversion error, or there was nothing left to convert */ + break; + } + /* + * else: + * _toUnicodeWithCallback() wrote into the pivot buffer, + * continue with fromUnicode conversion. + * + * Set the fromUnicode flush flag if we flush and if toUnicode has + * processed the end of the input. + */ + if( flush && toUArgs.source==sourceLimit && + sourceCnv->preToULength>=0 && + sourceCnv->UCharErrorBufferLength==0 + ) { + fromUArgs.flush=TRUE; + } + } + + /* + * The conversion loop is exited when one of the following is true: + * - the entire source text has been converted successfully to the target buffer + * - a target buffer overflow occurred + * - a conversion error occurred + */ + + *source=toUArgs.source; + *target=fromUArgs.target; + + /* terminate the target buffer if possible */ + if(flush && U_SUCCESS(*pErrorCode)) { + if(*target!=targetLimit) { + **target=0; + if(*pErrorCode==U_STRING_NOT_TERMINATED_WARNING) { + *pErrorCode=U_ZERO_ERROR; + } + } else { + *pErrorCode=U_STRING_NOT_TERMINATED_WARNING; + } + } +} + +/* internal implementation of ucnv_convert() etc. with preflighting */ +static int32_t +ucnv_internalConvert(UConverter *outConverter, UConverter *inConverter, + char *target, int32_t targetCapacity, + const char *source, int32_t sourceLength, + UErrorCode *pErrorCode) { + UChar pivotBuffer[CHUNK_SIZE]; + UChar *pivot, *pivot2; + + char *myTarget; + const char *sourceLimit; + const char *targetLimit; + int32_t targetLength=0; + + /* set up */ + if(sourceLength<0) { + sourceLimit=uprv_strchr(source, 0); + } else { + sourceLimit=source+sourceLength; + } + + /* if there is no input data, we're done */ + if(source==sourceLimit) { + return u_terminateChars(target, targetCapacity, 0, pErrorCode); + } + + pivot=pivot2=pivotBuffer; + myTarget=target; + targetLength=0; + + if(targetCapacity>0) { + /* perform real conversion */ + targetLimit=target+targetCapacity; + ucnv_convertEx(outConverter, inConverter, + &myTarget, targetLimit, + &source, sourceLimit, + pivotBuffer, &pivot, &pivot2, pivotBuffer+CHUNK_SIZE, + FALSE, + TRUE, + pErrorCode); + targetLength=(int32_t)(myTarget-target); + } + + /* + * If the output buffer is exhausted (or we are only "preflighting"), we need to stop writing + * to it but continue the conversion in order to store in targetCapacity + * the number of bytes that was required. + */ + if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR || targetCapacity==0) + { + char targetBuffer[CHUNK_SIZE]; + + targetLimit=targetBuffer+CHUNK_SIZE; + do { + *pErrorCode=U_ZERO_ERROR; + myTarget=targetBuffer; + ucnv_convertEx(outConverter, inConverter, + &myTarget, targetLimit, + &source, sourceLimit, + pivotBuffer, &pivot, &pivot2, pivotBuffer+CHUNK_SIZE, + FALSE, + TRUE, + pErrorCode); + targetLength+=(int32_t)(myTarget-targetBuffer); + } while(*pErrorCode==U_BUFFER_OVERFLOW_ERROR); + + /* done with preflighting, set warnings and errors as appropriate */ + return u_terminateChars(target, targetCapacity, targetLength, pErrorCode); + } + + /* no need to call u_terminateChars() because ucnv_convertEx() took care of that */ + return targetLength; +} + +U_CAPI int32_t U_EXPORT2 +ucnv_convert(const char *toConverterName, const char *fromConverterName, + char *target, int32_t targetCapacity, + const char *source, int32_t sourceLength, + UErrorCode *pErrorCode) { + UConverter in, out; /* stack-allocated */ + UConverter *inConverter, *outConverter; + int32_t targetLength; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + if( source==NULL || sourceLength<-1 || + targetCapacity<0 || (targetCapacity>0 && target==NULL) + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* if there is no input data, we're done */ + if(sourceLength==0 || (sourceLength<0 && *source==0)) { + return u_terminateChars(target, targetCapacity, 0, pErrorCode); + } + + /* create the converters */ + inConverter=ucnv_createConverter(&in, fromConverterName, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return 0; + } + + outConverter=ucnv_createConverter(&out, toConverterName, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + ucnv_close(inConverter); + return 0; + } + + targetLength=ucnv_internalConvert(outConverter, inConverter, + target, targetCapacity, + source, sourceLength, + pErrorCode); + + ucnv_close(inConverter); + ucnv_close(outConverter); + + return targetLength; +} + +/* @internal */ +static int32_t +ucnv_convertAlgorithmic(UBool convertToAlgorithmic, + UConverterType algorithmicType, + UConverter *cnv, + char *target, int32_t targetCapacity, + const char *source, int32_t sourceLength, + UErrorCode *pErrorCode) { + UConverter algoConverterStatic; /* stack-allocated */ + UConverter *algoConverter, *to, *from; + int32_t targetLength; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + if( cnv==NULL || source==NULL || sourceLength<-1 || + targetCapacity<0 || (targetCapacity>0 && target==NULL) + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* if there is no input data, we're done */ + if(sourceLength==0 || (sourceLength<0 && *source==0)) { + return u_terminateChars(target, targetCapacity, 0, pErrorCode); + } + + /* create the algorithmic converter */ + algoConverter=ucnv_createAlgorithmicConverter(&algoConverterStatic, algorithmicType, + "", 0, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return 0; + } + + /* reset the other converter */ + if(convertToAlgorithmic) { + /* cnv->Unicode->algo */ + ucnv_resetToUnicode(cnv); + to=algoConverter; + from=cnv; + } else { + /* algo->Unicode->cnv */ + ucnv_resetFromUnicode(cnv); + from=algoConverter; + to=cnv; + } + + targetLength=ucnv_internalConvert(to, from, + target, targetCapacity, + source, sourceLength, + pErrorCode); + + ucnv_close(algoConverter); + + return targetLength; +} + +U_CAPI int32_t U_EXPORT2 +ucnv_toAlgorithmic(UConverterType algorithmicType, + UConverter *cnv, + char *target, int32_t targetCapacity, + const char *source, int32_t sourceLength, + UErrorCode *pErrorCode) { + return ucnv_convertAlgorithmic(TRUE, algorithmicType, cnv, + target, targetCapacity, + source, sourceLength, + pErrorCode); +} + +U_CAPI int32_t U_EXPORT2 +ucnv_fromAlgorithmic(UConverter *cnv, + UConverterType algorithmicType, + char *target, int32_t targetCapacity, + const char *source, int32_t sourceLength, + UErrorCode *pErrorCode) { + return ucnv_convertAlgorithmic(FALSE, algorithmicType, cnv, + target, targetCapacity, + source, sourceLength, + pErrorCode); +} + +U_CAPI UConverterType U_EXPORT2 +ucnv_getType(const UConverter* converter) +{ + int8_t type = converter->sharedData->staticData->conversionType; +#if !UCONFIG_NO_LEGACY_CONVERSION + if(type == UCNV_MBCS) { + return ucnv_MBCSGetType(converter); + } +#endif + return (UConverterType)type; +} + +U_CAPI void U_EXPORT2 +ucnv_getStarters(const UConverter* converter, + UBool starters[256], + UErrorCode* err) +{ + if (err == NULL || U_FAILURE(*err)) { + return; + } + + if(converter->sharedData->impl->getStarters != NULL) { + converter->sharedData->impl->getStarters(converter, starters, err); + } else { + *err = U_ILLEGAL_ARGUMENT_ERROR; + } +} + +static const UAmbiguousConverter *ucnv_getAmbiguous(const UConverter *cnv) +{ + UErrorCode errorCode; + const char *name; + int32_t i; + + if(cnv==NULL) { + return NULL; + } + + errorCode=U_ZERO_ERROR; + name=ucnv_getName(cnv, &errorCode); + if(U_FAILURE(errorCode)) { + return NULL; + } + + for(i=0; i<(int32_t)(sizeof(ambiguousConverters)/sizeof(UAmbiguousConverter)); ++i) + { + if(0==uprv_strcmp(name, ambiguousConverters[i].name)) + { + return ambiguousConverters+i; + } + } + + return NULL; +} + +U_CAPI void U_EXPORT2 +ucnv_fixFileSeparator(const UConverter *cnv, + UChar* source, + int32_t sourceLength) { + const UAmbiguousConverter *a; + int32_t i; + UChar variant5c; + + if(cnv==NULL || source==NULL || sourceLength<=0 || (a=ucnv_getAmbiguous(cnv))==NULL) + { + return; + } + + variant5c=a->variant5c; + for(i=0; iuseFallback = usesFallback; +} + +U_CAPI UBool U_EXPORT2 +ucnv_usesFallback(const UConverter *cnv) +{ + return cnv->useFallback; +} + +U_CAPI void U_EXPORT2 +ucnv_getInvalidChars (const UConverter * converter, + char *errBytes, + int8_t * len, + UErrorCode * err) +{ + if (err == NULL || U_FAILURE(*err)) + { + return; + } + if (len == NULL || errBytes == NULL || converter == NULL) + { + *err = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + if (*len < converter->invalidCharLength) + { + *err = U_INDEX_OUTOFBOUNDS_ERROR; + return; + } + if ((*len = converter->invalidCharLength) > 0) + { + uprv_memcpy (errBytes, converter->invalidCharBuffer, *len); + } +} + +U_CAPI void U_EXPORT2 +ucnv_getInvalidUChars (const UConverter * converter, + UChar *errChars, + int8_t * len, + UErrorCode * err) +{ + if (err == NULL || U_FAILURE(*err)) + { + return; + } + if (len == NULL || errChars == NULL || converter == NULL) + { + *err = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + if (*len < converter->invalidUCharLength) + { + *err = U_INDEX_OUTOFBOUNDS_ERROR; + return; + } + if ((*len = converter->invalidUCharLength) > 0) + { + uprv_memcpy (errChars, converter->invalidUCharBuffer, sizeof(UChar) * (*len)); + } +} + +#define SIG_MAX_LEN 5 + +U_CAPI const char* U_EXPORT2 +ucnv_detectUnicodeSignature( const char* source, + int32_t sourceLength, + int32_t* signatureLength, + UErrorCode* pErrorCode) { + int32_t dummy; + + /* initial 0xa5 bytes: make sure that if we read preFromULength > 0){ + return U16_LENGTH(cnv->preFromUFirstCP)+cnv->preFromULength ; + }else if(cnv->preFromULength < 0){ + return -cnv->preFromULength ; + }else if(cnv->fromUChar32 > 0){ + return 1; + }else if(cnv->preFromUFirstCP >0){ + return U16_LENGTH(cnv->preFromUFirstCP); + } + return 0; + +} + +U_CAPI int32_t U_EXPORT2 +ucnv_toUCountPending(const UConverter* cnv, UErrorCode* status){ + + if(status == NULL || U_FAILURE(*status)){ + return -1; + } + if(cnv == NULL){ + *status = U_ILLEGAL_ARGUMENT_ERROR; + return -1; + } + + if(cnv->preToULength > 0){ + return cnv->preToULength ; + }else if(cnv->preToULength < 0){ + return -cnv->preToULength; + }else if(cnv->toULength > 0){ + return cnv->toULength; + } + return 0; +} +#endif + +/* + * Hey, Emacs, please set the following: + * + * Local Variables: + * indent-tabs-mode: nil + * End: + * + */ diff --git a/Build/source/libs/icu/icu-xetex/common/ucnv2022.c b/Build/source/libs/icu/icu-xetex/common/ucnv2022.c new file mode 100644 index 00000000000..dff4ba8340d --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnv2022.c @@ -0,0 +1,3463 @@ +/* +********************************************************************** +* Copyright (C) 2000-2007, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* file name: ucnv2022.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2000feb03 +* created by: Markus W. Scherer +* +* Change history: +* +* 06/29/2000 helena Major rewrite of the callback APIs. +* 08/08/2000 Ram Included support for ISO-2022-JP-2 +* Changed implementation of toUnicode +* function +* 08/21/2000 Ram Added support for ISO-2022-KR +* 08/29/2000 Ram Seperated implementation of EBCDIC to +* ucnvebdc.c +* 09/20/2000 Ram Added support for ISO-2022-CN +* Added implementations for getNextUChar() +* for specific 2022 country variants. +* 10/31/2000 Ram Implemented offsets logic functions +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION && !UCONFIG_NO_LEGACY_CONVERSION + +#include "unicode/ucnv.h" +#include "unicode/uset.h" +#include "unicode/ucnv_err.h" +#include "unicode/ucnv_cb.h" +#include "ucnv_imp.h" +#include "ucnv_bld.h" +#include "ucnv_cnv.h" +#include "ucnvmbcs.h" +#include "cstring.h" +#include "cmemory.h" + +#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) + +#ifdef U_ENABLE_GENERIC_ISO_2022 +/* + * I am disabling the generic ISO-2022 converter after proposing to do so on + * the icu mailing list two days ago. + * + * Reasons: + * 1. It does not fully support the ISO-2022/ECMA-35 specification with all of + * its designation sequences, single shifts with return to the previous state, + * switch-with-no-return to UTF-16BE or similar, etc. + * This is unlike the language-specific variants like ISO-2022-JP which + * require a much smaller repertoire of ISO-2022 features. + * These variants continue to be supported. + * 2. I believe that no one is really using the generic ISO-2022 converter + * but rather always one of the language-specific variants. + * Note that ICU's generic ISO-2022 converter has always output one escape + * sequence followed by UTF-8 for the whole stream. + * 3. Switching between subcharsets is extremely slow, because each time + * the previous converter is closed and a new one opened, + * without any kind of caching, least-recently-used list, etc. + * 4. The code is currently buggy, and given the above it does not seem + * reasonable to spend the time on maintenance. + * 5. ISO-2022 subcharsets should normally be used with 7-bit byte encodings. + * This means, for example, that when ISO-8859-7 is designated, the following + * ISO-2022 bytes 00..7f should be interpreted as ISO-8859-7 bytes 80..ff. + * The ICU ISO-2022 converter does not handle this - and has no information + * about which subconverter would have to be shifted vs. which is designed + * for 7-bit ISO-2022. + * + * Markus Scherer 2003-dec-03 + */ +#endif + +static const char SHIFT_IN_STR[] = "\x0F"; +static const char SHIFT_OUT_STR[] = "\x0E"; + +#define CR 0x0D +#define LF 0x0A +#define H_TAB 0x09 +#define V_TAB 0x0B +#define SPACE 0x20 + +enum { + HWKANA_START=0xff61, + HWKANA_END=0xff9f +}; + +/* + * 94-character sets with native byte values A1..FE are encoded in ISO 2022 + * as bytes 21..7E. (Subtract 0x80.) + * 96-character sets with native byte values A0..FF are encoded in ISO 2022 + * as bytes 20..7F. (Subtract 0x80.) + * Do not encode C1 control codes with native bytes 80..9F + * as bytes 00..1F (C0 control codes). + */ +enum { + GR94_START=0xa1, + GR94_END=0xfe, + GR96_START=0xa0, + GR96_END=0xff +}; + +/* + * ISO 2022 control codes must not be converted from Unicode + * because they would mess up the byte stream. + * The bit mask 0x0800c000 has bits set at bit positions 0xe, 0xf, 0x1b + * corresponding to SO, SI, and ESC. + */ +#define IS_2022_CONTROL(c) (((c)<0x20) && (((uint32_t)1<<(c))&0x0800c000)!=0) + +/* for ISO-2022-JP and -CN implementations */ +typedef enum { + /* shared values */ + INVALID_STATE=-1, + ASCII = 0, + + SS2_STATE=0x10, + SS3_STATE, + + /* JP */ + ISO8859_1 = 1 , + ISO8859_7 = 2 , + JISX201 = 3, + JISX208 = 4, + JISX212 = 5, + GB2312 =6, + KSC5601 =7, + HWKANA_7BIT=8, /* Halfwidth Katakana 7 bit */ + + /* CN */ + /* the first few enum constants must keep their values because they correspond to myConverterArray[] */ + GB2312_1=1, + ISO_IR_165=2, + CNS_11643=3, + + /* + * these are used in StateEnum and ISO2022State variables, + * but CNS_11643 must be used to index into myConverterArray[] + */ + CNS_11643_0=0x20, + CNS_11643_1, + CNS_11643_2, + CNS_11643_3, + CNS_11643_4, + CNS_11643_5, + CNS_11643_6, + CNS_11643_7 +} StateEnum; + +/* is the StateEnum charset value for a DBCS charset? */ +#define IS_JP_DBCS(cs) (JISX208<=(cs) && (cs)<=KSC5601) + +#define CSM(cs) ((uint16_t)1<<(cs)) + +/* + * Each of these charset masks (with index x) contains a bit for a charset in exact correspondence + * to whether that charset is used in the corresponding version x of ISO_2022,locale=ja,version=x + * + * Note: The converter uses some leniency: + * - The escape sequence ESC ( I for half-width 7-bit Katakana is recognized in + * all versions, not just JIS7 and JIS8. + * - ICU does not distinguish between different versions of JIS X 0208. + */ +static const uint16_t jpCharsetMasks[5]={ + CSM(ASCII)|CSM(JISX201)|CSM(JISX208)|CSM(HWKANA_7BIT), + CSM(ASCII)|CSM(JISX201)|CSM(JISX208)|CSM(HWKANA_7BIT)|CSM(JISX212), + CSM(ASCII)|CSM(JISX201)|CSM(JISX208)|CSM(HWKANA_7BIT)|CSM(JISX212)|CSM(GB2312)|CSM(KSC5601)|CSM(ISO8859_1)|CSM(ISO8859_7), + CSM(ASCII)|CSM(JISX201)|CSM(JISX208)|CSM(HWKANA_7BIT)|CSM(JISX212)|CSM(GB2312)|CSM(KSC5601)|CSM(ISO8859_1)|CSM(ISO8859_7), + CSM(ASCII)|CSM(JISX201)|CSM(JISX208)|CSM(HWKANA_7BIT)|CSM(JISX212)|CSM(GB2312)|CSM(KSC5601)|CSM(ISO8859_1)|CSM(ISO8859_7) +}; + +typedef enum { + ASCII1=0, + LATIN1, + SBCS, + DBCS, + MBCS, + HWKANA +}Cnv2022Type; + +typedef struct ISO2022State { + int8_t cs[4]; /* charset number for SI (G0)/SO (G1)/SS2 (G2)/SS3 (G3) */ + int8_t g; /* 0..3 for G0..G3 (SI/SO/SS2/SS3) */ + int8_t prevG; /* g before single shift (SS2 or SS3) */ +} ISO2022State; + +#define UCNV_OPTIONS_VERSION_MASK 0xf +#define UCNV_2022_MAX_CONVERTERS 10 + +typedef struct{ + UConverterSharedData *myConverterArray[UCNV_2022_MAX_CONVERTERS]; + UConverter *currentConverter; + Cnv2022Type currentType; + ISO2022State toU2022State, fromU2022State; + uint32_t key; + uint32_t version; +#ifdef U_ENABLE_GENERIC_ISO_2022 + UBool isFirstBuffer; +#endif + char name[30]; + char locale[3]; +}UConverterDataISO2022; + +/* Protos */ +/* ISO-2022 ----------------------------------------------------------------- */ + +/*Forward declaration */ +U_CFUNC void +ucnv_fromUnicode_UTF8(UConverterFromUnicodeArgs * args, + UErrorCode * err); +U_CFUNC void +ucnv_fromUnicode_UTF8_OFFSETS_LOGIC(UConverterFromUnicodeArgs * args, + UErrorCode * err); + +#define ESC_2022 0x1B /*ESC*/ + +typedef enum +{ + INVALID_2022 = -1, /*Doesn't correspond to a valid iso 2022 escape sequence*/ + VALID_NON_TERMINAL_2022 = 0, /*so far corresponds to a valid iso 2022 escape sequence*/ + VALID_TERMINAL_2022 = 1, /*corresponds to a valid iso 2022 escape sequence*/ + VALID_MAYBE_TERMINAL_2022 = 2 /*so far matches one iso 2022 escape sequence, but by adding more characters might match another escape sequence*/ +} UCNV_TableStates_2022; + +/* +* The way these state transition arrays work is: +* ex : ESC$B is the sequence for JISX208 +* a) First Iteration: char is ESC +* i) Get the value of ESC from normalize_esq_chars_2022[] with int value of ESC as index +* int x = normalize_esq_chars_2022[27] which is equal to 1 +* ii) Search for this value in escSeqStateTable_Key_2022[] +* value of x is stored at escSeqStateTable_Key_2022[0] +* iii) Save this index as offset +* iv) Get state of this sequence from escSeqStateTable_Value_2022[] +* escSeqStateTable_Value_2022[offset], which is VALID_NON_TERMINAL_2022 +* b) Switch on this state and continue to next char +* i) Get the value of $ from normalize_esq_chars_2022[] with int value of $ as index +* which is normalize_esq_chars_2022[36] == 4 +* ii) x is currently 1(from above) +* x<<=5 -- x is now 32 +* x+=normalize_esq_chars_2022[36] +* now x is 36 +* iii) Search for this value in escSeqStateTable_Key_2022[] +* value of x is stored at escSeqStateTable_Key_2022[2], so offset is 2 +* iv) Get state of this sequence from escSeqStateTable_Value_2022[] +* escSeqStateTable_Value_2022[offset], which is VALID_NON_TERMINAL_2022 +* c) Switch on this state and continue to next char +* i) Get the value of B from normalize_esq_chars_2022[] with int value of B as index +* ii) x is currently 36 (from above) +* x<<=5 -- x is now 1152 +* x+=normalize_esq_chars_2022[66] +* now x is 1161 +* iii) Search for this value in escSeqStateTable_Key_2022[] +* value of x is stored at escSeqStateTable_Key_2022[21], so offset is 21 +* iv) Get state of this sequence from escSeqStateTable_Value_2022[21] +* escSeqStateTable_Value_2022[offset], which is VALID_TERMINAL_2022 +* v) Get the converter name form escSeqStateTable_Result_2022[21] which is JISX208 +*/ + + +/*Below are the 3 arrays depicting a state transition table*/ +static const int8_t normalize_esq_chars_2022[256] = { +/* 0 1 2 3 4 5 6 7 8 9 */ + + 0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 + ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 + ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,1 ,0 ,0 + ,0 ,0 ,0 ,0 ,0 ,0 ,4 ,7 ,29 ,0 + ,2 ,24 ,26 ,27 ,0 ,3 ,23 ,6 ,0 ,0 + ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 + ,0 ,0 ,0 ,0 ,5 ,8 ,9 ,10 ,11 ,12 + ,13 ,14 ,15 ,16 ,17 ,18 ,19 ,20 ,25 ,28 + ,0 ,0 ,21 ,0 ,0 ,0 ,0 ,0 ,0 ,0 + ,22 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 + ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 + ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 + ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 + ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 + ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 + ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 + ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 + ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 + ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 + ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 + ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 + ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 + ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 + ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 + ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 + ,0 ,0 ,0 ,0 ,0 ,0 +}; + +#ifdef U_ENABLE_GENERIC_ISO_2022 +/* + * When the generic ISO-2022 converter is completely removed, not just disabled + * per #ifdef, then the following state table and the associated tables that are + * dimensioned with MAX_STATES_2022 should be trimmed. + * + * Especially, VALID_MAYBE_TERMINAL_2022 will not be used any more, and all of + * the associated escape sequences starting with ESC ( B should be removed. + * This includes the ones with key values 1097 and all of the ones above 1000000. + * + * For the latter, the tables can simply be truncated. + * For the former, since the tables must be kept parallel, it is probably best + * to simply duplicate an adjacent table cell, parallel in all tables. + * + * It may make sense to restructure the tables, especially by using small search + * tables for the variants instead of indexing them parallel to the table here. + */ +#endif + +#define MAX_STATES_2022 74 +static const int32_t escSeqStateTable_Key_2022[MAX_STATES_2022] = { +/* 0 1 2 3 4 5 6 7 8 9 */ + + 1 ,34 ,36 ,39 ,55 ,57 ,60 ,61 ,1093 ,1096 + ,1097 ,1098 ,1099 ,1100 ,1101 ,1102 ,1103 ,1104 ,1105 ,1106 + ,1109 ,1154 ,1157 ,1160 ,1161 ,1176 ,1178 ,1179 ,1254 ,1257 + ,1768 ,1773 ,1957 ,35105 ,36933 ,36936 ,36937 ,36938 ,36939 ,36940 + ,36942 ,36943 ,36944 ,36945 ,36946 ,36947 ,36948 ,37640 ,37642 ,37644 + ,37646 ,37711 ,37744 ,37745 ,37746 ,37747 ,37748 ,40133 ,40136 ,40138 + ,40139 ,40140 ,40141 ,1123363 ,35947624 ,35947625 ,35947626 ,35947627 ,35947629 ,35947630 + ,35947631 ,35947635 ,35947636 ,35947638 +}; + +#ifdef U_ENABLE_GENERIC_ISO_2022 + +static const char* const escSeqStateTable_Result_2022[MAX_STATES_2022] = { + /* 0 1 2 3 4 5 6 7 8 9 */ + + NULL ,NULL ,NULL ,NULL ,NULL ,NULL ,NULL ,NULL ,"latin1" ,"latin1" + ,"latin1" ,"ibm-865" ,"ibm-865" ,"ibm-865" ,"ibm-865" ,"ibm-865" ,"ibm-865" ,"JISX0201" ,"JISX0201" ,"latin1" + ,"latin1" ,NULL ,"JISX-208" ,"ibm-5478" ,"JISX-208" ,NULL ,NULL ,NULL ,NULL ,"UTF8" + ,"ISO-8859-1" ,"ISO-8859-7" ,"JIS-X-208" ,NULL ,"ibm-955" ,"ibm-367" ,"ibm-952" ,"ibm-949" ,"JISX-212" ,"ibm-1383" + ,"ibm-952" ,"ibm-964" ,"ibm-964" ,"ibm-964" ,"ibm-964" ,"ibm-964" ,"ibm-964" ,"ibm-5478" ,"ibm-949" ,"ISO-IR-165" + ,"CNS-11643-1992,1" ,"CNS-11643-1992,2" ,"CNS-11643-1992,3" ,"CNS-11643-1992,4" ,"CNS-11643-1992,5" ,"CNS-11643-1992,6" ,"CNS-11643-1992,7" ,"UTF16_PlatformEndian" ,"UTF16_PlatformEndian" ,"UTF16_PlatformEndian" + ,"UTF16_PlatformEndian" ,"UTF16_PlatformEndian" ,"UTF16_PlatformEndian" ,NULL ,"latin1" ,"ibm-912" ,"ibm-913" ,"ibm-914" ,"ibm-813" ,"ibm-1089" + ,"ibm-920" ,"ibm-915" ,"ibm-915" ,"latin1" +}; + +#endif + +static const UCNV_TableStates_2022 escSeqStateTable_Value_2022[MAX_STATES_2022] = { +/* 0 1 2 3 4 5 6 7 8 9 */ + VALID_NON_TERMINAL_2022 ,VALID_NON_TERMINAL_2022 ,VALID_NON_TERMINAL_2022 ,VALID_NON_TERMINAL_2022 ,VALID_NON_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_NON_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 + ,VALID_MAYBE_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 + ,VALID_TERMINAL_2022 ,VALID_NON_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_NON_TERMINAL_2022 ,VALID_NON_TERMINAL_2022 ,VALID_NON_TERMINAL_2022 ,VALID_NON_TERMINAL_2022 ,VALID_TERMINAL_2022 + ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_NON_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 + ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 + ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 + ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_NON_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 + ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 +}; + + +/* Type def for refactoring changeState_2022 code*/ +typedef enum{ +#ifdef U_ENABLE_GENERIC_ISO_2022 + ISO_2022=0, +#endif + ISO_2022_JP=1, + ISO_2022_KR=2, + ISO_2022_CN=3 +} Variant2022; + +/*********** ISO 2022 Converter Protos ***********/ +static void +_ISO2022Open(UConverter *cnv, const char *name, const char *locale,uint32_t options, UErrorCode *errorCode); + +static void + _ISO2022Close(UConverter *converter); + +static void +_ISO2022Reset(UConverter *converter, UConverterResetChoice choice); + +static const char* +_ISO2022getName(const UConverter* cnv); + +static void +_ISO_2022_WriteSub(UConverterFromUnicodeArgs *args, int32_t offsetIndex, UErrorCode *err); + +static UConverter * +_ISO_2022_SafeClone(const UConverter *cnv, void *stackBuffer, int32_t *pBufferSize, UErrorCode *status); + +#ifdef U_ENABLE_GENERIC_ISO_2022 +static void +T_UConverter_toUnicode_ISO_2022_OFFSETS_LOGIC(UConverterToUnicodeArgs* args, UErrorCode* err); +#endif + +/*const UConverterSharedData _ISO2022Data;*/ +static const UConverterSharedData _ISO2022JPData; +static const UConverterSharedData _ISO2022KRData; +static const UConverterSharedData _ISO2022CNData; + +/*************** Converter implementations ******************/ + +/* The purpose of this function is to get around gcc compiler warnings. */ +static U_INLINE void +fromUWriteUInt8(UConverter *cnv, + const char *bytes, int32_t length, + uint8_t **target, const char *targetLimit, + int32_t **offsets, + int32_t sourceIndex, + UErrorCode *pErrorCode) +{ + char *targetChars = (char *)*target; + ucnv_fromUWriteBytes(cnv, bytes, length, &targetChars, targetLimit, + offsets, sourceIndex, pErrorCode); + *target = (uint8_t*)targetChars; + +} + +static U_INLINE void +setInitialStateToUnicodeKR(UConverter* converter, UConverterDataISO2022 *myConverterData){ + if(myConverterData->version == 1) { + UConverter *cnv = myConverterData->currentConverter; + + cnv->toUnicodeStatus=0; /* offset */ + cnv->mode=0; /* state */ + cnv->toULength=0; /* byteIndex */ + } +} + +static U_INLINE void +setInitialStateFromUnicodeKR(UConverter* converter,UConverterDataISO2022 *myConverterData){ + /* in ISO-2022-KR the designator sequence appears only once + * in a file so we append it only once + */ + if( converter->charErrorBufferLength==0){ + + converter->charErrorBufferLength = 4; + converter->charErrorBuffer[0] = 0x1b; + converter->charErrorBuffer[1] = 0x24; + converter->charErrorBuffer[2] = 0x29; + converter->charErrorBuffer[3] = 0x43; + } + if(myConverterData->version == 1) { + UConverter *cnv = myConverterData->currentConverter; + + cnv->fromUChar32=0; + cnv->fromUnicodeStatus=1; /* prevLength */ + } +} + +static void +_ISO2022Open(UConverter *cnv, const char *name, const char *locale,uint32_t options, UErrorCode *errorCode){ + + char myLocale[6]={' ',' ',' ',' ',' ',' '}; + + cnv->extraInfo = uprv_malloc (sizeof (UConverterDataISO2022)); + if(cnv->extraInfo != NULL) { + UConverterDataISO2022 *myConverterData=(UConverterDataISO2022 *) cnv->extraInfo; + uint32_t version; + + uprv_memset(myConverterData, 0, sizeof(UConverterDataISO2022)); + myConverterData->currentType = ASCII1; + cnv->fromUnicodeStatus =FALSE; + if(locale){ + uprv_strncpy(myLocale, locale, sizeof(myLocale)); + } + version = options & UCNV_OPTIONS_VERSION_MASK; + myConverterData->version = version; + if(myLocale[0]=='j' && (myLocale[1]=='a'|| myLocale[1]=='p') && + (myLocale[2]=='_' || myLocale[2]=='\0')) + { + size_t len=0; + /* open the required converters and cache them */ + if(jpCharsetMasks[version]&CSM(ISO8859_7)) { + myConverterData->myConverterArray[ISO8859_7]= ucnv_loadSharedData("ISO8859_7", NULL, errorCode); + } + myConverterData->myConverterArray[JISX201] = ucnv_loadSharedData("JISX0201", NULL, errorCode); + myConverterData->myConverterArray[JISX208] = ucnv_loadSharedData("jisx-208", NULL, errorCode); + if(jpCharsetMasks[version]&CSM(JISX212)) { + myConverterData->myConverterArray[JISX212] = ucnv_loadSharedData("jisx-212", NULL, errorCode); + } + if(jpCharsetMasks[version]&CSM(GB2312)) { + myConverterData->myConverterArray[GB2312] = ucnv_loadSharedData("ibm-5478", NULL, errorCode); /* gb_2312_80-1 */ + } + if(jpCharsetMasks[version]&CSM(KSC5601)) { + myConverterData->myConverterArray[KSC5601] = ucnv_loadSharedData("ksc_5601", NULL, errorCode); + } + + /* set the function pointers to appropriate funtions */ + cnv->sharedData=(UConverterSharedData*)(&_ISO2022JPData); + uprv_strcpy(myConverterData->locale,"ja"); + + (void)uprv_strcpy(myConverterData->name,"ISO_2022,locale=ja,version="); + len = uprv_strlen(myConverterData->name); + myConverterData->name[len]=(char)(myConverterData->version+(int)'0'); + myConverterData->name[len+1]='\0'; + } + else if(myLocale[0]=='k' && (myLocale[1]=='o'|| myLocale[1]=='r') && + (myLocale[2]=='_' || myLocale[2]=='\0')) + { + if (version==1){ + myConverterData->currentConverter= + ucnv_open("icu-internal-25546",errorCode); + + if (U_FAILURE(*errorCode)) { + _ISO2022Close(cnv); + return; + } + + (void)uprv_strcpy(myConverterData->name,"ISO_2022,locale=ko,version=1"); + uprv_memcpy(cnv->subChars, myConverterData->currentConverter->subChars, 4); + cnv->subCharLen = myConverterData->currentConverter->subCharLen; + }else{ + myConverterData->currentConverter=ucnv_open("ibm-949",errorCode); + + if (U_FAILURE(*errorCode)) { + _ISO2022Close(cnv); + return; + } + + myConverterData->version = 0; + (void)uprv_strcpy(myConverterData->name,"ISO_2022,locale=ko,version=0"); + } + + /* initialize the state variables */ + setInitialStateToUnicodeKR(cnv, myConverterData); + setInitialStateFromUnicodeKR(cnv, myConverterData); + + /* set the function pointers to appropriate funtions */ + cnv->sharedData=(UConverterSharedData*)&_ISO2022KRData; + uprv_strcpy(myConverterData->locale,"ko"); + } + else if(((myLocale[0]=='z' && myLocale[1]=='h') || (myLocale[0]=='c'&& myLocale[1]=='n'))&& + (myLocale[2]=='_' || myLocale[2]=='\0')) + { + + /* open the required converters and cache them */ + myConverterData->myConverterArray[GB2312_1] = ucnv_loadSharedData("ibm-5478", NULL, errorCode); + if(version==1) { + myConverterData->myConverterArray[ISO_IR_165] = ucnv_loadSharedData("iso-ir-165", NULL, errorCode); + } + myConverterData->myConverterArray[CNS_11643] = ucnv_loadSharedData("cns-11643-1992", NULL, errorCode); + + + /* set the function pointers to appropriate funtions */ + cnv->sharedData=(UConverterSharedData*)&_ISO2022CNData; + uprv_strcpy(myConverterData->locale,"cn"); + + if (version==1){ + (void)uprv_strcpy(myConverterData->name,"ISO_2022,locale=zh,version=1"); + }else{ + myConverterData->version = 0; + (void)uprv_strcpy(myConverterData->name,"ISO_2022,locale=zh,version=0"); + } + } + else{ +#ifdef U_ENABLE_GENERIC_ISO_2022 + myConverterData->isFirstBuffer = TRUE; + + /* append the UTF-8 escape sequence */ + cnv->charErrorBufferLength = 3; + cnv->charErrorBuffer[0] = 0x1b; + cnv->charErrorBuffer[1] = 0x25; + cnv->charErrorBuffer[2] = 0x42; + + cnv->sharedData=(UConverterSharedData*)&_ISO2022Data; + /* initialize the state variables */ + uprv_strcpy(myConverterData->name,"ISO_2022"); +#else + *errorCode = U_UNSUPPORTED_ERROR; + return; +#endif + } + + cnv->maxBytesPerUChar=cnv->sharedData->staticData->maxBytesPerChar; + + if(U_FAILURE(*errorCode)) { + _ISO2022Close(cnv); + } + } else { + *errorCode = U_MEMORY_ALLOCATION_ERROR; + } +} + + +static void +_ISO2022Close(UConverter *converter) { + UConverterDataISO2022* myData =(UConverterDataISO2022 *) (converter->extraInfo); + UConverterSharedData **array = myData->myConverterArray; + int32_t i; + + if (converter->extraInfo != NULL) { + /*close the array of converter pointers and free the memory*/ + for (i=0; icurrentConverter); + + if(!converter->isExtraLocal){ + uprv_free (converter->extraInfo); + converter->extraInfo = NULL; + } + } +} + +static void +_ISO2022Reset(UConverter *converter, UConverterResetChoice choice) { + UConverterDataISO2022 *myConverterData=(UConverterDataISO2022 *) (converter->extraInfo); + if(choice<=UCNV_RESET_TO_UNICODE) { + uprv_memset(&myConverterData->toU2022State, 0, sizeof(ISO2022State)); + myConverterData->key = 0; + } + if(choice!=UCNV_RESET_TO_UNICODE) { + uprv_memset(&myConverterData->fromU2022State, 0, sizeof(ISO2022State)); + } +#ifdef U_ENABLE_GENERIC_ISO_2022 + if(myConverterData->locale[0] == 0){ + if(choice<=UCNV_RESET_TO_UNICODE) { + myConverterData->isFirstBuffer = TRUE; + myConverterData->key = 0; + if (converter->mode == UCNV_SO){ + ucnv_close (myConverterData->currentConverter); + myConverterData->currentConverter=NULL; + } + converter->mode = UCNV_SI; + } + if(choice!=UCNV_RESET_TO_UNICODE) { + /* re-append UTF-8 escape sequence */ + converter->charErrorBufferLength = 3; + converter->charErrorBuffer[0] = 0x1b; + converter->charErrorBuffer[1] = 0x28; + converter->charErrorBuffer[2] = 0x42; + } + } + else +#endif + { + /* reset the state variables */ + if(myConverterData->locale[0] == 'k'){ + if(choice<=UCNV_RESET_TO_UNICODE) { + setInitialStateToUnicodeKR(converter, myConverterData); + } + if(choice!=UCNV_RESET_TO_UNICODE) { + setInitialStateFromUnicodeKR(converter, myConverterData); + } + } + } +} + +static const char* +_ISO2022getName(const UConverter* cnv){ + if(cnv->extraInfo){ + UConverterDataISO2022* myData= (UConverterDataISO2022*)cnv->extraInfo; + return myData->name; + } + return NULL; +} + + +/*************** to unicode *******************/ +/**************************************************************************** + * Recognized escape sequences are + * (B ASCII + * .A ISO-8859-1 + * .F ISO-8859-7 + * (J JISX-201 + * (I JISX-201 + * $B JISX-208 + * $@ JISX-208 + * $(D JISX-212 + * $A GB2312 + * $(C KSC5601 + */ +static const StateEnum nextStateToUnicodeJP[MAX_STATES_2022]= { +/* 0 1 2 3 4 5 6 7 8 9 */ + INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,SS2_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE + ,ASCII ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,JISX201 ,HWKANA_7BIT ,JISX201 ,INVALID_STATE + ,INVALID_STATE ,INVALID_STATE ,JISX208 ,GB2312 ,JISX208 ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE + ,ISO8859_1 ,ISO8859_7 ,JISX208 ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,KSC5601 ,JISX212 ,INVALID_STATE + ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE + ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE + ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE + ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE +}; + +/*************** to unicode *******************/ +static const StateEnum nextStateToUnicodeCN[MAX_STATES_2022]= { +/* 0 1 2 3 4 5 6 7 8 9 */ + INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,SS2_STATE ,SS3_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE + ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE + ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE + ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE + ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,GB2312_1 ,INVALID_STATE ,ISO_IR_165 + ,CNS_11643_1 ,CNS_11643_2 ,CNS_11643_3 ,CNS_11643_4 ,CNS_11643_5 ,CNS_11643_6 ,CNS_11643_7 ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE + ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE + ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE +}; + + +static UCNV_TableStates_2022 +getKey_2022(char c,int32_t* key,int32_t* offset){ + int32_t togo; + int32_t low = 0; + int32_t hi = MAX_STATES_2022; + int32_t oldmid=0; + + togo = normalize_esq_chars_2022[(uint8_t)c]; + if(togo == 0) { + /* not a valid character anywhere in an escape sequence */ + *key = 0; + *offset = 0; + return INVALID_2022; + } + togo = (*key << 5) + togo; + + while (hi != low) /*binary search*/{ + + register int32_t mid = (hi+low) >> 1; /*Finds median*/ + + if (mid == oldmid) + break; + + if (escSeqStateTable_Key_2022[mid] > togo){ + hi = mid; + } + else if (escSeqStateTable_Key_2022[mid] < togo){ + low = mid; + } + else /*we found it*/{ + *key = togo; + *offset = mid; + return escSeqStateTable_Value_2022[mid]; + } + oldmid = mid; + + } + + *key = 0; + *offset = 0; + return INVALID_2022; +} + +/*runs through a state machine to determine the escape sequence - codepage correspondance + */ +static void +changeState_2022(UConverter* _this, + const char** source, + const char* sourceLimit, + Variant2022 var, + UErrorCode* err){ + UCNV_TableStates_2022 value; + UConverterDataISO2022* myData2022 = ((UConverterDataISO2022*)_this->extraInfo); + uint32_t key = myData2022->key; + int32_t offset = 0; + char c; + + value = VALID_NON_TERMINAL_2022; + while (*source < sourceLimit) { + c = *(*source)++; + _this->toUBytes[_this->toULength++]=(uint8_t)c; + value = getKey_2022(c,(int32_t *) &key, &offset); + + switch (value){ + + case VALID_NON_TERMINAL_2022 : + /* continue with the loop */ + break; + + case VALID_TERMINAL_2022: + key = 0; + goto DONE; + + case INVALID_2022: + goto DONE; + + case VALID_MAYBE_TERMINAL_2022: +#ifdef U_ENABLE_GENERIC_ISO_2022 + /* ESC ( B is ambiguous only for ISO_2022 itself */ + if(var == ISO_2022) { + /* discard toUBytes[] for ESC ( B because this sequence is correct and complete */ + _this->toULength = 0; + + /* TODO need to indicate that ESC ( B was seen; if failure, then need to replay from source or from MBCS-style replay */ + + /* continue with the loop */ + value = VALID_NON_TERMINAL_2022; + break; + } else +#endif + { + /* not ISO_2022 itself, finish here */ + value = VALID_TERMINAL_2022; + key = 0; + goto DONE; + } + } + } + +DONE: + myData2022->key = key; + + if (value == VALID_NON_TERMINAL_2022) { + /* indicate that the escape sequence is incomplete: key!=0 */ + return; + } else if (value == INVALID_2022 ) { + *err = U_ILLEGAL_ESCAPE_SEQUENCE; + return; + } else /* value == VALID_TERMINAL_2022 */ { + switch(var){ +#ifdef U_ENABLE_GENERIC_ISO_2022 + case ISO_2022: + { + const char *chosenConverterName = escSeqStateTable_Result_2022[offset]; + if(chosenConverterName == NULL) { + /* SS2 or SS3 */ + *err = U_UNSUPPORTED_ESCAPE_SEQUENCE; + return; + } + + _this->mode = UCNV_SI; + ucnv_close(myData2022->currentConverter); + myData2022->currentConverter = myUConverter = ucnv_open(chosenConverterName, err); + if(U_SUCCESS(*err)) { + myUConverter->fromCharErrorBehaviour = UCNV_TO_U_CALLBACK_STOP; + _this->mode = UCNV_SO; + } + break; + } +#endif + case ISO_2022_JP: + { + StateEnum tempState=nextStateToUnicodeJP[offset]; + switch(tempState) { + case INVALID_STATE: + *err = U_UNSUPPORTED_ESCAPE_SEQUENCE; + break; + case SS2_STATE: + if(myData2022->toU2022State.cs[2]!=0) { + if(myData2022->toU2022State.g<2) { + myData2022->toU2022State.prevG=myData2022->toU2022State.g; + } + myData2022->toU2022State.g=2; + } else { + /* illegal to have SS2 before a matching designator */ + *err = U_ILLEGAL_ESCAPE_SEQUENCE; + } + break; + /* case SS3_STATE: not used in ISO-2022-JP-x */ + case ISO8859_1: + case ISO8859_7: + if((jpCharsetMasks[myData2022->version] & CSM(tempState)) == 0) { + *err = U_UNSUPPORTED_ESCAPE_SEQUENCE; + } else { + /* G2 charset for SS2 */ + myData2022->toU2022State.cs[2]=(int8_t)tempState; + } + break; + default: + if((jpCharsetMasks[myData2022->version] & CSM(tempState)) == 0) { + *err = U_UNSUPPORTED_ESCAPE_SEQUENCE; + } else { + /* G0 charset */ + myData2022->toU2022State.cs[0]=(int8_t)tempState; + } + break; + } + } + break; + case ISO_2022_CN: + { + StateEnum tempState=nextStateToUnicodeCN[offset]; + switch(tempState) { + case INVALID_STATE: + *err = U_UNSUPPORTED_ESCAPE_SEQUENCE; + break; + case SS2_STATE: + if(myData2022->toU2022State.cs[2]!=0) { + if(myData2022->toU2022State.g<2) { + myData2022->toU2022State.prevG=myData2022->toU2022State.g; + } + myData2022->toU2022State.g=2; + } else { + /* illegal to have SS2 before a matching designator */ + *err = U_ILLEGAL_ESCAPE_SEQUENCE; + } + break; + case SS3_STATE: + if(myData2022->toU2022State.cs[3]!=0) { + if(myData2022->toU2022State.g<2) { + myData2022->toU2022State.prevG=myData2022->toU2022State.g; + } + myData2022->toU2022State.g=3; + } else { + /* illegal to have SS3 before a matching designator */ + *err = U_ILLEGAL_ESCAPE_SEQUENCE; + } + break; + case ISO_IR_165: + if(myData2022->version==0) { + *err = U_UNSUPPORTED_ESCAPE_SEQUENCE; + break; + } + /*fall through*/ + case GB2312_1: + /*fall through*/ + case CNS_11643_1: + myData2022->toU2022State.cs[1]=(int8_t)tempState; + break; + case CNS_11643_2: + myData2022->toU2022State.cs[2]=(int8_t)tempState; + break; + default: + /* other CNS 11643 planes */ + if(myData2022->version==0) { + *err = U_UNSUPPORTED_ESCAPE_SEQUENCE; + } else { + myData2022->toU2022State.cs[3]=(int8_t)tempState; + } + break; + } + } + break; + case ISO_2022_KR: + if(offset==0x30){ + /* nothing to be done, just accept this one escape sequence */ + } else { + *err = U_UNSUPPORTED_ESCAPE_SEQUENCE; + } + break; + + default: + *err = U_ILLEGAL_ESCAPE_SEQUENCE; + break; + } + } + if(U_SUCCESS(*err)) { + _this->toULength = 0; + } +} + +/*Checks the characters of the buffer against valid 2022 escape sequences +*if the match we return a pointer to the initial start of the sequence otherwise +*we return sourceLimit +*/ +/*for 2022 looks ahead in the stream + *to determine the longest possible convertible + *data stream + */ +static U_INLINE const char* +getEndOfBuffer_2022(const char** source, + const char* sourceLimit, + UBool flush){ + + const char* mySource = *source; + +#ifdef U_ENABLE_GENERIC_ISO_2022 + if (*source >= sourceLimit) + return sourceLimit; + + do{ + + if (*mySource == ESC_2022){ + int8_t i; + int32_t key = 0; + int32_t offset; + UCNV_TableStates_2022 value = VALID_NON_TERMINAL_2022; + + /* Kludge: I could not + * figure out the reason for validating an escape sequence + * twice - once here and once in changeState_2022(). + * is it possible to have an ESC character in a ISO2022 + * byte stream which is valid in a code page? Is it legal? + */ + for (i=0; + (mySource+i < sourceLimit)&&(value == VALID_NON_TERMINAL_2022); + i++) { + value = getKey_2022(*(mySource+i), &key, &offset); + } + if (value > 0 || *mySource==ESC_2022) + return mySource; + + if ((value == VALID_NON_TERMINAL_2022)&&(!flush) ) + return sourceLimit; + } + }while (++mySource < sourceLimit); + + return sourceLimit; +#else + while(mySource < sourceLimit && *mySource != ESC_2022) { + ++mySource; + } + return mySource; +#endif +} + + +/* This inline function replicates code in _MBCSFromUChar32() function in ucnvmbcs.c + * any future change in _MBCSFromUChar32() function should be reflected here. + * @return number of bytes in *value; negative number if fallback; 0 if no mapping + */ +static U_INLINE int32_t +MBCS_FROM_UCHAR32_ISO2022(UConverterSharedData* sharedData, + UChar32 c, + uint32_t* value, + UBool useFallback, + int outputType) +{ + const int32_t *cx; + const uint16_t *table; + uint32_t stage2Entry; + uint32_t myValue; + int32_t length; + const uint8_t *p; + /* + * TODO(markus): Use and require new, faster MBCS conversion table structures. + * Use internal version of ucnv_open() that verifies that the new structures are available, + * else U_INTERNAL_PROGRAM_ERROR. + */ + /* BMP-only codepages are stored without stage 1 entries for supplementary code points */ + if(c<0x10000 || (sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY)) { + table=sharedData->mbcs.fromUnicodeTable; + stage2Entry=MBCS_STAGE_2_FROM_U(table, c); + /* get the bytes and the length for the output */ + if(outputType==MBCS_OUTPUT_2){ + myValue=MBCS_VALUE_2_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); + if(myValue<=0xff) { + length=1; + } else { + length=2; + } + } else /* outputType==MBCS_OUTPUT_3 */ { + p=MBCS_POINTER_3_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); + myValue=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; + if(myValue<=0xff) { + length=1; + } else if(myValue<=0xffff) { + length=2; + } else { + length=3; + } + } + /* + * TODO(markus): Use Shift-JIS table for JIS X 0208, to save mapping table space. + * Pass in parameter for type of output bytes, for validation and shifting: + * - Direct: Pass bytes through, but forbid control codes 00-1F (except SI/SO/ESC) and space 20? + * (Need to allow some (TAB/LF/CR) or most of them for ASCII and maybe JIS X 0201.) + * - A1-FE: Subtract 80 after range check. + * - SJIS: Shift DBCS result to 21-7E x 21-7E. + */ + /* is this code point assigned, or do we use fallbacks? */ + if((stage2Entry&(1<<(16+(c&0xf))))!=0) { + /* assigned */ + *value=myValue; + return length; + } else if(FROM_U_USE_FALLBACK(useFallback, c) && myValue!=0) { + /* + * We allow a 0 byte output if the "assigned" bit is set for this entry. + * There is no way with this data structure for fallback output + * to be a zero byte. + */ + *value=myValue; + return -length; + } + } + + cx=sharedData->mbcs.extIndexes; + if(cx!=NULL) { + return ucnv_extSimpleMatchFromU(cx, c, value, useFallback); + } + + /* unassigned */ + return 0; +} + +/* This inline function replicates code in _MBCSSingleFromUChar32() function in ucnvmbcs.c + * any future change in _MBCSSingleFromUChar32() function should be reflected here. + * @param retval pointer to output byte + * @return 1 roundtrip byte 0 no mapping -1 fallback byte + */ +static U_INLINE int32_t +MBCS_SINGLE_FROM_UCHAR32(UConverterSharedData* sharedData, + UChar32 c, + uint32_t* retval, + UBool useFallback) +{ + const uint16_t *table; + int32_t value; + /* BMP-only codepages are stored without stage 1 entries for supplementary code points */ + if(c>=0x10000 && !(sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY)) { + return 0; + } + /* convert the Unicode code point in c into codepage bytes (same as in _MBCSFromUnicodeWithOffsets) */ + table=sharedData->mbcs.fromUnicodeTable; + /* get the byte for the output */ + value=MBCS_SINGLE_RESULT_FROM_U(table, (uint16_t *)sharedData->mbcs.fromUnicodeBytes, c); + /* is this code point assigned, or do we use fallbacks? */ + *retval=(uint32_t)(value&0xff); + if(value>=0xf00) { + return 1; /* roundtrip */ + } else if(useFallback ? value>=0x800 : value>=0xc00) { + return -1; /* fallback taken */ + } else { + return 0; /* no mapping */ + } +} + +#ifdef U_ENABLE_GENERIC_ISO_2022 + +/********************************************************************************** +* ISO-2022 Converter +* +* +*/ + +static void +T_UConverter_toUnicode_ISO_2022_OFFSETS_LOGIC(UConverterToUnicodeArgs* args, + UErrorCode* err){ + const char* mySourceLimit, *realSourceLimit; + const char* sourceStart; + const UChar* myTargetStart; + UConverter* saveThis; + UConverterDataISO2022* myData; + int8_t length; + + saveThis = args->converter; + myData=((UConverterDataISO2022*)(saveThis->extraInfo)); + + realSourceLimit = args->sourceLimit; + while (args->source < realSourceLimit) { + if(myData->key == 0) { /* are we in the middle of an escape sequence? */ + /*Find the end of the buffer e.g : Next Escape Seq | end of Buffer*/ + mySourceLimit = getEndOfBuffer_2022(&(args->source), realSourceLimit, args->flush); + + if(args->source < mySourceLimit) { + if(myData->currentConverter==NULL) { + myData->currentConverter = ucnv_open("ASCII",err); + if(U_FAILURE(*err)){ + return; + } + + myData->currentConverter->fromCharErrorBehaviour = UCNV_TO_U_CALLBACK_STOP; + saveThis->mode = UCNV_SO; + } + + /* convert to before the ESC or until the end of the buffer */ + myData->isFirstBuffer=FALSE; + sourceStart = args->source; + myTargetStart = args->target; + args->converter = myData->currentConverter; + ucnv_toUnicode(args->converter, + &args->target, + args->targetLimit, + &args->source, + mySourceLimit, + args->offsets, + (UBool)(args->flush && mySourceLimit == realSourceLimit), + err); + args->converter = saveThis; + + if (*err == U_BUFFER_OVERFLOW_ERROR) { + /* move the overflow buffer */ + length = saveThis->UCharErrorBufferLength = myData->currentConverter->UCharErrorBufferLength; + myData->currentConverter->UCharErrorBufferLength = 0; + if(length > 0) { + uprv_memcpy(saveThis->UCharErrorBuffer, + myData->currentConverter->UCharErrorBuffer, + length*U_SIZEOF_UCHAR); + } + return; + } + + /* + * At least one of: + * -Error while converting + * -Done with entire buffer + * -Need to write offsets or update the current offset + * (leave that up to the code in ucnv.c) + * + * or else we just stopped at an ESC byte and continue with changeState_2022() + */ + if (U_FAILURE(*err) || + (args->source == realSourceLimit) || + (args->offsets != NULL && (args->target != myTargetStart || args->source != sourceStart) || + (mySourceLimit < realSourceLimit && myData->currentConverter->toULength > 0)) + ) { + /* copy partial or error input for truncated detection and error handling */ + if(U_FAILURE(*err)) { + length = saveThis->invalidCharLength = myData->currentConverter->invalidCharLength; + if(length > 0) { + uprv_memcpy(saveThis->invalidCharBuffer, myData->currentConverter->invalidCharBuffer, length); + } + } else { + length = saveThis->toULength = myData->currentConverter->toULength; + if(length > 0) { + uprv_memcpy(saveThis->toUBytes, myData->currentConverter->toUBytes, length); + if(args->source < mySourceLimit) { + *err = U_TRUNCATED_CHAR_FOUND; /* truncated input before ESC */ + } + } + } + return; + } + } + } + + sourceStart = args->source; + changeState_2022(args->converter, + &(args->source), + realSourceLimit, + ISO_2022, + err); + if (U_FAILURE(*err) || (args->source != sourceStart && args->offsets != NULL)) { + /* let the ucnv.c code update its current offset */ + return; + } + } +} + +#endif + +/* + * To Unicode Callback helper function + */ +static void +toUnicodeCallback(UConverter *cnv, + const uint32_t sourceChar, const uint32_t targetUniChar, + UErrorCode* err){ + if(sourceChar>0xff){ + cnv->toUBytes[0] = (uint8_t)(sourceChar>>8); + cnv->toUBytes[1] = (uint8_t)sourceChar; + cnv->toULength = 2; + } + else{ + cnv->toUBytes[0] =(char) sourceChar; + cnv->toULength = 2; + } + + if(targetUniChar == (missingCharMarker-1/*0xfffe*/)){ + *err = U_INVALID_CHAR_FOUND; + } + else{ + *err = U_ILLEGAL_CHAR_FOUND; + } +} + +/**************************************ISO-2022-JP*************************************************/ + +/************************************** IMPORTANT ************************************************** +* The UConverter_fromUnicode_ISO2022_JP converter does not use ucnv_fromUnicode() functions for SBCS,DBCS and +* MBCS; instead, the values are obtained directly by calling _MBCSFromUChar32(). +* The converter iterates over each Unicode codepoint +* to obtain the equivalent codepoints from the codepages supported. Since the source buffer is +* processed one char at a time it would make sense to reduce the extra processing a canned converter +* would do as far as possible. +* +* If the implementation of these macros or structure of sharedData struct change in the future, make +* sure that ISO-2022 is also changed. +*************************************************************************************************** +*/ + +/*************************************************************************************************** +* Rules for ISO-2022-jp encoding +* (i) Escape sequences must be fully contained within a line they should not +* span new lines or CRs +* (ii) If the last character on a line is represented by two bytes then an ASCII or +* JIS-Roman character escape sequence should follow before the line terminates +* (iii) If the first character on the line is represented by two bytes then a two +* byte character escape sequence should precede it +* (iv) If no escape sequence is encountered then the characters are ASCII +* (v) Latin(ISO-8859-1) and Greek(ISO-8859-7) characters must be designated to G2, +* and invoked with SS2 (ESC N). +* (vi) If there is any G0 designation in text, there must be a switch to +* ASCII or to JIS X 0201-Roman before a space character (but not +* necessarily before "ESC 4/14 2/0" or "ESC N ' '") or control +* characters such as tab or CRLF. +* (vi) Supported encodings: +* ASCII, JISX201, JISX208, JISX212, GB2312, KSC5601, ISO-8859-1,ISO-8859-7 +* +* source : RFC-1554 +* +* JISX201, JISX208,JISX212 : new .cnv data files created +* KSC5601 : alias to ibm-949 mapping table +* GB2312 : alias to ibm-1386 mapping table +* ISO-8859-1 : Algorithmic implemented as LATIN1 case +* ISO-8859-7 : alisas to ibm-9409 mapping table +*/ + +/* preference order of JP charsets */ +static const StateEnum jpCharsetPref[]={ + ASCII, + JISX201, + ISO8859_1, + ISO8859_7, + JISX208, + JISX212, + GB2312, + KSC5601, + HWKANA_7BIT +}; + +/* + * The escape sequences must be in order of the enum constants like JISX201 = 3, + * not in order of jpCharsetPref[]! + */ +static const char escSeqChars[][6] ={ + "\x1B\x28\x42", /* (B ASCII */ + "\x1B\x2E\x41", /* .A ISO-8859-1 */ + "\x1B\x2E\x46", /* .F ISO-8859-7 */ + "\x1B\x28\x4A", /* (J JISX-201 */ + "\x1B\x24\x42", /* $B JISX-208 */ + "\x1B\x24\x28\x44", /* $(D JISX-212 */ + "\x1B\x24\x41", /* $A GB2312 */ + "\x1B\x24\x28\x43", /* $(C KSC5601 */ + "\x1B\x28\x49" /* (I HWKANA_7BIT */ + +}; +static const int32_t escSeqCharsLen[] ={ + 3, /* length of (B ASCII */ + 3, /* length of .A ISO-8859-1 */ + 3, /* length of .F ISO-8859-7 */ + 3, /* length of (J JISX-201 */ + 3, /* length of $B JISX-208 */ + 4, /* length of $(D JISX-212 */ + 3, /* length of $A GB2312 */ + 4, /* length of $(C KSC5601 */ + 3 /* length of (I HWKANA_7BIT */ +}; + +/* +* The iteration over various code pages works this way: +* i) Get the currentState from myConverterData->currentState +* ii) Check if the character is mapped to a valid character in the currentState +* Yes -> a) set the initIterState to currentState +* b) remain in this state until an invalid character is found +* No -> a) go to the next code page and find the character +* iii) Before changing the state increment the current state check if the current state +* is equal to the intitIteration state +* Yes -> A character that cannot be represented in any of the supported encodings +* break and return a U_INVALID_CHARACTER error +* No -> Continue and find the character in next code page +* +* +* TODO: Implement a priority technique where the users are allowed to set the priority of code pages +*/ + +static void +UConverter_fromUnicode_ISO_2022_JP_OFFSETS_LOGIC(UConverterFromUnicodeArgs* args, UErrorCode* err) { + UConverter *cnv = args->converter; + UConverterDataISO2022 *converterData; + ISO2022State *pFromU2022State; + uint8_t *target = (uint8_t *) args->target; + const uint8_t *targetLimit = (const uint8_t *) args->targetLimit; + const UChar* source = args->source; + const UChar* sourceLimit = args->sourceLimit; + int32_t* offsets = args->offsets; + UChar32 sourceChar; + char buffer[8]; + int32_t len, outLen; + int8_t choices[10]; + int32_t choiceCount; + uint32_t targetValue = 0; + UBool useFallback; + + int32_t i; + int8_t cs, g; + + /* set up the state */ + converterData = (UConverterDataISO2022*)cnv->extraInfo; + pFromU2022State = &converterData->fromU2022State; + + choiceCount = 0; + + /* check if the last codepoint of previous buffer was a lead surrogate*/ + if((sourceChar = cnv->fromUChar32)!=0 && target< targetLimit) { + goto getTrail; + } + + while(source < sourceLimit) { + if(target < targetLimit) { + + sourceChar = *(source++); + /*check if the char is a First surrogate*/ + if(UTF_IS_SURROGATE(sourceChar)) { + if(UTF_IS_SURROGATE_FIRST(sourceChar)) { +getTrail: + /*look ahead to find the trail surrogate*/ + if(source < sourceLimit) { + /* test the following code unit */ + UChar trail=(UChar) *source; + if(UTF_IS_SECOND_SURROGATE(trail)) { + source++; + sourceChar=UTF16_GET_PAIR_VALUE(sourceChar, trail); + cnv->fromUChar32=0x00; + /* convert this supplementary code point */ + /* exit this condition tree */ + } else { + /* this is an unmatched lead code unit (1st surrogate) */ + /* callback(illegal) */ + *err=U_ILLEGAL_CHAR_FOUND; + cnv->fromUChar32=sourceChar; + break; + } + } else { + /* no more input */ + cnv->fromUChar32=sourceChar; + break; + } + } else { + /* this is an unmatched trail code unit (2nd surrogate) */ + /* callback(illegal) */ + *err=U_ILLEGAL_CHAR_FOUND; + cnv->fromUChar32=sourceChar; + break; + } + } + + /* do not convert SO/SI/ESC */ + if(IS_2022_CONTROL(sourceChar)) { + /* callback(illegal) */ + *err=U_ILLEGAL_CHAR_FOUND; + cnv->fromUChar32=sourceChar; + break; + } + + /* do the conversion */ + + if(choiceCount == 0) { + uint16_t csm; + + /* + * The csm variable keeps track of which charsets are allowed + * and not used yet while building the choices[]. + */ + csm = jpCharsetMasks[converterData->version]; + choiceCount = 0; + + /* JIS7/8: try single-byte half-width Katakana before JISX208 */ + if(converterData->version == 3 || converterData->version == 4) { + choices[choiceCount++] = (int8_t)HWKANA_7BIT; + } + /* Do not try single-byte half-width Katakana for other versions. */ + csm &= ~CSM(HWKANA_7BIT); + + /* try the current G0 charset */ + choices[choiceCount++] = cs = pFromU2022State->cs[0]; + csm &= ~CSM(cs); + + /* try the current G2 charset */ + if((cs = pFromU2022State->cs[2]) != 0) { + choices[choiceCount++] = cs; + csm &= ~CSM(cs); + } + + /* try all the other possible charsets */ + for(i = 0; i < LENGTHOF(jpCharsetPref); ++i) { + cs = (int8_t)jpCharsetPref[i]; + if(CSM(cs) & csm) { + choices[choiceCount++] = cs; + csm &= ~CSM(cs); + } + } + } + + cs = g = 0; + /* + * len==0: no mapping found yet + * len<0: found a fallback result: continue looking for a roundtrip but no further fallbacks + * len>0: found a roundtrip result, done + */ + len = 0; + /* + * We will turn off useFallback after finding a fallback, + * but we still get fallbacks from PUA code points as usual. + * Therefore, we will also need to check that we don't overwrite + * an early fallback with a later one. + */ + useFallback = cnv->useFallback; + + for(i = 0; i < choiceCount && len <= 0; ++i) { + uint32_t value; + int32_t len2; + int8_t cs0 = choices[i]; + switch(cs0) { + case ASCII: + if(sourceChar <= 0x7f) { + targetValue = (uint32_t)sourceChar; + len = 1; + cs = cs0; + g = 0; + } + break; + case ISO8859_1: + if(GR96_START <= sourceChar && sourceChar <= GR96_END) { + targetValue = (uint32_t)sourceChar - 0x80; + len = 1; + cs = cs0; + g = 2; + } + break; + case HWKANA_7BIT: + if((uint32_t)(HWKANA_END-sourceChar)<=(HWKANA_END-HWKANA_START)) { + if(converterData->version==3) { + /* JIS7: use G1 (SO) */ + /* Shift U+FF61..U+FF9F to bytes 21..5F. */ + targetValue = (uint32_t)(sourceChar - (HWKANA_START - 0x21)); + len = 1; + pFromU2022State->cs[1] = cs = cs0; /* do not output an escape sequence */ + g = 1; + } else if(converterData->version==4) { + /* JIS8: use 8-bit bytes with any single-byte charset, see escape sequence output below */ + /* Shift U+FF61..U+FF9F to bytes A1..DF. */ + targetValue = (uint32_t)(sourceChar - (HWKANA_START - 0xa1)); + len = 1; + + cs = pFromU2022State->cs[0]; + if(IS_JP_DBCS(cs)) { + /* switch from a DBCS charset to JISX201 */ + cs = (int8_t)JISX201; + } + /* else stay in the current G0 charset */ + g = 0; + } + /* else do not use HWKANA_7BIT with other versions */ + } + break; + case JISX201: + /* G0 SBCS */ + len2 = MBCS_SINGLE_FROM_UCHAR32( + converterData->myConverterArray[cs0], + sourceChar, &value, + useFallback); + if(len2 != 0 && !(len2 < 0 && len != 0) && value <= 0x7f) { + targetValue = value; + len = len2; + cs = cs0; + g = 0; + useFallback = FALSE; + } + break; + case ISO8859_7: + /* G0 SBCS forced to 7-bit output */ + len2 = MBCS_SINGLE_FROM_UCHAR32( + converterData->myConverterArray[cs0], + sourceChar, &value, + useFallback); + if(len2 != 0 && !(len2 < 0 && len != 0) && GR96_START <= value && value <= GR96_END) { + targetValue = value - 0x80; + len = len2; + cs = cs0; + g = 2; + useFallback = FALSE; + } + break; + default: + /* G0 DBCS */ + len2 = MBCS_FROM_UCHAR32_ISO2022( + converterData->myConverterArray[cs0], + sourceChar, &value, + useFallback, MBCS_OUTPUT_2); + if(len2 == 2 || (len2 == -2 && len == 0)) { /* only accept DBCS: abs(len)==2 */ + if(cs0 == KSC5601) { + /* + * Check for valid bytes for the encoding scheme. + * This is necessary because the sub-converter (windows-949) + * has a broader encoding scheme than is valid for 2022. + * + * Check that the result is a 2-byte value with each byte in the range A1..FE + * (strict EUC-KR DBCS) before accepting it and subtracting 0x80 from each byte + * to move it to the ISO 2022 range 21..7E. + */ + if( (uint16_t)(value - 0xa1a1) <= (0xfefe - 0xa1a1) && + (uint8_t)(value - 0xa1) <= (0xfe - 0xa1) + ) { + value -= 0x8080; /* shift down to 21..7e byte range */ + } else { + break; /* not valid for ISO 2022 */ + } + } + targetValue = value; + len = len2; + cs = cs0; + g = 0; + useFallback = FALSE; + } + break; + } + } + + if(len != 0) { + if(len < 0) { + len = -len; /* fallback */ + } + outLen = 0; /* count output bytes */ + + /* write SI if necessary (only for JIS7) */ + if(pFromU2022State->g == 1 && g == 0) { + buffer[outLen++] = UCNV_SI; + pFromU2022State->g = 0; + } + + /* write the designation sequence if necessary */ + if(cs != pFromU2022State->cs[g]) { + int32_t escLen = escSeqCharsLen[cs]; + uprv_memcpy(buffer + outLen, escSeqChars[cs], escLen); + outLen += escLen; + pFromU2022State->cs[g] = cs; + + /* invalidate the choices[] */ + choiceCount = 0; + } + + /* write the shift sequence if necessary */ + if(g != pFromU2022State->g) { + switch(g) { + /* case 0 handled before writing escapes */ + case 1: + buffer[outLen++] = UCNV_SO; + pFromU2022State->g = 1; + break; + default: /* case 2 */ + buffer[outLen++] = 0x1b; + buffer[outLen++] = 0x4e; + break; + /* no case 3: no SS3 in ISO-2022-JP-x */ + } + } + + /* write the output bytes */ + if(len == 1) { + buffer[outLen++] = (char)targetValue; + } else /* len == 2 */ { + buffer[outLen++] = (char)(targetValue >> 8); + buffer[outLen++] = (char)targetValue; + } + } else { + /* + * if we cannot find the character after checking all codepages + * then this is an error + */ + *err = U_INVALID_CHAR_FOUND; + cnv->fromUChar32=sourceChar; + break; + } + + if(sourceChar == CR || sourceChar == LF) { + /* reset the G2 state at the end of a line (conversion got us into ASCII or JISX201 already) */ + pFromU2022State->cs[2] = 0; + choiceCount = 0; + } + + /* output outLen>0 bytes in buffer[] */ + if(outLen == 1) { + *target++ = buffer[0]; + if(offsets) { + *offsets++ = (int32_t)(source - args->source - 1); /* -1: known to be ASCII */ + } + } else if(outLen == 2 && (target + 2) <= targetLimit) { + *target++ = buffer[0]; + *target++ = buffer[1]; + if(offsets) { + int32_t sourceIndex = (int32_t)(source - args->source - U16_LENGTH(sourceChar)); + *offsets++ = sourceIndex; + *offsets++ = sourceIndex; + } + } else { + fromUWriteUInt8( + cnv, + buffer, outLen, + &target, (const char *)targetLimit, + &offsets, (int32_t)(source - args->source - U16_LENGTH(sourceChar)), + err); + if(U_FAILURE(*err)) { + break; + } + } + } /* end if(myTargetIndexg!=0 || pFromU2022State->cs[0]!=ASCII) && + args->flush && source>=sourceLimit && cnv->fromUChar32==0 + ) { + int32_t sourceIndex; + + outLen = 0; + + if(pFromU2022State->g != 0) { + buffer[outLen++] = UCNV_SI; + pFromU2022State->g = 0; + } + + if(pFromU2022State->cs[0] != ASCII) { + int32_t escLen = escSeqCharsLen[ASCII]; + uprv_memcpy(buffer + outLen, escSeqChars[ASCII], escLen); + outLen += escLen; + pFromU2022State->cs[0] = (int8_t)ASCII; + } + + /* get the source index of the last input character */ + /* + * TODO this would be simpler and more reliable if we used a pair + * of sourceIndex/prevSourceIndex like in ucnvmbcs.c + * so that we could simply use the prevSourceIndex here; + * this code gives an incorrect result for the rare case of an unmatched + * trail surrogate that is alone in the last buffer of the text stream + */ + sourceIndex=(int32_t)(source-args->source); + if(sourceIndex>0) { + --sourceIndex; + if( U16_IS_TRAIL(args->source[sourceIndex]) && + (sourceIndex==0 || U16_IS_LEAD(args->source[sourceIndex-1])) + ) { + --sourceIndex; + } + } else { + sourceIndex=-1; + } + + fromUWriteUInt8( + cnv, + buffer, outLen, + &target, (const char *)targetLimit, + &offsets, sourceIndex, + err); + } + + /*save the state and return */ + args->source = source; + args->target = (char*)target; +} + +/*************** to unicode *******************/ + +static void +UConverter_toUnicode_ISO_2022_JP_OFFSETS_LOGIC(UConverterToUnicodeArgs *args, + UErrorCode* err){ + char tempBuf[3]; + const char *mySource = (char *) args->source; + UChar *myTarget = args->target; + const char *mySourceLimit = args->sourceLimit; + uint32_t targetUniChar = 0x0000; + uint32_t mySourceChar = 0x0000; + UConverterDataISO2022* myData; + ISO2022State *pToU2022State; + StateEnum cs; + + myData=(UConverterDataISO2022*)(args->converter->extraInfo); + pToU2022State = &myData->toU2022State; + + if(myData->key != 0) { + /* continue with a partial escape sequence */ + goto escape; + } else if(args->converter->toULength == 1 && mySource < mySourceLimit && myTarget < args->targetLimit) { + /* continue with a partial double-byte character */ + mySourceChar = args->converter->toUBytes[0]; + args->converter->toULength = 0; + cs = (StateEnum)pToU2022State->cs[pToU2022State->g]; + goto getTrailByte; + } + + while(mySource < mySourceLimit){ + + targetUniChar =missingCharMarker; + + if(myTarget < args->targetLimit){ + + mySourceChar= (unsigned char) *mySource++; + + switch(mySourceChar) { + case UCNV_SI: + if(myData->version==3) { + pToU2022State->g=0; + continue; + } else { + /* only JIS7 uses SI/SO, not ISO-2022-JP-x */ + break; + } + + case UCNV_SO: + if(myData->version==3) { + /* JIS7: switch to G1 half-width Katakana */ + pToU2022State->cs[1] = (int8_t)HWKANA_7BIT; + pToU2022State->g=1; + continue; + } else { + /* only JIS7 uses SI/SO, not ISO-2022-JP-x */ + break; + } + + case ESC_2022: + mySource--; +escape: + changeState_2022(args->converter,&(mySource), + mySourceLimit, ISO_2022_JP,err); + + /* invalid or illegal escape sequence */ + if(U_FAILURE(*err)){ + args->target = myTarget; + args->source = mySource; + return; + } + continue; + + /* ISO-2022-JP does not use single-byte (C1) SS2 and SS3 */ + + case CR: + /*falls through*/ + case LF: + /* automatically reset to single-byte mode */ + if((StateEnum)pToU2022State->cs[0] != ASCII && (StateEnum)pToU2022State->cs[0] != JISX201) { + pToU2022State->cs[0] = (int8_t)ASCII; + } + pToU2022State->cs[2] = 0; + pToU2022State->g = 0; + /* falls through */ + default: + /* convert one or two bytes */ + cs = (StateEnum)pToU2022State->cs[pToU2022State->g]; + if( (uint8_t)(mySourceChar - 0xa1) <= (0xdf - 0xa1) && myData->version==4 && + !IS_JP_DBCS(cs) + ) { + /* 8-bit halfwidth katakana in any single-byte mode for JIS8 */ + targetUniChar = mySourceChar + (HWKANA_START - 0xa1); + + /* return from a single-shift state to the previous one */ + if(pToU2022State->g >= 2) { + pToU2022State->g=pToU2022State->prevG; + } + } else switch(cs) { + case ASCII: + if(mySourceChar <= 0x7f) { + targetUniChar = mySourceChar; + } + break; + case ISO8859_1: + if(mySourceChar <= 0x7f) { + targetUniChar = mySourceChar + 0x80; + } + /* return from a single-shift state to the previous one */ + pToU2022State->g=pToU2022State->prevG; + break; + case ISO8859_7: + if(mySourceChar <= 0x7f) { + /* convert mySourceChar+0x80 to use a normal 8-bit table */ + targetUniChar = + _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP( + myData->myConverterArray[cs], + mySourceChar + 0x80); + } + /* return from a single-shift state to the previous one */ + pToU2022State->g=pToU2022State->prevG; + break; + case JISX201: + if(mySourceChar <= 0x7f) { + targetUniChar = + _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP( + myData->myConverterArray[cs], + mySourceChar); + } + break; + case HWKANA_7BIT: + if((uint8_t)(mySourceChar - 0x21) <= (0x5f - 0x21)) { + /* 7-bit halfwidth Katakana */ + targetUniChar = mySourceChar + (HWKANA_START - 0x21); + } + break; + default: + /* G0 DBCS */ + if(mySource < mySourceLimit) { + char trailByte; +getTrailByte: + tempBuf[0] = (char) (mySourceChar); + tempBuf[1] = trailByte = *mySource++; + mySourceChar = (mySourceChar << 8) | (uint8_t)(trailByte); + targetUniChar = ucnv_MBCSSimpleGetNextUChar(myData->myConverterArray[cs], tempBuf, 2, FALSE); + } else { + args->converter->toUBytes[0] = (uint8_t)mySourceChar; + args->converter->toULength = 1; + goto endloop; + } + } /* End of inner switch */ + break; + } /* End of outer switch */ + if(targetUniChar < (missingCharMarker-1/*0xfffe*/)){ + if(args->offsets){ + args->offsets[myTarget - args->target] = (int32_t)(mySource - args->source - (mySourceChar <= 0xff ? 1 : 2)); + } + *(myTarget++)=(UChar)targetUniChar; + } + else if(targetUniChar > missingCharMarker){ + /* disassemble the surrogate pair and write to output*/ + targetUniChar-=0x0010000; + *myTarget = (UChar)(0xd800+(UChar)(targetUniChar>>10)); + if(args->offsets){ + args->offsets[myTarget - args->target] = (int32_t)(mySource - args->source - (mySourceChar <= 0xff ? 1 : 2)); + } + ++myTarget; + if(myTarget< args->targetLimit){ + *myTarget = (UChar)(0xdc00+(UChar)(targetUniChar&0x3ff)); + if(args->offsets){ + args->offsets[myTarget - args->target] = (int32_t)(mySource - args->source - (mySourceChar <= 0xff ? 1 : 2)); + } + ++myTarget; + }else{ + args->converter->UCharErrorBuffer[args->converter->UCharErrorBufferLength++]= + (UChar)(0xdc00+(UChar)(targetUniChar&0x3ff)); + } + + } + else{ + /* Call the callback function*/ + toUnicodeCallback(args->converter,mySourceChar,targetUniChar,err); + break; + } + } + else{ /* goes with "if(myTarget < args->targetLimit)" way up near top of function */ + *err =U_BUFFER_OVERFLOW_ERROR; + break; + } + } +endloop: + args->target = myTarget; + args->source = mySource; +} + + +/*************************************************************** +* Rules for ISO-2022-KR encoding +* i) The KSC5601 designator sequence should appear only once in a file, +* at the begining of a line before any KSC5601 characters. This usually +* means that it appears by itself on the first line of the file +* ii) There are only 2 shifting sequences SO to shift into double byte mode +* and SI to shift into single byte mode +*/ +static void +UConverter_fromUnicode_ISO_2022_KR_OFFSETS_LOGIC_IBM(UConverterFromUnicodeArgs* args, UErrorCode* err){ + + UConverter* saveConv = args->converter; + UConverterDataISO2022 *myConverterData=(UConverterDataISO2022*)saveConv->extraInfo; + args->converter=myConverterData->currentConverter; + + myConverterData->currentConverter->fromUChar32 = saveConv->fromUChar32; + ucnv_MBCSFromUnicodeWithOffsets(args,err); + saveConv->fromUChar32 = myConverterData->currentConverter->fromUChar32; + + if(*err == U_BUFFER_OVERFLOW_ERROR) { + if(myConverterData->currentConverter->charErrorBufferLength > 0) { + uprv_memcpy( + saveConv->charErrorBuffer, + myConverterData->currentConverter->charErrorBuffer, + myConverterData->currentConverter->charErrorBufferLength); + } + saveConv->charErrorBufferLength = myConverterData->currentConverter->charErrorBufferLength; + myConverterData->currentConverter->charErrorBufferLength = 0; + } + args->converter=saveConv; +} + +static void +UConverter_fromUnicode_ISO_2022_KR_OFFSETS_LOGIC(UConverterFromUnicodeArgs* args, UErrorCode* err){ + + const UChar *source = args->source; + const UChar *sourceLimit = args->sourceLimit; + unsigned char *target = (unsigned char *) args->target; + unsigned char *targetLimit = (unsigned char *) args->targetLimit; + int32_t* offsets = args->offsets; + uint32_t targetByteUnit = 0x0000; + UChar32 sourceChar = 0x0000; + UBool isTargetByteDBCS; + UBool oldIsTargetByteDBCS; + UConverterDataISO2022 *converterData; + UConverterSharedData* sharedData; + UBool useFallback; + int32_t length =0; + + converterData=(UConverterDataISO2022*)args->converter->extraInfo; + /* if the version is 1 then the user is requesting + * conversion with ibm-25546 pass the arguments to + * MBCS converter and return + */ + if(converterData->version==1){ + UConverter_fromUnicode_ISO_2022_KR_OFFSETS_LOGIC_IBM(args,err); + return; + } + + /* initialize data */ + sharedData = converterData->currentConverter->sharedData; + useFallback = args->converter->useFallback; + isTargetByteDBCS=(UBool)args->converter->fromUnicodeStatus; + oldIsTargetByteDBCS = isTargetByteDBCS; + + isTargetByteDBCS = (UBool) args->converter->fromUnicodeStatus; + if((sourceChar = args->converter->fromUChar32)!=0 && target targetLimit){ + sourceChar = *source++; + + /* do not convert SO/SI/ESC */ + if(IS_2022_CONTROL(sourceChar)) { + /* callback(illegal) */ + *err=U_ILLEGAL_CHAR_FOUND; + args->converter->fromUChar32=sourceChar; + break; + } + + length = MBCS_FROM_UCHAR32_ISO2022(sharedData,sourceChar,&targetByteUnit,useFallback,MBCS_OUTPUT_2); + if(length < 0) { + length = -length; /* fallback */ + } + /* only DBCS or SBCS characters are expected*/ + /* DB characters with high bit set to 1 are expected */ + if(length > 2 || length==0 ||(((targetByteUnit & 0x8080) != 0x8080)&& length==2)){ + targetByteUnit=missingCharMarker; + } + if (targetByteUnit != missingCharMarker){ + + oldIsTargetByteDBCS = isTargetByteDBCS; + isTargetByteDBCS = (UBool)(targetByteUnit>0x00FF); + /* append the shift sequence */ + if (oldIsTargetByteDBCS != isTargetByteDBCS ){ + + if (isTargetByteDBCS) + *target++ = UCNV_SO; + else + *target++ = UCNV_SI; + if(offsets) + *(offsets++) = (int32_t)(source - args->source-1); + } + /* write the targetUniChar to target */ + if(targetByteUnit <= 0x00FF){ + if( target < targetLimit){ + *(target++) = (unsigned char) targetByteUnit; + if(offsets){ + *(offsets++) = (int32_t)(source - args->source-1); + } + + }else{ + args->converter->charErrorBuffer[args->converter->charErrorBufferLength++] = (unsigned char) (targetByteUnit); + *err = U_BUFFER_OVERFLOW_ERROR; + } + }else{ + if(target < targetLimit){ + *(target++) =(unsigned char) ((targetByteUnit>>8) -0x80); + if(offsets){ + *(offsets++) = (int32_t)(source - args->source-1); + } + if(target < targetLimit){ + *(target++) =(unsigned char) (targetByteUnit -0x80); + if(offsets){ + *(offsets++) = (int32_t)(source - args->source-1); + } + }else{ + args->converter->charErrorBuffer[args->converter->charErrorBufferLength++] = (unsigned char) (targetByteUnit -0x80); + *err = U_BUFFER_OVERFLOW_ERROR; + } + }else{ + args->converter->charErrorBuffer[args->converter->charErrorBufferLength++] = (unsigned char) ((targetByteUnit>>8) -0x80); + args->converter->charErrorBuffer[args->converter->charErrorBufferLength++] = (unsigned char) (targetByteUnit-0x80); + *err = U_BUFFER_OVERFLOW_ERROR; + } + } + + } + else{ + /* oops.. the code point is unassingned + * set the error and reason + */ + + /*check if the char is a First surrogate*/ + if(UTF_IS_SURROGATE(sourceChar)) { + if(UTF_IS_SURROGATE_FIRST(sourceChar)) { +getTrail: + /*look ahead to find the trail surrogate*/ + if(source < sourceLimit) { + /* test the following code unit */ + UChar trail=(UChar) *source; + if(UTF_IS_SECOND_SURROGATE(trail)) { + source++; + sourceChar=UTF16_GET_PAIR_VALUE(sourceChar, trail); + *err = U_INVALID_CHAR_FOUND; + /* convert this surrogate code point */ + /* exit this condition tree */ + } else { + /* this is an unmatched lead code unit (1st surrogate) */ + /* callback(illegal) */ + *err=U_ILLEGAL_CHAR_FOUND; + } + } else { + /* no more input */ + *err = U_ZERO_ERROR; + } + } else { + /* this is an unmatched trail code unit (2nd surrogate) */ + /* callback(illegal) */ + *err=U_ILLEGAL_CHAR_FOUND; + } + } else { + /* callback(unassigned) for a BMP code point */ + *err = U_INVALID_CHAR_FOUND; + } + + args->converter->fromUChar32=sourceChar; + break; + } + } /* end if(myTargetIndexflush && source>=sourceLimit && args->converter->fromUChar32==0 + ) { + int32_t sourceIndex; + + /* we are switching to ASCII */ + isTargetByteDBCS=FALSE; + + /* get the source index of the last input character */ + /* + * TODO this would be simpler and more reliable if we used a pair + * of sourceIndex/prevSourceIndex like in ucnvmbcs.c + * so that we could simply use the prevSourceIndex here; + * this code gives an incorrect result for the rare case of an unmatched + * trail surrogate that is alone in the last buffer of the text stream + */ + sourceIndex=(int32_t)(source-args->source); + if(sourceIndex>0) { + --sourceIndex; + if( U16_IS_TRAIL(args->source[sourceIndex]) && + (sourceIndex==0 || U16_IS_LEAD(args->source[sourceIndex-1])) + ) { + --sourceIndex; + } + } else { + sourceIndex=-1; + } + + fromUWriteUInt8( + args->converter, + SHIFT_IN_STR, 1, + &target, (const char *)targetLimit, + &offsets, sourceIndex, + err); + } + + /*save the state and return */ + args->source = source; + args->target = (char*)target; + args->converter->fromUnicodeStatus = (uint32_t)isTargetByteDBCS; +} + +/************************ To Unicode ***************************************/ + +static void +UConverter_toUnicode_ISO_2022_KR_OFFSETS_LOGIC_IBM(UConverterToUnicodeArgs *args, + UErrorCode* err){ + char const* sourceStart; + UConverterDataISO2022* myData=(UConverterDataISO2022*)(args->converter->extraInfo); + + UConverterToUnicodeArgs subArgs; + int32_t minArgsSize; + + /* set up the subconverter arguments */ + if(args->sizesize; + } else { + minArgsSize = (int32_t)sizeof(UConverterToUnicodeArgs); + } + + uprv_memcpy(&subArgs, args, minArgsSize); + subArgs.size = (uint16_t)minArgsSize; + subArgs.converter = myData->currentConverter; + + /* remember the original start of the input for offsets */ + sourceStart = args->source; + + if(myData->key != 0) { + /* continue with a partial escape sequence */ + goto escape; + } + + while(U_SUCCESS(*err) && args->source < args->sourceLimit) { + /*Find the end of the buffer e.g : Next Escape Seq | end of Buffer*/ + subArgs.source = args->source; + subArgs.sourceLimit = getEndOfBuffer_2022(&(args->source), args->sourceLimit, args->flush); + if(subArgs.source != subArgs.sourceLimit) { + /* + * get the current partial byte sequence + * + * it needs to be moved between the public and the subconverter + * so that the conversion framework, which only sees the public + * converter, can handle truncated and illegal input etc. + */ + if(args->converter->toULength > 0) { + uprv_memcpy(subArgs.converter->toUBytes, args->converter->toUBytes, args->converter->toULength); + } + subArgs.converter->toULength = args->converter->toULength; + + /* + * Convert up to the end of the input, or to before the next escape character. + * Does not handle conversion extensions because the preToU[] state etc. + * is not copied. + */ + ucnv_MBCSToUnicodeWithOffsets(&subArgs, err); + + if(args->offsets != NULL && sourceStart != args->source) { + /* update offsets to base them on the actual start of the input */ + int32_t *offsets = args->offsets; + UChar *target = args->target; + int32_t delta = (int32_t)(args->source - sourceStart); + while(target < subArgs.target) { + if(*offsets >= 0) { + *offsets += delta; + } + ++offsets; + ++target; + } + } + args->source = subArgs.source; + args->target = subArgs.target; + args->offsets = subArgs.offsets; + + /* copy input/error/overflow buffers */ + if(subArgs.converter->toULength > 0) { + uprv_memcpy(args->converter->toUBytes, subArgs.converter->toUBytes, subArgs.converter->toULength); + } + args->converter->toULength = subArgs.converter->toULength; + + if(*err == U_BUFFER_OVERFLOW_ERROR) { + if(subArgs.converter->UCharErrorBufferLength > 0) { + uprv_memcpy(args->converter->UCharErrorBuffer, subArgs.converter->UCharErrorBuffer, + subArgs.converter->UCharErrorBufferLength); + } + args->converter->UCharErrorBufferLength=subArgs.converter->UCharErrorBufferLength; + subArgs.converter->UCharErrorBufferLength = 0; + } + } + + if (U_FAILURE(*err) || (args->source == args->sourceLimit)) { + return; + } + +escape: + changeState_2022(args->converter, + &(args->source), + args->sourceLimit, + ISO_2022_KR, + err); + } +} + +static void +UConverter_toUnicode_ISO_2022_KR_OFFSETS_LOGIC(UConverterToUnicodeArgs *args, + UErrorCode* err){ + char tempBuf[2]; + const char *mySource = ( char *) args->source; + UChar *myTarget = args->target; + const char *mySourceLimit = args->sourceLimit; + UChar32 targetUniChar = 0x0000; + UChar mySourceChar = 0x0000; + UConverterDataISO2022* myData; + UConverterSharedData* sharedData ; + UBool useFallback; + + myData=(UConverterDataISO2022*)(args->converter->extraInfo); + if(myData->version==1){ + UConverter_toUnicode_ISO_2022_KR_OFFSETS_LOGIC_IBM(args,err); + return; + } + + /* initialize state */ + sharedData = myData->currentConverter->sharedData; + useFallback = args->converter->useFallback; + + if(myData->key != 0) { + /* continue with a partial escape sequence */ + goto escape; + } else if(args->converter->toULength == 1 && mySource < mySourceLimit && myTarget < args->targetLimit) { + /* continue with a partial double-byte character */ + mySourceChar = args->converter->toUBytes[0]; + args->converter->toULength = 0; + goto getTrailByte; + } + + while(mySource< mySourceLimit){ + + if(myTarget < args->targetLimit){ + + mySourceChar= (unsigned char) *mySource++; + + if(mySourceChar==UCNV_SI){ + myData->toU2022State.g = 0; + /*consume the source */ + continue; + }else if(mySourceChar==UCNV_SO){ + myData->toU2022State.g = 1; + /*consume the source */ + continue; + }else if(mySourceChar==ESC_2022){ + mySource--; +escape: + changeState_2022(args->converter,&(mySource), + mySourceLimit, ISO_2022_KR, err); + if(U_FAILURE(*err)){ + args->target = myTarget; + args->source = mySource; + return; + } + continue; + } + + if(myData->toU2022State.g == 1) { + if(mySource < mySourceLimit) { + char trailByte; +getTrailByte: + trailByte = *mySource++; + tempBuf[0] = (char)(mySourceChar + 0x80); + tempBuf[1] = (char)(trailByte + 0x80); + mySourceChar = (mySourceChar << 8) | (uint8_t)(trailByte); + if((mySourceChar & 0x8080) == 0) { + targetUniChar = ucnv_MBCSSimpleGetNextUChar(sharedData, tempBuf, 2, useFallback); + } else { + /* illegal bytes > 0x7f */ + targetUniChar = missingCharMarker; + } + } else { + args->converter->toUBytes[0] = (uint8_t)mySourceChar; + args->converter->toULength = 1; + break; + } + } + else{ + targetUniChar = ucnv_MBCSSimpleGetNextUChar(sharedData, mySource - 1, 1, useFallback); + } + if(targetUniChar < 0xfffe){ + if(args->offsets) { + args->offsets[myTarget - args->target] = (int32_t)(mySource - args->source - (mySourceChar <= 0xff ? 1 : 2)); + } + *(myTarget++)=(UChar)targetUniChar; + } + else { + /* Call the callback function*/ + toUnicodeCallback(args->converter,mySourceChar,targetUniChar,err); + break; + } + } + else{ + *err =U_BUFFER_OVERFLOW_ERROR; + break; + } + } + args->target = myTarget; + args->source = mySource; +} + +/*************************** END ISO2022-KR *********************************/ + +/*************************** ISO-2022-CN ********************************* +* +* Rules for ISO-2022-CN Encoding: +* i) The designator sequence must appear once on a line before any instance +* of character set it designates. +* ii) If two lines contain characters from the same character set, both lines +* must include the designator sequence. +* iii) Once the designator sequence is known, a shifting sequence has to be found +* to invoke the shifting +* iv) All lines start in ASCII and end in ASCII. +* v) Four shifting sequences are employed for this purpose: +* +* Sequcence ASCII Eq Charsets +* ---------- ------- --------- +* SI US-ASCII +* SO CNS-11643-1992 Plane 1, GB2312, ISO-IR-165 +* SS2 N CNS-11643-1992 Plane 2 +* SS3 O CNS-11643-1992 Planes 3-7 +* +* vi) +* SOdesignator : ESC "$" ")" finalchar_for_SO +* SS2designator : ESC "$" "*" finalchar_for_SS2 +* SS3designator : ESC "$" "+" finalchar_for_SS3 +* +* ESC $ ) A Indicates the bytes following SO are Chinese +* characters as defined in GB 2312-80, until +* another SOdesignation appears +* +* +* ESC $ ) E Indicates the bytes following SO are as defined +* in ISO-IR-165 (for details, see section 2.1), +* until another SOdesignation appears +* +* ESC $ ) G Indicates the bytes following SO are as defined +* in CNS 11643-plane-1, until another +* SOdesignation appears +* +* ESC $ * H Indicates the two bytes immediately following +* SS2 is a Chinese character as defined in CNS +* 11643-plane-2, until another SS2designation +* appears +* (Meaning N must preceed every 2 byte +* sequence.) +* +* ESC $ + I Indicates the immediate two bytes following SS3 +* is a Chinese character as defined in CNS +* 11643-plane-3, until another SS3designation +* appears +* (Meaning O must preceed every 2 byte +* sequence.) +* +* ESC $ + J Indicates the immediate two bytes following SS3 +* is a Chinese character as defined in CNS +* 11643-plane-4, until another SS3designation +* appears +* (In English: O must preceed every 2 byte +* sequence.) +* +* ESC $ + K Indicates the immediate two bytes following SS3 +* is a Chinese character as defined in CNS +* 11643-plane-5, until another SS3designation +* appears +* +* ESC $ + L Indicates the immediate two bytes following SS3 +* is a Chinese character as defined in CNS +* 11643-plane-6, until another SS3designation +* appears +* +* ESC $ + M Indicates the immediate two bytes following SS3 +* is a Chinese character as defined in CNS +* 11643-plane-7, until another SS3designation +* appears +* +* As in ISO-2022-CN, each line starts in ASCII, and ends in ASCII, and +* has its own designation information before any Chinese characters +* appear +* +*/ + +/* The following are defined this way to make the strings truely readonly */ +static const char GB_2312_80_STR[] = "\x1B\x24\x29\x41"; +static const char ISO_IR_165_STR[] = "\x1B\x24\x29\x45"; +static const char CNS_11643_1992_Plane_1_STR[] = "\x1B\x24\x29\x47"; +static const char CNS_11643_1992_Plane_2_STR[] = "\x1B\x24\x2A\x48"; +static const char CNS_11643_1992_Plane_3_STR[] = "\x1B\x24\x2B\x49"; +static const char CNS_11643_1992_Plane_4_STR[] = "\x1B\x24\x2B\x4A"; +static const char CNS_11643_1992_Plane_5_STR[] = "\x1B\x24\x2B\x4B"; +static const char CNS_11643_1992_Plane_6_STR[] = "\x1B\x24\x2B\x4C"; +static const char CNS_11643_1992_Plane_7_STR[] = "\x1B\x24\x2B\x4D"; + +/********************** ISO2022-CN Data **************************/ +static const char* const escSeqCharsCN[10] ={ + SHIFT_IN_STR, /* ASCII */ + GB_2312_80_STR, + ISO_IR_165_STR, + CNS_11643_1992_Plane_1_STR, + CNS_11643_1992_Plane_2_STR, + CNS_11643_1992_Plane_3_STR, + CNS_11643_1992_Plane_4_STR, + CNS_11643_1992_Plane_5_STR, + CNS_11643_1992_Plane_6_STR, + CNS_11643_1992_Plane_7_STR +}; + +static void +UConverter_fromUnicode_ISO_2022_CN_OFFSETS_LOGIC(UConverterFromUnicodeArgs* args, UErrorCode* err){ + UConverter *cnv = args->converter; + UConverterDataISO2022 *converterData; + ISO2022State *pFromU2022State; + uint8_t *target = (uint8_t *) args->target; + const uint8_t *targetLimit = (const uint8_t *) args->targetLimit; + const UChar* source = args->source; + const UChar* sourceLimit = args->sourceLimit; + int32_t* offsets = args->offsets; + UChar32 sourceChar; + char buffer[8]; + int32_t len; + int8_t choices[3]; + int32_t choiceCount; + uint32_t targetValue = 0; + UBool useFallback; + + /* set up the state */ + converterData = (UConverterDataISO2022*)cnv->extraInfo; + pFromU2022State = &converterData->fromU2022State; + + choiceCount = 0; + + /* check if the last codepoint of previous buffer was a lead surrogate*/ + if((sourceChar = cnv->fromUChar32)!=0 && target< targetLimit) { + goto getTrail; + } + + while( source < sourceLimit){ + if(target < targetLimit){ + + sourceChar = *(source++); + /*check if the char is a First surrogate*/ + if(UTF_IS_SURROGATE(sourceChar)) { + if(UTF_IS_SURROGATE_FIRST(sourceChar)) { +getTrail: + /*look ahead to find the trail surrogate*/ + if(source < sourceLimit) { + /* test the following code unit */ + UChar trail=(UChar) *source; + if(UTF_IS_SECOND_SURROGATE(trail)) { + source++; + sourceChar=UTF16_GET_PAIR_VALUE(sourceChar, trail); + cnv->fromUChar32=0x00; + /* convert this supplementary code point */ + /* exit this condition tree */ + } else { + /* this is an unmatched lead code unit (1st surrogate) */ + /* callback(illegal) */ + *err=U_ILLEGAL_CHAR_FOUND; + cnv->fromUChar32=sourceChar; + break; + } + } else { + /* no more input */ + cnv->fromUChar32=sourceChar; + break; + } + } else { + /* this is an unmatched trail code unit (2nd surrogate) */ + /* callback(illegal) */ + *err=U_ILLEGAL_CHAR_FOUND; + cnv->fromUChar32=sourceChar; + break; + } + } + + /* do the conversion */ + if(sourceChar <= 0x007f ){ + /* do not convert SO/SI/ESC */ + if(IS_2022_CONTROL(sourceChar)) { + /* callback(illegal) */ + *err=U_ILLEGAL_CHAR_FOUND; + cnv->fromUChar32=sourceChar; + break; + } + + /* US-ASCII */ + if(pFromU2022State->g == 0) { + buffer[0] = (char)sourceChar; + len = 1; + } else { + buffer[0] = UCNV_SI; + buffer[1] = (char)sourceChar; + len = 2; + pFromU2022State->g = 0; + choiceCount = 0; + } + if(sourceChar == CR || sourceChar == LF) { + /* reset the state at the end of a line */ + uprv_memset(pFromU2022State, 0, sizeof(ISO2022State)); + choiceCount = 0; + } + } + else{ + /* convert U+0080..U+10ffff */ + int32_t i; + int8_t cs, g; + + if(choiceCount == 0) { + /* try the current SO/G1 converter first */ + choices[0] = pFromU2022State->cs[1]; + + /* default to GB2312_1 if none is designated yet */ + if(choices[0] == 0) { + choices[0] = GB2312_1; + } + + if(converterData->version == 0) { + /* ISO-2022-CN */ + + /* try the other SO/G1 converter; a CNS_11643_1 lookup may result in any plane */ + if(choices[0] == GB2312_1) { + choices[1] = (int8_t)CNS_11643_1; + } else { + choices[1] = (int8_t)GB2312_1; + } + + choiceCount = 2; + } else { + /* ISO-2022-CN-EXT */ + + /* try one of the other converters */ + switch(choices[0]) { + case GB2312_1: + choices[1] = (int8_t)CNS_11643_1; + choices[2] = (int8_t)ISO_IR_165; + break; + case ISO_IR_165: + choices[1] = (int8_t)GB2312_1; + choices[2] = (int8_t)CNS_11643_1; + break; + default: /* CNS_11643_x */ + choices[1] = (int8_t)GB2312_1; + choices[2] = (int8_t)ISO_IR_165; + break; + } + + choiceCount = 3; + } + } + + cs = g = 0; + /* + * len==0: no mapping found yet + * len<0: found a fallback result: continue looking for a roundtrip but no further fallbacks + * len>0: found a roundtrip result, done + */ + len = 0; + /* + * We will turn off useFallback after finding a fallback, + * but we still get fallbacks from PUA code points as usual. + * Therefore, we will also need to check that we don't overwrite + * an early fallback with a later one. + */ + useFallback = cnv->useFallback; + + for(i = 0; i < choiceCount && len <= 0; ++i) { + int8_t cs0 = choices[i]; + if(cs0 > 0) { + uint32_t value; + int32_t len2; + if(cs0 > CNS_11643_0) { + len2 = MBCS_FROM_UCHAR32_ISO2022( + converterData->myConverterArray[CNS_11643], + sourceChar, + &value, + useFallback, + MBCS_OUTPUT_3); + if(len2 == 3 || (len2 == -3 && len == 0)) { + targetValue = value; + cs = (int8_t)(CNS_11643_0 + (value >> 16) - 0x80); + if(len2 >= 0) { + len = 2; + } else { + len = -2; + useFallback = FALSE; + } + if(cs == CNS_11643_1) { + g = 1; + } else if(cs == CNS_11643_2) { + g = 2; + } else /* plane 3..7 */ if(converterData->version == 1) { + g = 3; + } else { + /* ISO-2022-CN (without -EXT) does not support plane 3..7 */ + len = 0; + } + } + } else { + /* GB2312_1 or ISO-IR-165 */ + len2 = MBCS_FROM_UCHAR32_ISO2022( + converterData->myConverterArray[cs0], + sourceChar, + &value, + useFallback, + MBCS_OUTPUT_2); + if(len2 == 2 || (len2 == -2 && len == 0)) { + targetValue = value; + len = len2; + cs = cs0; + g = 1; + useFallback = FALSE; + } + } + } + } + + if(len != 0) { + len = 0; /* count output bytes; it must have been abs(len) == 2 */ + + /* write the designation sequence if necessary */ + if(cs != pFromU2022State->cs[g]) { + if(cs < CNS_11643) { + uprv_memcpy(buffer, escSeqCharsCN[cs], 4); + } else { + uprv_memcpy(buffer, escSeqCharsCN[CNS_11643 + (cs - CNS_11643_1)], 4); + } + len = 4; + pFromU2022State->cs[g] = cs; + if(g == 1) { + /* changing the SO/G1 charset invalidates the choices[] */ + choiceCount = 0; + } + } + + /* write the shift sequence if necessary */ + if(g != pFromU2022State->g) { + switch(g) { + case 1: + buffer[len++] = UCNV_SO; + + /* set the new state only if it is the locking shift SO/G1, not for SS2 or SS3 */ + pFromU2022State->g = 1; + break; + case 2: + buffer[len++] = 0x1b; + buffer[len++] = 0x4e; + break; + default: /* case 3 */ + buffer[len++] = 0x1b; + buffer[len++] = 0x4f; + break; + } + } + + /* write the two output bytes */ + buffer[len++] = (char)(targetValue >> 8); + buffer[len++] = (char)targetValue; + } else { + /* if we cannot find the character after checking all codepages + * then this is an error + */ + *err = U_INVALID_CHAR_FOUND; + cnv->fromUChar32=sourceChar; + break; + } + } + + /* output len>0 bytes in buffer[] */ + if(len == 1) { + *target++ = buffer[0]; + if(offsets) { + *offsets++ = (int32_t)(source - args->source - 1); /* -1: known to be ASCII */ + } + } else if(len == 2 && (target + 2) <= targetLimit) { + *target++ = buffer[0]; + *target++ = buffer[1]; + if(offsets) { + int32_t sourceIndex = (int32_t)(source - args->source - U16_LENGTH(sourceChar)); + *offsets++ = sourceIndex; + *offsets++ = sourceIndex; + } + } else { + fromUWriteUInt8( + cnv, + buffer, len, + &target, (const char *)targetLimit, + &offsets, (int32_t)(source - args->source - U16_LENGTH(sourceChar)), + err); + if(U_FAILURE(*err)) { + break; + } + } + } /* end if(myTargetIndexg!=0 && + args->flush && source>=sourceLimit && cnv->fromUChar32==0 + ) { + int32_t sourceIndex; + + /* we are switching to ASCII */ + pFromU2022State->g=0; + + /* get the source index of the last input character */ + /* + * TODO this would be simpler and more reliable if we used a pair + * of sourceIndex/prevSourceIndex like in ucnvmbcs.c + * so that we could simply use the prevSourceIndex here; + * this code gives an incorrect result for the rare case of an unmatched + * trail surrogate that is alone in the last buffer of the text stream + */ + sourceIndex=(int32_t)(source-args->source); + if(sourceIndex>0) { + --sourceIndex; + if( U16_IS_TRAIL(args->source[sourceIndex]) && + (sourceIndex==0 || U16_IS_LEAD(args->source[sourceIndex-1])) + ) { + --sourceIndex; + } + } else { + sourceIndex=-1; + } + + fromUWriteUInt8( + cnv, + SHIFT_IN_STR, 1, + &target, (const char *)targetLimit, + &offsets, sourceIndex, + err); + } + + /*save the state and return */ + args->source = source; + args->target = (char*)target; +} + + +static void +UConverter_toUnicode_ISO_2022_CN_OFFSETS_LOGIC(UConverterToUnicodeArgs *args, + UErrorCode* err){ + char tempBuf[3]; + const char *mySource = (char *) args->source; + UChar *myTarget = args->target; + const char *mySourceLimit = args->sourceLimit; + uint32_t targetUniChar = 0x0000; + uint32_t mySourceChar = 0x0000; + UConverterDataISO2022* myData; + ISO2022State *pToU2022State; + + myData=(UConverterDataISO2022*)(args->converter->extraInfo); + pToU2022State = &myData->toU2022State; + + if(myData->key != 0) { + /* continue with a partial escape sequence */ + goto escape; + } else if(args->converter->toULength == 1 && mySource < mySourceLimit && myTarget < args->targetLimit) { + /* continue with a partial double-byte character */ + mySourceChar = args->converter->toUBytes[0]; + args->converter->toULength = 0; + goto getTrailByte; + } + + while(mySource < mySourceLimit){ + + targetUniChar =missingCharMarker; + + if(myTarget < args->targetLimit){ + + mySourceChar= (unsigned char) *mySource++; + + switch(mySourceChar){ + case UCNV_SI: + pToU2022State->g=0; + continue; + + case UCNV_SO: + if(pToU2022State->cs[1] != 0) { + pToU2022State->g=1; + continue; + } else { + /* illegal to have SO before a matching designator */ + break; + } + + case ESC_2022: + mySource--; +escape: + changeState_2022(args->converter,&(mySource), + mySourceLimit, ISO_2022_CN,err); + + /* invalid or illegal escape sequence */ + if(U_FAILURE(*err)){ + args->target = myTarget; + args->source = mySource; + return; + } + continue; + + /* ISO-2022-CN does not use single-byte (C1) SS2 and SS3 */ + + case CR: + /*falls through*/ + case LF: + uprv_memset(pToU2022State, 0, sizeof(ISO2022State)); + /* falls through */ + default: + /* convert one or two bytes */ + if(pToU2022State->g != 0) { + if(mySource < mySourceLimit) { + UConverterSharedData *cnv; + StateEnum tempState; + int32_t tempBufLen; + char trailByte; +getTrailByte: + trailByte = *mySource++; + tempState = (StateEnum)pToU2022State->cs[pToU2022State->g]; + if(tempState > CNS_11643_0) { + cnv = myData->myConverterArray[CNS_11643]; + tempBuf[0] = (char) (0x80+(tempState-CNS_11643_0)); + tempBuf[1] = (char) (mySourceChar); + tempBuf[2] = trailByte; + tempBufLen = 3; + + }else{ + cnv = myData->myConverterArray[tempState]; + tempBuf[0] = (char) (mySourceChar); + tempBuf[1] = trailByte; + tempBufLen = 2; + } + mySourceChar = (mySourceChar << 8) | (uint8_t)(trailByte); + if(pToU2022State->g>=2) { + /* return from a single-shift state to the previous one */ + pToU2022State->g=pToU2022State->prevG; + } + targetUniChar = ucnv_MBCSSimpleGetNextUChar(cnv, tempBuf, tempBufLen, FALSE); + } else { + args->converter->toUBytes[0] = (uint8_t)mySourceChar; + args->converter->toULength = 1; + goto endloop; + } + } + else{ + if(mySourceChar <= 0x7f) { + targetUniChar = (UChar) mySourceChar; + } + } + break; + } + if(targetUniChar < (missingCharMarker-1/*0xfffe*/)){ + if(args->offsets){ + args->offsets[myTarget - args->target] = (int32_t)(mySource - args->source - (mySourceChar <= 0xff ? 1 : 2)); + } + *(myTarget++)=(UChar)targetUniChar; + } + else if(targetUniChar > missingCharMarker){ + /* disassemble the surrogate pair and write to output*/ + targetUniChar-=0x0010000; + *myTarget = (UChar)(0xd800+(UChar)(targetUniChar>>10)); + if(args->offsets){ + args->offsets[myTarget - args->target] = (int32_t)(mySource - args->source - (mySourceChar <= 0xff ? 1 : 2)); + } + ++myTarget; + if(myTarget< args->targetLimit){ + *myTarget = (UChar)(0xdc00+(UChar)(targetUniChar&0x3ff)); + if(args->offsets){ + args->offsets[myTarget - args->target] = (int32_t)(mySource - args->source - (mySourceChar <= 0xff ? 1 : 2)); + } + ++myTarget; + }else{ + args->converter->UCharErrorBuffer[args->converter->UCharErrorBufferLength++]= + (UChar)(0xdc00+(UChar)(targetUniChar&0x3ff)); + } + + } + else{ + /* Call the callback function*/ + toUnicodeCallback(args->converter,mySourceChar,targetUniChar,err); + break; + } + } + else{ + *err =U_BUFFER_OVERFLOW_ERROR; + break; + } + } +endloop: + args->target = myTarget; + args->source = mySource; +} + +static void +_ISO_2022_WriteSub(UConverterFromUnicodeArgs *args, int32_t offsetIndex, UErrorCode *err) { + UConverter *cnv = args->converter; + UConverterDataISO2022 *myConverterData=(UConverterDataISO2022 *) cnv->extraInfo; + ISO2022State *pFromU2022State=&myConverterData->fromU2022State; + char *p, *subchar; + char buffer[8]; + int32_t length; + + subchar=(char *)cnv->subChars; + length=cnv->subCharLen; /* assume length==1 for most variants */ + + p = buffer; + switch(myConverterData->locale[0]){ + case 'j': + { + int8_t cs; + + if(pFromU2022State->g == 1) { + /* JIS7: switch from G1 to G0 */ + pFromU2022State->g = 0; + *p++ = UCNV_SI; + } + + cs = pFromU2022State->cs[0]; + if(cs != ASCII && cs != JISX201) { + /* not in ASCII or JIS X 0201: switch to ASCII */ + pFromU2022State->cs[0] = (int8_t)ASCII; + *p++ = '\x1b'; + *p++ = '\x28'; + *p++ = '\x42'; + } + + *p++ = subchar[0]; + break; + } + case 'c': + if(pFromU2022State->g != 0) { + /* not in ASCII mode: switch to ASCII */ + pFromU2022State->g = 0; + *p++ = UCNV_SI; + } + *p++ = subchar[0]; + break; + case 'k': + if(myConverterData->version == 0) { + if(length == 1) { + if((UBool)args->converter->fromUnicodeStatus) { + /* in DBCS mode: switch to SBCS */ + args->converter->fromUnicodeStatus = 0; + *p++ = UCNV_SI; + } + *p++ = subchar[0]; + } else /* length == 2*/ { + if(!(UBool)args->converter->fromUnicodeStatus) { + /* in SBCS mode: switch to DBCS */ + args->converter->fromUnicodeStatus = 1; + *p++ = UCNV_SO; + } + *p++ = subchar[0]; + *p++ = subchar[1]; + } + break; + } else { + /* save the subconverter's substitution string */ + uint8_t *currentSubChars = myConverterData->currentConverter->subChars; + int8_t currentSubCharLen = myConverterData->currentConverter->subCharLen; + + /* set our substitution string into the subconverter */ + myConverterData->currentConverter->subChars = (uint8_t *)subchar; + myConverterData->currentConverter->subCharLen = (int8_t)length; + + /* let the subconverter write the subchar, set/retrieve fromUChar32 state */ + args->converter = myConverterData->currentConverter; + myConverterData->currentConverter->fromUChar32 = cnv->fromUChar32; + ucnv_cbFromUWriteSub(args, 0, err); + cnv->fromUChar32 = myConverterData->currentConverter->fromUChar32; + args->converter = cnv; + + /* restore the subconverter's substitution string */ + myConverterData->currentConverter->subChars = currentSubChars; + myConverterData->currentConverter->subCharLen = currentSubCharLen; + + if(*err == U_BUFFER_OVERFLOW_ERROR) { + if(myConverterData->currentConverter->charErrorBufferLength > 0) { + uprv_memcpy( + cnv->charErrorBuffer, + myConverterData->currentConverter->charErrorBuffer, + myConverterData->currentConverter->charErrorBufferLength); + } + cnv->charErrorBufferLength = myConverterData->currentConverter->charErrorBufferLength; + myConverterData->currentConverter->charErrorBufferLength = 0; + } + return; + } + default: + /* not expected */ + break; + } + ucnv_cbFromUWriteBytes(args, + buffer, (int32_t)(p - buffer), + offsetIndex, err); +} + +/* + * Structure for cloning an ISO 2022 converter into a single memory block. + * ucnv_safeClone() of the converter will align the entire cloneStruct, + * and then ucnv_safeClone() of the sub-converter may additionally align + * currentConverter inside the cloneStruct, for which we need the deadSpace + * after currentConverter. + * This is because UAlignedMemory may be larger than the actually + * necessary alignment size for the platform. + * The other cloneStruct fields will not be moved around, + * and are aligned properly with cloneStruct's alignment. + */ +struct cloneStruct +{ + UConverter cnv; + UConverter currentConverter; + UAlignedMemory deadSpace; + UConverterDataISO2022 mydata; +}; + + +static UConverter * +_ISO_2022_SafeClone( + const UConverter *cnv, + void *stackBuffer, + int32_t *pBufferSize, + UErrorCode *status) +{ + struct cloneStruct * localClone; + UConverterDataISO2022 *cnvData; + int32_t i, size; + + if (*pBufferSize == 0) { /* 'preflighting' request - set needed size into *pBufferSize */ + *pBufferSize = (int32_t)sizeof(struct cloneStruct); + return NULL; + } + + cnvData = (UConverterDataISO2022 *)cnv->extraInfo; + localClone = (struct cloneStruct *)stackBuffer; + + /* ucnv.c/ucnv_safeClone() copied the main UConverter already */ + + uprv_memcpy(&localClone->mydata, cnvData, sizeof(UConverterDataISO2022)); + localClone->cnv.extraInfo = &localClone->mydata; /* set pointer to extra data */ + localClone->cnv.isExtraLocal = TRUE; + + /* share the subconverters */ + + if(cnvData->currentConverter != NULL) { + size = (int32_t)(sizeof(UConverter) + sizeof(UAlignedMemory)); /* include size of padding */ + localClone->mydata.currentConverter = + ucnv_safeClone(cnvData->currentConverter, + &localClone->currentConverter, + &size, status); + if(U_FAILURE(*status)) { + return NULL; + } + } + + for(i=0; imyConverterArray[i] != NULL) { + ucnv_incrementRefCount(cnvData->myConverterArray[i]); + } + } + + return &localClone->cnv; +} + +static void +_ISO_2022_GetUnicodeSet(const UConverter *cnv, + const USetAdder *sa, + UConverterUnicodeSet which, + UErrorCode *pErrorCode) +{ + int32_t i; + UConverterDataISO2022* cnvData; + + if (U_FAILURE(*pErrorCode)) { + return; + } +#ifdef U_ENABLE_GENERIC_ISO_2022 + if (cnv->sharedData == &_ISO2022Data) { + /* We use UTF-8 in this case */ + sa->addRange(sa->set, 0, 0xd7FF); + sa->addRange(sa->set, 0xE000, 0x10FFFF); + return; + } +#endif + + cnvData = (UConverterDataISO2022*)cnv->extraInfo; + + /* open a set and initialize it with code points that are algorithmically round-tripped */ + switch(cnvData->locale[0]){ + case 'j': + if(jpCharsetMasks[cnvData->version]&CSM(ISO8859_1)) { + /* include Latin-1 for some variants of JP */ + sa->addRange(sa->set, 0, 0xff); + } else { + /* include ASCII for JP */ + sa->addRange(sa->set, 0, 0x7f); + } + if(jpCharsetMasks[cnvData->version]&CSM(HWKANA_7BIT)) { + /* include half-width Katakana for JP */ + sa->addRange(sa->set, HWKANA_START, HWKANA_END); + } + break; + case 'c': + case 'z': + /* include ASCII for CN */ + sa->addRange(sa->set, 0, 0x7f); + break; + case 'k': + /* there is only one converter for KR, and it is not in the myConverterArray[] */ + cnvData->currentConverter->sharedData->impl->getUnicodeSet( + cnvData->currentConverter, sa, which, pErrorCode); + /* the loop over myConverterArray[] will simply not find another converter */ + break; + default: + break; + } + + /* + * Version-specific for CN: + * CN version 0 does not map CNS planes 3..7 although + * they are all available in the CNS conversion table; + * CN version 1 does map them all. + * The two versions create different Unicode sets. + */ + for (i=0; imyConverterArray[i]!=NULL) { + if( (cnvData->locale[0]=='c' || cnvData->locale[0]=='z') && + cnvData->version==0 && i==CNS_11643 + ) { + /* special handling for non-EXT ISO-2022-CN: add only code points for CNS planes 1 and 2 */ + ucnv_MBCSGetUnicodeSetForBytes( + cnvData->myConverterArray[i], + sa, UCNV_ROUNDTRIP_SET, + 0, 0x81, 0x82, + pErrorCode); + } else { + ucnv_MBCSGetUnicodeSetForUnicode(cnvData->myConverterArray[i], sa, which, pErrorCode); + } + } + } + + /* + * ISO 2022 converters must not convert SO/SI/ESC despite what + * sub-converters do by themselves. + * Remove these characters from the set. + */ + sa->remove(sa->set, 0x0e); + sa->remove(sa->set, 0x0f); + sa->remove(sa->set, 0x1b); +} + +static const UConverterImpl _ISO2022Impl={ + UCNV_ISO_2022, + + NULL, + NULL, + + _ISO2022Open, + _ISO2022Close, + _ISO2022Reset, + +#ifdef U_ENABLE_GENERIC_ISO_2022 + T_UConverter_toUnicode_ISO_2022_OFFSETS_LOGIC, + T_UConverter_toUnicode_ISO_2022_OFFSETS_LOGIC, + ucnv_fromUnicode_UTF8, + ucnv_fromUnicode_UTF8_OFFSETS_LOGIC, +#else + NULL, + NULL, + NULL, + NULL, +#endif + NULL, + + NULL, + _ISO2022getName, + _ISO_2022_WriteSub, + _ISO_2022_SafeClone, + _ISO_2022_GetUnicodeSet +}; +static const UConverterStaticData _ISO2022StaticData={ + sizeof(UConverterStaticData), + "ISO_2022", + 2022, + UCNV_IBM, + UCNV_ISO_2022, + 1, + 3, /* max 3 bytes per UChar from UTF-8 (4 bytes from surrogate _pair_) */ + { 0x1a, 0, 0, 0 }, + 1, + FALSE, + FALSE, + 0, + 0, + { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */ +}; +const UConverterSharedData _ISO2022Data={ + sizeof(UConverterSharedData), + ~((uint32_t) 0), + NULL, + NULL, + &_ISO2022StaticData, + FALSE, + &_ISO2022Impl, + 0 +}; + +/*************JP****************/ +static const UConverterImpl _ISO2022JPImpl={ + UCNV_ISO_2022, + + NULL, + NULL, + + _ISO2022Open, + _ISO2022Close, + _ISO2022Reset, + + UConverter_toUnicode_ISO_2022_JP_OFFSETS_LOGIC, + UConverter_toUnicode_ISO_2022_JP_OFFSETS_LOGIC, + UConverter_fromUnicode_ISO_2022_JP_OFFSETS_LOGIC, + UConverter_fromUnicode_ISO_2022_JP_OFFSETS_LOGIC, + NULL, + + NULL, + _ISO2022getName, + _ISO_2022_WriteSub, + _ISO_2022_SafeClone, + _ISO_2022_GetUnicodeSet +}; +static const UConverterStaticData _ISO2022JPStaticData={ + sizeof(UConverterStaticData), + "ISO_2022_JP", + 0, + UCNV_IBM, + UCNV_ISO_2022, + 1, + 6, /* max 6 bytes per UChar: 4-byte escape sequence + DBCS */ + { 0x1a, 0, 0, 0 }, + 1, + FALSE, + FALSE, + 0, + 0, + { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */ +}; +static const UConverterSharedData _ISO2022JPData={ + sizeof(UConverterSharedData), + ~((uint32_t) 0), + NULL, + NULL, + &_ISO2022JPStaticData, + FALSE, + &_ISO2022JPImpl, + 0 +}; + +/************* KR ***************/ +static const UConverterImpl _ISO2022KRImpl={ + UCNV_ISO_2022, + + NULL, + NULL, + + _ISO2022Open, + _ISO2022Close, + _ISO2022Reset, + + UConverter_toUnicode_ISO_2022_KR_OFFSETS_LOGIC, + UConverter_toUnicode_ISO_2022_KR_OFFSETS_LOGIC, + UConverter_fromUnicode_ISO_2022_KR_OFFSETS_LOGIC, + UConverter_fromUnicode_ISO_2022_KR_OFFSETS_LOGIC, + NULL, + + NULL, + _ISO2022getName, + _ISO_2022_WriteSub, + _ISO_2022_SafeClone, + _ISO_2022_GetUnicodeSet +}; +static const UConverterStaticData _ISO2022KRStaticData={ + sizeof(UConverterStaticData), + "ISO_2022_KR", + 0, + UCNV_IBM, + UCNV_ISO_2022, + 1, + 3, /* max 3 bytes per UChar: SO+DBCS */ + { 0x1a, 0, 0, 0 }, + 1, + FALSE, + FALSE, + 0, + 0, + { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */ +}; +static const UConverterSharedData _ISO2022KRData={ + sizeof(UConverterSharedData), + ~((uint32_t) 0), + NULL, + NULL, + &_ISO2022KRStaticData, + FALSE, + &_ISO2022KRImpl, + 0 +}; + +/*************** CN ***************/ +static const UConverterImpl _ISO2022CNImpl={ + + UCNV_ISO_2022, + + NULL, + NULL, + + _ISO2022Open, + _ISO2022Close, + _ISO2022Reset, + + UConverter_toUnicode_ISO_2022_CN_OFFSETS_LOGIC, + UConverter_toUnicode_ISO_2022_CN_OFFSETS_LOGIC, + UConverter_fromUnicode_ISO_2022_CN_OFFSETS_LOGIC, + UConverter_fromUnicode_ISO_2022_CN_OFFSETS_LOGIC, + NULL, + + NULL, + _ISO2022getName, + _ISO_2022_WriteSub, + _ISO_2022_SafeClone, + _ISO_2022_GetUnicodeSet +}; +static const UConverterStaticData _ISO2022CNStaticData={ + sizeof(UConverterStaticData), + "ISO_2022_CN", + 0, + UCNV_IBM, + UCNV_ISO_2022, + 1, + 8, /* max 8 bytes per UChar: 4-byte CNS designator + 2 bytes for SS2/SS3 + DBCS */ + { 0x1a, 0, 0, 0 }, + 1, + FALSE, + FALSE, + 0, + 0, + { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */ +}; +static const UConverterSharedData _ISO2022CNData={ + sizeof(UConverterSharedData), + ~((uint32_t) 0), + NULL, + NULL, + &_ISO2022CNStaticData, + FALSE, + &_ISO2022CNImpl, + 0 +}; + + + +#endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */ diff --git a/Build/source/libs/icu/icu-xetex/common/ucnv_bld.c b/Build/source/libs/icu/icu-xetex/common/ucnv_bld.c new file mode 100644 index 00000000000..f627d6b8239 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnv_bld.c @@ -0,0 +1,1583 @@ +/* + ******************************************************************** + * COPYRIGHT: + * Copyright (c) 1996-2007, International Business Machines Corporation and + * others. All Rights Reserved. + ******************************************************************** + * + * uconv_bld.c: + * + * Defines functions that are used in the creation/initialization/deletion + * of converters and related structures. + * uses uconv_io.h routines to access disk information + * is used by ucnv.h to implement public API create/delete/flushCache routines + * Modification History: + * + * Date Name Description + * + * 06/20/2000 helena OS/400 port changes; mostly typecast. + * 06/29/2000 helena Major rewrite of the callback interface. +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "unicode/putil.h" +#include "unicode/udata.h" +#include "unicode/ucnv.h" +#include "unicode/uloc.h" +#include "utracimp.h" +#include "ucnv_io.h" +#include "ucnv_bld.h" +#include "ucnvmbcs.h" +#include "ucnv_ext.h" +#include "ucnv_cnv.h" +#include "ucnv_imp.h" +#include "uhash.h" +#include "umutex.h" +#include "cstring.h" +#include "cmemory.h" +#include "ucln_cmn.h" +#include "ustr_cnv.h" + + + +#if 0 +#include +extern void UCNV_DEBUG_LOG(char *what, char *who, void *p, int l); +#define UCNV_DEBUG_LOG(x,y,z) UCNV_DEBUG_LOG(x,y,z,__LINE__) +#else +# define UCNV_DEBUG_LOG(x,y,z) +#endif + +static const UConverterSharedData * const +converterData[UCNV_NUMBER_OF_SUPPORTED_CONVERTER_TYPES]={ + NULL, NULL, + +#if UCONFIG_NO_LEGACY_CONVERSION + NULL, +#else + &_MBCSData, +#endif + + &_Latin1Data, + &_UTF8Data, &_UTF16BEData, &_UTF16LEData, &_UTF32BEData, &_UTF32LEData, + NULL, + +#if UCONFIG_NO_LEGACY_CONVERSION + NULL, + NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, + NULL, +#else + &_ISO2022Data, + &_LMBCSData1,&_LMBCSData2, &_LMBCSData3, &_LMBCSData4, &_LMBCSData5, &_LMBCSData6, + &_LMBCSData8,&_LMBCSData11,&_LMBCSData16,&_LMBCSData17,&_LMBCSData18,&_LMBCSData19, + &_HZData, +#endif + + &_SCSUData, + +#if UCONFIG_NO_LEGACY_CONVERSION + NULL, +#else + &_ISCIIData, +#endif + + &_ASCIIData, + &_UTF7Data, &_Bocu1Data, &_UTF16Data, &_UTF32Data, &_CESU8Data, &_IMAPData +}; + +/* Please keep this in binary sorted order for getAlgorithmicTypeFromName. + Also the name should be in lower case and all spaces, dashes and underscores + removed +*/ +static struct { + const char *name; + const UConverterType type; +} const cnvNameType[] = { + { "bocu1", UCNV_BOCU1 }, + { "cesu8", UCNV_CESU8 }, +#if !UCONFIG_NO_LEGACY_CONVERSION + { "hz",UCNV_HZ }, +#endif + { "imapmailboxname", UCNV_IMAP_MAILBOX }, +#if !UCONFIG_NO_LEGACY_CONVERSION + { "iscii", UCNV_ISCII }, + { "iso2022", UCNV_ISO_2022 }, +#endif + { "iso88591", UCNV_LATIN_1 }, +#if !UCONFIG_NO_LEGACY_CONVERSION + { "lmbcs1", UCNV_LMBCS_1 }, + { "lmbcs11",UCNV_LMBCS_11 }, + { "lmbcs16",UCNV_LMBCS_16 }, + { "lmbcs17",UCNV_LMBCS_17 }, + { "lmbcs18",UCNV_LMBCS_18 }, + { "lmbcs19",UCNV_LMBCS_19 }, + { "lmbcs2", UCNV_LMBCS_2 }, + { "lmbcs3", UCNV_LMBCS_3 }, + { "lmbcs4", UCNV_LMBCS_4 }, + { "lmbcs5", UCNV_LMBCS_5 }, + { "lmbcs6", UCNV_LMBCS_6 }, + { "lmbcs8", UCNV_LMBCS_8 }, +#endif + { "scsu", UCNV_SCSU }, + { "usascii", UCNV_US_ASCII }, + { "utf16", UCNV_UTF16 }, + { "utf16be", UCNV_UTF16_BigEndian }, + { "utf16le", UCNV_UTF16_LittleEndian }, +#if U_IS_BIG_ENDIAN + { "utf16oppositeendian", UCNV_UTF16_LittleEndian }, + { "utf16platformendian", UCNV_UTF16_BigEndian }, +#else + { "utf16oppositeendian", UCNV_UTF16_BigEndian}, + { "utf16platformendian", UCNV_UTF16_LittleEndian }, +#endif + { "utf32", UCNV_UTF32 }, + { "utf32be", UCNV_UTF32_BigEndian }, + { "utf32le", UCNV_UTF32_LittleEndian }, +#if U_IS_BIG_ENDIAN + { "utf32oppositeendian", UCNV_UTF32_LittleEndian }, + { "utf32platformendian", UCNV_UTF32_BigEndian }, +#else + { "utf32oppositeendian", UCNV_UTF32_BigEndian }, + { "utf32platformendian", UCNV_UTF32_LittleEndian }, +#endif + { "utf7", UCNV_UTF7 }, + { "utf8", UCNV_UTF8 } +}; + + +/*initializes some global variables */ +static UHashtable *SHARED_DATA_HASHTABLE = NULL; +static UMTX cnvCacheMutex = NULL; /* Mutex for synchronizing cnv cache access. */ + /* Note: the global mutex is used for */ + /* reference count updates. */ + +static const char **gAvailableConverters = NULL; +static uint16_t gAvailableConverterCount = 0; + +/* This contains the resolved converter name. So no further alias lookup is needed again. */ +static char gDefaultConverterNameBuffer[UCNV_MAX_CONVERTER_NAME_LENGTH + 1]; /* +1 for NULL */ +static const char *gDefaultConverterName = NULL; + +/* +If the default converter is an algorithmic converter, this is the cached value. +We don't cache a full UConverter and clone it because ucnv_clone doesn't have +less overhead than an algorithmic open. We don't cache non-algorithmic converters +because ucnv_flushCache must be able to unload the default converter and its table. +*/ +static const UConverterSharedData *gDefaultAlgorithmicSharedData = NULL; + +/* Does gDefaultConverterName have a converter option and require extra parsing? */ +static UBool gDefaultConverterContainsOption; + + +static const char DATA_TYPE[] = "cnv"; + +/* ucnv_cleanup - delete all storage held by the converter cache, except any */ +/* in use by open converters. */ +/* Not thread safe. */ +/* Not supported API. */ +static UBool U_CALLCONV ucnv_cleanup(void) { + ucnv_flushCache(); + if (SHARED_DATA_HASHTABLE != NULL && uhash_count(SHARED_DATA_HASHTABLE) == 0) { + uhash_close(SHARED_DATA_HASHTABLE); + SHARED_DATA_HASHTABLE = NULL; + } + + /* Called from ucnv_flushCache because it allocates the hashtable */ + /*ucnv_flushAvailableConverterCache();*/ + + gDefaultConverterName = NULL; + gDefaultConverterNameBuffer[0] = 0; + gDefaultConverterContainsOption = FALSE; + gDefaultAlgorithmicSharedData = NULL; + + umtx_destroy(&cnvCacheMutex); /* Don't worry about destroying the mutex even */ + /* if the hash table still exists. The mutex */ + /* will lazily re-init itself if needed. */ + return (SHARED_DATA_HASHTABLE == NULL); +} + +static UBool U_CALLCONV +isCnvAcceptable(void *context, + const char *type, const char *name, + const UDataInfo *pInfo) { + return (UBool)( + pInfo->size>=20 && + pInfo->isBigEndian==U_IS_BIG_ENDIAN && + pInfo->charsetFamily==U_CHARSET_FAMILY && + pInfo->sizeofUChar==U_SIZEOF_UCHAR && + pInfo->dataFormat[0]==0x63 && /* dataFormat="cnvt" */ + pInfo->dataFormat[1]==0x6e && + pInfo->dataFormat[2]==0x76 && + pInfo->dataFormat[3]==0x74 && + pInfo->formatVersion[0]==6); /* Everything will be version 6 */ +} + +/** + * Un flatten shared data from a UDATA.. + */ +static UConverterSharedData* +ucnv_data_unFlattenClone(UConverterLoadArgs *pArgs, UDataMemory *pData, UErrorCode *status) +{ + /* UDataInfo info; -- necessary only if some converters have different formatVersion */ + const uint8_t *raw = (const uint8_t *)udata_getMemory(pData); + const UConverterStaticData *source = (const UConverterStaticData *) raw; + UConverterSharedData *data; + UConverterType type = (UConverterType)source->conversionType; + + if(U_FAILURE(*status)) + return NULL; + + if( (uint16_t)type >= UCNV_NUMBER_OF_SUPPORTED_CONVERTER_TYPES || + converterData[type] == NULL || + converterData[type]->referenceCounter != 1 || + source->structSize != sizeof(UConverterStaticData)) + { + *status = U_INVALID_TABLE_FORMAT; + return NULL; + } + + data = (UConverterSharedData *)uprv_malloc(sizeof(UConverterSharedData)); + if(data == NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + + /* copy initial values from the static structure for this type */ + uprv_memcpy(data, converterData[type], sizeof(UConverterSharedData)); + +#if 0 /* made UConverterMBCSTable part of UConverterSharedData -- markus 20031107 */ + /* + * It would be much more efficient if the table were a direct member, not a pointer. + * However, that would add to the size of all UConverterSharedData objects + * even if they do not use this table (especially algorithmic ones). + * If this changes, then the static templates from converterData[type] + * need more entries. + * + * In principle, it would be cleaner if the load() function below + * allocated the table. + */ + data->table = (UConverterTable *)uprv_malloc(sizeof(UConverterTable)); + if(data->table == NULL) { + uprv_free(data); + *status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + uprv_memset(data->table, 0, sizeof(UConverterTable)); +#endif + + data->staticData = source; + + data->sharedDataCached = FALSE; + + /* fill in fields from the loaded data */ + data->dataMemory = (void*)pData; /* for future use */ + + if(data->impl->load != NULL) { + data->impl->load(data, pArgs, raw + source->structSize, status); + if(U_FAILURE(*status)) { + uprv_free(data->table); + uprv_free(data); + return NULL; + } + } + return data; +} + +/*Takes an alias name gets an actual converter file name + *goes to disk and opens it. + *allocates the memory and returns a new UConverter object + */ +static UConverterSharedData *createConverterFromFile(UConverterLoadArgs *pArgs, UErrorCode * err) +{ + UDataMemory *data; + UConverterSharedData *sharedData; + + UTRACE_ENTRY_OC(UTRACE_UCNV_LOAD); + + if (U_FAILURE (*err)) { + UTRACE_EXIT_STATUS(*err); + return NULL; + } + + UTRACE_DATA2(UTRACE_OPEN_CLOSE, "load converter %s from package %s", pArgs->name, pArgs->pkg); + + data = udata_openChoice(pArgs->pkg, DATA_TYPE, pArgs->name, isCnvAcceptable, NULL, err); + if(U_FAILURE(*err)) + { + UTRACE_EXIT_STATUS(*err); + return NULL; + } + + sharedData = ucnv_data_unFlattenClone(pArgs, data, err); + if(U_FAILURE(*err)) + { + udata_close(data); + UTRACE_EXIT_STATUS(*err); + return NULL; + } + + /* + * TODO Store pkg in a field in the shared data so that delta-only converters + * can load base converters from the same package. + * If the pkg name is longer than the field, then either do not load the converter + * in the first place, or just set the pkg field to "". + */ + + UTRACE_EXIT_PTR_STATUS(sharedData, *err); + return sharedData; +} + +/*returns a converter type from a string + */ +static const UConverterSharedData * +getAlgorithmicTypeFromName(const char *realName) +{ + uint32_t mid, start, limit; + uint32_t lastMid; + int result; + char strippedName[UCNV_MAX_CONVERTER_NAME_LENGTH]; + + /* Lower case and remove ignoreable characters. */ + ucnv_io_stripForCompare(strippedName, realName); + + /* do a binary search for the alias */ + start = 0; + limit = sizeof(cnvNameType)/sizeof(cnvNameType[0]); + mid = limit; + lastMid = UINT32_MAX; + + for (;;) { + mid = (uint32_t)((start + limit) / 2); + if (lastMid == mid) { /* Have we moved? */ + break; /* We haven't moved, and it wasn't found. */ + } + lastMid = mid; + result = uprv_strcmp(strippedName, cnvNameType[mid].name); + + if (result < 0) { + limit = mid; + } else if (result > 0) { + start = mid; + } else { + return converterData[cnvNameType[mid].type]; + } + } + + return NULL; +} + +/* +* Based on the number of known converters, this determines how many times larger +* the shared data hash table should be. When on small platforms, or just a couple +* of converters are used, this number should be 2. When memory is plentiful, or +* when ucnv_countAvailable is ever used with a lot of available converters, +* this should be 4. +* Larger numbers reduce the number of hash collisions, but use more memory. +*/ +#define UCNV_CACHE_LOAD_FACTOR 2 + +/* Puts the shared data in the static hashtable SHARED_DATA_HASHTABLE */ +/* Will always be called with the cnvCacheMutex alrady being held */ +/* by the calling function. */ +/* Stores the shared data in the SHARED_DATA_HASHTABLE + * @param data The shared data + */ +static void +ucnv_shareConverterData(UConverterSharedData * data) +{ + UErrorCode err = U_ZERO_ERROR; + /*Lazy evaluates the Hashtable itself */ + /*void *sanity = NULL;*/ + + if (SHARED_DATA_HASHTABLE == NULL) + { + SHARED_DATA_HASHTABLE = uhash_openSize(uhash_hashChars, uhash_compareChars, NULL, + ucnv_io_countKnownConverters(&err)*UCNV_CACHE_LOAD_FACTOR, + &err); + ucln_common_registerCleanup(UCLN_COMMON_UCNV, ucnv_cleanup); + + if (U_FAILURE(err)) + return; + } + + /* ### check to see if the element is not already there! */ + + /* + sanity = ucnv_getSharedConverterData (data->staticData->name); + if(sanity != NULL) + { + UCNV_DEBUG_LOG("put:overwrite!",data->staticData->name,sanity); + } + UCNV_DEBUG_LOG("put:chk",data->staticData->name,sanity); + */ + + /* Mark it shared */ + data->sharedDataCached = TRUE; + + uhash_put(SHARED_DATA_HASHTABLE, + (void*) data->staticData->name, /* Okay to cast away const as long as + keyDeleter == NULL */ + data, + &err); + UCNV_DEBUG_LOG("put", data->staticData->name,data); + +} + +/* Look up a converter name in the shared data cache. */ +/* cnvCacheMutex must be held by the caller to protect the hash table. */ +/* gets the shared data from the SHARED_DATA_HASHTABLE (might return NULL if it isn't there) + * @param name The name of the shared data + * @return the shared data from the SHARED_DATA_HASHTABLE + */ +static UConverterSharedData * +ucnv_getSharedConverterData(const char *name) +{ + /*special case when no Table has yet been created we return NULL */ + if (SHARED_DATA_HASHTABLE == NULL) + { + return NULL; + } + else + { + UConverterSharedData *rc; + + rc = (UConverterSharedData*)uhash_get(SHARED_DATA_HASHTABLE, name); + UCNV_DEBUG_LOG("get",name,rc); + return rc; + } +} + +/*frees the string of memory blocks associates with a sharedConverter + *if and only if the referenceCounter == 0 + */ +/* Deletes (frees) the Shared data it's passed. first it checks the referenceCounter to + * see if anyone is using it, if not it frees all the memory stemming from sharedConverterData and + * returns TRUE, + * otherwise returns FALSE + * @param sharedConverterData The shared data + * @return if not it frees all the memory stemming from sharedConverterData and + * returns TRUE, otherwise returns FALSE + */ +static UBool +ucnv_deleteSharedConverterData(UConverterSharedData * deadSharedData) +{ + UTRACE_ENTRY_OC(UTRACE_UCNV_UNLOAD); + UTRACE_DATA2(UTRACE_OPEN_CLOSE, "unload converter %s shared data %p", deadSharedData->staticData->name, deadSharedData); + + if (deadSharedData->referenceCounter > 0) { + UTRACE_EXIT_VALUE((int32_t)FALSE); + return FALSE; + } + + if (deadSharedData->impl->unload != NULL) { + deadSharedData->impl->unload(deadSharedData); + } + + if(deadSharedData->dataMemory != NULL) + { + UDataMemory *data = (UDataMemory*)deadSharedData->dataMemory; + udata_close(data); + } + + if(deadSharedData->table != NULL) + { + uprv_free(deadSharedData->table); + } + +#if 0 + /* if the static data is actually owned by the shared data */ + /* enable if we ever have this situation. */ + if(deadSharedData->staticDataOwned == TRUE) /* see ucnv_bld.h */ + { + uprv_free((void*)deadSharedData->staticData); + } +#endif + +#if 0 + /* Zap it ! */ + uprv_memset(deadSharedData->0, sizeof(*deadSharedData)); +#endif + + uprv_free(deadSharedData); + + UTRACE_EXIT_VALUE((int32_t)TRUE); + return TRUE; +} + +/** + * Load a non-algorithmic converter. + * If pkg==NULL, then this function must be called inside umtx_lock(&cnvCacheMutex). + */ +UConverterSharedData * +ucnv_load(UConverterLoadArgs *pArgs, UErrorCode *err) { + UConverterSharedData *mySharedConverterData; + + if(err == NULL || U_FAILURE(*err)) { + return NULL; + } + + if(pArgs->pkg != NULL && *pArgs->pkg != 0) { + /* application-provided converters are not currently cached */ + return createConverterFromFile(pArgs, err); + } + + mySharedConverterData = ucnv_getSharedConverterData(pArgs->name); + if (mySharedConverterData == NULL) + { + /*Not cached, we need to stream it in from file */ + mySharedConverterData = createConverterFromFile(pArgs, err); + if (U_FAILURE (*err) || (mySharedConverterData == NULL)) + { + return NULL; + } + else + { + /* share it with other library clients */ + ucnv_shareConverterData(mySharedConverterData); + } + } + else + { + /* The data for this converter was already in the cache. */ + /* Update the reference counter on the shared data: one more client */ + mySharedConverterData->referenceCounter++; + } + + return mySharedConverterData; +} + +/** + * Unload a non-algorithmic converter. + * It must be sharedData->referenceCounter != ~0 + * and this function must be called inside umtx_lock(&cnvCacheMutex). + */ +void +ucnv_unload(UConverterSharedData *sharedData) { + if(sharedData != NULL) { + if (sharedData->referenceCounter > 0) { + sharedData->referenceCounter--; + } + + if((sharedData->referenceCounter <= 0)&&(sharedData->sharedDataCached == FALSE)) { + ucnv_deleteSharedConverterData(sharedData); + } + } +} + +void +ucnv_unloadSharedDataIfReady(UConverterSharedData *sharedData) +{ + /* + Checking whether it's an algorithic converter is okay + in multithreaded applications because the value never changes. + Don't check referenceCounter for any other value. + */ + if(sharedData != NULL && sharedData->referenceCounter != ~0) { + umtx_lock(&cnvCacheMutex); + ucnv_unload(sharedData); + umtx_unlock(&cnvCacheMutex); + } +} + +void +ucnv_incrementRefCount(UConverterSharedData *sharedData) +{ + if(sharedData != NULL && sharedData->referenceCounter != ~0) { + umtx_lock(&cnvCacheMutex); + sharedData->referenceCounter++; + umtx_unlock(&cnvCacheMutex); + } +} + +static void +parseConverterOptions(const char *inName, + char *cnvName, + char *locale, + uint32_t *pFlags, + UErrorCode *err) +{ + char c; + int32_t len = 0; + + /* copy the converter name itself to cnvName */ + while((c=*inName)!=0 && c!=UCNV_OPTION_SEP_CHAR) { + if (++len>=UCNV_MAX_CONVERTER_NAME_LENGTH) { + *err = U_ILLEGAL_ARGUMENT_ERROR; /* bad name */ + *cnvName=0; + return; + } + *cnvName++=c; + inName++; + } + *cnvName=0; + + /* parse options. No more name copying should occur. */ + while((c=*inName)!=0) { + if(c==UCNV_OPTION_SEP_CHAR) { + ++inName; + } + + /* inName is behind an option separator */ + if(uprv_strncmp(inName, "locale=", 7)==0) { + /* do not modify locale itself in case we have multiple locale options */ + char *dest=locale; + + /* copy the locale option value */ + inName+=7; + len=0; + while((c=*inName)!=0 && c!=UCNV_OPTION_SEP_CHAR) { + ++inName; + + if(++len>=ULOC_FULLNAME_CAPACITY) { + *err=U_ILLEGAL_ARGUMENT_ERROR; /* bad name */ + *locale=0; + return; + } + + *dest++=c; + } + *dest=0; + } else if(uprv_strncmp(inName, "version=", 8)==0) { + /* copy the version option value into bits 3..0 of *pFlags */ + inName+=8; + c=*inName; + if(c==0) { + *pFlags&=~UCNV_OPTION_VERSION; + return; + } else if((uint8_t)(c-'0')<10) { + *pFlags=(*pFlags&~UCNV_OPTION_VERSION)|(uint32_t)(c-'0'); + ++inName; + } + } else if(uprv_strncmp(inName, "swaplfnl", 8)==0) { + inName+=8; + *pFlags|=UCNV_OPTION_SWAP_LFNL; + /* add processing for new options here with another } else if(uprv_strncmp(inName, "option-name=", XX)==0) { */ + } else { + /* ignore any other options until we define some */ + while(((c = *inName++) != 0) && (c != UCNV_OPTION_SEP_CHAR)) { + } + if(c==0) { + return; + } + } + } +} + +/*Logic determines if the converter is Algorithmic AND/OR cached + *depending on that: + * -we either go to get data from disk and cache it (Data=TRUE, Cached=False) + * -Get it from a Hashtable (Data=X, Cached=TRUE) + * -Call dataConverter initializer (Data=TRUE, Cached=TRUE) + * -Call AlgorithmicConverter initializer (Data=FALSE, Cached=TRUE) + */ +UConverterSharedData * +ucnv_loadSharedData(const char *converterName, UConverterLookupData *lookup, UErrorCode * err) { + UConverterLookupData stackLookup; + UConverterSharedData *mySharedConverterData = NULL; + UErrorCode internalErrorCode = U_ZERO_ERROR; + UBool mayContainOption = TRUE; + UBool checkForAlgorithmic = TRUE; + + if (U_FAILURE (*err)) { + return NULL; + } + + if(lookup == NULL) { + lookup = &stackLookup; + } + + lookup->locale[0] = 0; + lookup->options = 0; + + /* In case "name" is NULL we want to open the default converter. */ + if (converterName == NULL) { + /* Call ucnv_getDefaultName first to query the name from the OS. */ + lookup->realName = ucnv_getDefaultName(); + if (lookup->realName == NULL) { + *err = U_MISSING_RESOURCE_ERROR; + return NULL; + } + mySharedConverterData = (UConverterSharedData *)gDefaultAlgorithmicSharedData; + checkForAlgorithmic = FALSE; + mayContainOption = gDefaultConverterContainsOption; + /* the default converter name is already canonical */ + } + else if((converterName[0] == 'U' ? + ( converterName[1] == 'T' && converterName[2] == 'F') : + (converterName[0] == 'u' && converterName[1] == 't' && converterName[2] == 'f')) + && + (converterName[3] == '-' ? + (converterName[4] == '8' && converterName[5] == 0) : + (converterName[3] == '8' && converterName[4] == 0))) + { + /* fastpath for UTF-8 */ + return (UConverterSharedData *)converterData[UCNV_UTF8]; + } + else { + /* separate the converter name from the options */ + parseConverterOptions(converterName, lookup->cnvName, lookup->locale, &lookup->options, err); + if (U_FAILURE(*err)) { + /* Very bad name used. */ + return NULL; + } + + /* get the canonical converter name */ + lookup->realName = ucnv_io_getConverterName(lookup->cnvName, &mayContainOption, &internalErrorCode); + if (U_FAILURE(internalErrorCode) || lookup->realName == NULL) { + /* + * set the input name in case the converter was added + * without updating the alias table, or when there is no alias table + */ + lookup->realName = lookup->cnvName; + } + } + + /* separate the converter name from the options */ + if(mayContainOption && lookup->realName != lookup->cnvName) { + parseConverterOptions(lookup->realName, lookup->cnvName, lookup->locale, &lookup->options, err); + lookup->realName = lookup->cnvName; + } + + /* get the shared data for an algorithmic converter, if it is one */ + if (checkForAlgorithmic) { + mySharedConverterData = (UConverterSharedData *)getAlgorithmicTypeFromName(lookup->realName); + } + if (mySharedConverterData == NULL) + { + /* it is a data-based converter, get its shared data. */ + /* Hold the cnvCacheMutex through the whole process of checking the */ + /* converter data cache, and adding new entries to the cache */ + /* to prevent other threads from modifying the cache during the */ + /* process. */ + UConverterLoadArgs args={ 0 }; + + args.size=sizeof(UConverterLoadArgs); + args.nestedLoads=1; + args.options=lookup->options; + args.pkg=NULL; + args.name=lookup->realName; + + umtx_lock(&cnvCacheMutex); + mySharedConverterData = ucnv_load(&args, err); + umtx_unlock(&cnvCacheMutex); + if (U_FAILURE (*err) || (mySharedConverterData == NULL)) + { + return NULL; + } + } + + return mySharedConverterData; +} + +UConverter * +ucnv_createConverter(UConverter *myUConverter, const char *converterName, UErrorCode * err) +{ + UConverterLookupData stackLookup; + UConverterSharedData *mySharedConverterData; + + UTRACE_ENTRY_OC(UTRACE_UCNV_OPEN); + + if(U_SUCCESS(*err)) { + UTRACE_DATA1(UTRACE_OPEN_CLOSE, "open converter %s", converterName); + + mySharedConverterData = ucnv_loadSharedData(converterName, &stackLookup, err); + + if(U_SUCCESS(*err)) { + myUConverter = ucnv_createConverterFromSharedData( + myUConverter, mySharedConverterData, + stackLookup.realName, stackLookup.locale, stackLookup.options, + err); + + if(U_SUCCESS(*err)) { + UTRACE_EXIT_PTR_STATUS(myUConverter, *err); + return myUConverter; + } else { + ucnv_unloadSharedDataIfReady(mySharedConverterData); + } + } + } + + /* exit with error */ + UTRACE_EXIT_STATUS(*err); + return NULL; +} + +UConverter * +ucnv_createAlgorithmicConverter(UConverter *myUConverter, + UConverterType type, + const char *locale, uint32_t options, + UErrorCode *err) { + UConverter *cnv; + const UConverterSharedData *sharedData; + UBool isAlgorithmicConverter; + + UTRACE_ENTRY_OC(UTRACE_UCNV_OPEN_ALGORITHMIC); + UTRACE_DATA1(UTRACE_OPEN_CLOSE, "open algorithmic converter type %d", (int32_t)type); + + if(type<0 || UCNV_NUMBER_OF_SUPPORTED_CONVERTER_TYPES<=type) { + *err = U_ILLEGAL_ARGUMENT_ERROR; + UTRACE_EXIT_STATUS(U_ILLEGAL_ARGUMENT_ERROR); + return NULL; + } + + sharedData = converterData[type]; + umtx_lock(&cnvCacheMutex); + isAlgorithmicConverter = (UBool)(sharedData == NULL || sharedData->referenceCounter != ~0); + umtx_unlock(&cnvCacheMutex); + if (isAlgorithmicConverter) { + /* not a valid type, or not an algorithmic converter */ + *err = U_ILLEGAL_ARGUMENT_ERROR; + UTRACE_EXIT_STATUS(U_ILLEGAL_ARGUMENT_ERROR); + return NULL; + } + + cnv = ucnv_createConverterFromSharedData(myUConverter, (UConverterSharedData *)sharedData, "", + locale != NULL ? locale : "", options, err); + + UTRACE_EXIT_PTR_STATUS(cnv, *err); + return cnv; +} + +UConverter* +ucnv_createConverterFromPackage(const char *packageName, const char *converterName, UErrorCode * err) +{ + char cnvName[UCNV_MAX_CONVERTER_NAME_LENGTH], locale[ULOC_FULLNAME_CAPACITY]; + UConverter *myUConverter; + UConverterSharedData *mySharedConverterData; + + UConverterLoadArgs args={ 0 }; + + UTRACE_ENTRY_OC(UTRACE_UCNV_OPEN_PACKAGE); + + if(U_FAILURE(*err)) { + UTRACE_EXIT_STATUS(*err); + return NULL; + } + + UTRACE_DATA2(UTRACE_OPEN_CLOSE, "open converter %s from package %s", converterName, packageName); + + args.size=sizeof(UConverterLoadArgs); + args.nestedLoads=1; + args.pkg=packageName; + + /* first, get the options out of the converterName string */ + parseConverterOptions(converterName, cnvName, locale, &args.options, err); + if (U_FAILURE(*err)) { + /* Very bad name used. */ + UTRACE_EXIT_STATUS(*err); + return NULL; + } + args.name=cnvName; + + /* open the data, unflatten the shared structure */ + mySharedConverterData = createConverterFromFile(&args, err); + + if (U_FAILURE(*err)) { + UTRACE_EXIT_STATUS(*err); + return NULL; + } + + /* create the actual converter */ + myUConverter = ucnv_createConverterFromSharedData(NULL, mySharedConverterData, cnvName, locale, args.options, err); + + if (U_FAILURE(*err)) { + ucnv_close(myUConverter); + UTRACE_EXIT_STATUS(*err); + return NULL; + } + + UTRACE_EXIT_PTR_STATUS(myUConverter, *err); + return myUConverter; +} + + +UConverter* +ucnv_createConverterFromSharedData(UConverter *myUConverter, + UConverterSharedData *mySharedConverterData, + const char *realName, const char *locale, uint32_t options, + UErrorCode *err) +{ + UBool isCopyLocal; + + if(myUConverter == NULL) + { + myUConverter = (UConverter *) uprv_malloc (sizeof (UConverter)); + if(myUConverter == NULL) + { + *err = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + isCopyLocal = FALSE; + } else { + isCopyLocal = TRUE; + } + + /* initialize the converter */ + uprv_memset(myUConverter, 0, sizeof(UConverter)); + myUConverter->isCopyLocal = isCopyLocal; + /*myUConverter->isExtraLocal = FALSE;*/ /* Set by the memset call */ + myUConverter->sharedData = mySharedConverterData; + myUConverter->options = options; + myUConverter->preFromUFirstCP = U_SENTINEL; + myUConverter->fromCharErrorBehaviour = UCNV_TO_U_DEFAULT_CALLBACK; + myUConverter->fromUCharErrorBehaviour = UCNV_FROM_U_DEFAULT_CALLBACK; + myUConverter->toUnicodeStatus = mySharedConverterData->toUnicodeStatus; + myUConverter->maxBytesPerUChar = mySharedConverterData->staticData->maxBytesPerChar; + myUConverter->subChar1 = mySharedConverterData->staticData->subChar1; + myUConverter->subCharLen = mySharedConverterData->staticData->subCharLen; + myUConverter->subChars = (uint8_t *)myUConverter->subUChars; + uprv_memcpy(myUConverter->subChars, mySharedConverterData->staticData->subChar, myUConverter->subCharLen); + + if(mySharedConverterData->impl->open != NULL) { + mySharedConverterData->impl->open(myUConverter, realName, locale, options, err); + if(U_FAILURE(*err)) { + ucnv_close(myUConverter); + return NULL; + } + } + + return myUConverter; +} + +static void +ucnv_flushAvailableConverterCache() { + if (gAvailableConverters) { + umtx_lock(&cnvCacheMutex); + gAvailableConverterCount = 0; + uprv_free((char **)gAvailableConverters); + gAvailableConverters = NULL; + umtx_unlock(&cnvCacheMutex); + } +} + +/*Frees all shared immutable objects that aren't referred to (reference count = 0) + */ +U_CAPI int32_t U_EXPORT2 +ucnv_flushCache () +{ + UConverterSharedData *mySharedData = NULL; + int32_t pos; + int32_t tableDeletedNum = 0; + const UHashElement *e; + UErrorCode status = U_ILLEGAL_ARGUMENT_ERROR; + int32_t i, remaining; + + UTRACE_ENTRY_OC(UTRACE_UCNV_FLUSH_CACHE); + + /* Close the default converter without creating a new one so that everything will be flushed. */ + ucnv_close(u_getDefaultConverter(&status)); + + /*if shared data hasn't even been lazy evaluated yet + * return 0 + */ + if (SHARED_DATA_HASHTABLE == NULL) { + UTRACE_EXIT_VALUE((int32_t)0); + return 0; + } + + /*creates an enumeration to iterate through every element in the + * table + * + * Synchronization: holding cnvCacheMutex will prevent any other thread from + * accessing or modifying the hash table during the iteration. + * The reference count of an entry may be decremented by + * ucnv_close while the iteration is in process, but this is + * benign. It can't be incremented (in ucnv_createConverter()) + * because the sequence of looking up in the cache + incrementing + * is protected by cnvCacheMutex. + */ + umtx_lock(&cnvCacheMutex); + /* + * double loop: A delta/extension-only converter has a pointer to its base table's + * shared data; the first iteration of the outer loop may see the delta converter + * before the base converter, and unloading the delta converter may get the base + * converter's reference counter down to 0. + */ + i = 0; + do { + remaining = 0; + pos = -1; + while ((e = uhash_nextElement (SHARED_DATA_HASHTABLE, &pos)) != NULL) + { + mySharedData = (UConverterSharedData *) e->value.pointer; + /*deletes only if reference counter == 0 */ + if (mySharedData->referenceCounter == 0) + { + tableDeletedNum++; + + UCNV_DEBUG_LOG("del",mySharedData->staticData->name,mySharedData); + + uhash_removeElement(SHARED_DATA_HASHTABLE, e); + mySharedData->sharedDataCached = FALSE; + ucnv_deleteSharedConverterData (mySharedData); + } else { + ++remaining; + } + } + } while(++i == 1 && remaining > 0); + umtx_unlock(&cnvCacheMutex); + + UTRACE_DATA1(UTRACE_INFO, "ucnv_flushCache() exits with %d converters remaining", remaining); + + ucnv_flushAvailableConverterCache(); + + UTRACE_EXIT_VALUE(tableDeletedNum); + return tableDeletedNum; +} + +/* available converters list --------------------------------------------------- */ + +static UBool haveAvailableConverterList(UErrorCode *pErrorCode) { + int needInit; + UMTX_CHECK(&cnvCacheMutex, (gAvailableConverters == NULL), needInit); + if (needInit) { + UConverter tempConverter; + UEnumeration *allConvEnum = NULL; + uint16_t idx; + uint16_t localConverterCount; + uint16_t allConverterCount; + UErrorCode localStatus; + const char *converterName; + const char **localConverterList; + + allConvEnum = ucnv_openAllNames(pErrorCode); + allConverterCount = uenum_count(allConvEnum, pErrorCode); + if (U_FAILURE(*pErrorCode)) { + return FALSE; + } + + /* We can't have more than "*converterTable" converters to open */ + localConverterList = (const char **) uprv_malloc(allConverterCount * sizeof(char*)); + if (!localConverterList) { + *pErrorCode = U_MEMORY_ALLOCATION_ERROR; + return FALSE; + } + + /* Open the default converter to make sure that it has first dibs in the hash table. */ + localStatus = U_ZERO_ERROR; + ucnv_close(ucnv_createConverter(&tempConverter, NULL, &localStatus)); + + localConverterCount = 0; + + for (idx = 0; idx < allConverterCount; idx++) { + localStatus = U_ZERO_ERROR; + converterName = uenum_next(allConvEnum, NULL, &localStatus); + ucnv_close(ucnv_createConverter(&tempConverter, converterName, &localStatus)); + if (U_SUCCESS(localStatus)) { + localConverterList[localConverterCount++] = converterName; + } + } + uenum_close(allConvEnum); + + umtx_lock(&cnvCacheMutex); + if (gAvailableConverters == NULL) { + gAvailableConverters = localConverterList; + gAvailableConverterCount = localConverterCount; + ucln_common_registerCleanup(UCLN_COMMON_UCNV, ucnv_cleanup); + } + else { + uprv_free((char **)localConverterList); + } + umtx_unlock(&cnvCacheMutex); + } + return TRUE; +} + +U_CFUNC uint16_t +ucnv_bld_countAvailableConverters(UErrorCode *pErrorCode) { + if (haveAvailableConverterList(pErrorCode)) { + return gAvailableConverterCount; + } + return 0; +} + +U_CFUNC const char * +ucnv_bld_getAvailableConverter(uint16_t n, UErrorCode *pErrorCode) { + if (haveAvailableConverterList(pErrorCode)) { + if (n < gAvailableConverterCount) { + return gAvailableConverters[n]; + } + *pErrorCode = U_INDEX_OUTOFBOUNDS_ERROR; + } + return NULL; +} + +/* default converter name --------------------------------------------------- */ + +/* +Copy the canonical converter name. +ucnv_getDefaultName must be thread safe, which can call this function. + +ucnv_setDefaultName calls this function and it doesn't have to be +thread safe because there is no reliable/safe way to reset the +converter in use in all threads. If you did reset the converter, you +would not be sure that retrieving a default converter for one string +would be the same type of default converter for a successive string. +Since the name is a returned via ucnv_getDefaultName without copying, +you shouldn't be modifying or deleting the string from a separate thread. +*/ +static U_INLINE void +internalSetName(const char *name, UErrorCode *status) { + UConverterLookupData lookup; + int32_t length=(int32_t)(uprv_strlen(name)); + UBool containsOption = (UBool)(uprv_strchr(name, UCNV_OPTION_SEP_CHAR) != NULL); + const UConverterSharedData *algorithmicSharedData; + + lookup.locale[0] = 0; + lookup.options = 0; + lookup.realName = name; + if(containsOption) { + parseConverterOptions(lookup.realName, lookup.cnvName, lookup.locale, &lookup.options, status); + lookup.realName = lookup.cnvName; + } + algorithmicSharedData = getAlgorithmicTypeFromName(lookup.realName); + + umtx_lock(&cnvCacheMutex); + + gDefaultAlgorithmicSharedData = algorithmicSharedData; + gDefaultConverterContainsOption = containsOption; + uprv_memcpy(gDefaultConverterNameBuffer, name, length); + gDefaultConverterNameBuffer[length]=0; + gDefaultConverterName = gDefaultConverterNameBuffer; + + ucln_common_registerCleanup(UCLN_COMMON_UCNV, ucnv_cleanup); + + umtx_unlock(&cnvCacheMutex); +} + +/* + * In order to be really thread-safe, the get function would have to take + * a buffer parameter and copy the current string inside a mutex block. + * This implementation only tries to be really thread-safe while + * setting the name. + * It assumes that setting a pointer is atomic. + */ + +U_CAPI const char* U_EXPORT2 +ucnv_getDefaultName() { + /* local variable to be thread-safe */ + const char *name; + + /* + Multiple calls to ucnv_getDefaultName must be thread safe, + but ucnv_setDefaultName is not thread safe. + */ + UMTX_CHECK(&cnvCacheMutex, gDefaultConverterName, name); + if(name==NULL) { + UErrorCode errorCode = U_ZERO_ERROR; + UConverter *cnv = NULL; + + name = uprv_getDefaultCodepage(); + + /* if the name is there, test it out and get the canonical name with options */ + if(name != NULL) { + cnv = ucnv_open(name, &errorCode); + if(U_SUCCESS(errorCode) && cnv != NULL) { + name = ucnv_getName(cnv, &errorCode); + } + } + + if(name == NULL || name[0] == 0 + || U_FAILURE(errorCode) || cnv == NULL + || uprv_strlen(name)>=sizeof(gDefaultConverterNameBuffer)) + { + /* Panic time, let's use a fallback. */ +#if (U_CHARSET_FAMILY == U_ASCII_FAMILY) + name = "US-ASCII"; + /* there is no 'algorithmic' converter for EBCDIC */ +#elif defined(OS390) + name = "ibm-1047_P100-1995" UCNV_SWAP_LFNL_OPTION_STRING; +#else + name = "ibm-37_P100-1995"; +#endif + } + + internalSetName(name, &errorCode); + + /* The close may make the current name go away. */ + ucnv_close(cnv); + } + + return name; +} + +/* +This function is not thread safe, and it can't be thread safe. +See internalSetName or the API reference for details. +*/ +U_CAPI void U_EXPORT2 +ucnv_setDefaultName(const char *converterName) { + if(converterName==NULL) { + /* reset to the default codepage */ + gDefaultConverterName=NULL; + } else { + UErrorCode errorCode = U_ZERO_ERROR; + UConverter *cnv = NULL; + const char *name = NULL; + + /* if the name is there, test it out and get the canonical name with options */ + cnv = ucnv_open(converterName, &errorCode); + if(U_SUCCESS(errorCode) && cnv != NULL) { + name = ucnv_getName(cnv, &errorCode); + } + + if(U_SUCCESS(errorCode) && name!=NULL) { + internalSetName(name, &errorCode); + } + /* else this converter is bad to use. Don't change it to a bad value. */ + + /* The close may make the current name go away. */ + ucnv_close(cnv); + } +} + +/* data swapping ------------------------------------------------------------ */ + +/* most of this might belong more properly into ucnvmbcs.c, but that is so large */ + +#if !UCONFIG_NO_LEGACY_CONVERSION + +U_CAPI int32_t U_EXPORT2 +ucnv_swap(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode) { + const UDataInfo *pInfo; + int32_t headerSize; + + const uint8_t *inBytes; + uint8_t *outBytes; + + uint32_t offset, count, staticDataSize; + int32_t size; + + const UConverterStaticData *inStaticData; + UConverterStaticData *outStaticData; + + const _MBCSHeader *inMBCSHeader; + _MBCSHeader *outMBCSHeader; + _MBCSHeader mbcsHeader; + uint8_t outputType; + + int32_t maxFastUChar, mbcsIndexLength; + + const int32_t *inExtIndexes; + int32_t extOffset; + + /* udata_swapDataHeader checks the arguments */ + headerSize=udata_swapDataHeader(ds, inData, length, outData, pErrorCode); + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + /* check data format and format version */ + pInfo=(const UDataInfo *)((const char *)inData+4); + if(!( + pInfo->dataFormat[0]==0x63 && /* dataFormat="cnvt" */ + pInfo->dataFormat[1]==0x6e && + pInfo->dataFormat[2]==0x76 && + pInfo->dataFormat[3]==0x74 && + pInfo->formatVersion[0]==6 && + pInfo->formatVersion[1]>=2 + )) { + udata_printError(ds, "ucnv_swap(): data format %02x.%02x.%02x.%02x (format version %02x.%02x) is not recognized as an ICU .cnv conversion table\n", + pInfo->dataFormat[0], pInfo->dataFormat[1], + pInfo->dataFormat[2], pInfo->dataFormat[3], + pInfo->formatVersion[0], pInfo->formatVersion[1]); + *pErrorCode=U_UNSUPPORTED_ERROR; + return 0; + } + + inBytes=(const uint8_t *)inData+headerSize; + outBytes=(uint8_t *)outData+headerSize; + + /* read the initial UConverterStaticData structure after the UDataInfo header */ + inStaticData=(const UConverterStaticData *)inBytes; + outStaticData=(UConverterStaticData *)outBytes; + + if(length<0) { + staticDataSize=ds->readUInt32(inStaticData->structSize); + } else { + length-=headerSize; + if( lengthreadUInt32(inStaticData->structSize)) + ) { + udata_printError(ds, "ucnv_swap(): too few bytes (%d after header) for an ICU .cnv conversion table\n", + length); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + } + + if(length>=0) { + /* swap the static data */ + if(inStaticData!=outStaticData) { + uprv_memcpy(outStaticData, inStaticData, staticDataSize); + } + + ds->swapArray32(ds, &inStaticData->structSize, 4, + &outStaticData->structSize, pErrorCode); + ds->swapArray32(ds, &inStaticData->codepage, 4, + &outStaticData->codepage, pErrorCode); + + ds->swapInvChars(ds, inStaticData->name, (int32_t)uprv_strlen(inStaticData->name), + outStaticData->name, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + udata_printError(ds, "ucnv_swap(): error swapping converter name\n"); + return 0; + } + } + + inBytes+=staticDataSize; + outBytes+=staticDataSize; + if(length>=0) { + length-=(int32_t)staticDataSize; + } + + /* check for supported conversionType values */ + if(inStaticData->conversionType==UCNV_MBCS) { + /* swap MBCS data */ + inMBCSHeader=(const _MBCSHeader *)inBytes; + outMBCSHeader=(_MBCSHeader *)outBytes; + + if(0<=length && lengthversion[0]==4 && inMBCSHeader->version[1]>=1)) { + udata_printError(ds, "ucnv_swap(): unsupported _MBCSHeader.version %d.%d\n", + inMBCSHeader->version[0], inMBCSHeader->version[1]); + *pErrorCode=U_UNSUPPORTED_ERROR; + return 0; + } + + uprv_memcpy(mbcsHeader.version, inMBCSHeader->version, 4); + mbcsHeader.countStates= ds->readUInt32(inMBCSHeader->countStates); + mbcsHeader.countToUFallbacks= ds->readUInt32(inMBCSHeader->countToUFallbacks); + mbcsHeader.offsetToUCodeUnits= ds->readUInt32(inMBCSHeader->offsetToUCodeUnits); + mbcsHeader.offsetFromUTable= ds->readUInt32(inMBCSHeader->offsetFromUTable); + mbcsHeader.offsetFromUBytes= ds->readUInt32(inMBCSHeader->offsetFromUBytes); + mbcsHeader.flags= ds->readUInt32(inMBCSHeader->flags); + mbcsHeader.fromUBytesLength= ds->readUInt32(inMBCSHeader->fromUBytesLength); + + extOffset=(int32_t)(mbcsHeader.flags>>8); + outputType=(uint8_t)mbcsHeader.flags; + + /* make sure that the output type is known */ + switch(outputType) { + case MBCS_OUTPUT_1: + case MBCS_OUTPUT_2: + case MBCS_OUTPUT_3: + case MBCS_OUTPUT_4: + case MBCS_OUTPUT_3_EUC: + case MBCS_OUTPUT_4_EUC: + case MBCS_OUTPUT_2_SISO: + case MBCS_OUTPUT_EXT_ONLY: + /* OK */ + break; + default: + udata_printError(ds, "ucnv_swap(): unsupported MBCS output type 0x%x\n", + outputType); + *pErrorCode=U_UNSUPPORTED_ERROR; + return 0; + } + + /* calculate the length of the MBCS data */ + + /* + * utf8Friendly MBCS files (mbcsHeader.version 4.3) + * contain an additional mbcsIndex table: + * uint16_t[(maxFastUChar+1)>>6]; + * where maxFastUChar=((mbcsHeader.version[2]<<8)|0xff). + */ + maxFastUChar=0; + mbcsIndexLength=0; + if( outputType!=MBCS_OUTPUT_EXT_ONLY && outputType!=MBCS_OUTPUT_1 && + mbcsHeader.version[1]>=3 && (maxFastUChar=mbcsHeader.version[2])!=0 + ) { + maxFastUChar=(maxFastUChar<<8)|0xff; + mbcsIndexLength=((maxFastUChar+1)>>6)*2; /* number of bytes */ + } + + if(extOffset==0) { + size=(int32_t)(mbcsHeader.offsetFromUBytes+mbcsHeader.fromUBytesLength+mbcsIndexLength); + + /* avoid compiler warnings - not otherwise necessary, and the value does not matter */ + inExtIndexes=NULL; + } else { + /* there is extension data after the base data, see ucnv_ext.h */ + if(length>=0 && length<(extOffset+UCNV_EXT_INDEXES_MIN_LENGTH*4)) { + udata_printError(ds, "ucnv_swap(): too few bytes (%d after headers) for an ICU MBCS .cnv conversion table with extension data\n", + length); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + + inExtIndexes=(const int32_t *)(inBytes+extOffset); + size=extOffset+udata_readInt32(ds, inExtIndexes[UCNV_EXT_SIZE]); + } + + if(length>=0) { + if(lengthswapArray32(ds, &inMBCSHeader->countStates, 7*4, + &outMBCSHeader->countStates, pErrorCode); + + if(outputType==MBCS_OUTPUT_EXT_ONLY) { + /* + * extension-only file, + * contains a base name instead of normal base table data + */ + + /* swap the base name, between the header and the extension data */ + ds->swapInvChars(ds, inMBCSHeader+1, (int32_t)uprv_strlen((const char *)(inMBCSHeader+1)), + outMBCSHeader+1, pErrorCode); + } else { + /* normal file with base table data */ + + /* swap the state table, 1kB per state */ + ds->swapArray32(ds, inMBCSHeader+1, (int32_t)(mbcsHeader.countStates*1024), + outMBCSHeader+1, pErrorCode); + + /* swap the toUFallbacks[] */ + offset=sizeof(_MBCSHeader)+mbcsHeader.countStates*1024; + ds->swapArray32(ds, inBytes+offset, (int32_t)(mbcsHeader.countToUFallbacks*8), + outBytes+offset, pErrorCode); + + /* swap the unicodeCodeUnits[] */ + offset=mbcsHeader.offsetToUCodeUnits; + count=mbcsHeader.offsetFromUTable-offset; + ds->swapArray16(ds, inBytes+offset, (int32_t)count, + outBytes+offset, pErrorCode); + + /* offset to the stage 1 table, independent of the outputType */ + offset=mbcsHeader.offsetFromUTable; + + if(outputType==MBCS_OUTPUT_1) { + /* SBCS: swap the fromU tables, all 16 bits wide */ + count=(mbcsHeader.offsetFromUBytes-offset)+mbcsHeader.fromUBytesLength; + ds->swapArray16(ds, inBytes+offset, (int32_t)count, + outBytes+offset, pErrorCode); + } else { + /* otherwise: swap the stage tables separately */ + + /* stage 1 table: uint16_t[0x440 or 0x40] */ + if(inStaticData->unicodeMask&UCNV_HAS_SUPPLEMENTARY) { + count=0x440*2; /* for all of Unicode */ + } else { + count=0x40*2; /* only BMP */ + } + ds->swapArray16(ds, inBytes+offset, (int32_t)count, + outBytes+offset, pErrorCode); + + /* stage 2 table: uint32_t[] */ + offset+=count; + count=mbcsHeader.offsetFromUBytes-offset; + ds->swapArray32(ds, inBytes+offset, (int32_t)count, + outBytes+offset, pErrorCode); + + /* stage 3/result bytes: sometimes uint16_t[] or uint32_t[] */ + offset=mbcsHeader.offsetFromUBytes; + count=mbcsHeader.fromUBytesLength; + switch(outputType) { + case MBCS_OUTPUT_2: + case MBCS_OUTPUT_3_EUC: + case MBCS_OUTPUT_2_SISO: + ds->swapArray16(ds, inBytes+offset, (int32_t)count, + outBytes+offset, pErrorCode); + break; + case MBCS_OUTPUT_4: + ds->swapArray32(ds, inBytes+offset, (int32_t)count, + outBytes+offset, pErrorCode); + break; + default: + /* just uint8_t[], nothing to swap */ + break; + } + + if(mbcsIndexLength!=0) { + offset+=count; + count=mbcsIndexLength; + ds->swapArray16(ds, inBytes+offset, (int32_t)count, + outBytes+offset, pErrorCode); + } + } + } + + if(extOffset!=0) { + /* swap the extension data */ + inBytes+=extOffset; + outBytes+=extOffset; + + /* swap toUTable[] */ + offset=udata_readInt32(ds, inExtIndexes[UCNV_EXT_TO_U_INDEX]); + length=udata_readInt32(ds, inExtIndexes[UCNV_EXT_TO_U_LENGTH]); + ds->swapArray32(ds, inBytes+offset, length*4, outBytes+offset, pErrorCode); + + /* swap toUUChars[] */ + offset=udata_readInt32(ds, inExtIndexes[UCNV_EXT_TO_U_UCHARS_INDEX]); + length=udata_readInt32(ds, inExtIndexes[UCNV_EXT_TO_U_UCHARS_LENGTH]); + ds->swapArray16(ds, inBytes+offset, length*2, outBytes+offset, pErrorCode); + + /* swap fromUTableUChars[] */ + offset=udata_readInt32(ds, inExtIndexes[UCNV_EXT_FROM_U_UCHARS_INDEX]); + length=udata_readInt32(ds, inExtIndexes[UCNV_EXT_FROM_U_LENGTH]); + ds->swapArray16(ds, inBytes+offset, length*2, outBytes+offset, pErrorCode); + + /* swap fromUTableValues[] */ + offset=udata_readInt32(ds, inExtIndexes[UCNV_EXT_FROM_U_VALUES_INDEX]); + /* same length as for fromUTableUChars[] */ + ds->swapArray32(ds, inBytes+offset, length*4, outBytes+offset, pErrorCode); + + /* no need to swap fromUBytes[] */ + + /* swap fromUStage12[] */ + offset=udata_readInt32(ds, inExtIndexes[UCNV_EXT_FROM_U_STAGE_12_INDEX]); + length=udata_readInt32(ds, inExtIndexes[UCNV_EXT_FROM_U_STAGE_12_LENGTH]); + ds->swapArray16(ds, inBytes+offset, length*2, outBytes+offset, pErrorCode); + + /* swap fromUStage3[] */ + offset=udata_readInt32(ds, inExtIndexes[UCNV_EXT_FROM_U_STAGE_3_INDEX]); + length=udata_readInt32(ds, inExtIndexes[UCNV_EXT_FROM_U_STAGE_3_LENGTH]); + ds->swapArray16(ds, inBytes+offset, length*2, outBytes+offset, pErrorCode); + + /* swap fromUStage3b[] */ + offset=udata_readInt32(ds, inExtIndexes[UCNV_EXT_FROM_U_STAGE_3B_INDEX]); + length=udata_readInt32(ds, inExtIndexes[UCNV_EXT_FROM_U_STAGE_3B_LENGTH]); + ds->swapArray32(ds, inBytes+offset, length*4, outBytes+offset, pErrorCode); + + /* swap indexes[] */ + length=udata_readInt32(ds, inExtIndexes[UCNV_EXT_INDEXES_LENGTH]); + ds->swapArray32(ds, inBytes, length*4, outBytes, pErrorCode); + } + } + } else { + udata_printError(ds, "ucnv_swap(): unknown conversionType=%d!=UCNV_MBCS\n", + inStaticData->conversionType); + *pErrorCode=U_UNSUPPORTED_ERROR; + return 0; + } + + return headerSize+(int32_t)staticDataSize+size; +} + +#endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */ + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/ucnv_bld.h b/Build/source/libs/icu/icu-xetex/common/ucnv_bld.h new file mode 100644 index 00000000000..75aa4d183ab --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnv_bld.h @@ -0,0 +1,281 @@ +/* +********************************************************************** +* Copyright (C) 1999-2006, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* +* +* ucnv_bld.h: +* Contains internal data structure definitions +* Created by Bertrand A. Damiba +* +* Change history: +* +* 06/29/2000 helena Major rewrite of the callback APIs. +*/ + +#ifndef UCNV_BLD_H +#define UCNV_BLD_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "unicode/ucnv.h" +#include "unicode/ucnv_err.h" +#include "ucnv_cnv.h" +#include "ucnvmbcs.h" +#include "ucnv_ext.h" +#include "udataswp.h" + +/* size of the overflow buffers in UConverter, enough for escaping callbacks */ +#define UCNV_ERROR_BUFFER_LENGTH 32 + +/* at most 4 bytes per substitution character (part of .cnv file format! see UConverterStaticData) */ +#define UCNV_MAX_SUBCHAR_LEN 4 + +/* at most 8 bytes per character in toUBytes[] (UTF-8 uses up to 6) */ +#define UCNV_MAX_CHAR_LEN 8 + +/* converter options bits */ +#define UCNV_OPTION_VERSION 0xf +#define UCNV_OPTION_SWAP_LFNL 0x10 + + +U_CDECL_BEGIN /* We must declare the following as 'extern "C"' so that if ucnv + itself is compiled under C++, the linkage of the funcptrs will + work. + */ + +union UConverterTable { + UConverterMBCSTable mbcs; +}; + +typedef union UConverterTable UConverterTable; + +struct UConverterImpl; +typedef struct UConverterImpl UConverterImpl; + +/** values for the unicodeMask */ +#define UCNV_HAS_SUPPLEMENTARY 1 +#define UCNV_HAS_SURROGATES 2 + +typedef struct UConverterStaticData { /* +offset: size */ + uint32_t structSize; /* +0: 4 Size of this structure */ + + char name + [UCNV_MAX_CONVERTER_NAME_LENGTH]; /* +4: 60 internal name of the converter- invariant chars */ + + int32_t codepage; /* +64: 4 codepage # (now IBM-$codepage) */ + + int8_t platform; /* +68: 1 platform of the converter (only IBM now) */ + int8_t conversionType; /* +69: 1 conversion type */ + + int8_t minBytesPerChar; /* +70: 1 Minimum # bytes per char in this codepage */ + int8_t maxBytesPerChar; /* +71: 1 Maximum # bytes output per UChar in this codepage */ + + uint8_t subChar[UCNV_MAX_SUBCHAR_LEN]; /* +72: 4 [note: 4 and 8 byte boundary] */ + int8_t subCharLen; /* +76: 1 */ + + uint8_t hasToUnicodeFallback; /* +77: 1 UBool needs to be changed to UBool to be consistent across platform */ + uint8_t hasFromUnicodeFallback; /* +78: 1 */ + uint8_t unicodeMask; /* +79: 1 bit 0: has supplementary bit 1: has single surrogates */ + uint8_t subChar1; /* +80: 1 single-byte substitution character for IBM MBCS (0 if none) */ + uint8_t reserved[19]; /* +81: 19 to round out the structure */ + /* total size: 100 */ +} UConverterStaticData; + +/* + * Defines the UConverterSharedData struct, + * the immutable, shared part of UConverter. + */ +struct UConverterSharedData { + uint32_t structSize; /* Size of this structure */ + uint32_t referenceCounter; /* used to count number of clients, 0xffffffff for static SharedData */ + + const void *dataMemory; /* from udata_openChoice() - for cleanup */ + void *table; /* Unused. This used to be a UConverterTable - Pointer to conversion data - see mbcs below */ + + const UConverterStaticData *staticData; /* pointer to the static (non changing) data. */ + + UBool sharedDataCached; /* TRUE: shared data is in cache, don't destroy on ucnv_close() if 0 ref. FALSE: shared data isn't in the cache, do attempt to clean it up if the ref is 0 */ + /*UBool staticDataOwned; TRUE if static data owned by shared data & should be freed with it, NEVER true for udata() loaded statics. This ignored variable was removed to make space for sharedDataCached. */ + + const UConverterImpl *impl; /* vtable-style struct of mostly function pointers */ + + /*initial values of some members of the mutable part of object */ + uint32_t toUnicodeStatus; + + /* + * Shared data structures currently come in two flavors: + * - readonly for built-in algorithmic converters + * - allocated for MBCS, with a pointer to an allocated UConverterTable + * which always has a UConverterMBCSTable + * + * To eliminate one allocation, I am making the UConverterMBCSTable + * a member of the shared data. It is the last member so that static + * definitions of UConverterSharedData work as before. + * The table field above also remains to avoid updating all static + * definitions, but is now unused. + * + * markus 2003-nov-07 + */ + UConverterMBCSTable mbcs; +}; + +/* Defines a UConverter, the lightweight mutable part the user sees */ + +struct UConverter { + /* + * Error function pointer called when conversion issues + * occur during a ucnv_fromUnicode call + */ + void (U_EXPORT2 *fromUCharErrorBehaviour) (const void *context, + UConverterFromUnicodeArgs *args, + const UChar *codeUnits, + int32_t length, + UChar32 codePoint, + UConverterCallbackReason reason, + UErrorCode *); + /* + * Error function pointer called when conversion issues + * occur during a ucnv_toUnicode call + */ + void (U_EXPORT2 *fromCharErrorBehaviour) (const void *context, + UConverterToUnicodeArgs *args, + const char *codeUnits, + int32_t length, + UConverterCallbackReason reason, + UErrorCode *); + + /* + * Pointer to additional data that depends on the converter type. + * Used by ISO 2022, SCSU, GB 18030 converters, possibly more. + */ + void *extraInfo; + + const void *fromUContext; + const void *toUContext; + + /* + * Pointer to charset bytes for substitution string if subCharLen>0, + * or pointer to Unicode string (UChar *) if subCharLen<0. + * subCharLen==0 is equivalent to using a skip callback. + * If the pointer is !=subUChars then it is allocated with + * UCNV_ERROR_BUFFER_LENGTH * U_SIZEOF_UCHAR bytes. + * The subUChars field is declared as UChar[] not uint8_t[] to + * guarantee alignment for UChars. + */ + uint8_t *subChars; + + UConverterSharedData *sharedData; /* Pointer to the shared immutable part of the converter object */ + + uint32_t options; /* options flags from UConverterOpen, may contain additional bits */ + + UBool sharedDataIsCached; /* TRUE: shared data is in cache, don't destroy on ucnv_close() if 0 ref. FALSE: shared data isn't in the cache, do attempt to clean it up if the ref is 0 */ + UBool isCopyLocal; /* TRUE if UConverter is not owned and not released in ucnv_close() (stack-allocated, safeClone(), etc.) */ + UBool isExtraLocal; /* TRUE if extraInfo is not owned and not released in ucnv_close() (stack-allocated, safeClone(), etc.) */ + + UBool useFallback; + int8_t toULength; /* number of bytes in toUBytes */ + uint8_t toUBytes[UCNV_MAX_CHAR_LEN-1];/* more "toU status"; keeps the bytes of the current character */ + uint32_t toUnicodeStatus; /* Used to internalize stream status information */ + int32_t mode; + uint32_t fromUnicodeStatus; + + /* + * More fromUnicode() status. Serves 3 purposes: + * - keeps a lead surrogate between buffers (similar to toUBytes[]) + * - keeps a lead surrogate at the end of the stream, + * which the framework handles as truncated input + * - if the fromUnicode() implementation returns to the framework + * (ucnv.c ucnv_fromUnicode()), then the framework calls the callback + * for this code point + */ + UChar32 fromUChar32; + + /* + * value for ucnv_getMaxCharSize() + * + * usually simply copied from the static data, but ucnvmbcs.c modifies + * the value depending on the converter type and options + */ + int8_t maxBytesPerUChar; + + int8_t subCharLen; /* length of the codepage specific character sequence */ + int8_t invalidCharLength; + int8_t charErrorBufferLength; /* number of valid bytes in charErrorBuffer */ + + int8_t invalidUCharLength; + int8_t UCharErrorBufferLength; /* number of valid UChars in charErrorBuffer */ + + uint8_t subChar1; /* single-byte substitution character if different from subChar */ + UBool useSubChar1; + char invalidCharBuffer[UCNV_MAX_CHAR_LEN]; /* bytes from last error/callback situation */ + uint8_t charErrorBuffer[UCNV_ERROR_BUFFER_LENGTH]; /* codepage output from Error functions */ + UChar subUChars[UCNV_MAX_SUBCHAR_LEN/U_SIZEOF_UCHAR]; /* see subChars documentation */ + + UChar invalidUCharBuffer[U16_MAX_LENGTH]; /* UChars from last error/callback situation */ + UChar UCharErrorBuffer[UCNV_ERROR_BUFFER_LENGTH]; /* unicode output from Error functions */ + + /* fields for conversion extension */ + + /* store previous UChars/chars to continue partial matches */ + UChar32 preFromUFirstCP; /* >=0: partial match */ + UChar preFromU[UCNV_EXT_MAX_UCHARS]; + char preToU[UCNV_EXT_MAX_BYTES]; + int8_t preFromULength, preToULength; /* negative: replay */ + int8_t preToUFirstLength; /* length of first character */ +}; + +U_CDECL_END /* end of UConverter */ + +#define CONVERTER_FILE_EXTENSION ".cnv" + + +/** + * Return the number of all converter names. + * @param pErrorCode The error code + * @return the number of all converter names + */ +U_CFUNC uint16_t +ucnv_bld_countAvailableConverters(UErrorCode *pErrorCode); + +/** + * Return the (n)th converter name in mixed case, or NULL + * if there is none (typically, if the data cannot be loaded). + * 0<=indexreferenceCounter != ~0 + * and this function must be called inside umtx_lock(&cnvCacheMutex). + */ +void +ucnv_unload(UConverterSharedData *sharedData); + +/** + * Swap ICU .cnv conversion tables. See udataswp.h. + * @internal + */ +U_CAPI int32_t U_EXPORT2 +ucnv_swap(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode); + +#endif + +#endif /* _UCNV_BLD */ diff --git a/Build/source/libs/icu/icu-xetex/common/ucnv_cb.c b/Build/source/libs/icu/icu-xetex/common/ucnv_cb.c new file mode 100644 index 00000000000..fa34b658327 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnv_cb.c @@ -0,0 +1,259 @@ +/* +********************************************************************** +* Copyright (C) 2000-2006, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** + * ucnv_cb.c: + * External APIs for the ICU's codeset conversion library + * Helena Shih + * + * Modification History: + * + * Date Name Description + * 7/28/2000 srl Implementation + */ + +/** + * @name Character Conversion C API + * + */ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "unicode/ucnv_cb.h" +#include "ucnv_bld.h" +#include "ucnv_cnv.h" +#include "cmemory.h" + +/* need to update the offsets when the target moves. */ +/* Note: Recursion may occur in the cb functions, be sure to update the offsets correctly +if you don't use ucnv_cbXXX functions. Make sure you don't use the same callback within +the same call stack if the complexity arises. */ +U_CAPI void U_EXPORT2 +ucnv_cbFromUWriteBytes (UConverterFromUnicodeArgs *args, + const char* source, + int32_t length, + int32_t offsetIndex, + UErrorCode * err) +{ + if(U_FAILURE(*err)) { + return; + } + + ucnv_fromUWriteBytes( + args->converter, + source, length, + &args->target, args->targetLimit, + &args->offsets, offsetIndex, + err); +} + +U_CAPI void U_EXPORT2 +ucnv_cbFromUWriteUChars(UConverterFromUnicodeArgs *args, + const UChar** source, + const UChar* sourceLimit, + int32_t offsetIndex, + UErrorCode * err) +{ + /* + This is a fun one. Recursion can occur - we're basically going to + just retry shoving data through the same converter. Note, if you got + here through some kind of invalid sequence, you maybe should emit a + reset sequence of some kind and/or call ucnv_reset(). Since this + IS an actual conversion, take care that you've changed the callback + or the data, or you'll get an infinite loop. + + Please set the err value to something reasonable before calling + into this. + */ + + char *oldTarget; + + if(U_FAILURE(*err)) + { + return; + } + + oldTarget = args->target; + + ucnv_fromUnicode(args->converter, + &args->target, + args->targetLimit, + source, + sourceLimit, + NULL, /* no offsets */ + FALSE, /* no flush */ + err); + + if(args->offsets) + { + while (args->target != oldTarget) /* if it moved at all.. */ + { + *(args->offsets)++ = offsetIndex; + oldTarget++; + } + } + + /* + Note, if you did something like used a Stop subcallback, things would get interesting. + In fact, here's where we want to return the partially consumed in-source! + */ + if(*err == U_BUFFER_OVERFLOW_ERROR) + /* && (*source < sourceLimit && args->target >= args->targetLimit) + -- S. Hrcek */ + { + /* Overflowed the target. Now, we'll write into the charErrorBuffer. + It's a fixed size. If we overflow it... Hmm */ + char *newTarget; + const char *newTargetLimit; + UErrorCode err2 = U_ZERO_ERROR; + + int8_t errBuffLen; + + errBuffLen = args->converter->charErrorBufferLength; + + /* start the new target at the first free slot in the errbuff.. */ + newTarget = (char *)(args->converter->charErrorBuffer + errBuffLen); + + newTargetLimit = (char *)(args->converter->charErrorBuffer + + sizeof(args->converter->charErrorBuffer)); + + if(newTarget >= newTargetLimit) + { + *err = U_INTERNAL_PROGRAM_ERROR; + return; + } + + /* We're going to tell the converter that the errbuff len is empty. + This prevents the existing errbuff from being 'flushed' out onto + itself. If the errbuff is needed by the converter this time, + we're hosed - we're out of space! */ + + args->converter->charErrorBufferLength = 0; + + ucnv_fromUnicode(args->converter, + &newTarget, + newTargetLimit, + source, + sourceLimit, + NULL, + FALSE, + &err2); + + /* We can go ahead and overwrite the length here. We know just how + to recalculate it. */ + + args->converter->charErrorBufferLength = (int8_t)( + newTarget - (char*)args->converter->charErrorBuffer); + + if((newTarget >= newTargetLimit) || (err2 == U_BUFFER_OVERFLOW_ERROR)) + { + /* now we're REALLY in trouble. + Internal program error - callback shouldn't have written this much + data! + */ + *err = U_INTERNAL_PROGRAM_ERROR; + return; + } + /*else {*/ + /* sub errs could be invalid/truncated/illegal chars or w/e. + These might want to be passed on up.. But the problem is, we already + need to pass U_BUFFER_OVERFLOW_ERROR. That has to override these + other errs.. */ + + /* + if(U_FAILURE(err2)) + ?? + */ + /*}*/ + } +} + +U_CAPI void U_EXPORT2 +ucnv_cbFromUWriteSub (UConverterFromUnicodeArgs *args, + int32_t offsetIndex, + UErrorCode * err) +{ + UConverter *converter; + int32_t length; + + if(U_FAILURE(*err)) { + return; + } + converter = args->converter; + length = converter->subCharLen; + + if(length == 0) { + return; + } + + if(length < 0) { + /* + * Write/convert the substitution string. Its real length is -length. + * Unlike the escape callback, we need not change the converter's + * callback function because ucnv_setSubstString() verified that + * the string can be converted, so we will not get a conversion error + * and will not recurse. + * At worst we should get a U_BUFFER_OVERFLOW_ERROR. + */ + const UChar *source = (const UChar *)converter->subChars; + ucnv_cbFromUWriteUChars(args, &source, source - length, offsetIndex, err); + return; + } + + if(converter->sharedData->impl->writeSub!=NULL) { + converter->sharedData->impl->writeSub(args, offsetIndex, err); + } + else if(converter->subChar1!=0 && (uint16_t)converter->invalidUCharBuffer[0]<=(uint16_t)0xffu) { + /* + TODO: Is this untestable because the MBCS converter has a writeSub function to call + and the other converters don't use subChar1? + */ + ucnv_cbFromUWriteBytes(args, + (const char *)&converter->subChar1, 1, + offsetIndex, err); + } + else { + ucnv_cbFromUWriteBytes(args, + (const char *)converter->subChars, length, + offsetIndex, err); + } +} + +U_CAPI void U_EXPORT2 +ucnv_cbToUWriteUChars (UConverterToUnicodeArgs *args, + const UChar* source, + int32_t length, + int32_t offsetIndex, + UErrorCode * err) +{ + if(U_FAILURE(*err)) { + return; + } + + ucnv_toUWriteUChars( + args->converter, + source, length, + &args->target, args->targetLimit, + &args->offsets, offsetIndex, + err); +} + +U_CAPI void U_EXPORT2 +ucnv_cbToUWriteSub (UConverterToUnicodeArgs *args, + int32_t offsetIndex, + UErrorCode * err) +{ + static const UChar kSubstituteChar1 = 0x1A, kSubstituteChar = 0xFFFD; + + /* could optimize this case, just one uchar */ + if(args->converter->invalidCharLength == 1 && args->converter->subChar1 != 0) { + ucnv_cbToUWriteUChars(args, &kSubstituteChar1, 1, offsetIndex, err); + } else { + ucnv_cbToUWriteUChars(args, &kSubstituteChar, 1, offsetIndex, err); + } +} + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/ucnv_cnv.c b/Build/source/libs/icu/icu-xetex/common/ucnv_cnv.c new file mode 100644 index 00000000000..f8e2f830f91 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnv_cnv.c @@ -0,0 +1,174 @@ +/* +****************************************************************************** +* +* Copyright (C) 1999-2004, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* +* uconv_cnv.c: +* Implements all the low level conversion functions +* T_UnicodeConverter_{to,from}Unicode_$ConversionType +* +* Change history: +* +* 06/29/2000 helena Major rewrite of the callback APIs. +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "unicode/ucnv_err.h" +#include "unicode/ucnv.h" +#include "unicode/uset.h" +#include "ucnv_cnv.h" +#include "ucnv_bld.h" +#include "cmemory.h" + +U_CFUNC void +ucnv_getCompleteUnicodeSet(const UConverter *cnv, + const USetAdder *sa, + UConverterUnicodeSet which, + UErrorCode *pErrorCode) { + sa->addRange(sa->set, 0, 0x10ffff); +} + +U_CFUNC void +ucnv_getNonSurrogateUnicodeSet(const UConverter *cnv, + const USetAdder *sa, + UConverterUnicodeSet which, + UErrorCode *pErrorCode) { + sa->addRange(sa->set, 0, 0xd7ff); + sa->addRange(sa->set, 0xe000, 0x10ffff); +} + +U_CFUNC void +ucnv_fromUWriteBytes(UConverter *cnv, + const char *bytes, int32_t length, + char **target, const char *targetLimit, + int32_t **offsets, + int32_t sourceIndex, + UErrorCode *pErrorCode) { + char *t=*target; + int32_t *o; + + /* write bytes */ + if(offsets==NULL || (o=*offsets)==NULL) { + while(length>0 && t0 && t0) { + if(cnv!=NULL) { + t=(char *)cnv->charErrorBuffer; + cnv->charErrorBufferLength=(int8_t)length; + do { + *t++=(uint8_t)*bytes++; + } while(--length>0); + } + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } +} + +U_CFUNC void +ucnv_toUWriteUChars(UConverter *cnv, + const UChar *uchars, int32_t length, + UChar **target, const UChar *targetLimit, + int32_t **offsets, + int32_t sourceIndex, + UErrorCode *pErrorCode) { + UChar *t=*target; + int32_t *o; + + /* write UChars */ + if(offsets==NULL || (o=*offsets)==NULL) { + while(length>0 && t0 && t0) { + if(cnv!=NULL) { + t=cnv->UCharErrorBuffer; + cnv->UCharErrorBufferLength=(int8_t)length; + do { + *t++=*uchars++; + } while(--length>0); + } + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } +} + +U_CFUNC void +ucnv_toUWriteCodePoint(UConverter *cnv, + UChar32 c, + UChar **target, const UChar *targetLimit, + int32_t **offsets, + int32_t sourceIndex, + UErrorCode *pErrorCode) { + UChar *t; + int32_t *o; + + t=*target; + + if(t=0) { + if(cnv!=NULL) { + int8_t i=0; + U16_APPEND_UNSAFE(cnv->UCharErrorBuffer, i, c); + cnv->UCharErrorBufferLength=i; + } + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } +} + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/ucnv_cnv.h b/Build/source/libs/icu/icu-xetex/common/ucnv_cnv.h new file mode 100644 index 00000000000..cf612a754da --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnv_cnv.h @@ -0,0 +1,303 @@ +/* +********************************************************************** +* Copyright (C) 1999-2007, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* +* ucnv_cnv.h: +* Definitions for converter implementations. +* +* Modification History: +* +* Date Name Description +* 05/09/00 helena Added implementation to handle fallback mappings. +* 06/29/2000 helena Major rewrite of the callback APIs. +*/ + +#ifndef UCNV_CNV_H +#define UCNV_CNV_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "unicode/ucnv.h" +#include "unicode/ucnv_err.h" +#include "unicode/uset.h" +#include "uset_imp.h" + +U_CDECL_BEGIN + +/* this is used in fromUnicode DBCS tables as an "unassigned" marker */ +#define missingCharMarker 0xFFFF + +/* + * #define missingUCharMarker 0xfffe + * + * commented out because there are actually two values used in toUnicode tables: + * U+fffe "unassigned" + * U+ffff "illegal" + */ + +/** Forward declaration, see ucnv_bld.h */ +struct UConverterSharedData; +typedef struct UConverterSharedData UConverterSharedData; + +/* function types for UConverterImpl ---------------------------------------- */ + +/* struct with arguments for UConverterLoad and ucnv_load() */ +typedef struct { + int32_t size; /* sizeof(UConverterLoadArgs) */ + int32_t nestedLoads; /* count nested ucnv_load() calls */ + int32_t reserved; /* reserved - for good alignment of the pointers */ + uint32_t options; + const char *pkg, *name; +} UConverterLoadArgs; + +typedef void (*UConverterLoad) (UConverterSharedData *sharedData, + UConverterLoadArgs *pArgs, + const uint8_t *raw, UErrorCode *pErrorCode); +typedef void (*UConverterUnload) (UConverterSharedData *sharedData); + +typedef void (*UConverterOpen) (UConverter *cnv, const char *name, const char *locale,uint32_t options, UErrorCode *pErrorCode); +typedef void (*UConverterClose) (UConverter *cnv); + +typedef enum UConverterResetChoice { + UCNV_RESET_BOTH, + UCNV_RESET_TO_UNICODE, + UCNV_RESET_FROM_UNICODE +} UConverterResetChoice; + +typedef void (*UConverterReset) (UConverter *cnv, UConverterResetChoice choice); + +/* + * Converter implementation function(s) for ucnv_toUnicode(). + * If the toUnicodeWithOffsets function pointer is NULL, + * then the toUnicode function will be used and the offsets will be set to -1. + * + * Must maintain state across buffers. Use toUBytes[toULength] for partial input + * sequences; it will be checked in ucnv.c at the end of the input stream + * to detect truncated input. + * Some converters may need additional detection and may then set U_TRUNCATED_CHAR_FOUND. + * + * The toUnicodeWithOffsets must write exactly as many offset values as target + * units. Write offset values of -1 for when the source index corresponding to + * the output unit is not known (e.g., the character started in an earlier buffer). + * The pArgs->offsets pointer need not be moved forward. + * + * At function return, either one of the following conditions must be true: + * - U_BUFFER_OVERFLOW_ERROR and the target is full: target==targetLimit + * - another error code with toUBytes[toULength] set to the offending input + * - no error, and the source is consumed: source==sourceLimit + * + * The ucnv.c code will handle the end of the input (reset) + * (reset, and truncation detection) and callbacks. + */ +typedef void (*UConverterToUnicode) (UConverterToUnicodeArgs *, UErrorCode *); + +/* + * Same rules as for UConverterToUnicode. + * A lead surrogate is kept in fromUChar32 across buffers, and if an error + * occurs, then the offending input code point must be put into fromUChar32 + * as well. + */ +typedef void (*UConverterFromUnicode) (UConverterFromUnicodeArgs *, UErrorCode *); + +/* + * Converter implementation function for ucnv_convertEx(), for direct conversion + * between two charsets without pivoting through UTF-16. + * The rules are the same as for UConverterToUnicode and UConverterFromUnicode. + * In addition, + * - The toUnicode side must behave and keep state exactly like the + * UConverterToUnicode implementation for the same source charset. + * - A U_USING_DEFAULT_WARNING can be set to request to temporarily fall back + * to pivoting. When this function is called, the conversion framework makes + * sure that this warning is not set on input. + * - Continuing a partial match and flushing the toUnicode replay buffer + * are handled by pivoting, using the toUnicode and fromUnicode functions. + */ +typedef void (*UConverterConvert) (UConverterFromUnicodeArgs *pFromUArgs, + UConverterToUnicodeArgs *pToUArgs, + UErrorCode *pErrorCode); + +/* + * Converter implementation function for ucnv_getNextUChar(). + * If the function pointer is NULL, then the toUnicode function will be used. + * + * Will be called at a character boundary (toULength==0). + * May return with + * - U_INDEX_OUTOFBOUNDS_ERROR if there was no output for the input + * (the return value will be ignored) + * - U_TRUNCATED_CHAR_FOUND or another error code (never U_BUFFER_OVERFLOW_ERROR!) + * with toUBytes[toULength] set to the offending input + * (the return value will be ignored) + * - return UCNV_GET_NEXT_UCHAR_USE_TO_U, without moving the source pointer, + * to indicate that the ucnv.c code shall call the toUnicode function instead + * - return a real code point result + * + * Unless UCNV_GET_NEXT_UCHAR_USE_TO_U is returned, the source bytes must be consumed. + * + * The ucnv.c code will handle the end of the input (reset) + * (except for truncation detection!) and callbacks. + */ +typedef UChar32 (*UConverterGetNextUChar) (UConverterToUnicodeArgs *, UErrorCode *); + +typedef void (*UConverterGetStarters)(const UConverter* converter, + UBool starters[256], + UErrorCode *pErrorCode); + +/* If this function pointer is null or if the function returns null + * the name field in static data struct should be returned by + * ucnv_getName() API function + */ +typedef const char * (*UConverterGetName) (const UConverter *cnv); + +/** + * Write the codepage substitution character. + * If this function is not set, then ucnv_cbFromUWriteSub() writes + * the substitution character from UConverter. + * For stateful converters, it is typically necessary to handle this + * specificially for the converter in order to properly maintain the state. + */ +typedef void (*UConverterWriteSub) (UConverterFromUnicodeArgs *pArgs, int32_t offsetIndex, UErrorCode *pErrorCode); + +/** + * For converter-specific safeClone processing + * If this function is not set, then ucnv_safeClone assumes that the converter has no private data that changes + * after the converter is done opening. + * If this function is set, then it is called just after a memcpy() of + * converter data to the new, empty converter, and is expected to set up + * the initial state of the converter. It is not expected to increment the + * reference counts of the standard data types such as the shared data. + */ +typedef UConverter * (*UConverterSafeClone) (const UConverter *cnv, + void *stackBuffer, + int32_t *pBufferSize, + UErrorCode *status); + +/** + * Fills the set of Unicode code points that can be converted by an ICU converter. + * The API function ucnv_getUnicodeSet() clears the USet before calling + * the converter's getUnicodeSet() implementation; the converter should only + * add the appropriate code points to allow recursive use. + * For example, the ISO-2022-JP converter will call each subconverter's + * getUnicodeSet() implementation to consecutively add code points to + * the same USet, which will result in a union of the sets of all subconverters. + * + * For more documentation, see ucnv_getUnicodeSet() in ucnv.h. + */ +typedef void (*UConverterGetUnicodeSet) (const UConverter *cnv, + const USetAdder *sa, + UConverterUnicodeSet which, + UErrorCode *pErrorCode); + +UBool CONVERSION_U_SUCCESS (UErrorCode err); + +/** + * UConverterImpl contains all the data and functions for a converter type. + * Its function pointers work much like a C++ vtable. + * Many converter types need to define only a subset of the functions; + * when a function pointer is NULL, then a default action will be performed. + * + * Every converter type must implement toUnicode, fromUnicode, and getNextUChar, + * otherwise the converter may crash. + * Every converter type that has variable-length codepage sequences should + * also implement toUnicodeWithOffsets and fromUnicodeWithOffsets for + * correct offset handling. + * All other functions may or may not be implemented - it depends only on + * whether the converter type needs them. + * + * When open() fails, then close() will be called, if present. + */ +struct UConverterImpl { + UConverterType type; + + UConverterLoad load; + UConverterUnload unload; + + UConverterOpen open; + UConverterClose close; + UConverterReset reset; + + UConverterToUnicode toUnicode; + UConverterToUnicode toUnicodeWithOffsets; + UConverterFromUnicode fromUnicode; + UConverterFromUnicode fromUnicodeWithOffsets; + UConverterGetNextUChar getNextUChar; + + UConverterGetStarters getStarters; + UConverterGetName getName; + UConverterWriteSub writeSub; + UConverterSafeClone safeClone; + UConverterGetUnicodeSet getUnicodeSet; + + UConverterConvert toUTF8; + UConverterConvert fromUTF8; +}; + +extern const UConverterSharedData + _MBCSData, _Latin1Data, + _UTF8Data, _UTF16BEData, _UTF16LEData, _UTF32BEData, _UTF32LEData, + _ISO2022Data, + _LMBCSData1,_LMBCSData2, _LMBCSData3, _LMBCSData4, _LMBCSData5, _LMBCSData6, + _LMBCSData8,_LMBCSData11,_LMBCSData16,_LMBCSData17,_LMBCSData18,_LMBCSData19, + _HZData,_ISCIIData, _SCSUData, _ASCIIData, + _UTF7Data, _Bocu1Data, _UTF16Data, _UTF32Data, _CESU8Data, _IMAPData; + +U_CDECL_END + +/** Always use fallbacks from codepage to Unicode */ +#define TO_U_USE_FALLBACK(useFallback) TRUE +#define UCNV_TO_U_USE_FALLBACK(cnv) TRUE + +/** Use fallbacks from Unicode to codepage when cnv->useFallback or for private-use code points */ +#define IS_PRIVATE_USE(c) ((uint32_t)((c)-0xe000)<0x1900 || (uint32_t)((c)-0xf0000)<0x20000) +#define FROM_U_USE_FALLBACK(useFallback, c) ((useFallback) || IS_PRIVATE_USE(c)) +#define UCNV_FROM_U_USE_FALLBACK(cnv, c) FROM_U_USE_FALLBACK((cnv)->useFallback, c) + +/** + * Magic number for ucnv_getNextUChar(), returned by a + * getNextUChar() implementation to indicate to use the converter's toUnicode() + * instead of the native function. + * @internal + */ +#define UCNV_GET_NEXT_UCHAR_USE_TO_U -9 + +U_CFUNC void +ucnv_getCompleteUnicodeSet(const UConverter *cnv, + const USetAdder *sa, + UConverterUnicodeSet which, + UErrorCode *pErrorCode); + +U_CFUNC void +ucnv_getNonSurrogateUnicodeSet(const UConverter *cnv, + const USetAdder *sa, + UConverterUnicodeSet which, + UErrorCode *pErrorCode); + +U_CFUNC void +ucnv_fromUWriteBytes(UConverter *cnv, + const char *bytes, int32_t length, + char **target, const char *targetLimit, + int32_t **offsets, + int32_t sourceIndex, + UErrorCode *pErrorCode); +U_CFUNC void +ucnv_toUWriteUChars(UConverter *cnv, + const UChar *uchars, int32_t length, + UChar **target, const UChar *targetLimit, + int32_t **offsets, + int32_t sourceIndex, + UErrorCode *pErrorCode); + +U_CFUNC void +ucnv_toUWriteCodePoint(UConverter *cnv, + UChar32 c, + UChar **target, const UChar *targetLimit, + int32_t **offsets, + int32_t sourceIndex, + UErrorCode *pErrorCode); + +#endif + +#endif /* UCNV_CNV */ diff --git a/Build/source/libs/icu/icu-xetex/common/ucnv_err.c b/Build/source/libs/icu/icu-xetex/common/ucnv_err.c new file mode 100644 index 00000000000..9b2c2852acd --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnv_err.c @@ -0,0 +1,396 @@ +/* + ***************************************************************************** + * + * Copyright (C) 1998-2007, International Business Machines + * Corporation and others. All Rights Reserved. + * + ***************************************************************************** + * + * ucnv_err.c + * Implements error behaviour functions called by T_UConverter_{from,to}Unicode + * + * +* Change history: +* +* 06/29/2000 helena Major rewrite of the callback APIs. +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "unicode/ucnv_err.h" +#include "unicode/ucnv_cb.h" +#include "ucnv_cnv.h" +#include "cmemory.h" +#include "unicode/ucnv.h" +#include "ustrfmt.h" + +#define VALUE_STRING_LENGTH 32 +/*Magic # 32 = 4(number of char in value string) * 8(max number of bytes per char for any converter) */ +#define UNICODE_PERCENT_SIGN_CODEPOINT 0x0025 +#define UNICODE_U_CODEPOINT 0x0055 +#define UNICODE_X_CODEPOINT 0x0058 +#define UNICODE_RS_CODEPOINT 0x005C +#define UNICODE_U_LOW_CODEPOINT 0x0075 +#define UNICODE_X_LOW_CODEPOINT 0x0078 +#define UNICODE_AMP_CODEPOINT 0x0026 +#define UNICODE_HASH_CODEPOINT 0x0023 +#define UNICODE_SEMICOLON_CODEPOINT 0x003B +#define UNICODE_PLUS_CODEPOINT 0x002B +#define UNICODE_LEFT_CURLY_CODEPOINT 0x007B +#define UNICODE_RIGHT_CURLY_CODEPOINT 0x007D +#define UCNV_PRV_ESCAPE_ICU 0 +#define UCNV_PRV_ESCAPE_C 'C' +#define UCNV_PRV_ESCAPE_XML_DEC 'D' +#define UCNV_PRV_ESCAPE_XML_HEX 'X' +#define UCNV_PRV_ESCAPE_JAVA 'J' +#define UCNV_PRV_ESCAPE_UNICODE 'U' +#define UCNV_PRV_STOP_ON_ILLEGAL 'i' + +/*Function Pointer STOPS at the ILLEGAL_SEQUENCE */ +U_CAPI void U_EXPORT2 +UCNV_FROM_U_CALLBACK_STOP ( + const void *context, + UConverterFromUnicodeArgs *fromUArgs, + const UChar* codeUnits, + int32_t length, + UChar32 codePoint, + UConverterCallbackReason reason, + UErrorCode * err) +{ + /* the caller must have set the error code accordingly */ + return; +} + + +/*Function Pointer STOPS at the ILLEGAL_SEQUENCE */ +U_CAPI void U_EXPORT2 +UCNV_TO_U_CALLBACK_STOP ( + const void *context, + UConverterToUnicodeArgs *toUArgs, + const char* codePoints, + int32_t length, + UConverterCallbackReason reason, + UErrorCode * err) +{ + /* the caller must have set the error code accordingly */ + return; +} + +U_CAPI void U_EXPORT2 +UCNV_FROM_U_CALLBACK_SKIP ( + const void *context, + UConverterFromUnicodeArgs *fromUArgs, + const UChar* codeUnits, + int32_t length, + UChar32 codePoint, + UConverterCallbackReason reason, + UErrorCode * err) +{ + if (reason <= UCNV_IRREGULAR) + { + if (context == NULL || (*((char*)context) == UCNV_PRV_STOP_ON_ILLEGAL && reason == UCNV_UNASSIGNED)) + { + *err = U_ZERO_ERROR; + } + /* else the caller must have set the error code accordingly. */ + } + /* else ignore the reset, close and clone calls. */ +} + +U_CAPI void U_EXPORT2 +UCNV_FROM_U_CALLBACK_SUBSTITUTE ( + const void *context, + UConverterFromUnicodeArgs *fromArgs, + const UChar* codeUnits, + int32_t length, + UChar32 codePoint, + UConverterCallbackReason reason, + UErrorCode * err) +{ + if (reason <= UCNV_IRREGULAR) + { + if (context == NULL || (*((char*)context) == UCNV_PRV_STOP_ON_ILLEGAL && reason == UCNV_UNASSIGNED)) + { + *err = U_ZERO_ERROR; + ucnv_cbFromUWriteSub(fromArgs, 0, err); + } + /* else the caller must have set the error code accordingly. */ + } + /* else ignore the reset, close and clone calls. */ +} + +/*uses uprv_itou to get a unicode escape sequence of the offensive sequence, + *uses a clean copy (resetted) of the converter, to convert that unicode + *escape sequence to the target codepage (if conversion failure happens then + *we revert to substituting with subchar) + */ +U_CAPI void U_EXPORT2 +UCNV_FROM_U_CALLBACK_ESCAPE ( + const void *context, + UConverterFromUnicodeArgs *fromArgs, + const UChar *codeUnits, + int32_t length, + UChar32 codePoint, + UConverterCallbackReason reason, + UErrorCode * err) +{ + + UChar valueString[VALUE_STRING_LENGTH]; + int32_t valueStringLength = 0; + int32_t i = 0; + + const UChar *myValueSource = NULL; + UErrorCode err2 = U_ZERO_ERROR; + UConverterFromUCallback original = NULL; + const void *originalContext; + + UConverterFromUCallback ignoredCallback = NULL; + const void *ignoredContext; + + if (reason > UCNV_IRREGULAR) + { + return; + } + + ucnv_setFromUCallBack (fromArgs->converter, + (UConverterFromUCallback) UCNV_FROM_U_CALLBACK_SUBSTITUTE, + NULL, + &original, + &originalContext, + &err2); + + if (U_FAILURE (err2)) + { + *err = err2; + return; + } + if(context==NULL) + { + while (i < length) + { + valueString[valueStringLength++] = (UChar) UNICODE_PERCENT_SIGN_CODEPOINT; /* adding % */ + valueString[valueStringLength++] = (UChar) UNICODE_U_CODEPOINT; /* adding U */ + valueStringLength += uprv_itou (valueString + valueStringLength, VALUE_STRING_LENGTH - valueStringLength, (uint16_t)codeUnits[i++], 16, 4); + } + } + else + { + switch(*((char*)context)) + { + case UCNV_PRV_ESCAPE_JAVA: + while (i < length) + { + valueString[valueStringLength++] = (UChar) UNICODE_RS_CODEPOINT; /* adding \ */ + valueString[valueStringLength++] = (UChar) UNICODE_U_LOW_CODEPOINT; /* adding u */ + valueStringLength += uprv_itou (valueString + valueStringLength, VALUE_STRING_LENGTH - valueStringLength, (uint16_t)codeUnits[i++], 16, 4); + } + break; + + case UCNV_PRV_ESCAPE_C: + valueString[valueStringLength++] = (UChar) UNICODE_RS_CODEPOINT; /* adding \ */ + + if(length==2){ + valueString[valueStringLength++] = (UChar) UNICODE_U_CODEPOINT; /* adding U */ + valueStringLength += uprv_itou (valueString + valueStringLength, VALUE_STRING_LENGTH - valueStringLength, codePoint, 16, 8); + + } + else{ + valueString[valueStringLength++] = (UChar) UNICODE_U_LOW_CODEPOINT; /* adding u */ + valueStringLength += uprv_itou (valueString + valueStringLength, VALUE_STRING_LENGTH - valueStringLength, (uint16_t)codeUnits[0], 16, 4); + } + break; + + case UCNV_PRV_ESCAPE_XML_DEC: + + valueString[valueStringLength++] = (UChar) UNICODE_AMP_CODEPOINT; /* adding & */ + valueString[valueStringLength++] = (UChar) UNICODE_HASH_CODEPOINT; /* adding # */ + if(length==2){ + valueStringLength += uprv_itou (valueString + valueStringLength, VALUE_STRING_LENGTH - valueStringLength, codePoint, 10, 0); + } + else{ + valueStringLength += uprv_itou (valueString + valueStringLength, VALUE_STRING_LENGTH - valueStringLength, (uint16_t)codeUnits[0], 10, 0); + } + valueString[valueStringLength++] = (UChar) UNICODE_SEMICOLON_CODEPOINT; /* adding ; */ + break; + + case UCNV_PRV_ESCAPE_XML_HEX: + + valueString[valueStringLength++] = (UChar) UNICODE_AMP_CODEPOINT; /* adding & */ + valueString[valueStringLength++] = (UChar) UNICODE_HASH_CODEPOINT; /* adding # */ + valueString[valueStringLength++] = (UChar) UNICODE_X_LOW_CODEPOINT; /* adding x */ + if(length==2){ + valueStringLength += uprv_itou (valueString + valueStringLength, VALUE_STRING_LENGTH - valueStringLength, codePoint, 16, 0); + } + else{ + valueStringLength += uprv_itou (valueString + valueStringLength, VALUE_STRING_LENGTH - valueStringLength, (uint16_t)codeUnits[0], 16, 0); + } + valueString[valueStringLength++] = (UChar) UNICODE_SEMICOLON_CODEPOINT; /* adding ; */ + break; + + case UCNV_PRV_ESCAPE_UNICODE: + valueString[valueStringLength++] = (UChar) UNICODE_LEFT_CURLY_CODEPOINT; /* adding { */ + valueString[valueStringLength++] = (UChar) UNICODE_U_CODEPOINT; /* adding U */ + valueString[valueStringLength++] = (UChar) UNICODE_PLUS_CODEPOINT; /* adding + */ + if (length == 2) { + valueStringLength += uprv_itou (valueString + valueStringLength, VALUE_STRING_LENGTH - valueStringLength, codePoint, 16, 4); + } else { + valueStringLength += uprv_itou (valueString + valueStringLength, VALUE_STRING_LENGTH - valueStringLength, (uint16_t)codeUnits[0], 16, 4); + } + valueString[valueStringLength++] = (UChar) UNICODE_RIGHT_CURLY_CODEPOINT; /* adding } */ + break; + + default: + while (i < length) + { + valueString[valueStringLength++] = (UChar) UNICODE_PERCENT_SIGN_CODEPOINT; /* adding % */ + valueString[valueStringLength++] = (UChar) UNICODE_U_CODEPOINT; /* adding U */ + valueStringLength += uprv_itou (valueString + valueStringLength, VALUE_STRING_LENGTH - valueStringLength, (uint16_t)codeUnits[i++], 16, 4); + } + } + } + myValueSource = valueString; + + /* reset the error */ + *err = U_ZERO_ERROR; + + ucnv_cbFromUWriteUChars(fromArgs, &myValueSource, myValueSource+valueStringLength, 0, err); + + ucnv_setFromUCallBack (fromArgs->converter, + original, + originalContext, + &ignoredCallback, + &ignoredContext, + &err2); + if (U_FAILURE (err2)) + { + *err = err2; + return; + } + + return; +} + + + +U_CAPI void U_EXPORT2 +UCNV_TO_U_CALLBACK_SKIP ( + const void *context, + UConverterToUnicodeArgs *toArgs, + const char* codeUnits, + int32_t length, + UConverterCallbackReason reason, + UErrorCode * err) +{ + if (reason <= UCNV_IRREGULAR) + { + if (context == NULL || (*((char*)context) == UCNV_PRV_STOP_ON_ILLEGAL && reason == UCNV_UNASSIGNED)) + { + *err = U_ZERO_ERROR; + } + /* else the caller must have set the error code accordingly. */ + } + /* else ignore the reset, close and clone calls. */ +} + +U_CAPI void U_EXPORT2 +UCNV_TO_U_CALLBACK_SUBSTITUTE ( + const void *context, + UConverterToUnicodeArgs *toArgs, + const char* codeUnits, + int32_t length, + UConverterCallbackReason reason, + UErrorCode * err) +{ + if (reason <= UCNV_IRREGULAR) + { + if (context == NULL || (*((char*)context) == UCNV_PRV_STOP_ON_ILLEGAL && reason == UCNV_UNASSIGNED)) + { + *err = U_ZERO_ERROR; + ucnv_cbToUWriteSub(toArgs,0,err); + } + /* else the caller must have set the error code accordingly. */ + } + /* else ignore the reset, close and clone calls. */ +} + +/*uses uprv_itou to get a unicode escape sequence of the offensive sequence, + *and uses that as the substitution sequence + */ +U_CAPI void U_EXPORT2 +UCNV_TO_U_CALLBACK_ESCAPE ( + const void *context, + UConverterToUnicodeArgs *toArgs, + const char* codeUnits, + int32_t length, + UConverterCallbackReason reason, + UErrorCode * err) +{ + UChar uniValueString[VALUE_STRING_LENGTH]; + int32_t valueStringLength = 0; + int32_t i = 0; + + if (reason > UCNV_IRREGULAR) + { + return; + } + + if(context==NULL) + { + while (i < length) + { + uniValueString[valueStringLength++] = (UChar) UNICODE_PERCENT_SIGN_CODEPOINT; /* adding % */ + uniValueString[valueStringLength++] = (UChar) UNICODE_X_CODEPOINT; /* adding X */ + valueStringLength += uprv_itou (uniValueString + valueStringLength, VALUE_STRING_LENGTH - valueStringLength, (uint8_t) codeUnits[i++], 16, 2); + } + } + else + { + switch(*((char*)context)) + { + case UCNV_PRV_ESCAPE_XML_DEC: + while (i < length) + { + uniValueString[valueStringLength++] = (UChar) UNICODE_AMP_CODEPOINT; /* adding & */ + uniValueString[valueStringLength++] = (UChar) UNICODE_HASH_CODEPOINT; /* adding # */ + valueStringLength += uprv_itou (uniValueString + valueStringLength, VALUE_STRING_LENGTH - valueStringLength, (uint8_t)codeUnits[i++], 10, 0); + uniValueString[valueStringLength++] = (UChar) UNICODE_SEMICOLON_CODEPOINT; /* adding ; */ + } + break; + + case UCNV_PRV_ESCAPE_XML_HEX: + while (i < length) + { + uniValueString[valueStringLength++] = (UChar) UNICODE_AMP_CODEPOINT; /* adding & */ + uniValueString[valueStringLength++] = (UChar) UNICODE_HASH_CODEPOINT; /* adding # */ + uniValueString[valueStringLength++] = (UChar) UNICODE_X_LOW_CODEPOINT; /* adding x */ + valueStringLength += uprv_itou (uniValueString + valueStringLength, VALUE_STRING_LENGTH - valueStringLength, (uint8_t)codeUnits[i++], 16, 0); + uniValueString[valueStringLength++] = (UChar) UNICODE_SEMICOLON_CODEPOINT; /* adding ; */ + } + break; + case UCNV_PRV_ESCAPE_C: + while (i < length) + { + uniValueString[valueStringLength++] = (UChar) UNICODE_RS_CODEPOINT; /* adding \ */ + uniValueString[valueStringLength++] = (UChar) UNICODE_X_LOW_CODEPOINT; /* adding x */ + valueStringLength += uprv_itou (uniValueString + valueStringLength, VALUE_STRING_LENGTH - valueStringLength, (uint8_t)codeUnits[i++], 16, 2); + } + break; + default: + while (i < length) + { + uniValueString[valueStringLength++] = (UChar) UNICODE_PERCENT_SIGN_CODEPOINT; /* adding % */ + uniValueString[valueStringLength++] = (UChar) UNICODE_X_CODEPOINT; /* adding X */ + uprv_itou (uniValueString + valueStringLength, VALUE_STRING_LENGTH - valueStringLength, (uint8_t) codeUnits[i++], 16, 2); + valueStringLength += 2; + } + } + } + /* reset the error */ + *err = U_ZERO_ERROR; + + ucnv_cbToUWriteUChars(toArgs, uniValueString, valueStringLength, 0, err); +} + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/ucnv_ext.c b/Build/source/libs/icu/icu-xetex/common/ucnv_ext.c new file mode 100644 index 00000000000..d60127fc89a --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnv_ext.c @@ -0,0 +1,1088 @@ +/* +****************************************************************************** +* +* Copyright (C) 2003-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: ucnv_ext.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2003jun13 +* created by: Markus W. Scherer +* +* Conversion extensions +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION && !UCONFIG_NO_LEGACY_CONVERSION + +#include "unicode/uset.h" +#include "ucnv_bld.h" +#include "ucnv_cnv.h" +#include "ucnv_ext.h" +#include "cmemory.h" + +/* to Unicode --------------------------------------------------------------- */ + +/* + * @return lookup value for the byte, if found; else 0 + */ +static U_INLINE uint32_t +ucnv_extFindToU(const uint32_t *toUSection, int32_t length, uint8_t byte) { + uint32_t word0, word; + int32_t i, start, limit; + + /* check the input byte against the lowest and highest section bytes */ + start=(int32_t)UCNV_EXT_TO_U_GET_BYTE(toUSection[0]); + limit=(int32_t)UCNV_EXT_TO_U_GET_BYTE(toUSection[length-1]); + if(byte1) { + return 0; /* no match of a DBCS sequence in SBCS mode */ + } else if(preLength==1) { + srcLength=0; + } else /* preLength==0 */ { + if(srcLength>1) { + srcLength=1; + } + } + flush=TRUE; + } + + /* we must not remember fallback matches when not using fallbacks */ + + /* match input units until there is a full match or the input is consumed */ + for(;;) { + /* go to the next section */ + toUSection=toUTable+index; + + /* read first pair of the section */ + value=*toUSection++; + length=UCNV_EXT_TO_U_GET_BYTE(value); + value=UCNV_EXT_TO_U_GET_VALUE(value); + if( value!=0 && + (UCNV_EXT_TO_U_IS_ROUNDTRIP(value) || + TO_U_USE_FALLBACK(useFallback)) && + UCNV_EXT_TO_U_VERIFY_SISO_MATCH(sisoState, i+j) + ) { + /* remember longest match so far */ + matchValue=value; + matchLength=i+j; + } + + /* match pre[] then src[] */ + if(iUCNV_EXT_MAX_BYTES) { + /* + * end of the entire input stream, stop with the longest match so far + * or: partial match must not be longer than UCNV_EXT_MAX_BYTES + * because it must fit into state buffers + */ + break; + } else { + /* continue with more input next time */ + return -length; + } + } + + /* search for the current UChar */ + value=ucnv_extFindToU(toUSection, length, b); + if(value==0) { + /* no match here, stop with the longest match so far */ + break; + } else { + if(UCNV_EXT_TO_U_IS_PARTIAL(value)) { + /* partial match, continue */ + index=(int32_t)UCNV_EXT_TO_U_GET_PARTIAL_INDEX(value); + } else { + if( (UCNV_EXT_TO_U_IS_ROUNDTRIP(value) || + TO_U_USE_FALLBACK(useFallback)) && + UCNV_EXT_TO_U_VERIFY_SISO_MATCH(sisoState, i+j) + ) { + /* full match, stop with result */ + matchValue=value; + matchLength=i+j; + } else { + /* full match on fallback not taken, stop with the longest match so far */ + } + break; + } + } + } + + if(matchLength==0) { + /* no match at all */ + return 0; + } + + /* return result */ + *pMatchValue=UCNV_EXT_TO_U_MASK_ROUNDTRIP(matchValue); + return matchLength; +} + +static U_INLINE void +ucnv_extWriteToU(UConverter *cnv, const int32_t *cx, + uint32_t value, + UChar **target, const UChar *targetLimit, + int32_t **offsets, int32_t srcIndex, + UErrorCode *pErrorCode) { + /* output the result */ + if(UCNV_EXT_TO_U_IS_CODE_POINT(value)) { + /* output a single code point */ + ucnv_toUWriteCodePoint( + cnv, UCNV_EXT_TO_U_GET_CODE_POINT(value), + target, targetLimit, + offsets, srcIndex, + pErrorCode); + } else { + /* output a string - with correct data we have resultLength>0 */ + ucnv_toUWriteUChars( + cnv, + UCNV_EXT_ARRAY(cx, UCNV_EXT_TO_U_UCHARS_INDEX, UChar)+ + UCNV_EXT_TO_U_GET_INDEX(value), + UCNV_EXT_TO_U_GET_LENGTH(value), + target, targetLimit, + offsets, srcIndex, + pErrorCode); + } +} + +/* + * get the SI/SO toU state (state 0 is for SBCS, 1 for DBCS), + * or 1 for DBCS-only, + * or -1 if the converter is not SI/SO stateful + * + * Note: For SI/SO stateful converters getting here, + * cnv->mode==0 is equivalent to firstLength==1. + */ +#define UCNV_SISO_STATE(cnv) \ + ((cnv)->sharedData->mbcs.outputType==MBCS_OUTPUT_2_SISO ? (int8_t)(cnv)->mode : \ + (cnv)->sharedData->mbcs.outputType==MBCS_OUTPUT_DBCS_ONLY ? 1 : -1) + +/* + * targettoUBytes, firstLength, + *src, (int32_t)(srcLimit-*src), + &value, + cnv->useFallback, flush); + if(match>0) { + /* advance src pointer for the consumed input */ + *src+=match-firstLength; + + /* write result to target */ + ucnv_extWriteToU(cnv, cx, + value, + target, targetLimit, + offsets, srcIndex, + pErrorCode); + return TRUE; + } else if(match<0) { + /* save state for partial match */ + const char *s; + int32_t j; + + /* copy the first code point */ + s=(const char *)cnv->toUBytes; + cnv->preToUFirstLength=(int8_t)firstLength; + for(j=0; jpreToU[j]=*s++; + } + + /* now copy the newly consumed input */ + s=*src; + match=-match; + for(; jpreToU[j]=*s++; + } + *src=s; /* same as *src=srcLimit; because we reached the end of input */ + cnv->preToULength=(int8_t)match; + return TRUE; + } else /* match==0 no match */ { + return FALSE; + } +} + +U_CFUNC UChar32 +ucnv_extSimpleMatchToU(const int32_t *cx, + const char *source, int32_t length, + UBool useFallback) { + uint32_t value; + int32_t match; + + if(length<=0) { + return 0xffff; + } + + /* try to match */ + match=ucnv_extMatchToU(cx, -1, + source, length, + NULL, 0, + &value, + useFallback, TRUE); + if(match==length) { + /* write result for simple, single-character conversion */ + if(UCNV_EXT_TO_U_IS_CODE_POINT(value)) { + return UCNV_EXT_TO_U_GET_CODE_POINT(value); + } + } + + /* + * return no match because + * - match>0 && value points to string: simple conversion cannot handle multiple code points + * - match>0 && match!=length: not all input consumed, forbidden for this function + * - match==0: no match found in the first place + * - match<0: partial match, not supported for simple conversion (and flush==TRUE) + */ + return 0xfffe; +} + +/* + * continue partial match with new input + * never called for simple, single-character conversion + */ +U_CFUNC void +ucnv_extContinueMatchToU(UConverter *cnv, + UConverterToUnicodeArgs *pArgs, int32_t srcIndex, + UErrorCode *pErrorCode) { + uint32_t value; + int32_t match, length; + + match=ucnv_extMatchToU(cnv->sharedData->mbcs.extIndexes, (int8_t)UCNV_SISO_STATE(cnv), + cnv->preToU, cnv->preToULength, + pArgs->source, (int32_t)(pArgs->sourceLimit-pArgs->source), + &value, + cnv->useFallback, pArgs->flush); + if(match>0) { + if(match>=cnv->preToULength) { + /* advance src pointer for the consumed input */ + pArgs->source+=match-cnv->preToULength; + cnv->preToULength=0; + } else { + /* the match did not use all of preToU[] - keep the rest for replay */ + length=cnv->preToULength-match; + uprv_memmove(cnv->preToU, cnv->preToU+match, length); + cnv->preToULength=(int8_t)-length; + } + + /* write result */ + ucnv_extWriteToU(cnv, cnv->sharedData->mbcs.extIndexes, + value, + &pArgs->target, pArgs->targetLimit, + &pArgs->offsets, srcIndex, + pErrorCode); + } else if(match<0) { + /* save state for partial match */ + const char *s; + int32_t j; + + /* just _append_ the newly consumed input to preToU[] */ + s=pArgs->source; + match=-match; + for(j=cnv->preToULength; jpreToU[j]=*s++; + } + pArgs->source=s; /* same as *src=srcLimit; because we reached the end of input */ + cnv->preToULength=(int8_t)match; + } else /* match==0 */ { + /* + * no match + * + * We need to split the previous input into two parts: + * + * 1. The first codepage character is unmappable - that's how we got into + * trying the extension data in the first place. + * We need to move it from the preToU buffer + * to the error buffer, set an error code, + * and prepare the rest of the previous input for 2. + * + * 2. The rest of the previous input must be converted once we + * come back from the callback for the first character. + * At that time, we have to try again from scratch to convert + * these input characters. + * The replay will be handled by the ucnv.c conversion code. + */ + + /* move the first codepage character to the error field */ + uprv_memcpy(cnv->toUBytes, cnv->preToU, cnv->preToUFirstLength); + cnv->toULength=cnv->preToUFirstLength; + + /* move the rest up inside the buffer */ + length=cnv->preToULength-cnv->preToUFirstLength; + if(length>0) { + uprv_memmove(cnv->preToU, cnv->preToU+cnv->preToUFirstLength, length); + } + + /* mark preToU for replay */ + cnv->preToULength=(int8_t)-length; + + /* set the error code for unassigned */ + *pErrorCode=U_INVALID_CHAR_FOUND; + } +} + +/* from Unicode ------------------------------------------------------------- */ + +/* + * @return index of the UChar, if found; else <0 + */ +static U_INLINE int32_t +ucnv_extFindFromU(const UChar *fromUSection, int32_t length, UChar u) { + int32_t i, start, limit; + + /* binary search */ + start=0; + limit=length; + for(;;) { + i=limit-start; + if(i<=1) { + break; /* done */ + } + /* start=0 + * @param src UChars that can be used to complete a match + * @param srcLength length of src, >=0 + * @param pMatchValue [out] output result value for the match from the data structure + * @param useFallback "use fallback" flag, usually from cnv->useFallback + * @param flush TRUE if the end of the input stream is reached + * @return >1: matched, return value=total match length (number of input units matched) + * 1: matched, no mapping but request for + * (only for the first code point) + * 0: no match + * <0: partial match, return value=negative total match length + * (partial matches are never returned for flush==TRUE) + * (partial matches are never returned as being longer than UCNV_EXT_MAX_UCHARS) + * the matchLength is 2 if only firstCP matched, and >2 if firstCP and + * further code units matched + */ +static int32_t +ucnv_extMatchFromU(const int32_t *cx, + UChar32 firstCP, + const UChar *pre, int32_t preLength, + const UChar *src, int32_t srcLength, + uint32_t *pMatchValue, + UBool useFallback, UBool flush) { + const uint16_t *stage12, *stage3; + const uint32_t *stage3b; + + const UChar *fromUTableUChars, *fromUSectionUChars; + const uint32_t *fromUTableValues, *fromUSectionValues; + + uint32_t value, matchValue; + int32_t i, j, index, length, matchLength; + UChar c; + + if(cx==NULL) { + return 0; /* no extension data, no match */ + } + + /* trie lookup of firstCP */ + index=firstCP>>10; /* stage 1 index */ + if(index>=cx[UCNV_EXT_FROM_U_STAGE_1_LENGTH]) { + return 0; /* the first code point is outside the trie */ + } + + stage12=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_12_INDEX, uint16_t); + stage3=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_3_INDEX, uint16_t); + index=UCNV_EXT_FROM_U(stage12, stage3, index, firstCP); + + stage3b=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_3B_INDEX, uint32_t); + value=stage3b[index]; + if(value==0) { + return 0; + } + + /* + * Tests for (value&UCNV_EXT_FROM_U_RESERVED_MASK)==0: + * Do not interpret values with reserved bits used, for forward compatibility, + * and do not even remember intermediate results with reserved bits used. + */ + + if(UCNV_EXT_TO_U_IS_PARTIAL(value)) { + /* partial match, enter the loop below */ + index=(int32_t)UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value); + + /* initialize */ + fromUTableUChars=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_UCHARS_INDEX, UChar); + fromUTableValues=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_VALUES_INDEX, uint32_t); + + matchValue=0; + i=j=matchLength=0; + + /* we must not remember fallback matches when not using fallbacks */ + + /* match input units until there is a full match or the input is consumed */ + for(;;) { + /* go to the next section */ + fromUSectionUChars=fromUTableUChars+index; + fromUSectionValues=fromUTableValues+index; + + /* read first pair of the section */ + length=*fromUSectionUChars++; + value=*fromUSectionValues++; + if( value!=0 && + (UCNV_EXT_FROM_U_IS_ROUNDTRIP(value) || + FROM_U_USE_FALLBACK(useFallback, firstCP)) && + (value&UCNV_EXT_FROM_U_RESERVED_MASK)==0 + ) { + /* remember longest match so far */ + matchValue=value; + matchLength=2+i+j; + } + + /* match pre[] then src[] */ + if(iUCNV_EXT_MAX_UCHARS) { + /* + * end of the entire input stream, stop with the longest match so far + * or: partial match must not be longer than UCNV_EXT_MAX_UCHARS + * because it must fit into state buffers + */ + break; + } else { + /* continue with more input next time */ + return -(2+length); + } + } + + /* search for the current UChar */ + index=ucnv_extFindFromU(fromUSectionUChars, length, c); + if(index<0) { + /* no match here, stop with the longest match so far */ + break; + } else { + value=fromUSectionValues[index]; + if(UCNV_EXT_FROM_U_IS_PARTIAL(value)) { + /* partial match, continue */ + index=(int32_t)UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value); + } else { + if( (UCNV_EXT_FROM_U_IS_ROUNDTRIP(value) || + FROM_U_USE_FALLBACK(useFallback, firstCP)) && + (value&UCNV_EXT_FROM_U_RESERVED_MASK)==0 + ) { + /* full match, stop with result */ + matchValue=value; + matchLength=2+i+j; + } else { + /* full match on fallback not taken, stop with the longest match so far */ + } + break; + } + } + } + + if(matchLength==0) { + /* no match at all */ + return 0; + } + } else /* result from firstCP trie lookup */ { + if( (UCNV_EXT_FROM_U_IS_ROUNDTRIP(value) || + FROM_U_USE_FALLBACK(useFallback, firstCP)) && + (value&UCNV_EXT_FROM_U_RESERVED_MASK)==0 + ) { + /* full match, stop with result */ + matchValue=value; + matchLength=2; + } else { + /* fallback not taken */ + return 0; + } + } + + /* return result */ + if(matchValue==UCNV_EXT_FROM_U_SUBCHAR1) { + return 1; /* assert matchLength==2 */ + } + + *pMatchValue=matchValue; + return matchLength; +} + +/* + * @param value fromUnicode mapping table value; ignores roundtrip and reserved bits + */ +static U_INLINE void +ucnv_extWriteFromU(UConverter *cnv, const int32_t *cx, + uint32_t value, + char **target, const char *targetLimit, + int32_t **offsets, int32_t srcIndex, + UErrorCode *pErrorCode) { + uint8_t buffer[1+UCNV_EXT_MAX_BYTES]; + const uint8_t *result; + int32_t length, prevLength; + + length=UCNV_EXT_FROM_U_GET_LENGTH(value); + value=(uint32_t)UCNV_EXT_FROM_U_GET_DATA(value); + + /* output the result */ + if(length<=UCNV_EXT_FROM_U_MAX_DIRECT_LENGTH) { + /* + * Generate a byte array and then write it below. + * This is not the fastest possible way, but it should be ok for + * extension mappings, and it is much simpler. + * Offset and overflow handling are only done once this way. + */ + uint8_t *p=buffer+1; /* reserve buffer[0] for shiftByte below */ + switch(length) { + case 3: + *p++=(uint8_t)(value>>16); + case 2: + *p++=(uint8_t)(value>>8); + case 1: + *p++=(uint8_t)value; + default: + break; /* will never occur */ + } + result=buffer+1; + } else { + result=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_BYTES_INDEX, uint8_t)+value; + } + + /* with correct data we have length>0 */ + + if((prevLength=cnv->fromUnicodeStatus)!=0) { + /* handle SI/SO stateful output */ + uint8_t shiftByte; + + if(prevLength>1 && length==1) { + /* change from double-byte mode to single-byte */ + shiftByte=(uint8_t)UCNV_SI; + cnv->fromUnicodeStatus=1; + } else if(prevLength==1 && length>1) { + /* change from single-byte mode to double-byte */ + shiftByte=(uint8_t)UCNV_SO; + cnv->fromUnicodeStatus=2; + } else { + shiftByte=0; + } + + if(shiftByte!=0) { + /* prepend the shift byte to the result bytes */ + buffer[0]=shiftByte; + if(result!=buffer+1) { + uprv_memcpy(buffer+1, result, length); + } + result=buffer; + ++length; + } + } + + ucnv_fromUWriteBytes(cnv, (const char *)result, length, + target, targetLimit, + offsets, srcIndex, + pErrorCode); +} + +/* + * targetuseFallback, flush); + + /* reject a match if the result is a single byte for DBCS-only */ + if( match>=2 && + !(UCNV_EXT_FROM_U_GET_LENGTH(value)==1 && + cnv->sharedData->mbcs.outputType==MBCS_OUTPUT_DBCS_ONLY) + ) { + /* advance src pointer for the consumed input */ + *src+=match-2; /* remove 2 for the initial code point */ + + /* write result to target */ + ucnv_extWriteFromU(cnv, cx, + value, + target, targetLimit, + offsets, srcIndex, + pErrorCode); + return TRUE; + } else if(match<0) { + /* save state for partial match */ + const UChar *s; + int32_t j; + + /* copy the first code point */ + cnv->preFromUFirstCP=cp; + + /* now copy the newly consumed input */ + s=*src; + match=-match-2; /* remove 2 for the initial code point */ + for(j=0; jpreFromU[j]=*s++; + } + *src=s; /* same as *src=srcLimit; because we reached the end of input */ + cnv->preFromULength=(int8_t)match; + return TRUE; + } else if(match==1) { + /* matched, no mapping but request for */ + cnv->useSubChar1=TRUE; + return FALSE; + } else /* match==0 no match */ { + return FALSE; + } +} + +/* + * Used by ISO 2022 implementation. + * @return number of bytes in *pValue; negative number if fallback; 0 for no mapping + */ +U_CFUNC int32_t +ucnv_extSimpleMatchFromU(const int32_t *cx, + UChar32 cp, uint32_t *pValue, + UBool useFallback) { + uint32_t value; + int32_t match; + + /* try to match */ + match=ucnv_extMatchFromU(cx, + cp, + NULL, 0, + NULL, 0, + &value, + useFallback, TRUE); + if(match>=2) { + /* write result for simple, single-character conversion */ + int32_t length; + int isRoundtrip; + + isRoundtrip=UCNV_EXT_FROM_U_IS_ROUNDTRIP(value); + length=UCNV_EXT_FROM_U_GET_LENGTH(value); + value=(uint32_t)UCNV_EXT_FROM_U_GET_DATA(value); + + if(length<=UCNV_EXT_FROM_U_MAX_DIRECT_LENGTH) { + *pValue=value; + return isRoundtrip ? length : -length; +#if 0 /* not currently used */ + } else if(length==4) { + /* de-serialize a 4-byte result */ + const uint8_t *result=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_BYTES_INDEX, uint8_t)+value; + *pValue= + ((uint32_t)result[0]<<24)| + ((uint32_t)result[1]<<16)| + ((uint32_t)result[2]<<8)| + result[3]; + return isRoundtrip ? 4 : -4; +#endif + } + } + + /* + * return no match because + * - match>1 && resultLength>4: result too long for simple conversion + * - match==1: no match found, preferred + * - match==0: no match found in the first place + * - match<0: partial match, not supported for simple conversion (and flush==TRUE) + */ + return 0; +} + +/* + * continue partial match with new input, requires cnv->preFromUFirstCP>=0 + * never called for simple, single-character conversion + */ +U_CFUNC void +ucnv_extContinueMatchFromU(UConverter *cnv, + UConverterFromUnicodeArgs *pArgs, int32_t srcIndex, + UErrorCode *pErrorCode) { + uint32_t value; + int32_t match; + + match=ucnv_extMatchFromU(cnv->sharedData->mbcs.extIndexes, + cnv->preFromUFirstCP, + cnv->preFromU, cnv->preFromULength, + pArgs->source, (int32_t)(pArgs->sourceLimit-pArgs->source), + &value, + cnv->useFallback, pArgs->flush); + if(match>=2) { + match-=2; /* remove 2 for the initial code point */ + + if(match>=cnv->preFromULength) { + /* advance src pointer for the consumed input */ + pArgs->source+=match-cnv->preFromULength; + cnv->preFromULength=0; + } else { + /* the match did not use all of preFromU[] - keep the rest for replay */ + int32_t length=cnv->preFromULength-match; + uprv_memmove(cnv->preFromU, cnv->preFromU+match, length*U_SIZEOF_UCHAR); + cnv->preFromULength=(int8_t)-length; + } + + /* finish the partial match */ + cnv->preFromUFirstCP=U_SENTINEL; + + /* write result */ + ucnv_extWriteFromU(cnv, cnv->sharedData->mbcs.extIndexes, + value, + &pArgs->target, pArgs->targetLimit, + &pArgs->offsets, srcIndex, + pErrorCode); + } else if(match<0) { + /* save state for partial match */ + const UChar *s; + int32_t j; + + /* just _append_ the newly consumed input to preFromU[] */ + s=pArgs->source; + match=-match-2; /* remove 2 for the initial code point */ + for(j=cnv->preFromULength; jpreFromU[j]=*s++; + } + pArgs->source=s; /* same as *src=srcLimit; because we reached the end of input */ + cnv->preFromULength=(int8_t)match; + } else /* match==0 or 1 */ { + /* + * no match + * + * We need to split the previous input into two parts: + * + * 1. The first code point is unmappable - that's how we got into + * trying the extension data in the first place. + * We need to move it from the preFromU buffer + * to the error buffer, set an error code, + * and prepare the rest of the previous input for 2. + * + * 2. The rest of the previous input must be converted once we + * come back from the callback for the first code point. + * At that time, we have to try again from scratch to convert + * these input characters. + * The replay will be handled by the ucnv.c conversion code. + */ + + if(match==1) { + /* matched, no mapping but request for */ + cnv->useSubChar1=TRUE; + } + + /* move the first code point to the error field */ + cnv->fromUChar32=cnv->preFromUFirstCP; + cnv->preFromUFirstCP=U_SENTINEL; + + /* mark preFromU for replay */ + cnv->preFromULength=-cnv->preFromULength; + + /* set the error code for unassigned */ + *pErrorCode=U_INVALID_CHAR_FOUND; + } +} + +static void +ucnv_extGetUnicodeSetString(const UConverterSharedData *sharedData, + const int32_t *cx, + const USetAdder *sa, + UConverterUnicodeSet which, + int32_t minLength, + UChar32 c, + UChar s[UCNV_EXT_MAX_UCHARS], int32_t length, + int32_t sectionIndex, + UErrorCode *pErrorCode) { + const UChar *fromUSectionUChars; + const uint32_t *fromUSectionValues; + + uint32_t value; + int32_t i, count; + + fromUSectionUChars=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_UCHARS_INDEX, UChar)+sectionIndex; + fromUSectionValues=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_VALUES_INDEX, uint32_t)+sectionIndex; + + /* read first pair of the section */ + count=*fromUSectionUChars++; + value=*fromUSectionValues++; + + if( value!=0 && + UCNV_EXT_FROM_U_IS_ROUNDTRIP(value) && + UCNV_EXT_FROM_U_GET_LENGTH(value)>=minLength + ) { + if(c>=0) { + /* add the initial code point */ + sa->add(sa->set, c); + } else { + /* add the string so far */ + sa->addString(sa->set, s, length); + } + } + + for(i=0; i=minLength + ) { + sa->addString(sa->set, s, length+1); + } + } +} + +U_CFUNC void +ucnv_extGetUnicodeSet(const UConverterSharedData *sharedData, + const USetAdder *sa, + UConverterUnicodeSet which, + UErrorCode *pErrorCode) { + const int32_t *cx; + const uint16_t *stage12, *stage3, *ps2, *ps3; + const uint32_t *stage3b; + + uint32_t value; + int32_t st1, stage1Length, st2, st3, minLength; + + UChar s[UCNV_EXT_MAX_UCHARS]; + UChar32 c; + int32_t length; + + cx=sharedData->mbcs.extIndexes; + if(cx==NULL) { + return; + } + + stage12=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_12_INDEX, uint16_t); + stage3=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_3_INDEX, uint16_t); + stage3b=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_3B_INDEX, uint32_t); + + stage1Length=cx[UCNV_EXT_FROM_U_STAGE_1_LENGTH]; + + /* enumerate the from-Unicode trie table */ + c=0; /* keep track of the current code point while enumerating */ + + if(sharedData->mbcs.outputType==MBCS_OUTPUT_DBCS_ONLY) { + /* DBCS-only, ignore single-byte results */ + minLength=2; + } else { + minLength=1; + } + + /* + * the trie enumeration is almost the same as + * in MBCSGetUnicodeSet() for MBCS_OUTPUT_1 + */ + for(st1=0; st1stage1Length) { + ps2=stage12+st2; + for(st2=0; st2<64; ++st2) { + if((st3=(int32_t)ps2[st2]< entries or other (future?) pseudo-entries + * with an output length of 0, or entries with reserved bits set. + * Recurse for partial results. + */ + do { + value=stage3b[*ps3++]; + if(value==0) { + /* no mapping, do nothing */ + } else if(UCNV_EXT_FROM_U_IS_PARTIAL(value)) { + length=0; + U16_APPEND_UNSAFE(s, length, c); + ucnv_extGetUnicodeSetString( + sharedData, cx, sa, which, minLength, + c, s, length, + (int32_t)UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value), + pErrorCode); + } else if(((value&(UCNV_EXT_FROM_U_ROUNDTRIP_FLAG|UCNV_EXT_FROM_U_RESERVED_MASK))== + UCNV_EXT_FROM_U_ROUNDTRIP_FLAG) && + UCNV_EXT_FROM_U_GET_LENGTH(value)>=minLength + ) { + sa->add(sa->set, c); + } + } while((++c&0xf)!=0); + } else { + c+=16; /* empty stage 3 block */ + } + } + } else { + c+=1024; /* empty stage 2 block */ + } + } +} + +#endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */ diff --git a/Build/source/libs/icu/icu-xetex/common/ucnv_ext.h b/Build/source/libs/icu/icu-xetex/common/ucnv_ext.h new file mode 100644 index 00000000000..c90fcf6ab8a --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnv_ext.h @@ -0,0 +1,463 @@ +/* +****************************************************************************** +* +* Copyright (C) 2003-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: ucnv_ext.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2003jun13 +* created by: Markus W. Scherer +* +* Conversion extensions +*/ + +#ifndef __UCNV_EXT_H__ +#define __UCNV_EXT_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "unicode/ucnv.h" +#include "ucnv_cnv.h" + +/* + * See icuhtml/design/conversion/conversion_extensions.html + * + * Conversion extensions serve two purposes: + * 1. They support m:n mappings. + * 2. They support extension-only conversion files that are used together + * with the regular conversion data in base files. + * + * A base file may contain an extension table (explicitly requested or + * implicitly generated for m:n mappings), but its extension table is not + * used when an extension-only file is used. + * + * It is an error if a base file contains any regular (not extension) mapping + * from the same sequence as a mapping in the extension file + * because the base mapping would hide the extension mapping. + * + * + * Data for conversion extensions: + * + * One set of data structures per conversion direction (to/from Unicode). + * The data structures are sorted by input units to allow for binary search. + * Input sequences of more than one unit are handled like contraction tables + * in collation: + * The lookup value of a unit points to another table that is to be searched + * for the next unit, recursively. + * + * For conversion from Unicode, the initial code point is looked up in + * a 3-stage trie for speed, + * with an additional table of unique results to save space. + * + * Long output strings are stored in separate arrays, with length and index + * in the lookup tables. + * Output results also include a flag distinguishing roundtrip from + * (reverse) fallback mappings. + * + * Input Unicode strings must not begin or end with unpaired surrogates + * to avoid problems with matches on parts of surrogate pairs. + * + * Mappings from multiple characters (code points or codepage state + * table sequences) must be searched preferring the longest match. + * For this to work and be efficient, the variable-width table must contain + * all mappings that contain prefixes of the multiple characters. + * If an extension table is built on top of a base table in another file + * and a base table entry is a prefix of a multi-character mapping, then + * this is an error. + * + * + * Implementation note: + * + * Currently, the parser and several checks in the code limit the number + * of UChars or bytes in a mapping to + * UCNV_EXT_MAX_UCHARS and UCNV_EXT_MAX_BYTES, respectively, + * which are output value limits in the data structure. + * + * For input, this is not strictly necessary - it is a hard limit only for the + * buffers in UConverter that are used to store partial matches. + * + * Input sequences could otherwise be arbitrarily long if partial matches + * need not be stored (i.e., if a sequence does not span several buffers with too + * many units before the last buffer), although then results would differ + * depending on whether partial matches exceed the limits or not, + * which depends on the pattern of buffer sizes. + * + * + * Data structure: + * + * int32_t indexes[>=32]; + * + * Array of indexes and lengths etc. The length of the array is at least 32. + * The actual length is stored in indexes[0] to be forward compatible. + * + * Each index to another array is the number of bytes from indexes[]. + * Each length of an array is the number of array base units in that array. + * + * Some of the structures may not be present, in which case their indexes + * and lengths are 0. + * + * Usage of indexes[i]: + * [0] length of indexes[] + * + * // to Unicode table + * [1] index of toUTable[] (array of uint32_t) + * [2] length of toUTable[] + * [3] index of toUUChars[] (array of UChar) + * [4] length of toUUChars[] + * + * // from Unicode table, not for the initial code point + * [5] index of fromUTableUChars[] (array of UChar) + * [6] index of fromUTableValues[] (array of uint32_t) + * [7] length of fromUTableUChars[] and fromUTableValues[] + * [8] index of fromUBytes[] (array of char) + * [9] length of fromUBytes[] + * + * // from Unicode trie for initial-code point lookup + * [10] index of fromUStage12[] (combined array of uint16_t for stages 1 & 2) + * [11] length of stage 1 portion of fromUStage12[] + * [12] length of fromUStage12[] + * [13] index of fromUStage3[] (array of uint16_t indexes into fromUStage3b[]) + * [14] length of fromUStage3[] + * [15] index of fromUStage3b[] (array of uint32_t like fromUTableValues[]) + * [16] length of fromUStage3b[] + * + * [17] Bit field containing numbers of bytes: + * 31..24 reserved, 0 + * 23..16 maximum input bytes + * 15.. 8 maximum output bytes + * 7.. 0 maximum bytes per UChar + * + * [18] Bit field containing numbers of UChars: + * 31..24 reserved, 0 + * 23..16 maximum input UChars + * 15.. 8 maximum output UChars + * 7.. 0 maximum UChars per byte + * + * [19] Bit field containing flags: + * (extension table unicodeMask) + * 1 UCNV_HAS_SURROGATES flag for the extension table + * 0 UCNV_HAS_SUPPLEMENTARY flag for the extension table + * + * [20]..[30] reserved, 0 + * [31] number of bytes for the entire extension structure + * [>31] reserved; there are indexes[0] indexes + * + * + * uint32_t toUTable[]; + * + * Array of byte/value pairs for lookups for toUnicode conversion. + * The array is partitioned into sections like collation contraction tables. + * Each section contains one word with the number of following words and + * a default value for when the lookup in this section yields no match. + * + * A section is sorted in ascending order of input bytes, + * allowing for fast linear or binary searches. + * The builder may store entries for a contiguous range of byte values + * (compare difference between the first and last one with count), + * which then allows for direct array access. + * The builder should always do this for the initial table section. + * + * Entries may have 0 values, see below. + * No two entries in a section have the same byte values. + * + * Each uint32_t contains an input byte value in bits 31..24 and the + * corresponding lookup value in bits 23..0. + * Interpret the value as follows: + * if(value==0) { + * no match, see below + * } else if(value<0x1f0000) { + * partial match - use value as index to the next toUTable section + * and match the next unit; (value indexes toUTable[value]) + * } else { + * if(bit 23 set) { + * roundtrip; + * } else { + * fallback; + * } + * unset value bit 23; + * if(value<=0x2fffff) { + * (value-0x1f0000) is a code point; (BMP: value<=0x1fffff) + * } else { + * bits 17..0 (value&0x3ffff) is an index to + * the result UChars in toUUChars[]; (0 indexes toUUChars[0]) + * length of the result=((value>>18)-12); (length=0..19) + * } + * } + * + * The first word in a section contains the number of following words in the + * input byte position (bits 31..24, number=1..0xff). + * The value of the initial word is used when the current byte is not found + * in this section. + * If the value is not 0, then it represents a result as above. + * If the value is 0, then the search has to return a shorter match with an + * earlier default value as the result, or result in "unmappable" even for the + * initial bytes. + * If the value is 0 for the initial toUTable entry, then the initial byte + * does not start any mapping input. + * + * + * UChar toUUChars[]; + * + * Contains toUnicode mapping results, stored as sequences of UChars. + * Indexes and lengths stored in the toUTable[]. + * + * + * UChar fromUTableUChars[]; + * uint32_t fromUTableValues[]; + * + * The fromUTable is split into two arrays, but works otherwise much like + * the toUTable. The array is partitioned into sections like collation + * contraction tables and toUTable. + * A row in the table consists of same-index entries in fromUTableUChars[] + * and fromUTableValues[]. + * + * Interpret a value as follows: + * if(value==0) { + * no match, see below + * } else if(value<=0xffffff) { (bits 31..24 are 0) + * partial match - use value as index to the next fromUTable section + * and match the next unit; (value indexes fromUTable[value]) + * } else { + * if(value==0x80000001) { + * return no mapping, but request for ; + * } + * if(bit 31 set) { + * roundtrip; + * } else { + * fallback; + * } + * // bits 30..29 reserved, 0 + * length=(value>>24)&0x1f; (bits 28..24) + * if(length==1..3) { + * bits 23..0 contain 1..3 bytes, padded with 00s on the left; + * } else { + * bits 23..0 (value&0xffffff) is an index to + * the result bytes in fromUBytes[]; (0 indexes fromUBytes[0]) + * } + * } + * + * The first pair in a section contains the number of following pairs in the + * UChar position (16 bits, number=1..0xffff). + * The value of the initial pair is used when the current UChar is not found + * in this section. + * If the value is not 0, then it represents a result as above. + * If the value is 0, then the search has to return a shorter match with an + * earlier default value as the result, or result in "unmappable" even for the + * initial UChars. + * + * If the from Unicode trie is present, then the from Unicode search tables + * are not used for initial code points. + * In this case, the first entries (index 0) in the tables are not used + * (reserved, set to 0) because a value of 0 is used in trie results + * to indicate no mapping. + * + * + * uint16_t fromUStage12[]; + * + * Stages 1 & 2 of a trie that maps an initial code point. + * Indexes in stage 1 are all offset by the length of stage 1 so that the + * same array pointer can be used for both stages. + * If (c>>10)>=(length of stage 1) then c does not start any mapping. + * Same bit distribution as for regular conversion tries. + * + * + * uint16_t fromUStage3[]; + * uint32_t fromUStage3b[]; + * + * Stage 3 of the trie. The first array simply contains indexes to the second, + * which contains words in the same format as fromUTableValues[]. + * Use a stage 3 granularity of 4, which allows for 256k stage 3 entries, + * and 16-bit entries in stage 3 allow for 64k stage 3b entries. + * The stage 3 granularity means that the stage 2 entry needs to be left-shifted. + * + * Two arrays are used because it is expected that more than half of the stage 3 + * entries will be zero. The 16-bit index stage 3 array saves space even + * considering storing a total of 6 bytes per non-zero entry in both arrays + * together. + * Using a stage 3 granularity of >1 diminishes the compactability in that stage + * but provides a larger effective addressing space in stage 2. + * All but the final result stage use 16-bit entries to save space. + * + * fromUStage3b[] contains a zero for "no mapping" at its index 0, + * and may contain UCNV_EXT_FROM_U_SUBCHAR1 at index 1 for " SUB mapping" + * (i.e., "no mapping" with preference for rather than ), + * and all other items are unique non-zero results. + * + * The default value of a fromUTableValues[] section that is referenced + * _directly_ from a fromUStage3b[] item may also be UCNV_EXT_FROM_U_SUBCHAR1, + * but this value must not occur anywhere else in fromUTableValues[] + * because "no mapping" is always a property of a single code point, + * never of multiple. + * + * + * char fromUBytes[]; + * + * Contains fromUnicode mapping results, stored as sequences of chars. + * Indexes and lengths stored in the fromUTableValues[]. + */ +enum { + UCNV_EXT_INDEXES_LENGTH, /* 0 */ + + UCNV_EXT_TO_U_INDEX, /* 1 */ + UCNV_EXT_TO_U_LENGTH, + UCNV_EXT_TO_U_UCHARS_INDEX, + UCNV_EXT_TO_U_UCHARS_LENGTH, + + UCNV_EXT_FROM_U_UCHARS_INDEX, /* 5 */ + UCNV_EXT_FROM_U_VALUES_INDEX, + UCNV_EXT_FROM_U_LENGTH, + UCNV_EXT_FROM_U_BYTES_INDEX, + UCNV_EXT_FROM_U_BYTES_LENGTH, + + UCNV_EXT_FROM_U_STAGE_12_INDEX, /* 10 */ + UCNV_EXT_FROM_U_STAGE_1_LENGTH, + UCNV_EXT_FROM_U_STAGE_12_LENGTH, + UCNV_EXT_FROM_U_STAGE_3_INDEX, + UCNV_EXT_FROM_U_STAGE_3_LENGTH, + UCNV_EXT_FROM_U_STAGE_3B_INDEX, + UCNV_EXT_FROM_U_STAGE_3B_LENGTH, + + UCNV_EXT_COUNT_BYTES, /* 17 */ + UCNV_EXT_COUNT_UCHARS, + UCNV_EXT_FLAGS, + + UCNV_EXT_RESERVED_INDEX, /* 20, moves with additional indexes */ + + UCNV_EXT_SIZE=31, + UCNV_EXT_INDEXES_MIN_LENGTH=32 +}; + +/* get the pointer to an extension array from indexes[index] */ +#define UCNV_EXT_ARRAY(indexes, index, itemType) \ + ((const itemType *)((const char *)(indexes)+(indexes)[index])) + +#define UCNV_GET_MAX_BYTES_PER_UCHAR(indexes) \ + ((indexes)[UCNV_EXT_COUNT_BYTES]&0xff) + +/* internal API ------------------------------------------------------------- */ + +U_CFUNC UBool +ucnv_extInitialMatchToU(UConverter *cnv, const int32_t *cx, + int32_t firstLength, + const char **src, const char *srcLimit, + UChar **target, const UChar *targetLimit, + int32_t **offsets, int32_t srcIndex, + UBool flush, + UErrorCode *pErrorCode); + +U_CFUNC UChar32 +ucnv_extSimpleMatchToU(const int32_t *cx, + const char *source, int32_t length, + UBool useFallback); + +U_CFUNC void +ucnv_extContinueMatchToU(UConverter *cnv, + UConverterToUnicodeArgs *pArgs, int32_t srcIndex, + UErrorCode *pErrorCode); + + +U_CFUNC UBool +ucnv_extInitialMatchFromU(UConverter *cnv, const int32_t *cx, + UChar32 cp, + const UChar **src, const UChar *srcLimit, + char **target, const char *targetLimit, + int32_t **offsets, int32_t srcIndex, + UBool flush, + UErrorCode *pErrorCode); + +U_CFUNC int32_t +ucnv_extSimpleMatchFromU(const int32_t *cx, + UChar32 cp, uint32_t *pValue, + UBool useFallback); + +U_CFUNC void +ucnv_extContinueMatchFromU(UConverter *cnv, + UConverterFromUnicodeArgs *pArgs, int32_t srcIndex, + UErrorCode *pErrorCode); + +U_CFUNC void +ucnv_extGetUnicodeSet(const UConverterSharedData *sharedData, + const USetAdder *sa, + UConverterUnicodeSet which, + UErrorCode *pErrorCode); + +/* toUnicode helpers -------------------------------------------------------- */ + +#define UCNV_EXT_TO_U_BYTE_SHIFT 24 +#define UCNV_EXT_TO_U_VALUE_MASK 0xffffff +#define UCNV_EXT_TO_U_MIN_CODE_POINT 0x1f0000 +#define UCNV_EXT_TO_U_MAX_CODE_POINT 0x2fffff +#define UCNV_EXT_TO_U_ROUNDTRIP_FLAG ((uint32_t)1<<23) +#define UCNV_EXT_TO_U_INDEX_MASK 0x3ffff +#define UCNV_EXT_TO_U_LENGTH_SHIFT 18 +#define UCNV_EXT_TO_U_LENGTH_OFFSET 12 + +/* maximum number of indexed UChars */ +#define UCNV_EXT_MAX_UCHARS 19 + +#define UCNV_EXT_TO_U_MAKE_WORD(byte, value) (((uint32_t)(byte)<>UCNV_EXT_TO_U_BYTE_SHIFT) +#define UCNV_EXT_TO_U_GET_VALUE(word) ((word)&UCNV_EXT_TO_U_VALUE_MASK) + +#define UCNV_EXT_TO_U_IS_PARTIAL(value) ((value)>UCNV_EXT_TO_U_LENGTH_SHIFT)-UCNV_EXT_TO_U_LENGTH_OFFSET) + +/* fromUnicode helpers ------------------------------------------------------ */ + +/* most trie constants are shared with ucnvmbcs.h */ + +/* see similar utrie.h UTRIE_INDEX_SHIFT and UTRIE_DATA_GRANULARITY */ +#define UCNV_EXT_STAGE_2_LEFT_SHIFT 2 +#define UCNV_EXT_STAGE_3_GRANULARITY 4 + +/* trie access, returns the stage 3 value=index to stage 3b; s1Index=c>>10 */ +#define UCNV_EXT_FROM_U(stage12, stage3, s1Index, c) \ + (stage3)[ ((int32_t)(stage12)[ (stage12)[s1Index] +(((c)>>4)&0x3f) ]< (impossible roundtrip to 0 bytes, value 01) */ +#define UCNV_EXT_FROM_U_SUBCHAR1 0x80000001 + +/* at most 3 bytes in the lower part of the value */ +#define UCNV_EXT_FROM_U_MAX_DIRECT_LENGTH 3 + +/* maximum number of indexed bytes */ +#define UCNV_EXT_MAX_BYTES 0x1f + +#define UCNV_EXT_FROM_U_IS_PARTIAL(value) (((value)>>UCNV_EXT_FROM_U_LENGTH_SHIFT)==0) +#define UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value) (value) + +#define UCNV_EXT_FROM_U_IS_ROUNDTRIP(value) (((value)&UCNV_EXT_FROM_U_ROUNDTRIP_FLAG)!=0) +#define UCNV_EXT_FROM_U_MASK_ROUNDTRIP(value) ((value)&~UCNV_EXT_FROM_U_ROUNDTRIP_FLAG) + +/* get length; masks away all other bits */ +#define UCNV_EXT_FROM_U_GET_LENGTH(value) (int32_t)(((value)>>UCNV_EXT_FROM_U_LENGTH_SHIFT)&UCNV_EXT_MAX_BYTES) + +/* get bytes or bytes index */ +#define UCNV_EXT_FROM_U_GET_DATA(value) ((value)&UCNV_EXT_FROM_U_DATA_MASK) + +#endif + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/ucnv_imp.h b/Build/source/libs/icu/icu-xetex/common/ucnv_imp.h new file mode 100644 index 00000000000..c1f48c0e186 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnv_imp.h @@ -0,0 +1,95 @@ +/* +********************************************************************** +* Copyright (C) 1999-2006, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* +* +* ucnv_imp.h: +* Contains all internal and external data structure definitions +* Created & Maitained by Bertrand A. Damiba +* +* +* +* ATTENTION: +* --------- +* Although the data structures in this file are open and stack allocatable +* we reserve the right to hide them in further releases. +*/ + +#ifndef UCNV_IMP_H +#define UCNV_IMP_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "unicode/uloc.h" +#include "ucnv_bld.h" + +/* figures out if we need to go to file to read in the data tables. + * @param converterName The name of the converter + * @param err The error code + * @return the newly created converter + */ +UConverter *ucnv_createConverter (UConverter *myUConverter, const char *converterName, UErrorCode * err); + +/* + * Open a purely algorithmic converter, specified by a type constant. + * @param myUConverter NULL, or pre-allocated UConverter structure to avoid + * a memory allocation + * @param type requested converter type + * @param locale locale parameter, or "" + * @param options converter options bit set (default 0) + * @param err ICU error code, not tested for U_FAILURE on input + * because this is an internal function + * @internal + */ +U_CFUNC UConverter * +ucnv_createAlgorithmicConverter(UConverter *myUConverter, + UConverterType type, + const char *locale, uint32_t options, + UErrorCode *err); + +/* Creates a converter from shared data + */ +UConverter* +ucnv_createConverterFromSharedData(UConverter *myUConverter, UConverterSharedData *mySharedConverterData, const char *realName, const char *locale, uint32_t options, UErrorCode *err); + +UConverter* ucnv_createConverterFromPackage(const char *packageName, const char *converterName, + UErrorCode *err); + +typedef struct { + char cnvName[UCNV_MAX_CONVERTER_NAME_LENGTH], locale[ULOC_FULLNAME_CAPACITY]; + const char *realName; + uint32_t options; +} UConverterLookupData; + +/** + * Load a converter but do not create a UConverter object. + * Simply return the UConverterSharedData. + * Performs alias lookup etc. + * @internal + */ +UConverterSharedData * +ucnv_loadSharedData(const char *converterName, UConverterLookupData *lookup, UErrorCode * err); + +/** + * This may unload the shared data in a thread safe manner. + * This will only unload the data if no other converters are sharing it. + */ +void +ucnv_unloadSharedDataIfReady(UConverterSharedData *sharedData); + +/** + * This is a thread safe way to increment the reference count. + */ +void +ucnv_incrementRefCount(UConverterSharedData *sharedData); + +#define UCNV_TO_U_DEFAULT_CALLBACK ((UConverterToUCallback) UCNV_TO_U_CALLBACK_SUBSTITUTE) +#define UCNV_FROM_U_DEFAULT_CALLBACK ((UConverterFromUCallback) UCNV_FROM_U_CALLBACK_SUBSTITUTE) + +#endif + +#endif /* _UCNV_IMP */ diff --git a/Build/source/libs/icu/icu-xetex/common/ucnv_io.c b/Build/source/libs/icu/icu-xetex/common/ucnv_io.c new file mode 100644 index 00000000000..29876a6714f --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnv_io.c @@ -0,0 +1,1341 @@ +/* +****************************************************************************** +* +* Copyright (C) 1999-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* +* +* ucnv_io.c: +* initializes global variables and defines functions pertaining to converter +* name resolution aspect of the conversion code. +* +* new implementation: +* +* created on: 1999nov22 +* created by: Markus W. Scherer +* +* Use the binary cnvalias.icu (created from convrtrs.txt) to work +* with aliases for converter names. +* +* Date Name Description +* 11/22/1999 markus Created +* 06/28/2002 grhoten Major overhaul of the converter alias design. +* Now an alias can map to different converters +* depending on the specified standard. +******************************************************************************* +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "unicode/ucnv.h" +#include "unicode/udata.h" + +#include "umutex.h" +#include "uarrsort.h" +#include "udataswp.h" +#include "cstring.h" +#include "cmemory.h" +#include "ucnv_io.h" +#include "uenumimp.h" +#include "ucln_cmn.h" + +/* Format of cnvalias.icu ----------------------------------------------------- + * + * cnvalias.icu is a binary, memory-mappable form of convrtrs.txt. + * This binary form contains several tables. All indexes are to uint16_t + * units, and not to the bytes (uint8_t units). Addressing everything on + * 16-bit boundaries allows us to store more information with small index + * numbers, which are also 16-bit in size. The majority of the table (except + * the string table) are 16-bit numbers. + * + * First there is the size of the Table of Contents (TOC). The TOC + * entries contain the size of each section. In order to find the offset + * you just need to sum up the previous offsets. + * The TOC length and entries are an array of uint32_t values. + * The first section after the TOC starts immediately after the TOC. + * + * 1) This section contains a list of converters. This list contains indexes + * into the string table for the converter name. The index of this list is + * also used by other sections, which are mentioned later on. + * This list is not sorted. + * + * 2) This section contains a list of tags. This list contains indexes + * into the string table for the tag name. The index of this list is + * also used by other sections, which are mentioned later on. + * This list is in priority order of standards. + * + * 3) This section contains a list of sorted unique aliases. This + * list contains indexes into the string table for the alias name. The + * index of this list is also used by other sections, like the 4th section. + * The index for the 3rd and 4th section is used to get the + * alias -> converter name mapping. Section 3 and 4 form a two column table. + * Some of the most significant bits of each index may contain other + * information (see findConverter for details). + * + * 4) This section contains a list of mapped converter names. Consider this + * as a table that maps the 3rd section to the 1st section. This list contains + * indexes into the 1st section. The index of this list is the same index in + * the 3rd section. There is also some extra information in the high bits of + * each converter index in this table. Currently it's only used to say that + * an alias mapped to this converter is ambiguous. See UCNV_CONVERTER_INDEX_MASK + * and UCNV_AMBIGUOUS_ALIAS_MAP_BIT for more information. This section is + * the predigested form of the 5th section so that an alias lookup can be fast. + * + * 5) This section contains a 2D array with indexes to the 6th section. This + * section is the full form of all alias mappings. The column index is the + * index into the converter list (column header). The row index is the index + * to tag list (row header). This 2D array is the top part a 3D array. The + * third dimension is in the 6th section. + * + * 6) This is blob of variable length arrays. Each array starts with a size, + * and is followed by indexes to alias names in the string table. This is + * the third dimension to the section 5. No other section should be referencing + * this section. + * + * 7) Starting in ICU 3.6, this can be a UConverterAliasOptions struct. Its + * presence indicates that a section 9 exists. UConverterAliasOptions specifies + * what type of string normalization is used among other potential things in the + * future. + * + * 8) This is the string table. All strings are indexed on an even address. + * There are two reasons for this. First many chip architectures locate strings + * faster on even address boundaries. Second, since all indexes are 16-bit + * numbers, this string table can be 128KB in size instead of 64KB when we + * only have strings starting on an even address. + * + * 9) When present this is a set of prenormalized strings from section 8. This + * table contains normalized strings with the dashes and spaces stripped out, + * and all strings lowercased. In the future, the options in section 7 may state + * other types of normalization. + * + * Here is the concept of section 5 and 6. It's a 3D cube. Each tag + * has a unique alias among all converters. That same alias can + * be mentioned in other standards on different converters, + * but only one alias per tag can be unique. + * + * + * Converter Names (Usually in TR22 form) + * -------------------------------------------. + * T / /| + * a / / | + * g / / | + * s / / | + * / / | + * ------------------------------------------/ | + * A | | | + * l | | | + * i | | / + * a | | / + * s | | / + * e | | / + * s | |/ + * ------------------------------------------- + * + * + * + * Here is what it really looks like. It's like swiss cheese. + * There are holes. Some converters aren't recognized by + * a standard, or they are really old converters that the + * standard doesn't recognize anymore. + * + * Converter Names (Usually in TR22 form) + * -------------------------------------------. + * T /##########################################/| + * a / # # /# + * g / # ## ## ### # ### ### ### #/ + * s / # ##### #### ## ## #/# + * / ### # # ## # # # ### # # #/## + * ------------------------------------------/# # + * A |### # # ## # # # ### # # #|# # + * l |# # # # # ## # #|# # + * i |# # # # # # #|# + * a |# #|# + * s | #|# + * e + * s + * + */ + +/** + * Used by the UEnumeration API + */ +typedef struct UAliasContext { + uint32_t listOffset; + uint32_t listIdx; +} UAliasContext; + +static const char DATA_NAME[] = "cnvalias"; +static const char DATA_TYPE[] = "icu"; + +static UDataMemory *gAliasData=NULL; + +enum { + tocLengthIndex=0, + converterListIndex=1, + tagListIndex=2, + aliasListIndex=3, + untaggedConvArrayIndex=4, + taggedAliasArrayIndex=5, + taggedAliasListsIndex=6, + tableOptionsIndex=7, + stringTableIndex=8, + normalizedStringTableIndex=9, + offsetsCount, /* length of the swapper's temporary offsets[] */ + minTocLength=8 /* min. tocLength in the file, does not count the tocLengthIndex! */ +}; + +static const UConverterAliasOptions defaultTableOptions = { + UCNV_IO_UNNORMALIZED, + 0 /* containsCnvOptionInfo */ +}; +static UConverterAlias gMainTable; + +#define GET_STRING(idx) (const char *)(gMainTable.stringTable + (idx)) +#define GET_NORMALIZED_STRING(idx) (const char *)(gMainTable.normalizedStringTable + (idx)) + +static UBool U_CALLCONV +isAcceptable(void *context, + const char *type, const char *name, + const UDataInfo *pInfo) { + return (UBool)( + pInfo->size>=20 && + pInfo->isBigEndian==U_IS_BIG_ENDIAN && + pInfo->charsetFamily==U_CHARSET_FAMILY && + pInfo->dataFormat[0]==0x43 && /* dataFormat="CvAl" */ + pInfo->dataFormat[1]==0x76 && + pInfo->dataFormat[2]==0x41 && + pInfo->dataFormat[3]==0x6c && + pInfo->formatVersion[0]==3); +} + +static UBool U_CALLCONV ucnv_io_cleanup(void) +{ + if (gAliasData) { + udata_close(gAliasData); + gAliasData = NULL; + } + + uprv_memset(&gMainTable, 0, sizeof(gMainTable)); + + return TRUE; /* Everything was cleaned up */ +} + +static UBool +haveAliasData(UErrorCode *pErrorCode) { + int needInit; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return FALSE; + } + + UMTX_CHECK(NULL, (gAliasData==NULL), needInit); + + /* load converter alias data from file if necessary */ + if (needInit) { + UDataMemory *data; + const uint16_t *table; + const uint32_t *sectionSizes; + uint32_t tableStart; + uint32_t currOffset; + + data = udata_openChoice(NULL, DATA_TYPE, DATA_NAME, isAcceptable, NULL, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return FALSE; + } + + sectionSizes = (const uint32_t *)udata_getMemory(data); + table = (const uint16_t *)sectionSizes; + + tableStart = sectionSizes[0]; + if (tableStart < minTocLength) { + *pErrorCode = U_INVALID_FORMAT_ERROR; + udata_close(data); + return FALSE; + } + + umtx_lock(NULL); + if(gAliasData==NULL) { + gAliasData = data; + data=NULL; + + gMainTable.converterListSize = sectionSizes[1]; + gMainTable.tagListSize = sectionSizes[2]; + gMainTable.aliasListSize = sectionSizes[3]; + gMainTable.untaggedConvArraySize = sectionSizes[4]; + gMainTable.taggedAliasArraySize = sectionSizes[5]; + gMainTable.taggedAliasListsSize = sectionSizes[6]; + gMainTable.optionTableSize = sectionSizes[7]; + gMainTable.stringTableSize = sectionSizes[8]; + + if (tableStart > 8) { + gMainTable.normalizedStringTableSize = sectionSizes[9]; + } + + currOffset = tableStart * (sizeof(uint32_t)/sizeof(uint16_t)) + (sizeof(uint32_t)/sizeof(uint16_t)); + gMainTable.converterList = table + currOffset; + + currOffset += gMainTable.converterListSize; + gMainTable.tagList = table + currOffset; + + currOffset += gMainTable.tagListSize; + gMainTable.aliasList = table + currOffset; + + currOffset += gMainTable.aliasListSize; + gMainTable.untaggedConvArray = table + currOffset; + + currOffset += gMainTable.untaggedConvArraySize; + gMainTable.taggedAliasArray = table + currOffset; + + /* aliasLists is a 1's based array, but it has a padding character */ + currOffset += gMainTable.taggedAliasArraySize; + gMainTable.taggedAliasLists = table + currOffset; + + currOffset += gMainTable.taggedAliasListsSize; + if (gMainTable.optionTableSize > 0 + && ((const UConverterAliasOptions *)(table + currOffset))->stringNormalizationType < UCNV_IO_NORM_TYPE_COUNT) + { + /* Faster table */ + gMainTable.optionTable = (const UConverterAliasOptions *)(table + currOffset); + } + else { + /* Smaller table, or I can't handle this normalization mode! + Use the original slower table lookup. */ + gMainTable.optionTable = &defaultTableOptions; + } + + currOffset += gMainTable.optionTableSize; + gMainTable.stringTable = table + currOffset; + + currOffset += gMainTable.stringTableSize; + gMainTable.normalizedStringTable = ((gMainTable.optionTable->stringNormalizationType == UCNV_IO_UNNORMALIZED) + ? gMainTable.stringTable : (table + currOffset)); + + ucln_common_registerCleanup(UCLN_COMMON_UCNV_IO, ucnv_io_cleanup); + } + umtx_unlock(NULL); + + /* if a different thread set it first, then close the extra data */ + if(data!=NULL) { + udata_close(data); /* NULL if it was set correctly */ + } + } + + return TRUE; +} + +static U_INLINE UBool +isAlias(const char *alias, UErrorCode *pErrorCode) { + if(alias==NULL) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return FALSE; + } + return (UBool)(*alias!=0); +} + +static uint32_t getTagNumber(const char *tagname) { + if (gMainTable.tagList) { + uint32_t tagNum; + for (tagNum = 0; tagNum < gMainTable.tagListSize; tagNum++) { + if (!uprv_stricmp(GET_STRING(gMainTable.tagList[tagNum]), tagname)) { + return tagNum; + } + } + } + + return UINT32_MAX; +} + +/* character types relevant for ucnv_compareNames() */ +enum { + IGNORE, + ZERO, + NONZERO, + MINLETTER /* any values from here on are lowercase letter mappings */ +}; + +/* character types for ASCII 00..7F */ +static const uint8_t asciiTypes[128] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ZERO, NONZERO, NONZERO, NONZERO, NONZERO, NONZERO, NONZERO, NONZERO, NONZERO, NONZERO, 0, 0, 0, 0, 0, 0, + 0, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0, 0, 0, 0, 0, + 0, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0, 0, 0, 0, 0 +}; + +#define GET_ASCII_TYPE(c) ((int8_t)(c) >= 0 ? asciiTypes[(uint8_t)c] : (uint8_t)IGNORE) + +/* character types for EBCDIC 80..FF */ +static const uint8_t ebcdicTypes[128] = { + 0, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0, 0, 0, 0, 0, 0, + 0, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0, 0, 0, 0, 0, 0, + 0, 0, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0, 0, 0, 0, 0, 0, + 0, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0, 0, 0, 0, 0, 0, + 0, 0, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0, 0, 0, 0, 0, 0, + ZERO, NONZERO, NONZERO, NONZERO, NONZERO, NONZERO, NONZERO, NONZERO, NONZERO, NONZERO, 0, 0, 0, 0, 0, 0 +}; + +#define GET_EBCDIC_TYPE(c) ((int8_t)(c) < 0 ? ebcdicTypes[(c)&0x7f] : (uint8_t)IGNORE) + +#if U_CHARSET_FAMILY==U_ASCII_FAMILY +# define GET_CHAR_TYPE(c) GET_ASCII_TYPE(c) +#elif U_CHARSET_FAMILY==U_EBCDIC_FAMILY +# define GET_CHAR_TYPE(c) GET_EBCDIC_TYPE(c) +#else +# error U_CHARSET_FAMILY is not valid +#endif + +/* @see ucnv_compareNames */ +U_CFUNC char * U_EXPORT2 +ucnv_io_stripASCIIForCompare(char *dst, const char *name) { + char *dstItr = dst; + uint8_t type, nextType; + char c1; + UBool afterDigit = FALSE; + + while ((c1 = *name++) != 0) { + type = GET_ASCII_TYPE(c1); + switch (type) { + case IGNORE: + afterDigit = FALSE; + continue; /* ignore all but letters and digits */ + case ZERO: + if (!afterDigit) { + nextType = GET_ASCII_TYPE(*name); + if (nextType == ZERO || nextType == NONZERO) { + continue; /* ignore leading zero before another digit */ + } + } + break; + case NONZERO: + afterDigit = TRUE; + break; + default: + c1 = (char)type; /* lowercased letter */ + afterDigit = FALSE; + break; + } + *dstItr++ = c1; + } + *dstItr = 0; + return dst; +} + +U_CFUNC char * U_EXPORT2 +ucnv_io_stripEBCDICForCompare(char *dst, const char *name) { + char *dstItr = dst; + uint8_t type, nextType; + char c1; + UBool afterDigit = FALSE; + + while ((c1 = *name++) != 0) { + type = GET_EBCDIC_TYPE(c1); + switch (type) { + case IGNORE: + afterDigit = FALSE; + continue; /* ignore all but letters and digits */ + case ZERO: + if (!afterDigit) { + nextType = GET_EBCDIC_TYPE(*name); + if (nextType == ZERO || nextType == NONZERO) { + continue; /* ignore leading zero before another digit */ + } + } + break; + case NONZERO: + afterDigit = TRUE; + break; + default: + c1 = (char)type; /* lowercased letter */ + afterDigit = FALSE; + break; + } + *dstItr++ = c1; + } + *dstItr = 0; + return dst; +} + +/** + * Do a fuzzy compare of two converter/alias names. + * The comparison is case-insensitive, ignores leading zeroes if they are not + * followed by further digits, and ignores all but letters and digits. + * Thus the strings "UTF-8", "utf_8", "u*T@f08" and "Utf 8" are exactly equivalent. + * See section 1.4, Charset Alias Matching in Unicode Technical Standard #22 + * at http://www.unicode.org/reports/tr22/ + * + * This is a symmetrical (commutative) operation; order of arguments + * is insignificant. This is an important property for sorting the + * list (when the list is preprocessed into binary form) and for + * performing binary searches on it at run time. + * + * @param name1 a converter name or alias, zero-terminated + * @param name2 a converter name or alias, zero-terminated + * @return 0 if the names match, or a negative value if the name1 + * lexically precedes name2, or a positive value if the name1 + * lexically follows name2. + * + * @see ucnv_io_stripForCompare + */ +U_CAPI int U_EXPORT2 +ucnv_compareNames(const char *name1, const char *name2) { + int rc; + uint8_t type, nextType; + char c1, c2; + UBool afterDigit1 = FALSE, afterDigit2 = FALSE; + + for (;;) { + while ((c1 = *name1++) != 0) { + type = GET_CHAR_TYPE(c1); + switch (type) { + case IGNORE: + afterDigit1 = FALSE; + continue; /* ignore all but letters and digits */ + case ZERO: + if (!afterDigit1) { + nextType = GET_CHAR_TYPE(*name1); + if (nextType == ZERO || nextType == NONZERO) { + continue; /* ignore leading zero before another digit */ + } + } + break; + case NONZERO: + afterDigit1 = TRUE; + break; + default: + c1 = (char)type; /* lowercased letter */ + afterDigit1 = FALSE; + break; + } + break; /* deliver c1 */ + } + while ((c2 = *name2++) != 0) { + type = GET_CHAR_TYPE(c2); + switch (type) { + case IGNORE: + afterDigit2 = FALSE; + continue; /* ignore all but letters and digits */ + case ZERO: + if (!afterDigit2) { + nextType = GET_CHAR_TYPE(*name2); + if (nextType == ZERO || nextType == NONZERO) { + continue; /* ignore leading zero before another digit */ + } + } + break; + case NONZERO: + afterDigit2 = TRUE; + break; + default: + c2 = (char)type; /* lowercased letter */ + afterDigit2 = FALSE; + break; + } + break; /* deliver c2 */ + } + + /* If we reach the ends of both strings then they match */ + if ((c1|c2)==0) { + return 0; + } + + /* Case-insensitive comparison */ + rc = (int)(unsigned char)c1 - (int)(unsigned char)c2; + if (rc != 0) { + return rc; + } + } +} + +/* + * search for an alias + * return the converter number index for gConverterList + */ +static U_INLINE uint32_t +findConverter(const char *alias, UBool *containsOption, UErrorCode *pErrorCode) { + uint32_t mid, start, limit; + uint32_t lastMid; + int result; + int isUnnormalized = (gMainTable.optionTable->stringNormalizationType == UCNV_IO_UNNORMALIZED); + char strippedName[UCNV_MAX_CONVERTER_NAME_LENGTH]; + + if (!isUnnormalized) { + if (uprv_strlen(alias) >= UCNV_MAX_CONVERTER_NAME_LENGTH) { + *pErrorCode = U_BUFFER_OVERFLOW_ERROR; + return UINT32_MAX; + } + + /* Lower case and remove ignoreable characters. */ + ucnv_io_stripForCompare(strippedName, alias); + alias = strippedName; + } + + /* do a binary search for the alias */ + start = 0; + limit = gMainTable.untaggedConvArraySize; + mid = limit; + lastMid = UINT32_MAX; + + for (;;) { + mid = (uint32_t)((start + limit) / 2); + if (lastMid == mid) { /* Have we moved? */ + break; /* We haven't moved, and it wasn't found. */ + } + lastMid = mid; + if (isUnnormalized) { + result = ucnv_compareNames(alias, GET_STRING(gMainTable.aliasList[mid])); + } + else { + result = uprv_strcmp(alias, GET_NORMALIZED_STRING(gMainTable.aliasList[mid])); + } + + if (result < 0) { + limit = mid; + } else if (result > 0) { + start = mid; + } else { + /* Since the gencnval tool folds duplicates into one entry, + * this alias in gAliasList is unique, but different standards + * may map an alias to different converters. + */ + if (gMainTable.untaggedConvArray[mid] & UCNV_AMBIGUOUS_ALIAS_MAP_BIT) { + *pErrorCode = U_AMBIGUOUS_ALIAS_WARNING; + } + /* State whether the canonical converter name contains an option. + This information is contained in this list in order to maintain backward & forward compatibility. */ + if (containsOption) { + UBool containsCnvOptionInfo = (UBool)gMainTable.optionTable->containsCnvOptionInfo; + *containsOption = (UBool)((containsCnvOptionInfo + && ((gMainTable.untaggedConvArray[mid] & UCNV_CONTAINS_OPTION_BIT) != 0)) + || !containsCnvOptionInfo); + } + return gMainTable.untaggedConvArray[mid] & UCNV_CONVERTER_INDEX_MASK; + } + } + + return UINT32_MAX; +} + +/* + * Is this alias in this list? + * alias and listOffset should be non-NULL. + */ +static U_INLINE UBool +isAliasInList(const char *alias, uint32_t listOffset) { + if (listOffset) { + uint32_t currAlias; + uint32_t listCount = gMainTable.taggedAliasLists[listOffset]; + /* +1 to skip listCount */ + const uint16_t *currList = gMainTable.taggedAliasLists + listOffset + 1; + for (currAlias = 0; currAlias < listCount; currAlias++) { + if (currList[currAlias] + && ucnv_compareNames(alias, GET_STRING(currList[currAlias]))==0) + { + return TRUE; + } + } + } + return FALSE; +} + +/* + * Search for an standard name of an alias (what is the default name + * that this standard uses?) + * return the listOffset for gTaggedAliasLists. If it's 0, + * the it couldn't be found, but the parameters are valid. + */ +static uint32_t +findTaggedAliasListsOffset(const char *alias, const char *standard, UErrorCode *pErrorCode) { + uint32_t idx; + uint32_t listOffset; + uint32_t convNum; + UErrorCode myErr = U_ZERO_ERROR; + uint32_t tagNum = getTagNumber(standard); + + /* Make a quick guess. Hopefully they used a TR22 canonical alias. */ + convNum = findConverter(alias, NULL, &myErr); + if (myErr != U_ZERO_ERROR) { + *pErrorCode = myErr; + } + + if (tagNum < (gMainTable.tagListSize - UCNV_NUM_HIDDEN_TAGS) && convNum < gMainTable.converterListSize) { + listOffset = gMainTable.taggedAliasArray[tagNum*gMainTable.converterListSize + convNum]; + if (listOffset && gMainTable.taggedAliasLists[listOffset + 1]) { + return listOffset; + } + if (myErr == U_AMBIGUOUS_ALIAS_WARNING) { + /* Uh Oh! They used an ambiguous alias. + We have to search the whole swiss cheese starting + at the highest standard affinity. + This may take a while. + */ + for (idx = 0; idx < gMainTable.taggedAliasArraySize; idx++) { + listOffset = gMainTable.taggedAliasArray[idx]; + if (listOffset && isAliasInList(alias, listOffset)) { + uint32_t currTagNum = idx/gMainTable.converterListSize; + uint32_t currConvNum = (idx - currTagNum*gMainTable.converterListSize); + uint32_t tempListOffset = gMainTable.taggedAliasArray[tagNum*gMainTable.converterListSize + currConvNum]; + if (tempListOffset && gMainTable.taggedAliasLists[tempListOffset + 1]) { + return tempListOffset; + } + /* else keep on looking */ + /* We could speed this up by starting on the next row + because an alias is unique per row, right now. + This would change if alias versioning appears. */ + } + } + /* The standard doesn't know about the alias */ + } + /* else no default name */ + return 0; + } + /* else converter or tag not found */ + + return UINT32_MAX; +} + +/* Return the canonical name */ +static uint32_t +findTaggedConverterNum(const char *alias, const char *standard, UErrorCode *pErrorCode) { + uint32_t idx; + uint32_t listOffset; + uint32_t convNum; + UErrorCode myErr = U_ZERO_ERROR; + uint32_t tagNum = getTagNumber(standard); + + /* Make a quick guess. Hopefully they used a TR22 canonical alias. */ + convNum = findConverter(alias, NULL, &myErr); + if (myErr != U_ZERO_ERROR) { + *pErrorCode = myErr; + } + + if (tagNum < (gMainTable.tagListSize - UCNV_NUM_HIDDEN_TAGS) && convNum < gMainTable.converterListSize) { + listOffset = gMainTable.taggedAliasArray[tagNum*gMainTable.converterListSize + convNum]; + if (listOffset && isAliasInList(alias, listOffset)) { + return convNum; + } + if (myErr == U_AMBIGUOUS_ALIAS_WARNING) { + /* Uh Oh! They used an ambiguous alias. + We have to search one slice of the swiss cheese. + We search only in the requested tag, not the whole thing. + This may take a while. + */ + uint32_t convStart = (tagNum)*gMainTable.converterListSize; + uint32_t convLimit = (tagNum+1)*gMainTable.converterListSize; + for (idx = convStart; idx < convLimit; idx++) { + listOffset = gMainTable.taggedAliasArray[idx]; + if (listOffset && isAliasInList(alias, listOffset)) { + return idx-convStart; + } + } + /* The standard doesn't know about the alias */ + } + /* else no canonical name */ + } + /* else converter or tag not found */ + + return UINT32_MAX; +} + + + +U_CFUNC const char * +ucnv_io_getConverterName(const char *alias, UBool *containsOption, UErrorCode *pErrorCode) { + if(haveAliasData(pErrorCode) && isAlias(alias, pErrorCode)) { + uint32_t convNum = findConverter(alias, containsOption, pErrorCode); + if (convNum < gMainTable.converterListSize) { + return GET_STRING(gMainTable.converterList[convNum]); + } + /* else converter not found */ + } + return NULL; +} + +static int32_t U_CALLCONV +ucnv_io_countStandardAliases(UEnumeration *enumerator, UErrorCode *pErrorCode) { + int32_t value = 0; + UAliasContext *myContext = (UAliasContext *)(enumerator->context); + uint32_t listOffset = myContext->listOffset; + + if (listOffset) { + value = gMainTable.taggedAliasLists[listOffset]; + } + return value; +} + +static const char* U_CALLCONV +ucnv_io_nextStandardAliases(UEnumeration *enumerator, + int32_t* resultLength, + UErrorCode *pErrorCode) +{ + UAliasContext *myContext = (UAliasContext *)(enumerator->context); + uint32_t listOffset = myContext->listOffset; + + if (listOffset) { + uint32_t listCount = gMainTable.taggedAliasLists[listOffset]; + const uint16_t *currList = gMainTable.taggedAliasLists + listOffset + 1; + + if (myContext->listIdx < listCount) { + const char *myStr = GET_STRING(currList[myContext->listIdx++]); + if (resultLength) { + *resultLength = (int32_t)uprv_strlen(myStr); + } + return myStr; + } + } + /* Either we accessed a zero length list, or we enumerated too far. */ + if (resultLength) { + *resultLength = 0; + } + return NULL; +} + +static void U_CALLCONV +ucnv_io_resetStandardAliases(UEnumeration *enumerator, UErrorCode *pErrorCode) { + ((UAliasContext *)(enumerator->context))->listIdx = 0; +} + +static void U_CALLCONV +ucnv_io_closeUEnumeration(UEnumeration *enumerator) { + uprv_free(enumerator->context); + uprv_free(enumerator); +} + +/* Enumerate the aliases for the specified converter and standard tag */ +static const UEnumeration gEnumAliases = { + NULL, + NULL, + ucnv_io_closeUEnumeration, + ucnv_io_countStandardAliases, + uenum_unextDefault, + ucnv_io_nextStandardAliases, + ucnv_io_resetStandardAliases +}; + +U_CAPI UEnumeration * U_EXPORT2 +ucnv_openStandardNames(const char *convName, + const char *standard, + UErrorCode *pErrorCode) +{ + UEnumeration *myEnum = NULL; + if (haveAliasData(pErrorCode) && isAlias(convName, pErrorCode)) { + uint32_t listOffset = findTaggedAliasListsOffset(convName, standard, pErrorCode); + + /* When listOffset == 0, we want to acknowledge that the + converter name and standard are okay, but there + is nothing to enumerate. */ + if (listOffset < gMainTable.taggedAliasListsSize) { + UAliasContext *myContext; + + myEnum = uprv_malloc(sizeof(UEnumeration)); + if (myEnum == NULL) { + *pErrorCode = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + uprv_memcpy(myEnum, &gEnumAliases, sizeof(UEnumeration)); + myContext = uprv_malloc(sizeof(UAliasContext)); + if (myContext == NULL) { + *pErrorCode = U_MEMORY_ALLOCATION_ERROR; + uprv_free(myEnum); + return NULL; + } + myContext->listOffset = listOffset; + myContext->listIdx = 0; + myEnum->context = myContext; + } + /* else converter or tag not found */ + } + return myEnum; +} + +static uint16_t +ucnv_io_countAliases(const char *alias, UErrorCode *pErrorCode) { + if(haveAliasData(pErrorCode) && isAlias(alias, pErrorCode)) { + uint32_t convNum = findConverter(alias, NULL, pErrorCode); + if (convNum < gMainTable.converterListSize) { + /* tagListNum - 1 is the ALL tag */ + int32_t listOffset = gMainTable.taggedAliasArray[(gMainTable.tagListSize - 1)*gMainTable.converterListSize + convNum]; + + if (listOffset) { + return gMainTable.taggedAliasLists[listOffset]; + } + /* else this shouldn't happen. internal program error */ + } + /* else converter not found */ + } + return 0; +} + +static uint16_t +ucnv_io_getAliases(const char *alias, uint16_t start, const char **aliases, UErrorCode *pErrorCode) { + if(haveAliasData(pErrorCode) && isAlias(alias, pErrorCode)) { + uint32_t currAlias; + uint32_t convNum = findConverter(alias, NULL, pErrorCode); + if (convNum < gMainTable.converterListSize) { + /* tagListNum - 1 is the ALL tag */ + int32_t listOffset = gMainTable.taggedAliasArray[(gMainTable.tagListSize - 1)*gMainTable.converterListSize + convNum]; + + if (listOffset) { + uint32_t listCount = gMainTable.taggedAliasLists[listOffset]; + /* +1 to skip listCount */ + const uint16_t *currList = gMainTable.taggedAliasLists + listOffset + 1; + + for (currAlias = start; currAlias < listCount; currAlias++) { + aliases[currAlias] = GET_STRING(currList[currAlias]); + } + } + /* else this shouldn't happen. internal program error */ + } + /* else converter not found */ + } + return 0; +} + +static const char * +ucnv_io_getAlias(const char *alias, uint16_t n, UErrorCode *pErrorCode) { + if(haveAliasData(pErrorCode) && isAlias(alias, pErrorCode)) { + uint32_t convNum = findConverter(alias, NULL, pErrorCode); + if (convNum < gMainTable.converterListSize) { + /* tagListNum - 1 is the ALL tag */ + int32_t listOffset = gMainTable.taggedAliasArray[(gMainTable.tagListSize - 1)*gMainTable.converterListSize + convNum]; + + if (listOffset) { + uint32_t listCount = gMainTable.taggedAliasLists[listOffset]; + /* +1 to skip listCount */ + const uint16_t *currList = gMainTable.taggedAliasLists + listOffset + 1; + + if (n < listCount) { + return GET_STRING(currList[n]); + } + *pErrorCode = U_INDEX_OUTOFBOUNDS_ERROR; + } + /* else this shouldn't happen. internal program error */ + } + /* else converter not found */ + } + return NULL; +} + +static uint16_t +ucnv_io_countStandards(UErrorCode *pErrorCode) { + if (haveAliasData(pErrorCode)) { + /* Don't include the empty list */ + return (uint16_t)(gMainTable.tagListSize - UCNV_NUM_HIDDEN_TAGS); + } + + return 0; +} + +U_CAPI const char * U_EXPORT2 +ucnv_getStandard(uint16_t n, UErrorCode *pErrorCode) { + if (haveAliasData(pErrorCode)) { + if (n < gMainTable.tagListSize - UCNV_NUM_HIDDEN_TAGS) { + return GET_STRING(gMainTable.tagList[n]); + } + *pErrorCode = U_INDEX_OUTOFBOUNDS_ERROR; + } + + return NULL; +} + +U_CAPI const char * U_EXPORT2 +ucnv_getStandardName(const char *alias, const char *standard, UErrorCode *pErrorCode) { + if (haveAliasData(pErrorCode) && isAlias(alias, pErrorCode)) { + uint32_t listOffset = findTaggedAliasListsOffset(alias, standard, pErrorCode); + + if (0 < listOffset && listOffset < gMainTable.taggedAliasListsSize) { + const uint16_t *currList = gMainTable.taggedAliasLists + listOffset + 1; + + /* Get the preferred name from this list */ + if (currList[0]) { + return GET_STRING(currList[0]); + } + /* else someone screwed up the alias table. */ + /* *pErrorCode = U_INVALID_FORMAT_ERROR */ + } + } + + return NULL; +} + +U_CAPI uint16_t U_EXPORT2 +ucnv_countAliases(const char *alias, UErrorCode *pErrorCode) +{ + return ucnv_io_countAliases(alias, pErrorCode); +} + + +U_CAPI const char* U_EXPORT2 +ucnv_getAlias(const char *alias, uint16_t n, UErrorCode *pErrorCode) +{ + return ucnv_io_getAlias(alias, n, pErrorCode); +} + +U_CAPI void U_EXPORT2 +ucnv_getAliases(const char *alias, const char **aliases, UErrorCode *pErrorCode) +{ + ucnv_io_getAliases(alias, 0, aliases, pErrorCode); +} + +U_CAPI uint16_t U_EXPORT2 +ucnv_countStandards(void) +{ + UErrorCode err = U_ZERO_ERROR; + return ucnv_io_countStandards(&err); +} + +U_CAPI const char * U_EXPORT2 +ucnv_getCanonicalName(const char *alias, const char *standard, UErrorCode *pErrorCode) { + if (haveAliasData(pErrorCode) && isAlias(alias, pErrorCode)) { + uint32_t convNum = findTaggedConverterNum(alias, standard, pErrorCode); + + if (convNum < gMainTable.converterListSize) { + return GET_STRING(gMainTable.converterList[convNum]); + } + } + + return NULL; +} + +static int32_t U_CALLCONV +ucnv_io_countAllConverters(UEnumeration *enumerator, UErrorCode *pErrorCode) { + return gMainTable.converterListSize; +} + +static const char* U_CALLCONV +ucnv_io_nextAllConverters(UEnumeration *enumerator, + int32_t* resultLength, + UErrorCode *pErrorCode) +{ + uint16_t *myContext = (uint16_t *)(enumerator->context); + + if (*myContext < gMainTable.converterListSize) { + const char *myStr = GET_STRING(gMainTable.converterList[(*myContext)++]); + if (resultLength) { + *resultLength = (int32_t)uprv_strlen(myStr); + } + return myStr; + } + /* Either we accessed a zero length list, or we enumerated too far. */ + if (resultLength) { + *resultLength = 0; + } + return NULL; +} + +static void U_CALLCONV +ucnv_io_resetAllConverters(UEnumeration *enumerator, UErrorCode *pErrorCode) { + *((uint16_t *)(enumerator->context)) = 0; +} + +static const UEnumeration gEnumAllConverters = { + NULL, + NULL, + ucnv_io_closeUEnumeration, + ucnv_io_countAllConverters, + uenum_unextDefault, + ucnv_io_nextAllConverters, + ucnv_io_resetAllConverters +}; + +U_CAPI UEnumeration * U_EXPORT2 +ucnv_openAllNames(UErrorCode *pErrorCode) { + UEnumeration *myEnum = NULL; + if (haveAliasData(pErrorCode)) { + uint16_t *myContext; + + myEnum = uprv_malloc(sizeof(UEnumeration)); + if (myEnum == NULL) { + *pErrorCode = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + uprv_memcpy(myEnum, &gEnumAllConverters, sizeof(UEnumeration)); + myContext = uprv_malloc(sizeof(uint16_t)); + if (myContext == NULL) { + *pErrorCode = U_MEMORY_ALLOCATION_ERROR; + uprv_free(myEnum); + return NULL; + } + *myContext = 0; + myEnum->context = myContext; + } + return myEnum; +} + +U_CFUNC uint16_t +ucnv_io_countKnownConverters(UErrorCode *pErrorCode) { + if (haveAliasData(pErrorCode)) { + return (uint16_t)gMainTable.converterListSize; + } + return 0; +} + +/* alias table swapping ----------------------------------------------------- */ + +typedef char * U_CALLCONV StripForCompareFn(char *dst, const char *name); + +/* + * row of a temporary array + * + * gets platform-endian charset string indexes and sorting indexes; + * after sorting this array by strings, the actual arrays are permutated + * according to the sorting indexes + */ +typedef struct TempRow { + uint16_t strIndex, sortIndex; +} TempRow; + +typedef struct TempAliasTable { + const char *chars; + TempRow *rows; + uint16_t *resort; + StripForCompareFn *stripForCompare; +} TempAliasTable; + +enum { + STACK_ROW_CAPACITY=500 +}; + +static int32_t +io_compareRows(const void *context, const void *left, const void *right) { + char strippedLeft[UCNV_MAX_CONVERTER_NAME_LENGTH], + strippedRight[UCNV_MAX_CONVERTER_NAME_LENGTH]; + + TempAliasTable *tempTable=(TempAliasTable *)context; + const char *chars=tempTable->chars; + + return (int32_t)uprv_strcmp(tempTable->stripForCompare(strippedLeft, chars+2*((const TempRow *)left)->strIndex), + tempTable->stripForCompare(strippedRight, chars+2*((const TempRow *)right)->strIndex)); +} + +U_CAPI int32_t U_EXPORT2 +ucnv_swapAliases(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode) { + const UDataInfo *pInfo; + int32_t headerSize; + + const uint16_t *inTable; + const uint32_t *inSectionSizes; + uint32_t toc[offsetsCount]; + uint32_t offsets[offsetsCount]; /* 16-bit-addressed offsets from inTable/outTable */ + uint32_t i, count, tocLength, topOffset; + + TempRow rows[STACK_ROW_CAPACITY]; + uint16_t resort[STACK_ROW_CAPACITY]; + TempAliasTable tempTable; + + /* udata_swapDataHeader checks the arguments */ + headerSize=udata_swapDataHeader(ds, inData, length, outData, pErrorCode); + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + /* check data format and format version */ + pInfo=(const UDataInfo *)((const char *)inData+4); + if(!( + pInfo->dataFormat[0]==0x43 && /* dataFormat="CvAl" */ + pInfo->dataFormat[1]==0x76 && + pInfo->dataFormat[2]==0x41 && + pInfo->dataFormat[3]==0x6c && + pInfo->formatVersion[0]==3 + )) { + udata_printError(ds, "ucnv_swapAliases(): data format %02x.%02x.%02x.%02x (format version %02x) is not an alias table\n", + pInfo->dataFormat[0], pInfo->dataFormat[1], + pInfo->dataFormat[2], pInfo->dataFormat[3], + pInfo->formatVersion[0]); + *pErrorCode=U_UNSUPPORTED_ERROR; + return 0; + } + + /* an alias table must contain at least the table of contents array */ + if(length>=0 && (length-headerSize)<4*(1+minTocLength)) { + udata_printError(ds, "ucnv_swapAliases(): too few bytes (%d after header) for an alias table\n", + length-headerSize); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + + inSectionSizes=(const uint32_t *)((const char *)inData+headerSize); + inTable=(const uint16_t *)inSectionSizes; + uprv_memset(toc, 0, sizeof(toc)); + toc[tocLengthIndex]=tocLength=ds->readUInt32(inSectionSizes[tocLengthIndex]); + if(tocLengthreadUInt32(inSectionSizes[i]); + } + + /* compute offsets */ + uprv_memset(offsets, 0, sizeof(offsets)); + offsets[converterListIndex]=2*(1+tocLength); /* count two 16-bit units per toc entry */ + for(i=tagListIndex; i<=tocLength; ++i) { + offsets[i]=offsets[i-1]+toc[i-1]; + } + + /* compute the overall size of the after-header data, in numbers of 16-bit units */ + topOffset=offsets[i-1]+toc[i-1]; + + if(length>=0) { + uint16_t *outTable; + const uint16_t *p, *p2; + uint16_t *q, *q2; + uint16_t oldIndex; + + if((length-headerSize)<(2*(int32_t)topOffset)) { + udata_printError(ds, "ucnv_swapAliases(): too few bytes (%d after header) for an alias table\n", + length-headerSize); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + + outTable=(uint16_t *)((char *)outData+headerSize); + + /* swap the entire table of contents */ + ds->swapArray32(ds, inTable, 4*(1+tocLength), outTable, pErrorCode); + + /* swap unormalized strings & normalized strings */ + ds->swapInvChars(ds, inTable+offsets[stringTableIndex], 2*(int32_t)(toc[stringTableIndex]+toc[normalizedStringTableIndex]), + outTable+offsets[stringTableIndex], pErrorCode); + if(U_FAILURE(*pErrorCode)) { + udata_printError(ds, "ucnv_swapAliases().swapInvChars(charset names) failed\n"); + return 0; + } + + if(ds->inCharset==ds->outCharset) { + /* no need to sort, just swap all 16-bit values together */ + ds->swapArray16(ds, + inTable+offsets[converterListIndex], + 2*(int32_t)(offsets[stringTableIndex]-offsets[converterListIndex]), + outTable+offsets[converterListIndex], + pErrorCode); + } else { + /* allocate the temporary table for sorting */ + count=toc[aliasListIndex]; + + tempTable.chars=(const char *)(outTable+offsets[stringTableIndex]); /* sort by outCharset */ + + if(count<=STACK_ROW_CAPACITY) { + tempTable.rows=rows; + tempTable.resort=resort; + } else { + tempTable.rows=(TempRow *)uprv_malloc(count*sizeof(TempRow)+count*2); + if(tempTable.rows==NULL) { + udata_printError(ds, "ucnv_swapAliases(): unable to allocate memory for sorting tables (max length: %u)\n", + count); + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return 0; + } + tempTable.resort=(uint16_t *)(tempTable.rows+count); + } + + if(ds->outCharset==U_ASCII_FAMILY) { + tempTable.stripForCompare=ucnv_io_stripASCIIForCompare; + } else /* U_EBCDIC_FAMILY */ { + tempTable.stripForCompare=ucnv_io_stripEBCDICForCompare; + } + + /* + * Sort unique aliases+mapped names. + * + * We need to sort the list again by outCharset strings because they + * sort differently for different charset families. + * First we set up a temporary table with the string indexes and + * sorting indexes and sort that. + * Then we permutate and copy/swap the actual values. + */ + p=inTable+offsets[aliasListIndex]; + q=outTable+offsets[aliasListIndex]; + + p2=inTable+offsets[untaggedConvArrayIndex]; + q2=outTable+offsets[untaggedConvArrayIndex]; + + for(i=0; ireadUInt16(p[i]); + tempTable.rows[i].sortIndex=(uint16_t)i; + } + + uprv_sortArray(tempTable.rows, (int32_t)count, sizeof(TempRow), + io_compareRows, &tempTable, + FALSE, pErrorCode); + + if(U_SUCCESS(*pErrorCode)) { + /* copy/swap/permutate items */ + if(p!=q) { + for(i=0; iswapArray16(ds, p+oldIndex, 2, q+i, pErrorCode); + ds->swapArray16(ds, p2+oldIndex, 2, q2+i, pErrorCode); + } + } else { + /* + * If we swap in-place, then the permutation must use another + * temporary array (tempTable.resort) + * before the results are copied to the outBundle. + */ + uint16_t *r=tempTable.resort; + + for(i=0; iswapArray16(ds, p+oldIndex, 2, r+i, pErrorCode); + } + uprv_memcpy(q, r, 2*count); + + for(i=0; iswapArray16(ds, p2+oldIndex, 2, r+i, pErrorCode); + } + uprv_memcpy(q2, r, 2*count); + } + } + + if(tempTable.rows!=rows) { + uprv_free(tempTable.rows); + } + + if(U_FAILURE(*pErrorCode)) { + udata_printError(ds, "ucnv_swapAliases().uprv_sortArray(%u items) failed\n", + count); + return 0; + } + + /* swap remaining 16-bit values */ + ds->swapArray16(ds, + inTable+offsets[converterListIndex], + 2*(int32_t)(offsets[aliasListIndex]-offsets[converterListIndex]), + outTable+offsets[converterListIndex], + pErrorCode); + ds->swapArray16(ds, + inTable+offsets[taggedAliasArrayIndex], + 2*(int32_t)(offsets[stringTableIndex]-offsets[taggedAliasArrayIndex]), + outTable+offsets[taggedAliasArrayIndex], + pErrorCode); + } + } + + return headerSize+2*(int32_t)topOffset; +} + +#endif + +/* + * Hey, Emacs, please set the following: + * + * Local Variables: + * indent-tabs-mode: nil + * End: + * + */ diff --git a/Build/source/libs/icu/icu-xetex/common/ucnv_io.h b/Build/source/libs/icu/icu-xetex/common/ucnv_io.h new file mode 100644 index 00000000000..060ffd05953 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnv_io.h @@ -0,0 +1,125 @@ +/* + ********************************************************************** + * Copyright (C) 1999-2006, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + * + * + * ucnv_io.h: + * defines variables and functions pertaining to converter name resolution + * aspect of the conversion code + */ + +#ifndef UCNV_IO_H +#define UCNV_IO_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "udataswp.h" + +#define UCNV_AMBIGUOUS_ALIAS_MAP_BIT 0x8000 +#define UCNV_CONTAINS_OPTION_BIT 0x4000 +#define UCNV_CONVERTER_INDEX_MASK 0xFFF +#define UCNV_NUM_RESERVED_TAGS 2 +#define UCNV_NUM_HIDDEN_TAGS 1 + +enum { + UCNV_IO_UNNORMALIZED, + UCNV_IO_STD_NORMALIZED, + UCNV_IO_NORM_TYPE_COUNT +}; + +typedef struct { + uint16_t stringNormalizationType; + uint16_t containsCnvOptionInfo; +} UConverterAliasOptions; + +typedef struct UConverterAlias { + const uint16_t *converterList; + const uint16_t *tagList; + const uint16_t *aliasList; + const uint16_t *untaggedConvArray; + const uint16_t *taggedAliasArray; + const uint16_t *taggedAliasLists; + const UConverterAliasOptions *optionTable; + const uint16_t *stringTable; + const uint16_t *normalizedStringTable; + + uint32_t converterListSize; + uint32_t tagListSize; + uint32_t aliasListSize; + uint32_t untaggedConvArraySize; + uint32_t taggedAliasArraySize; + uint32_t taggedAliasListsSize; + uint32_t optionTableSize; + uint32_t stringTableSize; + uint32_t normalizedStringTableSize; +} UConverterAlias; + +/** + * \var ucnv_io_stripForCompare + * Remove the underscores, dashes and spaces from the name, and convert + * the name to lower case. + * @param dst The destination buffer, which is <= the buffer of name. + * @param dst The destination buffer, which is <= the buffer of name. + * @see ucnv_compareNames + * @return the destination buffer. + */ +#if U_CHARSET_FAMILY==U_ASCII_FAMILY +# define ucnv_io_stripForCompare ucnv_io_stripASCIIForCompare +#elif U_CHARSET_FAMILY==U_EBCDIC_FAMILY +# define ucnv_io_stripForCompare ucnv_io_stripEBCDICForCompare +#else +# error U_CHARSET_FAMILY is not valid +#endif + +U_CAPI char * U_EXPORT2 +ucnv_io_stripASCIIForCompare(char *dst, const char *name); + +U_CAPI char * U_EXPORT2 +ucnv_io_stripEBCDICForCompare(char *dst, const char *name); + +/** + * Map a converter alias name to a canonical converter name. + * The alias is searched for case-insensitively, the converter name + * is returned in mixed-case. + * Returns NULL if the alias is not found. + * @param alias The alias name to be searched. + * @param containsOption A return value stating whether the returned converter name contains an option (a comma) + * @param pErrorCode The error code + * @return the converter name in mixed-case, return NULL if the alias is not found. + */ +U_CFUNC const char * +ucnv_io_getConverterName(const char *alias, UBool *containsOption, UErrorCode *pErrorCode); + +/** + * Return the number of all known converter names (no aliases). + * @param pErrorCode The error code + * @return the number of all aliases + */ +U_CFUNC uint16_t +ucnv_io_countKnownConverters(UErrorCode *pErrorCode); + +/** + * Swap an ICU converter alias table. See implementation for details. + * @internal + */ +U_CAPI int32_t U_EXPORT2 +ucnv_swapAliases(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode); + +#endif + +#endif /* _UCNV_IO */ + +/* + * Hey, Emacs, please set the following: + * + * Local Variables: + * indent-tabs-mode: nil + * End: + * + */ diff --git a/Build/source/libs/icu/icu-xetex/common/ucnv_lmb.c b/Build/source/libs/icu/icu-xetex/common/ucnv_lmb.c new file mode 100644 index 00000000000..79606af2894 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnv_lmb.c @@ -0,0 +1,1281 @@ +/* +********************************************************************** +* Copyright (C) 2000-2006, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* file name: ucnv_lmb.cpp +* encoding: US-ASCII +* tab size: 4 (not used) +* indentation:4 +* +* created on: 2000feb09 +* created by: Brendan Murray +* extensively hacked up by: Jim Snyder-Grant +* +* Modification History: +* +* Date Name Description +* +* 06/20/2000 helena OS/400 port changes; mostly typecast. +* 06/27/2000 Jim Snyder-Grant Deal with partial characters and small buffers. +* Add comments to document LMBCS format and implementation +* restructured order & breakdown of functions +* 06/28/2000 helena Major rewrite for the callback API changes. +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION && !UCONFIG_NO_LEGACY_CONVERSION + +#include "unicode/ucnv_err.h" +#include "unicode/ucnv.h" +#include "unicode/uset.h" +#include "cmemory.h" +#include "cstring.h" +#include "uassert.h" +#include "ucnv_imp.h" +#include "ucnv_bld.h" +#include "ucnv_cnv.h" + +#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) + +/* + LMBCS + + (Lotus Multi-Byte Character Set) + + LMBCS was invented in the late 1980's and is primarily used in Lotus Notes + databases and in Lotus 1-2-3 files. Programmers who work with the APIs + into these products will sometimes need to deal with strings in this format. + + The code in this file provides an implementation for an ICU converter of + LMBCS to and from Unicode. + + Since the LMBCS character set is only sparsely documented in existing + printed or online material, we have added extensive annotation to this + file to serve as a guide to understanding LMBCS. + + LMBCS was originally designed with these four sometimes-competing design goals: + + -Provide encodings for the characters in 12 existing national standards + (plus a few other characters) + -Minimal memory footprint + -Maximal speed of conversion into the existing national character sets + -No need to track a changing state as you interpret a string. + + + All of the national character sets LMBCS was trying to encode are 'ANSI' + based, in that the bytes from 0x20 - 0x7F are almost exactly the + same common Latin unaccented characters and symbols in all character sets. + + So, in order to help meet the speed & memory design goals, the common ANSI + bytes from 0x20-0x7F are represented by the same single-byte values in LMBCS. + + The general LMBCS code unit is from 1-3 bytes. We can describe the 3 bytes as + follows: + + [G] D1 [D2] + + That is, a sometimes-optional 'group' byte, followed by 1 and sometimes 2 + data bytes. The maximum size of a LMBCS chjaracter is 3 bytes: +*/ +#define ULMBCS_CHARSIZE_MAX 3 +/* + The single-byte values from 0x20 to 0x7F are examples of single D1 bytes. + We often have to figure out if byte values are below or above this, so we + use the ANSI nomenclature 'C0' and 'C1' to refer to the range of control + characters just above & below the common lower-ANSI range */ +#define ULMBCS_C0END 0x1F +#define ULMBCS_C1START 0x80 +/* + Since LMBCS is always dealing in byte units. we create a local type here for + dealing with these units of LMBCS code units: + +*/ +typedef uint8_t ulmbcs_byte_t; + +/* + Most of the values less than 0x20 are reserved in LMBCS to announce + which national character standard is being used for the 'D' bytes. + In the comments we show the common name and the IBM character-set ID + for these character-set announcers: +*/ + +#define ULMBCS_GRP_L1 0x01 /* Latin-1 :ibm-850 */ +#define ULMBCS_GRP_GR 0x02 /* Greek :ibm-851 */ +#define ULMBCS_GRP_HE 0x03 /* Hebrew :ibm-1255 */ +#define ULMBCS_GRP_AR 0x04 /* Arabic :ibm-1256 */ +#define ULMBCS_GRP_RU 0x05 /* Cyrillic :ibm-1251 */ +#define ULMBCS_GRP_L2 0x06 /* Latin-2 :ibm-852 */ +#define ULMBCS_GRP_TR 0x08 /* Turkish :ibm-1254 */ +#define ULMBCS_GRP_TH 0x0B /* Thai :ibm-874 */ +#define ULMBCS_GRP_JA 0x10 /* Japanese :ibm-943 */ +#define ULMBCS_GRP_KO 0x11 /* Korean :ibm-1261 */ +#define ULMBCS_GRP_TW 0x12 /* Chinese SC :ibm-950 */ +#define ULMBCS_GRP_CN 0x13 /* Chinese TC :ibm-1386 */ + +/* + So, the beginning of understanding LMBCS is that IF the first byte of a LMBCS + character is one of those 12 values, you can interpret the remaining bytes of + that character as coming from one of those character sets. Since the lower + ANSI bytes already are represented in single bytes, using one of the character + set announcers is used to announce a character that starts with a byte of + 0x80 or greater. + + The character sets are arranged so that the single byte sets all appear + before the multi-byte character sets. When we need to tell whether a + group byte is for a single byte char set or not we use this define: */ + +#define ULMBCS_DOUBLEOPTGROUP_START 0x10 + +/* +However, to fully understand LMBCS, you must also understand a series of +exceptions & optimizations made in service of the design goals. + +First, those of you who are character set mavens may have noticed that +the 'double-byte' character sets are actually multi-byte character sets +that can have 1 or two bytes, even in the upper-ascii range. To force +each group byte to introduce a fixed-width encoding (to make it faster to +count characters), we use a convention of doubling up on the group byte +to introduce any single-byte character > 0x80 in an otherwise double-byte +character set. So, for example, the LMBCS sequence x10 x10 xAE is the +same as '0xAE' in the Japanese code page 943. + +Next, you will notice that the list of group bytes has some gaps. +These are used in various ways. + +We reserve a few special single byte values for common control +characters. These are in the same place as their ANSI eqivalents for speed. +*/ + +#define ULMBCS_HT 0x09 /* Fixed control char - Horizontal Tab */ +#define ULMBCS_LF 0x0A /* Fixed control char - Line Feed */ +#define ULMBCS_CR 0x0D /* Fixed control char - Carriage Return */ + +/* Then, 1-2-3 reserved a special single-byte character to put at the +beginning of internal 'system' range names: */ + +#define ULMBCS_123SYSTEMRANGE 0x19 + +/* Then we needed a place to put all the other ansi control characters +that must be moved to different values because LMBCS reserves those +values for other purposes. To represent the control characters, we start +with a first byte of 0xF & add the control chaarcter value as the +second byte */ +#define ULMBCS_GRP_CTRL 0x0F + +/* For the C0 controls (less than 0x20), we add 0x20 to preserve the +useful doctrine that any byte less than 0x20 in a LMBCS char must be +the first byte of a character:*/ +#define ULMBCS_CTRLOFFSET 0x20 + +/* +Where to put the characters that aren't part of any of the 12 national +character sets? The first thing that was done, in the earlier years of +LMBCS, was to use up the spaces of the form + + [G] D1, + + where 'G' was one of the single-byte character groups, and + D1 was less than 0x80. These sequences are gathered together + into a Lotus-invented doublebyte character set to represent a + lot of stray values. Internally, in this implementation, we track this + as group '0', as a place to tuck this exceptions list.*/ + +#define ULMBCS_GRP_EXCEPT 0x00 +/* + Finally, as the durability and usefulness of UNICODE became clear, + LOTUS added a new group 0x14 to hold Unicode values not otherwise + represented in LMBCS: */ +#define ULMBCS_GRP_UNICODE 0x14 +/* The two bytes appearing after a 0x14 are intrepreted as UFT-16 BE +(Big-Endian) characters. The exception comes when the UTF16 +representation would have a zero as the second byte. In that case, +'F6' is used in its place, and the bytes are swapped. (This prevents +LMBCS from encoding any Unicode values of the form U+F6xx, but that's OK: +0xF6xx is in the middle of the Private Use Area.)*/ +#define ULMBCS_UNICOMPATZERO 0xF6 + +/* It is also useful in our code to have a constant for the size of +a LMBCS char that holds a literal Unicode value */ +#define ULMBCS_UNICODE_SIZE 3 + +/* +To squish the LMBCS representations down even further, and to make +translations even faster,sometimes the optimization group byte can be dropped +from a LMBCS character. This is decided on a process-by-process basis. The +group byte that is dropped is called the 'optimization group'. + +For Notes, the optimzation group is always 0x1.*/ +#define ULMBCS_DEFAULTOPTGROUP 0x1 +/* For 1-2-3 files, the optimzation group is stored in the header of the 1-2-3 +file. + + In any case, when using ICU, you either pass in the +optimization group as part of the name of the converter (LMBCS-1, LMBCS-2, +etc.). Using plain 'LMBCS' as the name of the converter will give you +LMBCS-1. + + +*** Implementation strategy *** + + +Because of the extensive use of other character sets, the LMBCS converter +keeps a mapping between optimization groups and IBM character sets, so that +ICU converters can be created and used as needed. */ + +/* As you can see, even though any byte below 0x20 could be an optimization +byte, only those at 0x13 or below can map to an actual converter. To limit +some loops and searches, we define a value for that last group converter:*/ + +#define ULMBCS_GRP_LAST 0x13 /* last LMBCS group that has a converter */ + +static const char * const OptGroupByteToCPName[ULMBCS_GRP_LAST + 1] = { + /* 0x0000 */ "lmb-excp", /* internal home for the LOTUS exceptions list */ + /* 0x0001 */ "ibm-850", + /* 0x0002 */ "ibm-851", + /* 0x0003 */ "windows-1255", + /* 0x0004 */ "windows-1256", + /* 0x0005 */ "windows-1251", + /* 0x0006 */ "ibm-852", + /* 0x0007 */ NULL, /* Unused */ + /* 0x0008 */ "windows-1254", + /* 0x0009 */ NULL, /* Control char HT */ + /* 0x000A */ NULL, /* Control char LF */ + /* 0x000B */ "windows-874", + /* 0x000C */ NULL, /* Unused */ + /* 0x000D */ NULL, /* Control char CR */ + /* 0x000E */ NULL, /* Unused */ + /* 0x000F */ NULL, /* Control chars: 0x0F20 + C0/C1 character: algorithmic */ + /* 0x0010 */ "windows-932", + /* 0x0011 */ "windows-949", + /* 0x0012 */ "windows-950", + /* 0x0013 */ "windows-936" + + /* The rest are null, including the 0x0014 Unicode compatibility region + and 0x0019, the 1-2-3 system range control char */ +}; + + +/* That's approximately all the data that's needed for translating + LMBCS to Unicode. + + +However, to translate Unicode to LMBCS, we need some more support. + +That's because there are often more than one possible mappings from a Unicode +code point back into LMBCS. The first thing we do is look up into a table +to figure out if there are more than one possible mappings. This table, +arranged by Unicode values (including ranges) either lists which group +to use, or says that it could go into one or more of the SBCS sets, or +into one or more of the DBCS sets. (If the character exists in both DBCS & +SBCS, the table will place it in the SBCS sets, to make the LMBCS code point +length as small as possible. Here's the two special markers we use to indicate +ambiguous mappings: */ + +#define ULMBCS_AMBIGUOUS_SBCS 0x80 /* could fit in more than one + LMBCS sbcs native encoding + (example: most accented latin) */ +#define ULMBCS_AMBIGUOUS_MBCS 0x81 /* could fit in more than one + LMBCS mbcs native encoding + (example: Unihan) */ + +/* And here's a simple way to see if a group falls in an appropriate range */ +#define ULMBCS_AMBIGUOUS_MATCH(agroup, xgroup) \ + ((((agroup) == ULMBCS_AMBIGUOUS_SBCS) && \ + (xgroup) < ULMBCS_DOUBLEOPTGROUP_START) || \ + (((agroup) == ULMBCS_AMBIGUOUS_MBCS) && \ + (xgroup) >= ULMBCS_DOUBLEOPTGROUP_START)) + + +/* The table & some code to use it: */ + + +static const struct _UniLMBCSGrpMap +{ + const UChar uniStartRange; + const UChar uniEndRange; + const ulmbcs_byte_t GrpType; +} UniLMBCSGrpMap[] += +{ + + {0x0001, 0x001F, ULMBCS_GRP_CTRL}, + {0x0080, 0x009F, ULMBCS_GRP_CTRL}, + {0x00A0, 0x01CD, ULMBCS_AMBIGUOUS_SBCS}, + {0x01CE, 0x01CE, ULMBCS_GRP_TW }, + {0x01CF, 0x02B9, ULMBCS_AMBIGUOUS_SBCS}, + {0x02BA, 0x02BA, ULMBCS_GRP_CN}, + {0x02BC, 0x02C8, ULMBCS_AMBIGUOUS_SBCS}, + {0x02C9, 0x02D0, ULMBCS_AMBIGUOUS_MBCS}, + {0x02D8, 0x02DD, ULMBCS_AMBIGUOUS_SBCS}, + {0x0384, 0x03CE, ULMBCS_AMBIGUOUS_SBCS}, + {0x0400, 0x044E, ULMBCS_GRP_RU}, + {0x044F, 0x044F, ULMBCS_AMBIGUOUS_MBCS}, + {0x0450, 0x0491, ULMBCS_GRP_RU}, + {0x05B0, 0x05F2, ULMBCS_GRP_HE}, + {0x060C, 0x06AF, ULMBCS_GRP_AR}, + {0x0E01, 0x0E5B, ULMBCS_GRP_TH}, + {0x200C, 0x200F, ULMBCS_AMBIGUOUS_SBCS}, + {0x2010, 0x2010, ULMBCS_AMBIGUOUS_MBCS}, + {0x2013, 0x2015, ULMBCS_AMBIGUOUS_SBCS}, + {0x2016, 0x2016, ULMBCS_AMBIGUOUS_MBCS}, + {0x2017, 0x2024, ULMBCS_AMBIGUOUS_SBCS}, + {0x2025, 0x2025, ULMBCS_AMBIGUOUS_MBCS}, + {0x2026, 0x2026, ULMBCS_AMBIGUOUS_SBCS}, + {0x2027, 0x2027, ULMBCS_GRP_CN}, + {0x2030, 0x2033, ULMBCS_AMBIGUOUS_SBCS}, + {0x2035, 0x2035, ULMBCS_AMBIGUOUS_MBCS}, + {0x2039, 0x203A, ULMBCS_AMBIGUOUS_SBCS}, + {0x203B, 0x203B, ULMBCS_AMBIGUOUS_MBCS}, + {0x2074, 0x2074, ULMBCS_GRP_KO}, + {0x207F, 0x207F, ULMBCS_GRP_EXCEPT}, + {0x2081, 0x2084, ULMBCS_GRP_KO}, + {0x20A4, 0x20AC, ULMBCS_AMBIGUOUS_SBCS}, + {0x2103, 0x2109, ULMBCS_AMBIGUOUS_MBCS}, + {0x2111, 0x2126, ULMBCS_AMBIGUOUS_SBCS}, + {0x212B, 0x212B, ULMBCS_AMBIGUOUS_MBCS}, + {0x2135, 0x2135, ULMBCS_AMBIGUOUS_SBCS}, + {0x2153, 0x2154, ULMBCS_GRP_KO}, + {0x215B, 0x215E, ULMBCS_GRP_EXCEPT}, + {0x2160, 0x2179, ULMBCS_AMBIGUOUS_MBCS}, + {0x2190, 0x2195, ULMBCS_GRP_EXCEPT}, + {0x2196, 0x2199, ULMBCS_AMBIGUOUS_MBCS}, + {0x21A8, 0x21A8, ULMBCS_GRP_EXCEPT}, + {0x21B8, 0x21B9, ULMBCS_GRP_CN}, + {0x21D0, 0x21D5, ULMBCS_GRP_EXCEPT}, + {0x21E7, 0x21E7, ULMBCS_GRP_CN}, + {0x2200, 0x220B, ULMBCS_GRP_EXCEPT}, + {0x220F, 0x2215, ULMBCS_AMBIGUOUS_MBCS}, + {0x2219, 0x2220, ULMBCS_GRP_EXCEPT}, + {0x2223, 0x2228, ULMBCS_AMBIGUOUS_MBCS}, + {0x2229, 0x222B, ULMBCS_GRP_EXCEPT}, + {0x222C, 0x223D, ULMBCS_AMBIGUOUS_MBCS}, + {0x2245, 0x2248, ULMBCS_GRP_EXCEPT}, + {0x224C, 0x224C, ULMBCS_GRP_TW}, + {0x2252, 0x2252, ULMBCS_AMBIGUOUS_MBCS}, + {0x2260, 0x2265, ULMBCS_GRP_EXCEPT}, + {0x2266, 0x226F, ULMBCS_AMBIGUOUS_MBCS}, + {0x2282, 0x2297, ULMBCS_GRP_EXCEPT}, + {0x2299, 0x22BF, ULMBCS_AMBIGUOUS_MBCS}, + {0x22C0, 0x22C0, ULMBCS_GRP_EXCEPT}, + {0x2310, 0x2310, ULMBCS_GRP_EXCEPT}, + {0x2312, 0x2312, ULMBCS_AMBIGUOUS_MBCS}, + {0x2318, 0x2321, ULMBCS_GRP_EXCEPT}, + {0x2318, 0x2321, ULMBCS_GRP_CN}, + {0x2460, 0x24E9, ULMBCS_AMBIGUOUS_MBCS}, + {0x2500, 0x2500, ULMBCS_AMBIGUOUS_SBCS}, + {0x2501, 0x2501, ULMBCS_AMBIGUOUS_MBCS}, + {0x2502, 0x2502, ULMBCS_AMBIGUOUS_SBCS}, + {0x2503, 0x2503, ULMBCS_AMBIGUOUS_MBCS}, + {0x2504, 0x2505, ULMBCS_GRP_TW}, + {0x2506, 0x2665, ULMBCS_AMBIGUOUS_MBCS}, + {0x2666, 0x2666, ULMBCS_GRP_EXCEPT}, + {0x2666, 0x2666, ULMBCS_GRP_EXCEPT}, + {0x2667, 0x2E7F, ULMBCS_AMBIGUOUS_SBCS}, + {0x2E80, 0xF861, ULMBCS_AMBIGUOUS_MBCS}, + {0xF862, 0xF8FF, ULMBCS_GRP_EXCEPT}, + {0xF900, 0xFA2D, ULMBCS_AMBIGUOUS_MBCS}, + {0xFB00, 0xFEFF, ULMBCS_AMBIGUOUS_SBCS}, + {0xFF01, 0xFFEE, ULMBCS_AMBIGUOUS_MBCS}, + {0xFFFF, 0xFFFF, ULMBCS_GRP_UNICODE} +}; + +static ulmbcs_byte_t +FindLMBCSUniRange(UChar uniChar) +{ + const struct _UniLMBCSGrpMap * pTable = UniLMBCSGrpMap; + + while (uniChar > pTable->uniEndRange) + { + pTable++; + } + + if (uniChar >= pTable->uniStartRange) + { + return pTable->GrpType; + } + return ULMBCS_GRP_UNICODE; +} + +/* +We also ask the creator of a converter to send in a preferred locale +that we can use in resolving ambiguous mappings. They send the locale +in as a string, and we map it, if possible, to one of the +LMBCS groups. We use this table, and the associated code, to +do the lookup: */ + +/************************************************** + This table maps locale ID's to LMBCS opt groups. + The default return is group 0x01. Note that for + performance reasons, the table is sorted in + increasing alphabetic order, with the notable + exception of zhTW. This is to force the check + for Traditonal Chinese before dropping back to + Simplified. + + Note too that the Latin-1 groups have been + commented out because it's the default, and + this shortens the table, allowing a serial + search to go quickly. + *************************************************/ + +static const struct _LocaleLMBCSGrpMap +{ + const char *LocaleID; + const ulmbcs_byte_t OptGroup; +} LocaleLMBCSGrpMap[] = +{ + {"ar", ULMBCS_GRP_AR}, + {"be", ULMBCS_GRP_RU}, + {"bg", ULMBCS_GRP_L2}, + /* {"ca", ULMBCS_GRP_L1}, */ + {"cs", ULMBCS_GRP_L2}, + /* {"da", ULMBCS_GRP_L1}, */ + /* {"de", ULMBCS_GRP_L1}, */ + {"el", ULMBCS_GRP_GR}, + /* {"en", ULMBCS_GRP_L1}, */ + /* {"es", ULMBCS_GRP_L1}, */ + /* {"et", ULMBCS_GRP_L1}, */ + /* {"fi", ULMBCS_GRP_L1}, */ + /* {"fr", ULMBCS_GRP_L1}, */ + {"he", ULMBCS_GRP_HE}, + {"hu", ULMBCS_GRP_L2}, + /* {"is", ULMBCS_GRP_L1}, */ + /* {"it", ULMBCS_GRP_L1}, */ + {"iw", ULMBCS_GRP_HE}, + {"ja", ULMBCS_GRP_JA}, + {"ko", ULMBCS_GRP_KO}, + /* {"lt", ULMBCS_GRP_L1}, */ + /* {"lv", ULMBCS_GRP_L1}, */ + {"mk", ULMBCS_GRP_RU}, + /* {"nl", ULMBCS_GRP_L1}, */ + /* {"no", ULMBCS_GRP_L1}, */ + {"pl", ULMBCS_GRP_L2}, + /* {"pt", ULMBCS_GRP_L1}, */ + {"ro", ULMBCS_GRP_L2}, + {"ru", ULMBCS_GRP_RU}, + {"sh", ULMBCS_GRP_L2}, + {"sk", ULMBCS_GRP_L2}, + {"sl", ULMBCS_GRP_L2}, + {"sq", ULMBCS_GRP_L2}, + {"sr", ULMBCS_GRP_RU}, + /* {"sv", ULMBCS_GRP_L1}, */ + {"th", ULMBCS_GRP_TH}, + {"tr", ULMBCS_GRP_TR}, + {"uk", ULMBCS_GRP_RU}, + /* {"vi", ULMBCS_GRP_L1}, */ + {"zhTW", ULMBCS_GRP_TW}, + {"zh", ULMBCS_GRP_CN}, + {NULL, ULMBCS_GRP_L1} +}; + + +static ulmbcs_byte_t +FindLMBCSLocale(const char *LocaleID) +{ + const struct _LocaleLMBCSGrpMap *pTable = LocaleLMBCSGrpMap; + + if ((!LocaleID) || (!*LocaleID)) + { + return 0; + } + + while (pTable->LocaleID) + { + if (*pTable->LocaleID == *LocaleID) /* Check only first char for speed */ + { + /* First char matches - check whole name, for entry-length */ + if (uprv_strncmp(pTable->LocaleID, LocaleID, strlen(pTable->LocaleID)) == 0) + return pTable->OptGroup; + } + else + if (*pTable->LocaleID > *LocaleID) /* Sorted alphabetically - exit */ + break; + pTable++; + } + return ULMBCS_GRP_L1; +} + + +/* + Before we get to the main body of code, here's how we hook up to the rest + of ICU. ICU converters are required to define a structure that includes + some function pointers, and some common data, in the style of a C++ + vtable. There is also room in there for converter-specific data. LMBCS + uses that converter-specific data to keep track of the 12 subconverters + we use, the optimization group, and the group (if any) that matches the + locale. We have one structure instantiated for each of the 12 possible + optimization groups. To avoid typos & to avoid boring the reader, we + put the declarations of these structures and functions into macros. To see + the definitions of these structures, see unicode\ucnv_bld.h +*/ + +typedef struct + { + UConverterSharedData *OptGrpConverter[ULMBCS_GRP_LAST+1]; /* Converter per Opt. grp. */ + uint8_t OptGroup; /* default Opt. grp. for this LMBCS session */ + uint8_t localeConverterIndex; /* reasonable locale match for index */ + } +UConverterDataLMBCS; + + +#define DECLARE_LMBCS_DATA(n) \ +static const UConverterImpl _LMBCSImpl##n={\ + UCNV_LMBCS_##n,\ + NULL,NULL,\ + _LMBCSOpen##n,\ + _LMBCSClose,\ + NULL,\ + _LMBCSToUnicodeWithOffsets,\ + _LMBCSToUnicodeWithOffsets,\ + _LMBCSFromUnicode,\ + _LMBCSFromUnicode,\ + NULL,\ + NULL,\ + NULL,\ + NULL,\ + _LMBCSSafeClone,\ + _LMBCSGetUnicodeSet\ +};\ +static const UConverterStaticData _LMBCSStaticData##n={\ + sizeof(UConverterStaticData),\ + "LMBCS-" #n,\ + 0, UCNV_IBM, UCNV_LMBCS_##n, 1, 3,\ + { 0x3f, 0, 0, 0 },1,FALSE,FALSE,0,0,{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0} \ +};\ +const UConverterSharedData _LMBCSData##n={\ + sizeof(UConverterSharedData), ~((uint32_t) 0),\ + NULL, NULL, &_LMBCSStaticData##n, FALSE, &_LMBCSImpl##n, \ + 0 \ +}; + + /* The only function we needed to duplicate 12 times was the 'open' +function, which will do basically the same thing except set a different +optimization group. So, we put the common stuff into a worker function, +and set up another macro to stamp out the 12 open functions:*/ +#define DEFINE_LMBCS_OPEN(n) \ +static void \ + _LMBCSOpen##n(UConverter* _this,const char* name,const char* locale,uint32_t options,UErrorCode* err) \ +{ _LMBCSOpenWorker(_this, name,locale,options, err, n);} + + + +/* Here's the open worker & the common close function */ +static void +_LMBCSOpenWorker(UConverter* _this, + const char* name, + const char* locale, + uint32_t options, + UErrorCode* err, + ulmbcs_byte_t OptGroup + ) +{ + UConverterDataLMBCS * extraInfo = (UConverterDataLMBCS*)uprv_malloc (sizeof (UConverterDataLMBCS)); + if(extraInfo != NULL) + { + ulmbcs_byte_t i; + + uprv_memset(extraInfo, 0, sizeof(UConverterDataLMBCS)); + + for (i=0; i <= ULMBCS_GRP_LAST && U_SUCCESS(*err); i++) + { + if(OptGroupByteToCPName[i] != NULL) { + extraInfo->OptGrpConverter[i] = ucnv_loadSharedData(OptGroupByteToCPName[i], NULL, err); + } + } + + if(U_SUCCESS(*err)) { + extraInfo->OptGroup = OptGroup; + extraInfo->localeConverterIndex = FindLMBCSLocale(locale); + } else { + /* one of the subconverters could not be loaded, unload the previous ones */ + while(i > 0) { + if(extraInfo->OptGrpConverter[--i] != NULL) { + ucnv_unloadSharedDataIfReady(extraInfo->OptGrpConverter[i]); + extraInfo->OptGrpConverter[i] = NULL; + } + } + } + } + else + { + *err = U_MEMORY_ALLOCATION_ERROR; + } + _this->extraInfo = extraInfo; +} + +static void +_LMBCSClose(UConverter * _this) +{ + if (_this->extraInfo != NULL) + { + ulmbcs_byte_t Ix; + UConverterDataLMBCS * extraInfo = (UConverterDataLMBCS *) _this->extraInfo; + + for (Ix=0; Ix <= ULMBCS_GRP_LAST; Ix++) + { + if (extraInfo->OptGrpConverter[Ix] != NULL) + ucnv_unloadSharedDataIfReady(extraInfo->OptGrpConverter[Ix]); + } + if (!_this->isExtraLocal) { + uprv_free (_this->extraInfo); + } + } +} + +typedef struct LMBCSClone { + UConverter cnv; + UConverterDataLMBCS lmbcs; +} LMBCSClone; + +static UConverter * +_LMBCSSafeClone(const UConverter *cnv, + void *stackBuffer, + int32_t *pBufferSize, + UErrorCode *status) { + LMBCSClone *newLMBCS; + UConverterDataLMBCS *extraInfo; + int32_t i; + + if(*pBufferSize<=0) { + *pBufferSize=(int32_t)sizeof(LMBCSClone); + return NULL; + } + + extraInfo=(UConverterDataLMBCS *)cnv->extraInfo; + newLMBCS=(LMBCSClone *)stackBuffer; + + /* ucnv.c/ucnv_safeClone() copied the main UConverter already */ + + uprv_memcpy(&newLMBCS->lmbcs, extraInfo, sizeof(UConverterDataLMBCS)); + + /* share the subconverters */ + for(i = 0; i <= ULMBCS_GRP_LAST; ++i) { + if(extraInfo->OptGrpConverter[i] != NULL) { + ucnv_incrementRefCount(extraInfo->OptGrpConverter[i]); + } + } + + newLMBCS->cnv.extraInfo = &newLMBCS->lmbcs; + newLMBCS->cnv.isExtraLocal = TRUE; + return &newLMBCS->cnv; +} + +static void +_LMBCSGetUnicodeSet(const UConverter *cnv, + const USetAdder *sa, + UConverterUnicodeSet which, + UErrorCode *pErrorCode) { + /* all but U+F6xx, see LMBCS explanation above (search for F6xx) */ + sa->addRange(sa->set, 0, 0xf5ff); + sa->addRange(sa->set, 0xf700, 0x10ffff); +} + +/* + Here's the basic helper function that we use when converting from + Unicode to LMBCS, and we suspect that a Unicode character will fit into + one of the 12 groups. The return value is the number of bytes written + starting at pStartLMBCS (if any). +*/ + +static size_t +LMBCSConversionWorker ( + UConverterDataLMBCS * extraInfo, /* subconverters, opt & locale groups */ + ulmbcs_byte_t group, /* The group to try */ + ulmbcs_byte_t * pStartLMBCS, /* where to put the results */ + UChar * pUniChar, /* The input unicode character */ + ulmbcs_byte_t * lastConverterIndex, /* output: track last successful group used */ + UBool * groups_tried /* output: track any unsuccessful groups */ +) +{ + ulmbcs_byte_t * pLMBCS = pStartLMBCS; + UConverterSharedData * xcnv = extraInfo->OptGrpConverter[group]; + + int bytesConverted; + uint32_t value; + ulmbcs_byte_t firstByte; + + U_ASSERT(xcnv); + U_ASSERT(group 0) { + firstByte = (ulmbcs_byte_t)(value >> ((bytesConverted - 1) * 8)); + } else { + /* most common failure mode is an unassigned character */ + groups_tried[group] = TRUE; + return 0; + } + + *lastConverterIndex = group; + + /* All initial byte values in lower ascii range should have been caught by now, + except with the exception group. + */ + U_ASSERT((firstByte <= ULMBCS_C0END) || (firstByte >= ULMBCS_C1START) || (group == ULMBCS_GRP_EXCEPT)); + + /* use converted data: first write 0, 1 or two group bytes */ + if (group != ULMBCS_GRP_EXCEPT && extraInfo->OptGroup != group) + { + *pLMBCS++ = group; + if (bytesConverted == 1 && group >= ULMBCS_DOUBLEOPTGROUP_START) + { + *pLMBCS++ = group; + } + } + + /* don't emit control chars */ + if ( bytesConverted == 1 && firstByte < 0x20 ) + return 0; + + + /* then move over the converted data */ + switch(bytesConverted) + { + case 4: + *pLMBCS++ = (ulmbcs_byte_t)(value >> 24); + case 3: + *pLMBCS++ = (ulmbcs_byte_t)(value >> 16); + case 2: + *pLMBCS++ = (ulmbcs_byte_t)(value >> 8); + case 1: + *pLMBCS++ = (ulmbcs_byte_t)value; + default: + /* will never occur */ + break; + } + + return (pLMBCS - pStartLMBCS); +} + + +/* This is a much simpler version of above, when we +know we are writing LMBCS using the Unicode group +*/ +static size_t +LMBCSConvertUni(ulmbcs_byte_t * pLMBCS, UChar uniChar) +{ + /* encode into LMBCS Unicode range */ + uint8_t LowCh = (uint8_t)(uniChar & 0x00FF); + uint8_t HighCh = (uint8_t)(uniChar >> 8); + + *pLMBCS++ = ULMBCS_GRP_UNICODE; + + if (LowCh == 0) + { + *pLMBCS++ = ULMBCS_UNICOMPATZERO; + *pLMBCS++ = HighCh; + } + else + { + *pLMBCS++ = HighCh; + *pLMBCS++ = LowCh; + } + return ULMBCS_UNICODE_SIZE; +} + + + +/* The main Unicode to LMBCS conversion function */ +static void +_LMBCSFromUnicode(UConverterFromUnicodeArgs* args, + UErrorCode* err) +{ + ulmbcs_byte_t lastConverterIndex = 0; + UChar uniChar; + ulmbcs_byte_t LMBCS[ULMBCS_CHARSIZE_MAX]; + ulmbcs_byte_t * pLMBCS; + int bytes_written; + UBool groups_tried[ULMBCS_GRP_LAST+1]; + UConverterDataLMBCS * extraInfo = (UConverterDataLMBCS *) args->converter->extraInfo; + int sourceIndex = 0; + + + /* Basic strategy: attempt to fill in local LMBCS 1-char buffer.(LMBCS) + If that succeeds, see if it will all fit into the target & copy it over + if it does. + + We try conversions in the following order: + + 1. Single-byte ascii & special fixed control chars (&null) + 2. Look up group in table & try that (could be + A) Unicode group + B) control group, + C) national encoding, + or ambiguous SBCS or MBCS group (on to step 4...) + + 3. If its ambiguous, try this order: + A) The optimization group + B) The locale group + C) The last group that succeeded with this string. + D) every other group that's relevent (single or double) + E) If its single-byte ambiguous, try the exceptions group + + 4. And as a grand fallback: Unicode + */ + + while (args->source < args->sourceLimit && !U_FAILURE(*err)) + { + if (args->target >= args->targetLimit) + { + *err = U_BUFFER_OVERFLOW_ERROR; + break; + } + uniChar = *(args->source); + bytes_written = 0; + pLMBCS = LMBCS; + + /* check cases in rough order of how common they are, for speed */ + + /* single byte matches: strategy 1 */ + + if (((uniChar > ULMBCS_C0END) && (uniChar < ULMBCS_C1START)) || + uniChar == 0 || uniChar == ULMBCS_HT || uniChar == ULMBCS_CR || + uniChar == ULMBCS_LF || uniChar == ULMBCS_123SYSTEMRANGE + ) + { + *pLMBCS++ = (ulmbcs_byte_t ) uniChar; + bytes_written = 1; + } + + + if (!bytes_written) + { + /* Check by UNICODE range (Strategy 2) */ + ulmbcs_byte_t group = FindLMBCSUniRange(uniChar); + + if (group == ULMBCS_GRP_UNICODE) /* (Strategy 2A) */ + { + pLMBCS += LMBCSConvertUni(pLMBCS,uniChar); + + bytes_written = pLMBCS - LMBCS; + } + else if (group == ULMBCS_GRP_CTRL) /* (Strategy 2B) */ + { + /* Handle control characters here */ + if (uniChar <= ULMBCS_C0END) + { + *pLMBCS++ = ULMBCS_GRP_CTRL; + *pLMBCS++ = (ulmbcs_byte_t)(ULMBCS_CTRLOFFSET + uniChar); + } + else if (uniChar >= ULMBCS_C1START && uniChar <= ULMBCS_C1START + ULMBCS_CTRLOFFSET) + { + *pLMBCS++ = ULMBCS_GRP_CTRL; + *pLMBCS++ = (ulmbcs_byte_t ) (uniChar & 0x00FF); + } + bytes_written = pLMBCS - LMBCS; + } + else if (group < ULMBCS_GRP_UNICODE) /* (Strategy 2C) */ + { + /* a specific converter has been identified - use it */ + bytes_written = LMBCSConversionWorker ( + extraInfo, group, pLMBCS, &uniChar, + &lastConverterIndex, groups_tried); + } + if (!bytes_written) /* the ambiguous group cases (Strategy 3) */ + { + uprv_memset(groups_tried, 0, sizeof(groups_tried)); + + /* check for non-default optimization group (Strategy 3A )*/ + if (extraInfo->OptGroup != 1 + && ULMBCS_AMBIGUOUS_MATCH(group, extraInfo->OptGroup)) + { + bytes_written = LMBCSConversionWorker (extraInfo, + extraInfo->OptGroup, pLMBCS, &uniChar, + &lastConverterIndex, groups_tried); + } + /* check for locale optimization group (Strategy 3B) */ + if (!bytes_written + && (extraInfo->localeConverterIndex) + && (ULMBCS_AMBIGUOUS_MATCH(group, extraInfo->localeConverterIndex))) + { + bytes_written = LMBCSConversionWorker (extraInfo, + extraInfo->localeConverterIndex, pLMBCS, &uniChar, + &lastConverterIndex, groups_tried); + } + /* check for last optimization group used for this string (Strategy 3C) */ + if (!bytes_written + && (lastConverterIndex) + && (ULMBCS_AMBIGUOUS_MATCH(group, lastConverterIndex))) + { + bytes_written = LMBCSConversionWorker (extraInfo, + lastConverterIndex, pLMBCS, &uniChar, + &lastConverterIndex, groups_tried); + + } + if (!bytes_written) + { + /* just check every possible matching converter (Strategy 3D) */ + ulmbcs_byte_t grp_start; + ulmbcs_byte_t grp_end; + ulmbcs_byte_t grp_ix; + grp_start = (ulmbcs_byte_t)((group == ULMBCS_AMBIGUOUS_MBCS) + ? ULMBCS_DOUBLEOPTGROUP_START + : ULMBCS_GRP_L1); + grp_end = (ulmbcs_byte_t)((group == ULMBCS_AMBIGUOUS_MBCS) + ? ULMBCS_GRP_LAST + : ULMBCS_GRP_TH); + for (grp_ix = grp_start; + grp_ix <= grp_end && !bytes_written; + grp_ix++) + { + if (extraInfo->OptGrpConverter [grp_ix] && !groups_tried [grp_ix]) + { + bytes_written = LMBCSConversionWorker (extraInfo, + grp_ix, pLMBCS, &uniChar, + &lastConverterIndex, groups_tried); + } + } + /* a final conversion fallback to the exceptions group if its likely + to be single byte (Strategy 3E) */ + if (!bytes_written && grp_start == ULMBCS_GRP_L1) + { + bytes_written = LMBCSConversionWorker (extraInfo, + ULMBCS_GRP_EXCEPT, pLMBCS, &uniChar, + &lastConverterIndex, groups_tried); + } + } + /* all of our other strategies failed. Fallback to Unicode. (Strategy 4)*/ + if (!bytes_written) + { + + pLMBCS += LMBCSConvertUni(pLMBCS, uniChar); + bytes_written = pLMBCS - LMBCS; + } + } + } + + /* we have a translation. increment source and write as much as posible to target */ + args->source++; + pLMBCS = LMBCS; + while (args->target < args->targetLimit && bytes_written--) + { + *(args->target)++ = *pLMBCS++; + if (args->offsets) + { + *(args->offsets)++ = sourceIndex; + } + } + sourceIndex++; + if (bytes_written > 0) + { + /* write any bytes that didn't fit in target to the error buffer, + common code will move this to target if we get called back with + enough target room + */ + uint8_t * pErrorBuffer = args->converter->charErrorBuffer; + *err = U_BUFFER_OVERFLOW_ERROR; + args->converter->charErrorBufferLength = (int8_t)bytes_written; + while (bytes_written--) + { + *pErrorBuffer++ = *pLMBCS++; + } + } + } +} + + +/* Now, the Unicode from LMBCS section */ + + +/* A function to call when we are looking at the Unicode group byte in LMBCS */ +static UChar +GetUniFromLMBCSUni(char const ** ppLMBCSin) /* Called with LMBCS-style Unicode byte stream */ +{ + uint8_t HighCh = *(*ppLMBCSin)++; /* Big-endian Unicode in LMBCS compatibility group*/ + uint8_t LowCh = *(*ppLMBCSin)++; + + if (HighCh == ULMBCS_UNICOMPATZERO ) + { + HighCh = LowCh; + LowCh = 0; /* zero-byte in LSB special character */ + } + return (UChar)((HighCh << 8) | LowCh); +} + + + +/* CHECK_SOURCE_LIMIT: Helper macro to verify that there are at least'index' + bytes left in source up to sourceLimit.Errors appropriately if not. + If we reach the limit, then update the source pointer to there to consume + all input as required by ICU converter semantics. +*/ + +#define CHECK_SOURCE_LIMIT(index) \ + if (args->source+index > args->sourceLimit){\ + *err = U_TRUNCATED_CHAR_FOUND;\ + args->source = args->sourceLimit;\ + return 0xffff;} + +/* Return the Unicode representation for the current LMBCS character */ + +static UChar32 +_LMBCSGetNextUCharWorker(UConverterToUnicodeArgs* args, + UErrorCode* err) +{ + UChar32 uniChar = 0; /* an output UNICODE char */ + ulmbcs_byte_t CurByte; /* A byte from the input stream */ + + /* error check */ + if (args->source >= args->sourceLimit) + { + *err = U_ILLEGAL_ARGUMENT_ERROR; + return 0xffff; + } + /* Grab first byte & save address for error recovery */ + CurByte = *((ulmbcs_byte_t *) (args->source++)); + + /* + * at entry of each if clause: + * 1. 'CurByte' points at the first byte of a LMBCS character + * 2. '*source'points to the next byte of the source stream after 'CurByte' + * + * the job of each if clause is: + * 1. set '*source' to point at the beginning of next char (nop if LMBCS char is only 1 byte) + * 2. set 'uniChar' up with the right Unicode value, or set 'err' appropriately + */ + + /* First lets check the simple fixed values. */ + + if(((CurByte > ULMBCS_C0END) && (CurByte < ULMBCS_C1START)) /* ascii range */ + || (CurByte == 0) + || CurByte == ULMBCS_HT || CurByte == ULMBCS_CR + || CurByte == ULMBCS_LF || CurByte == ULMBCS_123SYSTEMRANGE) + { + uniChar = CurByte; + } + else + { + UConverterDataLMBCS * extraInfo; + ulmbcs_byte_t group; + UConverterSharedData *cnv; + + if (CurByte == ULMBCS_GRP_CTRL) /* Control character group - no opt group update */ + { + ulmbcs_byte_t C0C1byte; + CHECK_SOURCE_LIMIT(1); + C0C1byte = *(args->source)++; + uniChar = (C0C1byte < ULMBCS_C1START) ? C0C1byte - ULMBCS_CTRLOFFSET : C0C1byte; + } + else + if (CurByte == ULMBCS_GRP_UNICODE) /* Unicode compatibility group: BigEndian UTF16 */ + { + CHECK_SOURCE_LIMIT(2); + + /* don't check for error indicators fffe/ffff below */ + return GetUniFromLMBCSUni(&(args->source)); + } + else if (CurByte <= ULMBCS_CTRLOFFSET) + { + group = CurByte; /* group byte is in the source */ + extraInfo = (UConverterDataLMBCS *) args->converter->extraInfo; + if (group > ULMBCS_GRP_LAST || (cnv = extraInfo->OptGrpConverter[group]) == NULL) + { + /* this is not a valid group byte - no converter*/ + *err = U_INVALID_CHAR_FOUND; + } + else if (group >= ULMBCS_DOUBLEOPTGROUP_START) /* double byte conversion */ + { + + CHECK_SOURCE_LIMIT(2); + + /* check for LMBCS doubled-group-byte case */ + if (*args->source == group) { + /* single byte */ + ++args->source; + uniChar = ucnv_MBCSSimpleGetNextUChar(cnv, args->source, 1, FALSE); + ++args->source; + } else { + /* double byte */ + uniChar = ucnv_MBCSSimpleGetNextUChar(cnv, args->source, 2, FALSE); + args->source += 2; + } + } + else { /* single byte conversion */ + CHECK_SOURCE_LIMIT(1); + CurByte = *(args->source)++; + + if (CurByte >= ULMBCS_C1START) + { + uniChar = _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP(cnv, CurByte); + } + else + { + /* The non-optimizable oddballs where there is an explicit byte + * AND the second byte is not in the upper ascii range + */ + char bytes[2]; + + extraInfo = (UConverterDataLMBCS *) args->converter->extraInfo; + cnv = extraInfo->OptGrpConverter [ULMBCS_GRP_EXCEPT]; + + /* Lookup value must include opt group */ + bytes[0] = group; + bytes[1] = CurByte; + uniChar = ucnv_MBCSSimpleGetNextUChar(cnv, bytes, 2, FALSE); + } + } + } + else if (CurByte >= ULMBCS_C1START) /* group byte is implicit */ + { + extraInfo = (UConverterDataLMBCS *) args->converter->extraInfo; + group = extraInfo->OptGroup; + cnv = extraInfo->OptGrpConverter[group]; + if (group >= ULMBCS_DOUBLEOPTGROUP_START) /* double byte conversion */ + { + if (!ucnv_MBCSIsLeadByte(cnv, CurByte)) + { + CHECK_SOURCE_LIMIT(0); + + /* let the MBCS conversion consume CurByte again */ + uniChar = ucnv_MBCSSimpleGetNextUChar(cnv, args->source - 1, 1, FALSE); + } + else + { + CHECK_SOURCE_LIMIT(1); + /* let the MBCS conversion consume CurByte again */ + uniChar = ucnv_MBCSSimpleGetNextUChar(cnv, args->source - 1, 2, FALSE); + ++args->source; + } + } + else /* single byte conversion */ + { + uniChar = _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP(cnv, CurByte); + } + } + } + return uniChar; +} + + +/* The exported function that converts lmbcs to one or more + UChars - currently UTF-16 +*/ +static void +_LMBCSToUnicodeWithOffsets(UConverterToUnicodeArgs* args, + UErrorCode* err) +{ + char LMBCS [ULMBCS_CHARSIZE_MAX]; + UChar uniChar; /* one output UNICODE char */ + const char * saveSource; /* beginning of current code point */ + const char * pStartLMBCS = args->source; /* beginning of whole string */ + const char * errSource = NULL; /* pointer to actual input in case an error occurs */ + int8_t savebytes = 0; + + /* Process from source to limit, or until error */ + while (U_SUCCESS(*err) && args->sourceLimit > args->source && args->targetLimit > args->target) + { + saveSource = args->source; /* beginning of current code point */ + + if (args->converter->toULength) /* reassemble char from previous call */ + { + const char *saveSourceLimit; + size_t size_old = args->converter->toULength; + + /* limit from source is either remainder of temp buffer, or user limit on source */ + size_t size_new_maybe_1 = sizeof(LMBCS) - size_old; + size_t size_new_maybe_2 = args->sourceLimit - args->source; + size_t size_new = (size_new_maybe_1 < size_new_maybe_2) ? size_new_maybe_1 : size_new_maybe_2; + + + uprv_memcpy(LMBCS, args->converter->toUBytes, size_old); + uprv_memcpy(LMBCS + size_old, args->source, size_new); + saveSourceLimit = args->sourceLimit; + args->source = errSource = LMBCS; + args->sourceLimit = LMBCS+size_old+size_new; + savebytes = (int8_t)(size_old+size_new); + uniChar = (UChar) _LMBCSGetNextUCharWorker(args, err); + args->source = saveSource + ((args->source - LMBCS) - size_old); + args->sourceLimit = saveSourceLimit; + + if (*err == U_TRUNCATED_CHAR_FOUND) + { + /* evil special case: source buffers so small a char spans more than 2 buffers */ + args->converter->toULength = savebytes; + uprv_memcpy(args->converter->toUBytes, LMBCS, savebytes); + args->source = args->sourceLimit; + *err = U_ZERO_ERROR; + return; + } + else + { + /* clear the partial-char marker */ + args->converter->toULength = 0; + } + } + else + { + errSource = saveSource; + uniChar = (UChar) _LMBCSGetNextUCharWorker(args, err); + savebytes = (int8_t)(args->source - saveSource); + } + if (U_SUCCESS(*err)) + { + if (uniChar < 0xfffe) + { + *(args->target)++ = uniChar; + if(args->offsets) + { + *(args->offsets)++ = saveSource - pStartLMBCS; + } + } + else if (uniChar == 0xfffe) + { + *err = U_INVALID_CHAR_FOUND; + } + else /* if (uniChar == 0xffff) */ + { + *err = U_ILLEGAL_CHAR_FOUND; + } + } + } + /* if target ran out before source, return U_BUFFER_OVERFLOW_ERROR */ + if (U_SUCCESS(*err) && args->sourceLimit > args->source && args->targetLimit <= args->target) + { + *err = U_BUFFER_OVERFLOW_ERROR; + } + else if (U_FAILURE(*err)) + { + /* If character incomplete or unmappable/illegal, store it in toUBytes[] */ + args->converter->toULength = savebytes; + if (savebytes > 0) { + uprv_memcpy(args->converter->toUBytes, errSource, savebytes); + } + if (*err == U_TRUNCATED_CHAR_FOUND) { + *err = U_ZERO_ERROR; + } + } +} + +/* And now, the macroized declarations of data & functions: */ +DEFINE_LMBCS_OPEN(1) +DEFINE_LMBCS_OPEN(2) +DEFINE_LMBCS_OPEN(3) +DEFINE_LMBCS_OPEN(4) +DEFINE_LMBCS_OPEN(5) +DEFINE_LMBCS_OPEN(6) +DEFINE_LMBCS_OPEN(8) +DEFINE_LMBCS_OPEN(11) +DEFINE_LMBCS_OPEN(16) +DEFINE_LMBCS_OPEN(17) +DEFINE_LMBCS_OPEN(18) +DEFINE_LMBCS_OPEN(19) + + +DECLARE_LMBCS_DATA(1) +DECLARE_LMBCS_DATA(2) +DECLARE_LMBCS_DATA(3) +DECLARE_LMBCS_DATA(4) +DECLARE_LMBCS_DATA(5) +DECLARE_LMBCS_DATA(6) +DECLARE_LMBCS_DATA(8) +DECLARE_LMBCS_DATA(11) +DECLARE_LMBCS_DATA(16) +DECLARE_LMBCS_DATA(17) +DECLARE_LMBCS_DATA(18) +DECLARE_LMBCS_DATA(19) + +#endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */ diff --git a/Build/source/libs/icu/icu-xetex/common/ucnv_set.c b/Build/source/libs/icu/icu-xetex/common/ucnv_set.c new file mode 100644 index 00000000000..772e7ed8176 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnv_set.c @@ -0,0 +1,67 @@ +/* +******************************************************************************* +* +* Copyright (C) 2003-2005, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: ucnv_set.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2004sep07 +* created by: Markus W. Scherer +* +* Conversion API functions using USet (ucnv_getUnicodeSet()) +* moved here from ucnv.c for removing the dependency of other ucnv_ +* implementation functions on the USet implementation. +*/ + +#include "unicode/utypes.h" +#include "unicode/uset.h" +#include "unicode/ucnv.h" +#include "ucnv_bld.h" +#include "uset_imp.h" + +#if !UCONFIG_NO_CONVERSION + +U_CAPI void U_EXPORT2 +ucnv_getUnicodeSet(const UConverter *cnv, + USet *setFillIn, + UConverterUnicodeSet whichSet, + UErrorCode *pErrorCode) { + /* argument checking */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return; + } + if(cnv==NULL || setFillIn==NULL || whichSetsharedData->impl->getUnicodeSet==NULL) { + *pErrorCode=U_UNSUPPORTED_ERROR; + return; + } + + { + USetAdder sa={ + NULL, + uset_add, + uset_addRange, + uset_addString, + uset_remove + }; + sa.set=setFillIn; + + /* empty the set */ + uset_clear(setFillIn); + + /* call the converter to add the code points it supports */ + cnv->sharedData->impl->getUnicodeSet(cnv, &sa, whichSet, pErrorCode); + } +} + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/ucnv_u16.c b/Build/source/libs/icu/icu-xetex/common/ucnv_u16.c new file mode 100644 index 00000000000..6fc9e2a4493 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnv_u16.c @@ -0,0 +1,1372 @@ +/* +********************************************************************** +* Copyright (C) 2002-2006, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* file name: ucnv_u16.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2002jul01 +* created by: Markus W. Scherer +* +* UTF-16 converter implementation. Used to be in ucnv_utf.c. +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "unicode/ucnv.h" +#include "ucnv_bld.h" +#include "ucnv_cnv.h" +#include "cmemory.h" + +enum { + UCNV_NEED_TO_WRITE_BOM=1 +}; + +/* UTF-16BE ----------------------------------------------------------------- */ + +#if U_IS_BIG_ENDIAN +# define _UTF16PEFromUnicodeWithOffsets _UTF16BEFromUnicodeWithOffsets +#else +# define _UTF16PEFromUnicodeWithOffsets _UTF16LEFromUnicodeWithOffsets +#endif + + +static void +_UTF16BEFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv; + const UChar *source; + char *target; + int32_t *offsets; + + uint32_t targetCapacity, length, sourceIndex; + UChar c, trail; + char overflow[4]; + + source=pArgs->source; + length=(int32_t)(pArgs->sourceLimit-source); + if(length<=0) { + /* no input, nothing to do */ + return; + } + + cnv=pArgs->converter; + + /* write the BOM if necessary */ + if(cnv->fromUnicodeStatus==UCNV_NEED_TO_WRITE_BOM) { + static const char bom[]={ (char)0xfe, (char)0xff }; + ucnv_fromUWriteBytes(cnv, + bom, 2, + &pArgs->target, pArgs->targetLimit, + &pArgs->offsets, -1, + pErrorCode); + cnv->fromUnicodeStatus=0; + } + + target=pArgs->target; + if(target >= pArgs->targetLimit) { + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + return; + } + + targetCapacity=(uint32_t)(pArgs->targetLimit-target); + offsets=pArgs->offsets; + sourceIndex=0; + + /* c!=0 indicates in several places outside the main loops that a surrogate was found */ + + if((c=(UChar)cnv->fromUChar32)!=0 && U16_IS_TRAIL(trail=*source) && targetCapacity>=4) { + /* the last buffer ended with a lead surrogate, output the surrogate pair */ + ++source; + --length; + target[0]=(uint8_t)(c>>8); + target[1]=(uint8_t)c; + target[2]=(uint8_t)(trail>>8); + target[3]=(uint8_t)trail; + target+=4; + targetCapacity-=4; + if(offsets!=NULL) { + *offsets++=-1; + *offsets++=-1; + *offsets++=-1; + *offsets++=-1; + } + sourceIndex=1; + cnv->fromUChar32=c=0; + } + + if(c==0) { + /* copy an even number of bytes for complete UChars */ + uint32_t count=2*length; + if(count>targetCapacity) { + count=targetCapacity&~1; + } + /* count is even */ + targetCapacity-=count; + count>>=1; + length-=count; + + if(offsets==NULL) { + while(count>0) { + c=*source++; + if(U16_IS_SINGLE(c)) { + target[0]=(uint8_t)(c>>8); + target[1]=(uint8_t)c; + target+=2; + } else if(U16_IS_SURROGATE_LEAD(c) && count>=2 && U16_IS_TRAIL(trail=*source)) { + ++source; + --count; + target[0]=(uint8_t)(c>>8); + target[1]=(uint8_t)c; + target[2]=(uint8_t)(trail>>8); + target[3]=(uint8_t)trail; + target+=4; + } else { + break; + } + --count; + } + } else { + while(count>0) { + c=*source++; + if(U16_IS_SINGLE(c)) { + target[0]=(uint8_t)(c>>8); + target[1]=(uint8_t)c; + target+=2; + *offsets++=sourceIndex; + *offsets++=sourceIndex++; + } else if(U16_IS_SURROGATE_LEAD(c) && count>=2 && U16_IS_TRAIL(trail=*source)) { + ++source; + --count; + target[0]=(uint8_t)(c>>8); + target[1]=(uint8_t)c; + target[2]=(uint8_t)(trail>>8); + target[3]=(uint8_t)trail; + target+=4; + *offsets++=sourceIndex; + *offsets++=sourceIndex; + *offsets++=sourceIndex; + *offsets++=sourceIndex; + sourceIndex+=2; + } else { + break; + } + --count; + } + } + + if(count==0) { + /* done with the loop for complete UChars */ + if(length>0 && targetCapacity>0) { + /* + * there is more input and some target capacity - + * it must be targetCapacity==1 because otherwise + * the above would have copied more; + * prepare for overflow output + */ + if(U16_IS_SINGLE(c=*source++)) { + overflow[0]=(char)(c>>8); + overflow[1]=(char)c; + length=2; /* 2 bytes to output */ + c=0; + /* } else { keep c for surrogate handling, length will be set there */ + } + } else { + length=0; + c=0; + } + } else { + /* keep c for surrogate handling, length will be set there */ + targetCapacity+=2*count; + } + } else { + length=0; /* from here on, length counts the bytes in overflow[] */ + } + + if(c!=0) { + /* + * c is a surrogate, and + * - source or target too short + * - or the surrogate is unmatched + */ + length=0; + if(U16_IS_SURROGATE_LEAD(c)) { + if(sourcesourceLimit) { + if(U16_IS_TRAIL(trail=*source)) { + /* output the surrogate pair, will overflow (see conditions comment above) */ + ++source; + overflow[0]=(char)(c>>8); + overflow[1]=(char)c; + overflow[2]=(char)(trail>>8); + overflow[3]=(char)trail; + length=4; /* 4 bytes to output */ + c=0; + } else { + /* unmatched lead surrogate */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + } + } else { + /* see if the trail surrogate is in the next buffer */ + } + } else { + /* unmatched trail surrogate */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + } + cnv->fromUChar32=c; + } + + if(length>0) { + /* output length bytes with overflow (length>targetCapacity>0) */ + ucnv_fromUWriteBytes(cnv, + overflow, length, + (char **)&target, pArgs->targetLimit, + &offsets, sourceIndex, + pErrorCode); + targetCapacity=(uint32_t)(pArgs->targetLimit-(char *)target); + } + + if(U_SUCCESS(*pErrorCode) && sourcesourceLimit && targetCapacity==0) { + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + + /* write back the updated pointers */ + pArgs->source=source; + pArgs->target=(char *)target; + pArgs->offsets=offsets; +} + +static void +_UTF16BEToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv; + const uint8_t *source; + UChar *target; + int32_t *offsets; + + uint32_t targetCapacity, length, count, sourceIndex; + UChar c, trail; + + cnv=pArgs->converter; + source=(const uint8_t *)pArgs->source; + length=(int32_t)((const uint8_t *)pArgs->sourceLimit-source); + if(length<=0 && cnv->toUnicodeStatus==0) { + /* no input, nothing to do */ + return; + } + + target=pArgs->target; + if(target >= pArgs->targetLimit) { + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + return; + } + + targetCapacity=(uint32_t)(pArgs->targetLimit-target); + offsets=pArgs->offsets; + sourceIndex=0; + c=0; + + /* complete a partial UChar or pair from the last call */ + if(cnv->toUnicodeStatus!=0) { + /* + * special case: single byte from a previous buffer, + * where the byte turned out not to belong to a trail surrogate + * and the preceding, unmatched lead surrogate was put into toUBytes[] + * for error handling + */ + cnv->toUBytes[0]=(uint8_t)cnv->toUnicodeStatus; + cnv->toULength=1; + cnv->toUnicodeStatus=0; + } + if((count=cnv->toULength)!=0) { + uint8_t *p=cnv->toUBytes; + do { + p[count++]=*source++; + ++sourceIndex; + --length; + if(count==2) { + c=((UChar)p[0]<<8)|p[1]; + if(U16_IS_SINGLE(c)) { + /* output the BMP code point */ + *target++=c; + if(offsets!=NULL) { + *offsets++=-1; + } + --targetCapacity; + count=0; + c=0; + break; + } else if(U16_IS_SURROGATE_LEAD(c)) { + /* continue collecting bytes for the trail surrogate */ + c=0; /* avoid unnecessary surrogate handling below */ + } else { + /* fall through to error handling for an unmatched trail surrogate */ + break; + } + } else if(count==4) { + c=((UChar)p[0]<<8)|p[1]; + trail=((UChar)p[2]<<8)|p[3]; + if(U16_IS_TRAIL(trail)) { + /* output the surrogate pair */ + *target++=c; + if(targetCapacity>=2) { + *target++=trail; + if(offsets!=NULL) { + *offsets++=-1; + *offsets++=-1; + } + targetCapacity-=2; + } else /* targetCapacity==1 */ { + targetCapacity=0; + cnv->UCharErrorBuffer[0]=trail; + cnv->UCharErrorBufferLength=1; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + count=0; + c=0; + break; + } else { + /* unmatched lead surrogate, handle here for consistent toUBytes[] */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + + /* back out reading the code unit after it */ + if(((const uint8_t *)pArgs->source-source)>=2) { + source-=2; + } else { + /* + * if the trail unit's first byte was in a previous buffer, then + * we need to put it into a special place because toUBytes[] will be + * used for the lead unit's bytes + */ + cnv->toUnicodeStatus=0x100|p[2]; + --source; + } + cnv->toULength=2; + + /* write back the updated pointers */ + pArgs->source=(const char *)source; + pArgs->target=target; + pArgs->offsets=offsets; + return; + } + } + } while(length>0); + cnv->toULength=(int8_t)count; + } + + /* copy an even number of bytes for complete UChars */ + count=2*targetCapacity; + if(count>length) { + count=length&~1; + } + if(c==0 && count>0) { + length-=count; + count>>=1; + targetCapacity-=count; + if(offsets==NULL) { + do { + c=((UChar)source[0]<<8)|source[1]; + source+=2; + if(U16_IS_SINGLE(c)) { + *target++=c; + } else if(U16_IS_SURROGATE_LEAD(c) && count>=2 && + U16_IS_TRAIL(trail=((UChar)source[0]<<8)|source[1]) + ) { + source+=2; + --count; + *target++=c; + *target++=trail; + } else { + break; + } + } while(--count>0); + } else { + do { + c=((UChar)source[0]<<8)|source[1]; + source+=2; + if(U16_IS_SINGLE(c)) { + *target++=c; + *offsets++=sourceIndex; + sourceIndex+=2; + } else if(U16_IS_SURROGATE_LEAD(c) && count>=2 && + U16_IS_TRAIL(trail=((UChar)source[0]<<8)|source[1]) + ) { + source+=2; + --count; + *target++=c; + *target++=trail; + *offsets++=sourceIndex; + *offsets++=sourceIndex; + sourceIndex+=4; + } else { + break; + } + } while(--count>0); + } + + if(count==0) { + /* done with the loop for complete UChars */ + c=0; + } else { + /* keep c for surrogate handling, trail will be set there */ + length+=2*(count-1); /* one more byte pair was consumed than count decremented */ + targetCapacity+=count; + } + } + + if(c!=0) { + /* + * c is a surrogate, and + * - source or target too short + * - or the surrogate is unmatched + */ + cnv->toUBytes[0]=(uint8_t)(c>>8); + cnv->toUBytes[1]=(uint8_t)c; + cnv->toULength=2; + + if(U16_IS_SURROGATE_LEAD(c)) { + if(length>=2) { + if(U16_IS_TRAIL(trail=((UChar)source[0]<<8)|source[1])) { + /* output the surrogate pair, will overflow (see conditions comment above) */ + source+=2; + length-=2; + *target++=c; + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + cnv->UCharErrorBuffer[0]=trail; + cnv->UCharErrorBufferLength=1; + cnv->toULength=0; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } else { + /* unmatched lead surrogate */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + } + } else { + /* see if the trail surrogate is in the next buffer */ + } + } else { + /* unmatched trail surrogate */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + } + } + + if(U_SUCCESS(*pErrorCode)) { + /* check for a remaining source byte */ + if(length>0) { + if(targetCapacity==0) { + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } else { + /* it must be length==1 because otherwise the above would have copied more */ + cnv->toUBytes[cnv->toULength++]=*source++; + } + } + } + + /* write back the updated pointers */ + pArgs->source=(const char *)source; + pArgs->target=target; + pArgs->offsets=offsets; +} + +static UChar32 +_UTF16BEGetNextUChar(UConverterToUnicodeArgs *pArgs, UErrorCode *err) { + const uint8_t *s, *sourceLimit; + UChar32 c; + + s=(const uint8_t *)pArgs->source; + sourceLimit=(const uint8_t *)pArgs->sourceLimit; + + if(s>=sourceLimit) { + /* no input */ + *err=U_INDEX_OUTOFBOUNDS_ERROR; + return 0xffff; + } + + if(s+2>sourceLimit) { + /* only one byte: truncated UChar */ + pArgs->converter->toUBytes[0]=*s++; + pArgs->converter->toULength=1; + pArgs->source=(const char *)s; + *err = U_TRUNCATED_CHAR_FOUND; + return 0xffff; + } + + /* get one UChar */ + c=((UChar32)*s<<8)|s[1]; + s+=2; + + /* check for a surrogate pair */ + if(U_IS_SURROGATE(c)) { + if(U16_IS_SURROGATE_LEAD(c)) { + if(s+2<=sourceLimit) { + UChar trail; + + /* get a second UChar and see if it is a trail surrogate */ + trail=((UChar)*s<<8)|s[1]; + if(U16_IS_TRAIL(trail)) { + c=U16_GET_SUPPLEMENTARY(c, trail); + s+=2; + } else { + /* unmatched lead surrogate */ + c=-2; + } + } else { + /* too few (2 or 3) bytes for a surrogate pair: truncated code point */ + uint8_t *bytes=pArgs->converter->toUBytes; + s-=2; + pArgs->converter->toULength=(int8_t)(sourceLimit-s); + do { + *bytes++=*s++; + } while(sconverter->toUBytes; + pArgs->converter->toULength=2; + *bytes=*(s-2); + bytes[1]=*(s-1); + + c=0xffff; + *err=U_ILLEGAL_CHAR_FOUND; + } + } + + pArgs->source=(const char *)s; + return c; +} + +static const UConverterImpl _UTF16BEImpl={ + UCNV_UTF16_BigEndian, + + NULL, + NULL, + + NULL, + NULL, + NULL, + + _UTF16BEToUnicodeWithOffsets, + _UTF16BEToUnicodeWithOffsets, + _UTF16BEFromUnicodeWithOffsets, + _UTF16BEFromUnicodeWithOffsets, + _UTF16BEGetNextUChar, + + NULL, + NULL, + NULL, + NULL, + ucnv_getNonSurrogateUnicodeSet +}; + +static const UConverterStaticData _UTF16BEStaticData={ + sizeof(UConverterStaticData), + "UTF-16BE", + 1200, UCNV_IBM, UCNV_UTF16_BigEndian, 2, 2, + { 0xff, 0xfd, 0, 0 },2,FALSE,FALSE, + 0, + 0, + { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */ +}; + + +const UConverterSharedData _UTF16BEData={ + sizeof(UConverterSharedData), ~((uint32_t) 0), + NULL, NULL, &_UTF16BEStaticData, FALSE, &_UTF16BEImpl, + 0 +}; + +/* UTF-16LE ----------------------------------------------------------------- */ + +static void +_UTF16LEFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv; + const UChar *source; + char *target; + int32_t *offsets; + + uint32_t targetCapacity, length, sourceIndex; + UChar c, trail; + char overflow[4]; + + source=pArgs->source; + length=(int32_t)(pArgs->sourceLimit-source); + if(length<=0) { + /* no input, nothing to do */ + return; + } + + cnv=pArgs->converter; + + /* write the BOM if necessary */ + if(cnv->fromUnicodeStatus==UCNV_NEED_TO_WRITE_BOM) { + static const char bom[]={ (char)0xff, (char)0xfe }; + ucnv_fromUWriteBytes(cnv, + bom, 2, + &pArgs->target, pArgs->targetLimit, + &pArgs->offsets, -1, + pErrorCode); + cnv->fromUnicodeStatus=0; + } + + target=pArgs->target; + if(target >= pArgs->targetLimit) { + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + return; + } + + targetCapacity=(uint32_t)(pArgs->targetLimit-pArgs->target); + offsets=pArgs->offsets; + sourceIndex=0; + + /* c!=0 indicates in several places outside the main loops that a surrogate was found */ + + if((c=(UChar)cnv->fromUChar32)!=0 && U16_IS_TRAIL(trail=*source) && targetCapacity>=4) { + /* the last buffer ended with a lead surrogate, output the surrogate pair */ + ++source; + --length; + target[0]=(uint8_t)c; + target[1]=(uint8_t)(c>>8); + target[2]=(uint8_t)trail; + target[3]=(uint8_t)(trail>>8); + target+=4; + targetCapacity-=4; + if(offsets!=NULL) { + *offsets++=-1; + *offsets++=-1; + *offsets++=-1; + *offsets++=-1; + } + sourceIndex=1; + cnv->fromUChar32=c=0; + } + + if(c==0) { + /* copy an even number of bytes for complete UChars */ + uint32_t count=2*length; + if(count>targetCapacity) { + count=targetCapacity&~1; + } + /* count is even */ + targetCapacity-=count; + count>>=1; + length-=count; + + if(offsets==NULL) { + while(count>0) { + c=*source++; + if(U16_IS_SINGLE(c)) { + target[0]=(uint8_t)c; + target[1]=(uint8_t)(c>>8); + target+=2; + } else if(U16_IS_SURROGATE_LEAD(c) && count>=2 && U16_IS_TRAIL(trail=*source)) { + ++source; + --count; + target[0]=(uint8_t)c; + target[1]=(uint8_t)(c>>8); + target[2]=(uint8_t)trail; + target[3]=(uint8_t)(trail>>8); + target+=4; + } else { + break; + } + --count; + } + } else { + while(count>0) { + c=*source++; + if(U16_IS_SINGLE(c)) { + target[0]=(uint8_t)c; + target[1]=(uint8_t)(c>>8); + target+=2; + *offsets++=sourceIndex; + *offsets++=sourceIndex++; + } else if(U16_IS_SURROGATE_LEAD(c) && count>=2 && U16_IS_TRAIL(trail=*source)) { + ++source; + --count; + target[0]=(uint8_t)c; + target[1]=(uint8_t)(c>>8); + target[2]=(uint8_t)trail; + target[3]=(uint8_t)(trail>>8); + target+=4; + *offsets++=sourceIndex; + *offsets++=sourceIndex; + *offsets++=sourceIndex; + *offsets++=sourceIndex; + sourceIndex+=2; + } else { + break; + } + --count; + } + } + + if(count==0) { + /* done with the loop for complete UChars */ + if(length>0 && targetCapacity>0) { + /* + * there is more input and some target capacity - + * it must be targetCapacity==1 because otherwise + * the above would have copied more; + * prepare for overflow output + */ + if(U16_IS_SINGLE(c=*source++)) { + overflow[0]=(char)c; + overflow[1]=(char)(c>>8); + length=2; /* 2 bytes to output */ + c=0; + /* } else { keep c for surrogate handling, length will be set there */ + } + } else { + length=0; + c=0; + } + } else { + /* keep c for surrogate handling, length will be set there */ + targetCapacity+=2*count; + } + } else { + length=0; /* from here on, length counts the bytes in overflow[] */ + } + + if(c!=0) { + /* + * c is a surrogate, and + * - source or target too short + * - or the surrogate is unmatched + */ + length=0; + if(U16_IS_SURROGATE_LEAD(c)) { + if(sourcesourceLimit) { + if(U16_IS_TRAIL(trail=*source)) { + /* output the surrogate pair, will overflow (see conditions comment above) */ + ++source; + overflow[0]=(char)c; + overflow[1]=(char)(c>>8); + overflow[2]=(char)trail; + overflow[3]=(char)(trail>>8); + length=4; /* 4 bytes to output */ + c=0; + } else { + /* unmatched lead surrogate */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + } + } else { + /* see if the trail surrogate is in the next buffer */ + } + } else { + /* unmatched trail surrogate */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + } + cnv->fromUChar32=c; + } + + if(length>0) { + /* output length bytes with overflow (length>targetCapacity>0) */ + ucnv_fromUWriteBytes(cnv, + overflow, length, + &target, pArgs->targetLimit, + &offsets, sourceIndex, + pErrorCode); + targetCapacity=(uint32_t)(pArgs->targetLimit-(char *)target); + } + + if(U_SUCCESS(*pErrorCode) && sourcesourceLimit && targetCapacity==0) { + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + + /* write back the updated pointers */ + pArgs->source=source; + pArgs->target=target; + pArgs->offsets=offsets; +} + +static void +_UTF16LEToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv; + const uint8_t *source; + UChar *target; + int32_t *offsets; + + uint32_t targetCapacity, length, count, sourceIndex; + UChar c, trail; + + cnv=pArgs->converter; + source=(const uint8_t *)pArgs->source; + length=(int32_t)((const uint8_t *)pArgs->sourceLimit-source); + if(length<=0 && cnv->toUnicodeStatus==0) { + /* no input, nothing to do */ + return; + } + + target=pArgs->target; + if(target >= pArgs->targetLimit) { + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + return; + } + + targetCapacity=(uint32_t)(pArgs->targetLimit-pArgs->target); + offsets=pArgs->offsets; + sourceIndex=0; + c=0; + + /* complete a partial UChar or pair from the last call */ + if(cnv->toUnicodeStatus!=0) { + /* + * special case: single byte from a previous buffer, + * where the byte turned out not to belong to a trail surrogate + * and the preceding, unmatched lead surrogate was put into toUBytes[] + * for error handling + */ + cnv->toUBytes[0]=(uint8_t)cnv->toUnicodeStatus; + cnv->toULength=1; + cnv->toUnicodeStatus=0; + } + if((count=cnv->toULength)!=0) { + uint8_t *p=cnv->toUBytes; + do { + p[count++]=*source++; + ++sourceIndex; + --length; + if(count==2) { + c=((UChar)p[1]<<8)|p[0]; + if(U16_IS_SINGLE(c)) { + /* output the BMP code point */ + *target++=c; + if(offsets!=NULL) { + *offsets++=-1; + } + --targetCapacity; + count=0; + c=0; + break; + } else if(U16_IS_SURROGATE_LEAD(c)) { + /* continue collecting bytes for the trail surrogate */ + c=0; /* avoid unnecessary surrogate handling below */ + } else { + /* fall through to error handling for an unmatched trail surrogate */ + break; + } + } else if(count==4) { + c=((UChar)p[1]<<8)|p[0]; + trail=((UChar)p[3]<<8)|p[2]; + if(U16_IS_TRAIL(trail)) { + /* output the surrogate pair */ + *target++=c; + if(targetCapacity>=2) { + *target++=trail; + if(offsets!=NULL) { + *offsets++=-1; + *offsets++=-1; + } + targetCapacity-=2; + } else /* targetCapacity==1 */ { + targetCapacity=0; + cnv->UCharErrorBuffer[0]=trail; + cnv->UCharErrorBufferLength=1; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + count=0; + c=0; + break; + } else { + /* unmatched lead surrogate, handle here for consistent toUBytes[] */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + + /* back out reading the code unit after it */ + if(((const uint8_t *)pArgs->source-source)>=2) { + source-=2; + } else { + /* + * if the trail unit's first byte was in a previous buffer, then + * we need to put it into a special place because toUBytes[] will be + * used for the lead unit's bytes + */ + cnv->toUnicodeStatus=0x100|p[2]; + --source; + } + cnv->toULength=2; + + /* write back the updated pointers */ + pArgs->source=(const char *)source; + pArgs->target=target; + pArgs->offsets=offsets; + return; + } + } + } while(length>0); + cnv->toULength=(int8_t)count; + } + + /* copy an even number of bytes for complete UChars */ + count=2*targetCapacity; + if(count>length) { + count=length&~1; + } + if(c==0 && count>0) { + length-=count; + count>>=1; + targetCapacity-=count; + if(offsets==NULL) { + do { + c=((UChar)source[1]<<8)|source[0]; + source+=2; + if(U16_IS_SINGLE(c)) { + *target++=c; + } else if(U16_IS_SURROGATE_LEAD(c) && count>=2 && + U16_IS_TRAIL(trail=((UChar)source[1]<<8)|source[0]) + ) { + source+=2; + --count; + *target++=c; + *target++=trail; + } else { + break; + } + } while(--count>0); + } else { + do { + c=((UChar)source[1]<<8)|source[0]; + source+=2; + if(U16_IS_SINGLE(c)) { + *target++=c; + *offsets++=sourceIndex; + sourceIndex+=2; + } else if(U16_IS_SURROGATE_LEAD(c) && count>=2 && + U16_IS_TRAIL(trail=((UChar)source[1]<<8)|source[0]) + ) { + source+=2; + --count; + *target++=c; + *target++=trail; + *offsets++=sourceIndex; + *offsets++=sourceIndex; + sourceIndex+=4; + } else { + break; + } + } while(--count>0); + } + + if(count==0) { + /* done with the loop for complete UChars */ + c=0; + } else { + /* keep c for surrogate handling, trail will be set there */ + length+=2*(count-1); /* one more byte pair was consumed than count decremented */ + targetCapacity+=count; + } + } + + if(c!=0) { + /* + * c is a surrogate, and + * - source or target too short + * - or the surrogate is unmatched + */ + cnv->toUBytes[0]=(uint8_t)c; + cnv->toUBytes[1]=(uint8_t)(c>>8); + cnv->toULength=2; + + if(U16_IS_SURROGATE_LEAD(c)) { + if(length>=2) { + if(U16_IS_TRAIL(trail=((UChar)source[1]<<8)|source[0])) { + /* output the surrogate pair, will overflow (see conditions comment above) */ + source+=2; + length-=2; + *target++=c; + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + cnv->UCharErrorBuffer[0]=trail; + cnv->UCharErrorBufferLength=1; + cnv->toULength=0; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } else { + /* unmatched lead surrogate */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + } + } else { + /* see if the trail surrogate is in the next buffer */ + } + } else { + /* unmatched trail surrogate */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + } + } + + if(U_SUCCESS(*pErrorCode)) { + /* check for a remaining source byte */ + if(length>0) { + if(targetCapacity==0) { + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } else { + /* it must be length==1 because otherwise the above would have copied more */ + cnv->toUBytes[cnv->toULength++]=*source++; + } + } + } + + /* write back the updated pointers */ + pArgs->source=(const char *)source; + pArgs->target=target; + pArgs->offsets=offsets; +} + +static UChar32 +_UTF16LEGetNextUChar(UConverterToUnicodeArgs *pArgs, UErrorCode *err) { + const uint8_t *s, *sourceLimit; + UChar32 c; + + s=(const uint8_t *)pArgs->source; + sourceLimit=(const uint8_t *)pArgs->sourceLimit; + + if(s>=sourceLimit) { + /* no input */ + *err=U_INDEX_OUTOFBOUNDS_ERROR; + return 0xffff; + } + + if(s+2>sourceLimit) { + /* only one byte: truncated UChar */ + pArgs->converter->toUBytes[0]=*s++; + pArgs->converter->toULength=1; + pArgs->source=(const char *)s; + *err = U_TRUNCATED_CHAR_FOUND; + return 0xffff; + } + + /* get one UChar */ + c=((UChar32)s[1]<<8)|*s; + s+=2; + + /* check for a surrogate pair */ + if(U_IS_SURROGATE(c)) { + if(U16_IS_SURROGATE_LEAD(c)) { + if(s+2<=sourceLimit) { + UChar trail; + + /* get a second UChar and see if it is a trail surrogate */ + trail=((UChar)s[1]<<8)|*s; + if(U16_IS_TRAIL(trail)) { + c=U16_GET_SUPPLEMENTARY(c, trail); + s+=2; + } else { + /* unmatched lead surrogate */ + c=-2; + } + } else { + /* too few (2 or 3) bytes for a surrogate pair: truncated code point */ + uint8_t *bytes=pArgs->converter->toUBytes; + s-=2; + pArgs->converter->toULength=(int8_t)(sourceLimit-s); + do { + *bytes++=*s++; + } while(sconverter->toUBytes; + pArgs->converter->toULength=2; + *bytes=*(s-2); + bytes[1]=*(s-1); + + c=0xffff; + *err=U_ILLEGAL_CHAR_FOUND; + } + } + + pArgs->source=(const char *)s; + return c; +} + +static const UConverterImpl _UTF16LEImpl={ + UCNV_UTF16_LittleEndian, + + NULL, + NULL, + + NULL, + NULL, + NULL, + + _UTF16LEToUnicodeWithOffsets, + _UTF16LEToUnicodeWithOffsets, + _UTF16LEFromUnicodeWithOffsets, + _UTF16LEFromUnicodeWithOffsets, + _UTF16LEGetNextUChar, + + NULL, + NULL, + NULL, + NULL, + ucnv_getNonSurrogateUnicodeSet +}; + + +static const UConverterStaticData _UTF16LEStaticData={ + sizeof(UConverterStaticData), + "UTF-16LE", + 1202, UCNV_IBM, UCNV_UTF16_LittleEndian, 2, 2, + { 0xfd, 0xff, 0, 0 },2,FALSE,FALSE, + 0, + 0, + { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */ +}; + + +const UConverterSharedData _UTF16LEData={ + sizeof(UConverterSharedData), ~((uint32_t) 0), + NULL, NULL, &_UTF16LEStaticData, FALSE, &_UTF16LEImpl, + 0 +}; + +/* UTF-16 (Detect BOM) ------------------------------------------------------ */ + +/* + * Detect a BOM at the beginning of the stream and select UTF-16BE or UTF-16LE + * accordingly. + * This is a simpler version of the UTF-32 converter below, with + * fewer states for shorter BOMs. + * + * State values: + * 0 initial state + * 1 saw FE + * 2..4 - + * 5 saw FF + * 6..7 - + * 8 UTF-16BE mode + * 9 UTF-16LE mode + * + * During detection: state&3==number of matching bytes so far. + * + * On output, emit U+FEFF as the first code point. + */ + +static void +_UTF16Reset(UConverter *cnv, UConverterResetChoice choice) { + if(choice<=UCNV_RESET_TO_UNICODE) { + /* reset toUnicode: state=0 */ + cnv->mode=0; + } + if(choice!=UCNV_RESET_TO_UNICODE) { + /* reset fromUnicode: prepare to output the UTF-16PE BOM */ + cnv->fromUnicodeStatus=UCNV_NEED_TO_WRITE_BOM; + } +} + +static void +_UTF16Open(UConverter *cnv, + const char *name, + const char *locale, + uint32_t options, + UErrorCode *pErrorCode) { + _UTF16Reset(cnv, UCNV_RESET_BOTH); +} + +static const char utf16BOM[8]={ (char)0xfe, (char)0xff, 0, 0, (char)0xff, (char)0xfe, 0, 0 }; + +static void +_UTF16ToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv=pArgs->converter; + const char *source=pArgs->source; + const char *sourceLimit=pArgs->sourceLimit; + int32_t *offsets=pArgs->offsets; + + int32_t state, offsetDelta; + char b; + + state=cnv->mode; + + /* + * If we detect a BOM in this buffer, then we must add the BOM size to the + * offsets because the actual converter function will not see and count the BOM. + * offsetDelta will have the number of the BOM bytes that are in the current buffer. + */ + offsetDelta=0; + + while(sourcesource); + } else if(state==5) { + state=9; /* detect UTF-16LE */ + offsetDelta=(int32_t)(source-pArgs->source); + } + } else { + /* switch to UTF-16BE and pass the previous bytes */ + if(source!=pArgs->source) { + /* just reset the source */ + source=pArgs->source; + } else { + UBool oldFlush=pArgs->flush; + + /* the first byte is from a previous buffer, replay it first */ + pArgs->source=utf16BOM+(state&4); /* select the correct BOM */ + pArgs->sourceLimit=pArgs->source+1; /* replay previous byte */ + pArgs->flush=FALSE; /* this sourceLimit is not the real source stream limit */ + + _UTF16BEToUnicodeWithOffsets(pArgs, pErrorCode); + + /* restore real pointers; pArgs->source will be set in case 8/9 */ + pArgs->sourceLimit=sourceLimit; + pArgs->flush=oldFlush; + } + state=8; + continue; + } + break; + case 8: + /* call UTF-16BE */ + pArgs->source=source; + _UTF16BEToUnicodeWithOffsets(pArgs, pErrorCode); + source=pArgs->source; + break; + case 9: + /* call UTF-16LE */ + pArgs->source=source; + _UTF16LEToUnicodeWithOffsets(pArgs, pErrorCode); + source=pArgs->source; + break; + default: + break; /* does not occur */ + } + } + + /* add BOM size to offsets - see comment at offsetDelta declaration */ + if(offsets!=NULL && offsetDelta!=0) { + int32_t *offsetsLimit=pArgs->offsets; + while(offsetssource=source; + + if(source==sourceLimit && pArgs->flush) { + /* handle truncated input */ + switch(state) { + case 0: + break; /* no input at all, nothing to do */ + case 8: + _UTF16BEToUnicodeWithOffsets(pArgs, pErrorCode); + break; + case 9: + _UTF16LEToUnicodeWithOffsets(pArgs, pErrorCode); + break; + default: + /* handle 0source=utf16BOM+(state&4); /* select the correct BOM */ + pArgs->sourceLimit=pArgs->source+(state&3); /* replay bytes */ + + /* no offsets: not enough for output */ + _UTF16BEToUnicodeWithOffsets(pArgs, pErrorCode); + pArgs->source=source; + pArgs->sourceLimit=sourceLimit; + state=8; + break; + } + } + + cnv->mode=state; +} + +static UChar32 +_UTF16GetNextUChar(UConverterToUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + switch(pArgs->converter->mode) { + case 8: + return _UTF16BEGetNextUChar(pArgs, pErrorCode); + case 9: + return _UTF16LEGetNextUChar(pArgs, pErrorCode); + default: + return UCNV_GET_NEXT_UCHAR_USE_TO_U; + } +} + +static const UConverterImpl _UTF16Impl = { + UCNV_UTF16, + + NULL, + NULL, + + _UTF16Open, + NULL, + _UTF16Reset, + + _UTF16ToUnicodeWithOffsets, + _UTF16ToUnicodeWithOffsets, + _UTF16PEFromUnicodeWithOffsets, + _UTF16PEFromUnicodeWithOffsets, + _UTF16GetNextUChar, + + NULL, /* ### TODO implement getStarters for all Unicode encodings?! */ + NULL, + NULL, + NULL, + ucnv_getNonSurrogateUnicodeSet +}; + +static const UConverterStaticData _UTF16StaticData = { + sizeof(UConverterStaticData), + "UTF-16", + 1204, /* CCSID for BOM sensitive UTF-16 */ + UCNV_IBM, UCNV_UTF16, 2, 2, +#if U_IS_BIG_ENDIAN + { 0xff, 0xfd, 0, 0 }, 2, +#else + { 0xfd, 0xff, 0, 0 }, 2, +#endif + FALSE, FALSE, + 0, + 0, + { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */ +}; + +const UConverterSharedData _UTF16Data = { + sizeof(UConverterSharedData), ~((uint32_t) 0), + NULL, NULL, &_UTF16StaticData, FALSE, &_UTF16Impl, + 0 +}; + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/ucnv_u32.c b/Build/source/libs/icu/icu-xetex/common/ucnv_u32.c new file mode 100644 index 00000000000..034bf9bbcb9 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnv_u32.c @@ -0,0 +1,1250 @@ +/* +********************************************************************** +* Copyright (C) 2002-2006, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* file name: ucnv_u32.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2002jul01 +* created by: Markus W. Scherer +* +* UTF-32 converter implementation. Used to be in ucnv_utf.c. +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "unicode/ucnv.h" +#include "ucnv_bld.h" +#include "ucnv_cnv.h" +#include "cmemory.h" + +#define MAXIMUM_UCS2 0x0000FFFF +#define MAXIMUM_UTF 0x0010FFFF +#define HALF_SHIFT 10 +#define HALF_BASE 0x0010000 +#define HALF_MASK 0x3FF +#define SURROGATE_HIGH_START 0xD800 +#define SURROGATE_LOW_START 0xDC00 + +/* -SURROGATE_LOW_START + HALF_BASE */ +#define SURROGATE_LOW_BASE 9216 + +enum { + UCNV_NEED_TO_WRITE_BOM=1 +}; + +/* UTF-32BE ----------------------------------------------------------------- */ + +static void +T_UConverter_toUnicode_UTF32_BE(UConverterToUnicodeArgs * args, + UErrorCode * err) +{ + const unsigned char *mySource = (unsigned char *) args->source; + UChar *myTarget = args->target; + const unsigned char *sourceLimit = (unsigned char *) args->sourceLimit; + const UChar *targetLimit = args->targetLimit; + unsigned char *toUBytes = args->converter->toUBytes; + uint32_t ch, i; + + /* Restore state of current sequence */ + if (args->converter->toUnicodeStatus && myTarget < targetLimit) { + i = args->converter->toULength; /* restore # of bytes consumed */ + args->converter->toULength = 0; + + ch = args->converter->toUnicodeStatus - 1;/*Stores the previously calculated ch from a previous call*/ + args->converter->toUnicodeStatus = 0; + goto morebytes; + } + + while (mySource < sourceLimit && myTarget < targetLimit) { + i = 0; + ch = 0; +morebytes: + while (i < sizeof(uint32_t)) { + if (mySource < sourceLimit) { + ch = (ch << 8) | (uint8_t)(*mySource); + toUBytes[i++] = (char) *(mySource++); + } + else { + /* stores a partially calculated target*/ + /* + 1 to make 0 a valid character */ + args->converter->toUnicodeStatus = ch + 1; + args->converter->toULength = (int8_t) i; + goto donefornow; + } + } + + if (ch <= MAXIMUM_UTF && !U_IS_SURROGATE(ch)) { + /* Normal valid byte when the loop has not prematurely terminated (i < inBytes) */ + if (ch <= MAXIMUM_UCS2) + { + /* fits in 16 bits */ + *(myTarget++) = (UChar) ch; + } + else { + /* write out the surrogates */ + *(myTarget++) = U16_LEAD(ch); + ch = U16_TRAIL(ch); + if (myTarget < targetLimit) { + *(myTarget++) = (UChar)ch; + } + else { + /* Put in overflow buffer (not handled here) */ + args->converter->UCharErrorBuffer[0] = (UChar) ch; + args->converter->UCharErrorBufferLength = 1; + *err = U_BUFFER_OVERFLOW_ERROR; + break; + } + } + } + else { + args->converter->toULength = (int8_t)i; + *err = U_ILLEGAL_CHAR_FOUND; + break; + } + } + +donefornow: + if (mySource < sourceLimit && myTarget >= targetLimit && U_SUCCESS(*err)) { + /* End of target buffer */ + *err = U_BUFFER_OVERFLOW_ERROR; + } + + args->target = myTarget; + args->source = (const char *) mySource; +} + +static void +T_UConverter_toUnicode_UTF32_BE_OFFSET_LOGIC(UConverterToUnicodeArgs * args, + UErrorCode * err) +{ + const unsigned char *mySource = (unsigned char *) args->source; + UChar *myTarget = args->target; + int32_t *myOffsets = args->offsets; + const unsigned char *sourceLimit = (unsigned char *) args->sourceLimit; + const UChar *targetLimit = args->targetLimit; + unsigned char *toUBytes = args->converter->toUBytes; + uint32_t ch, i; + int32_t offsetNum = 0; + + /* Restore state of current sequence */ + if (args->converter->toUnicodeStatus && myTarget < targetLimit) { + i = args->converter->toULength; /* restore # of bytes consumed */ + args->converter->toULength = 0; + + ch = args->converter->toUnicodeStatus - 1;/*Stores the previously calculated ch from a previous call*/ + args->converter->toUnicodeStatus = 0; + goto morebytes; + } + + while (mySource < sourceLimit && myTarget < targetLimit) { + i = 0; + ch = 0; +morebytes: + while (i < sizeof(uint32_t)) { + if (mySource < sourceLimit) { + ch = (ch << 8) | (uint8_t)(*mySource); + toUBytes[i++] = (char) *(mySource++); + } + else { + /* stores a partially calculated target*/ + /* + 1 to make 0 a valid character */ + args->converter->toUnicodeStatus = ch + 1; + args->converter->toULength = (int8_t) i; + goto donefornow; + } + } + + if (ch <= MAXIMUM_UTF && !U_IS_SURROGATE(ch)) { + /* Normal valid byte when the loop has not prematurely terminated (i < inBytes) */ + if (ch <= MAXIMUM_UCS2) { + /* fits in 16 bits */ + *(myTarget++) = (UChar) ch; + *(myOffsets++) = offsetNum; + } + else { + /* write out the surrogates */ + *(myTarget++) = U16_LEAD(ch); + *myOffsets++ = offsetNum; + ch = U16_TRAIL(ch); + if (myTarget < targetLimit) + { + *(myTarget++) = (UChar)ch; + *(myOffsets++) = offsetNum; + } + else { + /* Put in overflow buffer (not handled here) */ + args->converter->UCharErrorBuffer[0] = (UChar) ch; + args->converter->UCharErrorBufferLength = 1; + *err = U_BUFFER_OVERFLOW_ERROR; + break; + } + } + } + else { + args->converter->toULength = (int8_t)i; + *err = U_ILLEGAL_CHAR_FOUND; + break; + } + offsetNum += i; + } + +donefornow: + if (mySource < sourceLimit && myTarget >= targetLimit && U_SUCCESS(*err)) + { + /* End of target buffer */ + *err = U_BUFFER_OVERFLOW_ERROR; + } + + args->target = myTarget; + args->source = (const char *) mySource; + args->offsets = myOffsets; +} + +static void +T_UConverter_fromUnicode_UTF32_BE(UConverterFromUnicodeArgs * args, + UErrorCode * err) +{ + const UChar *mySource = args->source; + unsigned char *myTarget; + const UChar *sourceLimit = args->sourceLimit; + const unsigned char *targetLimit = (unsigned char *) args->targetLimit; + UChar32 ch, ch2; + unsigned int indexToWrite; + unsigned char temp[sizeof(uint32_t)]; + + if(mySource >= sourceLimit) { + /* no input, nothing to do */ + return; + } + + /* write the BOM if necessary */ + if(args->converter->fromUnicodeStatus==UCNV_NEED_TO_WRITE_BOM) { + static const char bom[]={ 0, 0, (char)0xfe, (char)0xff }; + ucnv_fromUWriteBytes(args->converter, + bom, 4, + &args->target, args->targetLimit, + &args->offsets, -1, + err); + args->converter->fromUnicodeStatus=0; + } + + myTarget = (unsigned char *) args->target; + temp[0] = 0; + + if (args->converter->fromUChar32) { + ch = args->converter->fromUChar32; + args->converter->fromUChar32 = 0; + goto lowsurogate; + } + + while (mySource < sourceLimit && myTarget < targetLimit) { + ch = *(mySource++); + + if (UTF_IS_SURROGATE(ch)) { + if (U_IS_LEAD(ch)) { +lowsurogate: + if (mySource < sourceLimit) { + ch2 = *mySource; + if (U_IS_TRAIL(ch2)) { + ch = ((ch - SURROGATE_HIGH_START) << HALF_SHIFT) + ch2 + SURROGATE_LOW_BASE; + mySource++; + } + else { + /* this is an unmatched trail code unit (2nd surrogate) */ + /* callback(illegal) */ + args->converter->fromUChar32 = ch; + *err = U_ILLEGAL_CHAR_FOUND; + break; + } + } + else { + /* ran out of source */ + args->converter->fromUChar32 = ch; + if (args->flush) { + /* this is an unmatched trail code unit (2nd surrogate) */ + /* callback(illegal) */ + *err = U_ILLEGAL_CHAR_FOUND; + } + break; + } + } + else { + /* this is an unmatched trail code unit (2nd surrogate) */ + /* callback(illegal) */ + args->converter->fromUChar32 = ch; + *err = U_ILLEGAL_CHAR_FOUND; + break; + } + } + + /* We cannot get any larger than 10FFFF because we are coming from UTF-16 */ + temp[1] = (uint8_t) (ch >> 16 & 0x1F); + temp[2] = (uint8_t) (ch >> 8); /* unsigned cast implicitly does (ch & FF) */ + temp[3] = (uint8_t) (ch); /* unsigned cast implicitly does (ch & FF) */ + + for (indexToWrite = 0; indexToWrite <= sizeof(uint32_t) - 1; indexToWrite++) { + if (myTarget < targetLimit) { + *(myTarget++) = temp[indexToWrite]; + } + else { + args->converter->charErrorBuffer[args->converter->charErrorBufferLength++] = temp[indexToWrite]; + *err = U_BUFFER_OVERFLOW_ERROR; + } + } + } + + if (mySource < sourceLimit && myTarget >= targetLimit && U_SUCCESS(*err)) { + *err = U_BUFFER_OVERFLOW_ERROR; + } + + args->target = (char *) myTarget; + args->source = mySource; +} + +static void +T_UConverter_fromUnicode_UTF32_BE_OFFSET_LOGIC(UConverterFromUnicodeArgs * args, + UErrorCode * err) +{ + const UChar *mySource = args->source; + unsigned char *myTarget; + int32_t *myOffsets; + const UChar *sourceLimit = args->sourceLimit; + const unsigned char *targetLimit = (unsigned char *) args->targetLimit; + UChar32 ch, ch2; + int32_t offsetNum = 0; + unsigned int indexToWrite; + unsigned char temp[sizeof(uint32_t)]; + + if(mySource >= sourceLimit) { + /* no input, nothing to do */ + return; + } + + /* write the BOM if necessary */ + if(args->converter->fromUnicodeStatus==UCNV_NEED_TO_WRITE_BOM) { + static const char bom[]={ 0, 0, (char)0xfe, (char)0xff }; + ucnv_fromUWriteBytes(args->converter, + bom, 4, + &args->target, args->targetLimit, + &args->offsets, -1, + err); + args->converter->fromUnicodeStatus=0; + } + + myTarget = (unsigned char *) args->target; + myOffsets = args->offsets; + temp[0] = 0; + + if (args->converter->fromUChar32) { + ch = args->converter->fromUChar32; + args->converter->fromUChar32 = 0; + goto lowsurogate; + } + + while (mySource < sourceLimit && myTarget < targetLimit) { + ch = *(mySource++); + + if (UTF_IS_SURROGATE(ch)) { + if (U_IS_LEAD(ch)) { +lowsurogate: + if (mySource < sourceLimit) { + ch2 = *mySource; + if (U_IS_TRAIL(ch2)) { + ch = ((ch - SURROGATE_HIGH_START) << HALF_SHIFT) + ch2 + SURROGATE_LOW_BASE; + mySource++; + } + else { + /* this is an unmatched trail code unit (2nd surrogate) */ + /* callback(illegal) */ + args->converter->fromUChar32 = ch; + *err = U_ILLEGAL_CHAR_FOUND; + break; + } + } + else { + /* ran out of source */ + args->converter->fromUChar32 = ch; + if (args->flush) { + /* this is an unmatched trail code unit (2nd surrogate) */ + /* callback(illegal) */ + *err = U_ILLEGAL_CHAR_FOUND; + } + break; + } + } + else { + /* this is an unmatched trail code unit (2nd surrogate) */ + /* callback(illegal) */ + args->converter->fromUChar32 = ch; + *err = U_ILLEGAL_CHAR_FOUND; + break; + } + } + + /* We cannot get any larger than 10FFFF because we are coming from UTF-16 */ + temp[1] = (uint8_t) (ch >> 16 & 0x1F); + temp[2] = (uint8_t) (ch >> 8); /* unsigned cast implicitly does (ch & FF) */ + temp[3] = (uint8_t) (ch); /* unsigned cast implicitly does (ch & FF) */ + + for (indexToWrite = 0; indexToWrite <= sizeof(uint32_t) - 1; indexToWrite++) { + if (myTarget < targetLimit) { + *(myTarget++) = temp[indexToWrite]; + *(myOffsets++) = offsetNum; + } + else { + args->converter->charErrorBuffer[args->converter->charErrorBufferLength++] = temp[indexToWrite]; + *err = U_BUFFER_OVERFLOW_ERROR; + } + } + offsetNum = offsetNum + 1 + (temp[1] != 0); + } + + if (mySource < sourceLimit && myTarget >= targetLimit && U_SUCCESS(*err)) { + *err = U_BUFFER_OVERFLOW_ERROR; + } + + args->target = (char *) myTarget; + args->source = mySource; + args->offsets = myOffsets; +} + +static UChar32 +T_UConverter_getNextUChar_UTF32_BE(UConverterToUnicodeArgs* args, + UErrorCode* err) +{ + const uint8_t *mySource; + UChar32 myUChar; + int32_t length; + + mySource = (const uint8_t *)args->source; + if (mySource >= (const uint8_t *)args->sourceLimit) + { + /* no input */ + *err = U_INDEX_OUTOFBOUNDS_ERROR; + return 0xffff; + } + + length = (int32_t)((const uint8_t *)args->sourceLimit - mySource); + if (length < 4) + { + /* got a partial character */ + uprv_memcpy(args->converter->toUBytes, mySource, length); + args->converter->toULength = (int8_t)length; + args->source = (const char *)(mySource + length); + *err = U_TRUNCATED_CHAR_FOUND; + return 0xffff; + } + + /* Don't even try to do a direct cast because the value may be on an odd address. */ + myUChar = ((UChar32)mySource[0] << 24) + | ((UChar32)mySource[1] << 16) + | ((UChar32)mySource[2] << 8) + | ((UChar32)mySource[3]); + + args->source = (const char *)(mySource + 4); + if ((uint32_t)myUChar <= MAXIMUM_UTF && !U_IS_SURROGATE(myUChar)) { + return myUChar; + } + + uprv_memcpy(args->converter->toUBytes, mySource, 4); + args->converter->toULength = 4; + + *err = U_ILLEGAL_CHAR_FOUND; + return 0xffff; +} + +static const UConverterImpl _UTF32BEImpl = { + UCNV_UTF32_BigEndian, + + NULL, + NULL, + + NULL, + NULL, + NULL, + + T_UConverter_toUnicode_UTF32_BE, + T_UConverter_toUnicode_UTF32_BE_OFFSET_LOGIC, + T_UConverter_fromUnicode_UTF32_BE, + T_UConverter_fromUnicode_UTF32_BE_OFFSET_LOGIC, + T_UConverter_getNextUChar_UTF32_BE, + + NULL, + NULL, + NULL, + NULL, + ucnv_getNonSurrogateUnicodeSet +}; + +/* The 1232 CCSID refers to any version of Unicode with any endianess of UTF-32 */ +static const UConverterStaticData _UTF32BEStaticData = { + sizeof(UConverterStaticData), + "UTF-32BE", + 1232, + UCNV_IBM, UCNV_UTF32_BigEndian, 4, 4, + { 0, 0, 0xff, 0xfd }, 4, FALSE, FALSE, + 0, + 0, + { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */ +}; + +const UConverterSharedData _UTF32BEData = { + sizeof(UConverterSharedData), ~((uint32_t) 0), + NULL, NULL, &_UTF32BEStaticData, FALSE, &_UTF32BEImpl, + 0 +}; + +/* UTF-32LE ---------------------------------------------------------- */ + +static void +T_UConverter_toUnicode_UTF32_LE(UConverterToUnicodeArgs * args, + UErrorCode * err) +{ + const unsigned char *mySource = (unsigned char *) args->source; + UChar *myTarget = args->target; + const unsigned char *sourceLimit = (unsigned char *) args->sourceLimit; + const UChar *targetLimit = args->targetLimit; + unsigned char *toUBytes = args->converter->toUBytes; + uint32_t ch, i; + + /* Restore state of current sequence */ + if (args->converter->toUnicodeStatus && myTarget < targetLimit) + { + i = args->converter->toULength; /* restore # of bytes consumed */ + args->converter->toULength = 0; + + /* Stores the previously calculated ch from a previous call*/ + ch = args->converter->toUnicodeStatus - 1; + args->converter->toUnicodeStatus = 0; + goto morebytes; + } + + while (mySource < sourceLimit && myTarget < targetLimit) + { + i = 0; + ch = 0; +morebytes: + while (i < sizeof(uint32_t)) + { + if (mySource < sourceLimit) + { + ch |= ((uint8_t)(*mySource)) << (i * 8); + toUBytes[i++] = (char) *(mySource++); + } + else + { + /* stores a partially calculated target*/ + /* + 1 to make 0 a valid character */ + args->converter->toUnicodeStatus = ch + 1; + args->converter->toULength = (int8_t) i; + goto donefornow; + } + } + + if (ch <= MAXIMUM_UTF && !U_IS_SURROGATE(ch)) { + /* Normal valid byte when the loop has not prematurely terminated (i < inBytes) */ + if (ch <= MAXIMUM_UCS2) { + /* fits in 16 bits */ + *(myTarget++) = (UChar) ch; + } + else { + /* write out the surrogates */ + *(myTarget++) = U16_LEAD(ch); + ch = U16_TRAIL(ch); + if (myTarget < targetLimit) { + *(myTarget++) = (UChar)ch; + } + else { + /* Put in overflow buffer (not handled here) */ + args->converter->UCharErrorBuffer[0] = (UChar) ch; + args->converter->UCharErrorBufferLength = 1; + *err = U_BUFFER_OVERFLOW_ERROR; + break; + } + } + } + else { + args->converter->toULength = (int8_t)i; + *err = U_ILLEGAL_CHAR_FOUND; + break; + } + } + +donefornow: + if (mySource < sourceLimit && myTarget >= targetLimit && U_SUCCESS(*err)) + { + /* End of target buffer */ + *err = U_BUFFER_OVERFLOW_ERROR; + } + + args->target = myTarget; + args->source = (const char *) mySource; +} + +static void +T_UConverter_toUnicode_UTF32_LE_OFFSET_LOGIC(UConverterToUnicodeArgs * args, + UErrorCode * err) +{ + const unsigned char *mySource = (unsigned char *) args->source; + UChar *myTarget = args->target; + int32_t *myOffsets = args->offsets; + const unsigned char *sourceLimit = (unsigned char *) args->sourceLimit; + const UChar *targetLimit = args->targetLimit; + unsigned char *toUBytes = args->converter->toUBytes; + uint32_t ch, i; + int32_t offsetNum = 0; + + /* Restore state of current sequence */ + if (args->converter->toUnicodeStatus && myTarget < targetLimit) + { + i = args->converter->toULength; /* restore # of bytes consumed */ + args->converter->toULength = 0; + + /* Stores the previously calculated ch from a previous call*/ + ch = args->converter->toUnicodeStatus - 1; + args->converter->toUnicodeStatus = 0; + goto morebytes; + } + + while (mySource < sourceLimit && myTarget < targetLimit) + { + i = 0; + ch = 0; +morebytes: + while (i < sizeof(uint32_t)) + { + if (mySource < sourceLimit) + { + ch |= ((uint8_t)(*mySource)) << (i * 8); + toUBytes[i++] = (char) *(mySource++); + } + else + { + /* stores a partially calculated target*/ + /* + 1 to make 0 a valid character */ + args->converter->toUnicodeStatus = ch + 1; + args->converter->toULength = (int8_t) i; + goto donefornow; + } + } + + if (ch <= MAXIMUM_UTF && !U_IS_SURROGATE(ch)) + { + /* Normal valid byte when the loop has not prematurely terminated (i < inBytes) */ + if (ch <= MAXIMUM_UCS2) + { + /* fits in 16 bits */ + *(myTarget++) = (UChar) ch; + *(myOffsets++) = offsetNum; + } + else { + /* write out the surrogates */ + *(myTarget++) = U16_LEAD(ch); + *(myOffsets++) = offsetNum; + ch = U16_TRAIL(ch); + if (myTarget < targetLimit) + { + *(myTarget++) = (UChar)ch; + *(myOffsets++) = offsetNum; + } + else + { + /* Put in overflow buffer (not handled here) */ + args->converter->UCharErrorBuffer[0] = (UChar) ch; + args->converter->UCharErrorBufferLength = 1; + *err = U_BUFFER_OVERFLOW_ERROR; + break; + } + } + } + else + { + args->converter->toULength = (int8_t)i; + *err = U_ILLEGAL_CHAR_FOUND; + break; + } + offsetNum += i; + } + +donefornow: + if (mySource < sourceLimit && myTarget >= targetLimit && U_SUCCESS(*err)) + { + /* End of target buffer */ + *err = U_BUFFER_OVERFLOW_ERROR; + } + + args->target = myTarget; + args->source = (const char *) mySource; + args->offsets = myOffsets; +} + +static void +T_UConverter_fromUnicode_UTF32_LE(UConverterFromUnicodeArgs * args, + UErrorCode * err) +{ + const UChar *mySource = args->source; + unsigned char *myTarget; + const UChar *sourceLimit = args->sourceLimit; + const unsigned char *targetLimit = (unsigned char *) args->targetLimit; + UChar32 ch, ch2; + unsigned int indexToWrite; + unsigned char temp[sizeof(uint32_t)]; + + if(mySource >= sourceLimit) { + /* no input, nothing to do */ + return; + } + + /* write the BOM if necessary */ + if(args->converter->fromUnicodeStatus==UCNV_NEED_TO_WRITE_BOM) { + static const char bom[]={ (char)0xff, (char)0xfe, 0, 0 }; + ucnv_fromUWriteBytes(args->converter, + bom, 4, + &args->target, args->targetLimit, + &args->offsets, -1, + err); + args->converter->fromUnicodeStatus=0; + } + + myTarget = (unsigned char *) args->target; + temp[3] = 0; + + if (args->converter->fromUChar32) + { + ch = args->converter->fromUChar32; + args->converter->fromUChar32 = 0; + goto lowsurogate; + } + + while (mySource < sourceLimit && myTarget < targetLimit) + { + ch = *(mySource++); + + if (UTF_IS_SURROGATE(ch)) { + if (U_IS_LEAD(ch)) + { +lowsurogate: + if (mySource < sourceLimit) + { + ch2 = *mySource; + if (U_IS_TRAIL(ch2)) { + ch = ((ch - SURROGATE_HIGH_START) << HALF_SHIFT) + ch2 + SURROGATE_LOW_BASE; + mySource++; + } + else { + /* this is an unmatched trail code unit (2nd surrogate) */ + /* callback(illegal) */ + args->converter->fromUChar32 = ch; + *err = U_ILLEGAL_CHAR_FOUND; + break; + } + } + else { + /* ran out of source */ + args->converter->fromUChar32 = ch; + if (args->flush) { + /* this is an unmatched trail code unit (2nd surrogate) */ + /* callback(illegal) */ + *err = U_ILLEGAL_CHAR_FOUND; + } + break; + } + } + else { + /* this is an unmatched trail code unit (2nd surrogate) */ + /* callback(illegal) */ + args->converter->fromUChar32 = ch; + *err = U_ILLEGAL_CHAR_FOUND; + break; + } + } + + /* We cannot get any larger than 10FFFF because we are coming from UTF-16 */ + temp[2] = (uint8_t) (ch >> 16 & 0x1F); + temp[1] = (uint8_t) (ch >> 8); /* unsigned cast implicitly does (ch & FF) */ + temp[0] = (uint8_t) (ch); /* unsigned cast implicitly does (ch & FF) */ + + for (indexToWrite = 0; indexToWrite <= sizeof(uint32_t) - 1; indexToWrite++) + { + if (myTarget < targetLimit) + { + *(myTarget++) = temp[indexToWrite]; + } + else + { + args->converter->charErrorBuffer[args->converter->charErrorBufferLength++] = temp[indexToWrite]; + *err = U_BUFFER_OVERFLOW_ERROR; + } + } + } + + if (mySource < sourceLimit && myTarget >= targetLimit && U_SUCCESS(*err)) + { + *err = U_BUFFER_OVERFLOW_ERROR; + } + + args->target = (char *) myTarget; + args->source = mySource; +} + +static void +T_UConverter_fromUnicode_UTF32_LE_OFFSET_LOGIC(UConverterFromUnicodeArgs * args, + UErrorCode * err) +{ + const UChar *mySource = args->source; + unsigned char *myTarget; + int32_t *myOffsets; + const UChar *sourceLimit = args->sourceLimit; + const unsigned char *targetLimit = (unsigned char *) args->targetLimit; + UChar32 ch, ch2; + unsigned int indexToWrite; + unsigned char temp[sizeof(uint32_t)]; + int32_t offsetNum = 0; + + if(mySource >= sourceLimit) { + /* no input, nothing to do */ + return; + } + + /* write the BOM if necessary */ + if(args->converter->fromUnicodeStatus==UCNV_NEED_TO_WRITE_BOM) { + static const char bom[]={ (char)0xff, (char)0xfe, 0, 0 }; + ucnv_fromUWriteBytes(args->converter, + bom, 4, + &args->target, args->targetLimit, + &args->offsets, -1, + err); + args->converter->fromUnicodeStatus=0; + } + + myTarget = (unsigned char *) args->target; + myOffsets = args->offsets; + temp[3] = 0; + + if (args->converter->fromUChar32) + { + ch = args->converter->fromUChar32; + args->converter->fromUChar32 = 0; + goto lowsurogate; + } + + while (mySource < sourceLimit && myTarget < targetLimit) + { + ch = *(mySource++); + + if (UTF_IS_SURROGATE(ch)) { + if (U_IS_LEAD(ch)) + { +lowsurogate: + if (mySource < sourceLimit) + { + ch2 = *mySource; + if (U_IS_TRAIL(ch2)) + { + ch = ((ch - SURROGATE_HIGH_START) << HALF_SHIFT) + ch2 + SURROGATE_LOW_BASE; + mySource++; + } + else { + /* this is an unmatched trail code unit (2nd surrogate) */ + /* callback(illegal) */ + args->converter->fromUChar32 = ch; + *err = U_ILLEGAL_CHAR_FOUND; + break; + } + } + else { + /* ran out of source */ + args->converter->fromUChar32 = ch; + if (args->flush) { + /* this is an unmatched trail code unit (2nd surrogate) */ + /* callback(illegal) */ + *err = U_ILLEGAL_CHAR_FOUND; + } + break; + } + } + else { + /* this is an unmatched trail code unit (2nd surrogate) */ + /* callback(illegal) */ + args->converter->fromUChar32 = ch; + *err = U_ILLEGAL_CHAR_FOUND; + break; + } + } + + /* We cannot get any larger than 10FFFF because we are coming from UTF-16 */ + temp[2] = (uint8_t) (ch >> 16 & 0x1F); + temp[1] = (uint8_t) (ch >> 8); /* unsigned cast implicitly does (ch & FF) */ + temp[0] = (uint8_t) (ch); /* unsigned cast implicitly does (ch & FF) */ + + for (indexToWrite = 0; indexToWrite <= sizeof(uint32_t) - 1; indexToWrite++) + { + if (myTarget < targetLimit) + { + *(myTarget++) = temp[indexToWrite]; + *(myOffsets++) = offsetNum; + } + else + { + args->converter->charErrorBuffer[args->converter->charErrorBufferLength++] = temp[indexToWrite]; + *err = U_BUFFER_OVERFLOW_ERROR; + } + } + offsetNum = offsetNum + 1 + (temp[2] != 0); + } + + if (mySource < sourceLimit && myTarget >= targetLimit && U_SUCCESS(*err)) + { + *err = U_BUFFER_OVERFLOW_ERROR; + } + + args->target = (char *) myTarget; + args->source = mySource; + args->offsets = myOffsets; +} + +static UChar32 +T_UConverter_getNextUChar_UTF32_LE(UConverterToUnicodeArgs* args, + UErrorCode* err) +{ + const uint8_t *mySource; + UChar32 myUChar; + int32_t length; + + mySource = (const uint8_t *)args->source; + if (mySource >= (const uint8_t *)args->sourceLimit) + { + /* no input */ + *err = U_INDEX_OUTOFBOUNDS_ERROR; + return 0xffff; + } + + length = (int32_t)((const uint8_t *)args->sourceLimit - mySource); + if (length < 4) + { + /* got a partial character */ + uprv_memcpy(args->converter->toUBytes, mySource, length); + args->converter->toULength = (int8_t)length; + args->source = (const char *)(mySource + length); + *err = U_TRUNCATED_CHAR_FOUND; + return 0xffff; + } + + /* Don't even try to do a direct cast because the value may be on an odd address. */ + myUChar = ((UChar32)mySource[3] << 24) + | ((UChar32)mySource[2] << 16) + | ((UChar32)mySource[1] << 8) + | ((UChar32)mySource[0]); + + args->source = (const char *)(mySource + 4); + if ((uint32_t)myUChar <= MAXIMUM_UTF && !U_IS_SURROGATE(myUChar)) { + return myUChar; + } + + uprv_memcpy(args->converter->toUBytes, mySource, 4); + args->converter->toULength = 4; + + *err = U_ILLEGAL_CHAR_FOUND; + return 0xffff; +} + +static const UConverterImpl _UTF32LEImpl = { + UCNV_UTF32_LittleEndian, + + NULL, + NULL, + + NULL, + NULL, + NULL, + + T_UConverter_toUnicode_UTF32_LE, + T_UConverter_toUnicode_UTF32_LE_OFFSET_LOGIC, + T_UConverter_fromUnicode_UTF32_LE, + T_UConverter_fromUnicode_UTF32_LE_OFFSET_LOGIC, + T_UConverter_getNextUChar_UTF32_LE, + + NULL, + NULL, + NULL, + NULL, + ucnv_getNonSurrogateUnicodeSet +}; + +/* The 1232 CCSID refers to any version of Unicode with any endianess of UTF-32 */ +static const UConverterStaticData _UTF32LEStaticData = { + sizeof(UConverterStaticData), + "UTF-32LE", + 1234, + UCNV_IBM, UCNV_UTF32_LittleEndian, 4, 4, + { 0xfd, 0xff, 0, 0 }, 4, FALSE, FALSE, + 0, + 0, + { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */ +}; + + +const UConverterSharedData _UTF32LEData = { + sizeof(UConverterSharedData), ~((uint32_t) 0), + NULL, NULL, &_UTF32LEStaticData, FALSE, &_UTF32LEImpl, + 0 +}; + +/* UTF-32 (Detect BOM) ------------------------------------------------------ */ + +/* + * Detect a BOM at the beginning of the stream and select UTF-32BE or UTF-32LE + * accordingly. + * + * State values: + * 0 initial state + * 1 saw 00 + * 2 saw 00 00 + * 3 saw 00 00 FE + * 4 - + * 5 saw FF + * 6 saw FF FE + * 7 saw FF FE 00 + * 8 UTF-32BE mode + * 9 UTF-32LE mode + * + * During detection: state&3==number of matching bytes so far. + * + * On output, emit U+FEFF as the first code point. + */ + +static void +_UTF32Reset(UConverter *cnv, UConverterResetChoice choice) { + if(choice<=UCNV_RESET_TO_UNICODE) { + /* reset toUnicode: state=0 */ + cnv->mode=0; + } + if(choice!=UCNV_RESET_TO_UNICODE) { + /* reset fromUnicode: prepare to output the UTF-32PE BOM */ + cnv->fromUnicodeStatus=UCNV_NEED_TO_WRITE_BOM; + } +} + +static void +_UTF32Open(UConverter *cnv, + const char *name, + const char *locale, + uint32_t options, + UErrorCode *pErrorCode) { + _UTF32Reset(cnv, UCNV_RESET_BOTH); +} + +static const char utf32BOM[8]={ 0, 0, (char)0xfe, (char)0xff, (char)0xff, (char)0xfe, 0, 0 }; + +static void +_UTF32ToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv=pArgs->converter; + const char *source=pArgs->source; + const char *sourceLimit=pArgs->sourceLimit; + int32_t *offsets=pArgs->offsets; + + int32_t state, offsetDelta; + char b; + + state=cnv->mode; + + /* + * If we detect a BOM in this buffer, then we must add the BOM size to the + * offsets because the actual converter function will not see and count the BOM. + * offsetDelta will have the number of the BOM bytes that are in the current buffer. + */ + offsetDelta=0; + + while(sourcesource); + } else if(state==8) { + state=9; /* detect UTF-32LE */ + offsetDelta=(int32_t)(source-pArgs->source); + } + } else { + /* switch to UTF-32BE and pass the previous bytes */ + int32_t count=(int32_t)(source-pArgs->source); /* number of bytes from this buffer */ + + /* reset the source */ + source=pArgs->source; + + if(count==(state&3)) { + /* simple: all in the same buffer, just reset source */ + } else { + UBool oldFlush=pArgs->flush; + + /* some of the bytes are from a previous buffer, replay those first */ + pArgs->source=utf32BOM+(state&4); /* select the correct BOM */ + pArgs->sourceLimit=pArgs->source+((state&3)-count); /* replay previous bytes */ + pArgs->flush=FALSE; /* this sourceLimit is not the real source stream limit */ + + /* no offsets: bytes from previous buffer, and not enough for output */ + T_UConverter_toUnicode_UTF32_BE(pArgs, pErrorCode); + + /* restore real pointers; pArgs->source will be set in case 8/9 */ + pArgs->sourceLimit=sourceLimit; + pArgs->flush=oldFlush; + } + state=8; + continue; + } + break; + case 8: + /* call UTF-32BE */ + pArgs->source=source; + if(offsets==NULL) { + T_UConverter_toUnicode_UTF32_BE(pArgs, pErrorCode); + } else { + T_UConverter_toUnicode_UTF32_BE_OFFSET_LOGIC(pArgs, pErrorCode); + } + source=pArgs->source; + break; + case 9: + /* call UTF-32LE */ + pArgs->source=source; + if(offsets==NULL) { + T_UConverter_toUnicode_UTF32_LE(pArgs, pErrorCode); + } else { + T_UConverter_toUnicode_UTF32_LE_OFFSET_LOGIC(pArgs, pErrorCode); + } + source=pArgs->source; + break; + default: + break; /* does not occur */ + } + } + + /* add BOM size to offsets - see comment at offsetDelta declaration */ + if(offsets!=NULL && offsetDelta!=0) { + int32_t *offsetsLimit=pArgs->offsets; + while(offsetssource=source; + + if(source==sourceLimit && pArgs->flush) { + /* handle truncated input */ + switch(state) { + case 0: + break; /* no input at all, nothing to do */ + case 8: + T_UConverter_toUnicode_UTF32_BE(pArgs, pErrorCode); + break; + case 9: + T_UConverter_toUnicode_UTF32_LE(pArgs, pErrorCode); + break; + default: + /* handle 0source=utf32BOM+(state&4); /* select the correct BOM */ + pArgs->sourceLimit=pArgs->source+(state&3); /* replay bytes */ + + /* no offsets: not enough for output */ + T_UConverter_toUnicode_UTF32_BE(pArgs, pErrorCode); + pArgs->source=source; + pArgs->sourceLimit=sourceLimit; + state=8; + break; + } + } + + cnv->mode=state; +} + +static UChar32 +_UTF32GetNextUChar(UConverterToUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + switch(pArgs->converter->mode) { + case 8: + return T_UConverter_getNextUChar_UTF32_BE(pArgs, pErrorCode); + case 9: + return T_UConverter_getNextUChar_UTF32_LE(pArgs, pErrorCode); + default: + return UCNV_GET_NEXT_UCHAR_USE_TO_U; + } +} + +static const UConverterImpl _UTF32Impl = { + UCNV_UTF32, + + NULL, + NULL, + + _UTF32Open, + NULL, + _UTF32Reset, + + _UTF32ToUnicodeWithOffsets, + _UTF32ToUnicodeWithOffsets, +#if U_IS_BIG_ENDIAN + T_UConverter_fromUnicode_UTF32_BE, + T_UConverter_fromUnicode_UTF32_BE_OFFSET_LOGIC, +#else + T_UConverter_fromUnicode_UTF32_LE, + T_UConverter_fromUnicode_UTF32_LE_OFFSET_LOGIC, +#endif + _UTF32GetNextUChar, + + NULL, /* ### TODO implement getStarters for all Unicode encodings?! */ + NULL, + NULL, + NULL, + ucnv_getNonSurrogateUnicodeSet +}; + +/* The 1236 CCSID refers to any version of Unicode with a BOM sensitive endianess of UTF-32 */ +static const UConverterStaticData _UTF32StaticData = { + sizeof(UConverterStaticData), + "UTF-32", + 1236, + UCNV_IBM, UCNV_UTF32, 4, 4, +#if U_IS_BIG_ENDIAN + { 0, 0, 0xff, 0xfd }, 4, +#else + { 0xfd, 0xff, 0, 0 }, 4, +#endif + FALSE, FALSE, + 0, + 0, + { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */ +}; + +const UConverterSharedData _UTF32Data = { + sizeof(UConverterSharedData), ~((uint32_t) 0), + NULL, NULL, &_UTF32StaticData, FALSE, &_UTF32Impl, + 0 +}; + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/ucnv_u7.c b/Build/source/libs/icu/icu-xetex/common/ucnv_u7.c new file mode 100644 index 00000000000..62e2792922c --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnv_u7.c @@ -0,0 +1,1459 @@ +/* +********************************************************************** +* Copyright (C) 2002-2004, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* file name: ucnv_u7.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2002jul01 +* created by: Markus W. Scherer +* +* UTF-7 converter implementation. Used to be in ucnv_utf.c. +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "unicode/ucnv.h" +#include "ucnv_bld.h" +#include "ucnv_cnv.h" + +/* UTF-7 -------------------------------------------------------------------- */ + +/* + * UTF-7 is a stateful encoding of Unicode. + * It is defined in RFC 2152. (http://www.ietf.org/rfc/rfc2152.txt) + * It was intended for use in Internet email systems, using in its bytewise + * encoding only a subset of 7-bit US-ASCII. + * UTF-7 is deprecated in favor of UTF-8/16/32 and SCSU, but still + * occasionally used. + * + * For converting Unicode to UTF-7, the RFC allows to encode some US-ASCII + * characters directly or in base64. Especially, the characters in set O + * as defined in the RFC (see below) may be encoded directly but are not + * allowed in, e.g., email headers. + * By default, the ICU UTF-7 converter encodes set O directly. + * By choosing the option "version=1", set O will be escaped instead. + * For example: + * utf7Converter=ucnv_open("UTF-7,version=1"); + * + * For details about email headers see RFC 2047. + */ + +/* + * Tests for US-ASCII characters belonging to character classes + * defined in UTF-7. + * + * Set D (directly encoded characters) consists of the following + * characters: the upper and lower case letters A through Z + * and a through z, the 10 digits 0-9, and the following nine special + * characters (note that "+" and "=" are omitted): + * '(),-./:? + * + * Set O (optional direct characters) consists of the following + * characters (note that "\" and "~" are omitted): + * !"#$%&*;<=>@[]^_`{|} + * + * According to the rules in RFC 2152, the byte values for the following + * US-ASCII characters are not used in UTF-7 and are therefore illegal: + * - all C0 control codes except for CR LF TAB + * - BACKSLASH + * - TILDE + * - DEL + * - all codes beyond US-ASCII, i.e. all >127 + */ +#define inSetD(c) \ + ((uint8_t)((c)-97)<26 || (uint8_t)((c)-65)<26 || /* letters */ \ + (uint8_t)((c)-48)<10 || /* digits */ \ + (uint8_t)((c)-39)<3 || /* '() */ \ + (uint8_t)((c)-44)<4 || /* ,-./ */ \ + (c)==58 || (c)==63 /* :? */ \ + ) + +#define inSetO(c) \ + ((uint8_t)((c)-33)<6 || /* !"#$%& */ \ + (uint8_t)((c)-59)<4 || /* ;<=> */ \ + (uint8_t)((c)-93)<4 || /* ]^_` */ \ + (uint8_t)((c)-123)<3 || /* {|} */ \ + (c)==42 || (c)==64 || (c)==91 /* *@[ */ \ + ) + +#define isCRLFTAB(c) ((c)==13 || (c)==10 || (c)==9) +#define isCRLFSPTAB(c) ((c)==32 || (c)==13 || (c)==10 || (c)==9) + +#define PLUS 43 +#define MINUS 45 +#define BACKSLASH 92 +#define TILDE 126 + +/* legal byte values: all US-ASCII graphic characters from space to before tilde, and CR LF TAB */ +#define isLegalUTF7(c) (((uint8_t)((c)-32)<94 && (c)!=BACKSLASH) || isCRLFTAB(c)) + +/* encode directly sets D and O and CR LF SP TAB */ +static const UBool encodeDirectlyMaximum[128]={ + /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0 +}; + +/* encode directly set D and CR LF SP TAB but not set O */ +static const UBool encodeDirectlyRestricted[128]={ + /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, + + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, + + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 +}; + +static const uint8_t +toBase64[64]={ + /* A-Z */ + 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, + 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, + /* a-z */ + 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, + /* 0-9 */ + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, + /* +/ */ + 43, 47 +}; + +static const int8_t +fromBase64[128]={ + /* C0 controls, -1 for legal ones (CR LF TAB), -3 for illegal ones */ + -3, -3, -3, -3, -3, -3, -3, -3, -3, -1, -1, -3, -3, -1, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + + /* general punctuation with + and / and a special value (-2) for - */ + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -2, -1, 63, + /* digits */ + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, + + /* A-Z */ + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -3, -1, -1, -1, + + /* a-z */ + -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -3, -3 +}; + +/* + * converter status values: + * + * toUnicodeStatus: + * 24 inDirectMode (boolean) + * 23..16 base64Counter (-1..7) + * 15..0 bits (up to 14 bits incoming base64) + * + * fromUnicodeStatus: + * 31..28 version (0: set O direct 1: set O escaped) + * 24 inDirectMode (boolean) + * 23..16 base64Counter (0..2) + * 7..0 bits (6 bits outgoing base64) + * + */ + +static void +_UTF7Reset(UConverter *cnv, UConverterResetChoice choice) { + if(choice<=UCNV_RESET_TO_UNICODE) { + /* reset toUnicode */ + cnv->toUnicodeStatus=0x1000000; /* inDirectMode=TRUE */ + cnv->toULength=0; + } + if(choice!=UCNV_RESET_TO_UNICODE) { + /* reset fromUnicode */ + cnv->fromUnicodeStatus=(cnv->fromUnicodeStatus&0xf0000000)|0x1000000; /* keep version, inDirectMode=TRUE */ + } +} + +static void +_UTF7Open(UConverter *cnv, + const char *name, + const char *locale, + uint32_t options, + UErrorCode *pErrorCode) { + if((options&0xf)<=1) { + cnv->fromUnicodeStatus=(options&0xf)<<28; + _UTF7Reset(cnv, UCNV_RESET_BOTH); + } else { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + } +} + +static void +_UTF7ToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv; + const uint8_t *source, *sourceLimit; + UChar *target; + const UChar *targetLimit; + int32_t *offsets; + + uint8_t *bytes; + uint8_t byteIndex; + + int32_t length, targetCapacity; + + /* UTF-7 state */ + uint16_t bits; + int8_t base64Counter; + UBool inDirectMode; + + int8_t base64Value; + + int32_t sourceIndex, nextSourceIndex; + + uint8_t b; + /* set up the local pointers */ + cnv=pArgs->converter; + + source=(const uint8_t *)pArgs->source; + sourceLimit=(const uint8_t *)pArgs->sourceLimit; + target=pArgs->target; + targetLimit=pArgs->targetLimit; + offsets=pArgs->offsets; + /* get the state machine state */ + { + uint32_t status=cnv->toUnicodeStatus; + inDirectMode=(UBool)((status>>24)&1); + base64Counter=(int8_t)(status>>16); + bits=(uint16_t)status; + } + bytes=cnv->toUBytes; + byteIndex=cnv->toULength; + + /* sourceIndex=-1 if the current character began in the previous buffer */ + sourceIndex=byteIndex==0 ? 0 : -1; + nextSourceIndex=0; + + if(inDirectMode) { +directMode: + /* + * In Direct Mode, most US-ASCII characters are encoded directly, i.e., + * with their US-ASCII byte values. + * Backslash and Tilde and most control characters are not allowed in UTF-7. + * A plus sign starts Unicode (or "escape") Mode. + * + * In Direct Mode, only the sourceIndex is used. + */ + byteIndex=0; + length=(int32_t)(sourceLimit-source); + targetCapacity=(int32_t)(targetLimit-target); + if(length>targetCapacity) { + length=targetCapacity; + } + while(length>0) { + b=*source++; + if(!isLegalUTF7(b)) { + /* illegal */ + bytes[0]=b; + byteIndex=1; + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + break; + } else if(b!=PLUS) { + /* write directly encoded character */ + *target++=b; + if(offsets!=NULL) { + *offsets++=sourceIndex++; + } + } else /* PLUS */ { + /* switch to Unicode mode */ + nextSourceIndex=++sourceIndex; + inDirectMode=FALSE; + byteIndex=0; + bits=0; + base64Counter=-1; + goto unicodeMode; + } + --length; + } + if(source=targetLimit) { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + } else { +unicodeMode: + /* + * In Unicode (or "escape") Mode, UTF-16BE is base64-encoded. + * The base64 sequence ends with any character that is not in the base64 alphabet. + * A terminating minus sign is consumed. + * + * In Unicode Mode, the sourceIndex has the index to the start of the current + * base64 bytes, while nextSourceIndex is precisely parallel to source, + * keeping the index to the following byte. + * Note that in 2 out of 3 cases, UChars overlap within a base64 byte. + */ + while(source=126) { + /* illegal - test other illegal US-ASCII values by base64Value==-3 */ + inDirectMode=TRUE; + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + break; + } else if((base64Value=fromBase64[b])>=0) { + /* collect base64 bytes into UChars */ + switch(base64Counter) { + case -1: /* -1 is immediately after the + */ + case 0: + bits=base64Value; + base64Counter=1; + break; + case 1: + case 3: + case 4: + case 6: + bits=(uint16_t)((bits<<6)|base64Value); + ++base64Counter; + break; + case 2: + *target++=(UChar)((bits<<4)|(base64Value>>2)); + if(offsets!=NULL) { + *offsets++=sourceIndex; + sourceIndex=nextSourceIndex-1; + } + bytes[0]=b; /* keep this byte in case an error occurs */ + byteIndex=1; + bits=(uint16_t)(base64Value&3); + base64Counter=3; + break; + case 5: + *target++=(UChar)((bits<<2)|(base64Value>>4)); + if(offsets!=NULL) { + *offsets++=sourceIndex; + sourceIndex=nextSourceIndex-1; + } + bytes[0]=b; /* keep this byte in case an error occurs */ + byteIndex=1; + bits=(uint16_t)(base64Value&15); + base64Counter=6; + break; + case 7: + *target++=(UChar)((bits<<6)|base64Value); + if(offsets!=NULL) { + *offsets++=sourceIndex; + sourceIndex=nextSourceIndex; + } + byteIndex=0; + bits=0; + base64Counter=0; + break; + default: + /* will never occur */ + break; + } + } else if(base64Value==-2) { + /* minus sign terminates the base64 sequence */ + inDirectMode=TRUE; + if(base64Counter==-1) { + /* +- i.e. a minus immediately following a plus */ + *target++=PLUS; + if(offsets!=NULL) { + *offsets++=sourceIndex-1; + } + } else { + /* absorb the minus and leave the Unicode Mode */ + if(bits!=0) { + /* bits are illegally left over, a UChar is incomplete */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + break; + } + } + sourceIndex=nextSourceIndex; + goto directMode; + } else if(base64Value==-1) /* for any legal character except base64 and minus sign */ { + /* leave the Unicode Mode */ + inDirectMode=TRUE; + if(base64Counter==-1) { + /* illegal: + immediately followed by something other than base64 or minus sign */ + /* include the plus sign in the reported sequence */ + --sourceIndex; + bytes[0]=PLUS; + bytes[1]=b; + byteIndex=2; + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + break; + } else if(bits==0) { + /* un-read the character in case it is a plus sign */ + --source; + sourceIndex=nextSourceIndex-1; + goto directMode; + } else { + /* bits are illegally left over, a UChar is incomplete */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + break; + } + } else /* base64Value==-3 for illegal characters */ { + /* illegal */ + inDirectMode=TRUE; + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + break; + } + } else { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + } + } + + if(U_SUCCESS(*pErrorCode) && pArgs->flush && source==sourceLimit && bits==0) { + /* + * if we are in Unicode mode, then the byteIndex might not be 0, + * but that is ok if bits==0 + * -> we set byteIndex=0 at the end of the stream to avoid a truncated error + * (not true for IMAP-mailbox-name where we must end in direct mode) + */ + byteIndex=0; + } + + /* set the converter state back into UConverter */ + cnv->toUnicodeStatus=((uint32_t)inDirectMode<<24)|((uint32_t)((uint8_t)base64Counter)<<16)|(uint32_t)bits; + cnv->toULength=byteIndex; + + /* write back the updated pointers */ + pArgs->source=(const char *)source; + pArgs->target=target; + pArgs->offsets=offsets; + return; +} + +static void +_UTF7FromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv; + const UChar *source, *sourceLimit; + uint8_t *target, *targetLimit; + int32_t *offsets; + + int32_t length, targetCapacity, sourceIndex; + UChar c; + + /* UTF-7 state */ + const UBool *encodeDirectly; + uint8_t bits; + int8_t base64Counter; + UBool inDirectMode; + + /* set up the local pointers */ + cnv=pArgs->converter; + + /* set up the local pointers */ + source=pArgs->source; + sourceLimit=pArgs->sourceLimit; + target=(uint8_t *)pArgs->target; + targetLimit=(uint8_t *)pArgs->targetLimit; + offsets=pArgs->offsets; + + /* get the state machine state */ + { + uint32_t status=cnv->fromUnicodeStatus; + encodeDirectly= status<0x10000000 ? encodeDirectlyMaximum : encodeDirectlyRestricted; + inDirectMode=(UBool)((status>>24)&1); + base64Counter=(int8_t)(status>>16); + bits=(uint8_t)status; + } + + /* UTF-7 always encodes UTF-16 code units, therefore we need only a simple sourceIndex */ + sourceIndex=0; + + if(inDirectMode) { +directMode: + length=(int32_t)(sourceLimit-source); + targetCapacity=(int32_t)(targetLimit-target); + if(length>targetCapacity) { + length=targetCapacity; + } + while(length>0) { + c=*source++; + /* currently always encode CR LF SP TAB directly */ + if(c<=127 && encodeDirectly[c]) { + /* encode directly */ + *target++=(uint8_t)c; + if(offsets!=NULL) { + *offsets++=sourceIndex++; + } + } else if(c==PLUS) { + /* output +- for + */ + *target++=PLUS; + if(targetcharErrorBuffer[0]=MINUS; + cnv->charErrorBufferLength=1; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + } else { + /* un-read this character and switch to Unicode Mode */ + --source; + *target++=PLUS; + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + inDirectMode=FALSE; + base64Counter=0; + goto unicodeMode; + } + --length; + } + if(source=targetLimit) { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + } else { +unicodeMode: + while(sourcecharErrorBuffer[0]=MINUS; + cnv->charErrorBufferLength=1; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + } + goto directMode; + } else { + /* + * base64 this character: + * Output 2 or 3 base64 bytes for the remaining bits of the previous character + * and the bits of this character, each implicitly in UTF-16BE. + * + * Here, bits is an 8-bit variable because only 6 bits need to be kept from one + * character to the next. The actual 2 or 4 bits are shifted to the left edge + * of the 6-bits field 5..0 to make the termination of the base64 sequence easier. + */ + switch(base64Counter) { + case 0: + *target++=toBase64[c>>10]; + if(target>4)&0x3f]; + if(offsets!=NULL) { + *offsets++=sourceIndex; + *offsets++=sourceIndex++; + } + } else { + if(offsets!=NULL) { + *offsets++=sourceIndex++; + } + cnv->charErrorBuffer[0]=toBase64[(c>>4)&0x3f]; + cnv->charErrorBufferLength=1; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + bits=(uint8_t)((c&15)<<2); + base64Counter=1; + break; + case 1: + *target++=toBase64[bits|(c>>14)]; + if(target>8)&0x3f]; + if(target>2)&0x3f]; + if(offsets!=NULL) { + *offsets++=sourceIndex; + *offsets++=sourceIndex; + *offsets++=sourceIndex++; + } + } else { + if(offsets!=NULL) { + *offsets++=sourceIndex; + *offsets++=sourceIndex++; + } + cnv->charErrorBuffer[0]=toBase64[(c>>2)&0x3f]; + cnv->charErrorBufferLength=1; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + } else { + if(offsets!=NULL) { + *offsets++=sourceIndex++; + } + cnv->charErrorBuffer[0]=toBase64[(c>>8)&0x3f]; + cnv->charErrorBuffer[1]=toBase64[(c>>2)&0x3f]; + cnv->charErrorBufferLength=2; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + bits=(uint8_t)((c&3)<<4); + base64Counter=2; + break; + case 2: + *target++=toBase64[bits|(c>>12)]; + if(target>6)&0x3f]; + if(targetcharErrorBuffer[0]=toBase64[c&0x3f]; + cnv->charErrorBufferLength=1; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + } else { + if(offsets!=NULL) { + *offsets++=sourceIndex++; + } + cnv->charErrorBuffer[0]=toBase64[(c>>6)&0x3f]; + cnv->charErrorBuffer[1]=toBase64[c&0x3f]; + cnv->charErrorBufferLength=2; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + bits=0; + base64Counter=0; + break; + default: + /* will never occur */ + break; + } + } + } else { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + } + } + + if(pArgs->flush && source>=sourceLimit) { + /* flush remaining bits to the target */ + if(!inDirectMode && base64Counter!=0) { + if(targetcharErrorBuffer[cnv->charErrorBufferLength++]=toBase64[bits]; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + } + /* reset the state for the next conversion */ + cnv->fromUnicodeStatus=(cnv->fromUnicodeStatus&0xf0000000)|0x1000000; /* keep version, inDirectMode=TRUE */ + } else { + /* set the converter state back into UConverter */ + cnv->fromUnicodeStatus= + (cnv->fromUnicodeStatus&0xf0000000)| /* keep version*/ + ((uint32_t)inDirectMode<<24)|((uint32_t)base64Counter<<16)|(uint32_t)bits; + } + + /* write back the updated pointers */ + pArgs->source=source; + pArgs->target=(char *)target; + pArgs->offsets=offsets; + return; +} + +static const char * +_UTF7GetName(const UConverter *cnv) { + switch(cnv->fromUnicodeStatus>>28) { + case 1: + return "UTF-7,version=1"; + default: + return "UTF-7"; + } +} + +static const UConverterImpl _UTF7Impl={ + UCNV_UTF7, + + NULL, + NULL, + + _UTF7Open, + NULL, + _UTF7Reset, + + _UTF7ToUnicodeWithOffsets, + _UTF7ToUnicodeWithOffsets, + _UTF7FromUnicodeWithOffsets, + _UTF7FromUnicodeWithOffsets, + NULL, + + NULL, + _UTF7GetName, + NULL, /* we don't need writeSub() because we never call a callback at fromUnicode() */ + NULL, + ucnv_getCompleteUnicodeSet +}; + +static const UConverterStaticData _UTF7StaticData={ + sizeof(UConverterStaticData), + "UTF-7", + 0, /* TODO CCSID for UTF-7 */ + UCNV_IBM, UCNV_UTF7, + 1, 4, + { 0x3f, 0, 0, 0 }, 1, /* the subchar is not used */ + FALSE, FALSE, + 0, + 0, + { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */ +}; + +const UConverterSharedData _UTF7Data={ + sizeof(UConverterSharedData), ~((uint32_t)0), + NULL, NULL, &_UTF7StaticData, FALSE, &_UTF7Impl, + 0 +}; + +/* IMAP mailbox name encoding ----------------------------------------------- */ + +/* + * RFC 2060: INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1 + * http://www.ietf.org/rfc/rfc2060.txt + * + * 5.1.3. Mailbox International Naming Convention + * + * By convention, international mailbox names are specified using a + * modified version of the UTF-7 encoding described in [UTF-7]. The + * purpose of these modifications is to correct the following problems + * with UTF-7: + * + * 1) UTF-7 uses the "+" character for shifting; this conflicts with + * the common use of "+" in mailbox names, in particular USENET + * newsgroup names. + * + * 2) UTF-7's encoding is BASE64 which uses the "/" character; this + * conflicts with the use of "/" as a popular hierarchy delimiter. + * + * 3) UTF-7 prohibits the unencoded usage of "\"; this conflicts with + * the use of "\" as a popular hierarchy delimiter. + * + * 4) UTF-7 prohibits the unencoded usage of "~"; this conflicts with + * the use of "~" in some servers as a home directory indicator. + * + * 5) UTF-7 permits multiple alternate forms to represent the same + * string; in particular, printable US-ASCII chararacters can be + * represented in encoded form. + * + * In modified UTF-7, printable US-ASCII characters except for "&" + * represent themselves; that is, characters with octet values 0x20-0x25 + * and 0x27-0x7e. The character "&" (0x26) is represented by the two- + * octet sequence "&-". + * + * All other characters (octet values 0x00-0x1f, 0x7f-0xff, and all + * Unicode 16-bit octets) are represented in modified BASE64, with a + * further modification from [UTF-7] that "," is used instead of "/". + * Modified BASE64 MUST NOT be used to represent any printing US-ASCII + * character which can represent itself. + * + * "&" is used to shift to modified BASE64 and "-" to shift back to US- + * ASCII. All names start in US-ASCII, and MUST end in US-ASCII (that + * is, a name that ends with a Unicode 16-bit octet MUST end with a "- + * "). + * + * For example, here is a mailbox name which mixes English, Japanese, + * and Chinese text: ~peter/mail/&ZeVnLIqe-/&U,BTFw- + */ + +/* + * Tests for US-ASCII characters belonging to character classes + * defined in UTF-7. + * + * Set D (directly encoded characters) consists of the following + * characters: the upper and lower case letters A through Z + * and a through z, the 10 digits 0-9, and the following nine special + * characters (note that "+" and "=" are omitted): + * '(),-./:? + * + * Set O (optional direct characters) consists of the following + * characters (note that "\" and "~" are omitted): + * !"#$%&*;<=>@[]^_`{|} + * + * According to the rules in RFC 2152, the byte values for the following + * US-ASCII characters are not used in UTF-7 and are therefore illegal: + * - all C0 control codes except for CR LF TAB + * - BACKSLASH + * - TILDE + * - DEL + * - all codes beyond US-ASCII, i.e. all >127 + */ + +/* uses '&' not '+' to start a base64 sequence */ +#define AMPERSAND 0x26 +#define COMMA 0x2c +#define SLASH 0x2f + +/* legal byte values: all US-ASCII graphic characters 0x20..0x7e */ +#define isLegalIMAP(c) (0x20<=(c) && (c)<=0x7e) + +/* direct-encode all of printable ASCII 0x20..0x7e except '&' 0x26 */ +#define inSetDIMAP(c) (isLegalIMAP(c) && c!=AMPERSAND) + +#define TO_BASE64_IMAP(n) ((n)<63 ? toBase64[n] : COMMA) +#define FROM_BASE64_IMAP(c) ((c)==COMMA ? 63 : (c)==SLASH ? -1 : fromBase64[c]) + +/* + * converter status values: + * + * toUnicodeStatus: + * 24 inDirectMode (boolean) + * 23..16 base64Counter (-1..7) + * 15..0 bits (up to 14 bits incoming base64) + * + * fromUnicodeStatus: + * 24 inDirectMode (boolean) + * 23..16 base64Counter (0..2) + * 7..0 bits (6 bits outgoing base64) + * + * ignore bits 31..25 + */ + +static void +_IMAPToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv; + const uint8_t *source, *sourceLimit; + UChar *target; + const UChar *targetLimit; + int32_t *offsets; + + uint8_t *bytes; + uint8_t byteIndex; + + int32_t length, targetCapacity; + + /* UTF-7 state */ + uint16_t bits; + int8_t base64Counter; + UBool inDirectMode; + + int8_t base64Value; + + int32_t sourceIndex, nextSourceIndex; + + UChar c; + uint8_t b; + + /* set up the local pointers */ + cnv=pArgs->converter; + + source=(const uint8_t *)pArgs->source; + sourceLimit=(const uint8_t *)pArgs->sourceLimit; + target=pArgs->target; + targetLimit=pArgs->targetLimit; + offsets=pArgs->offsets; + /* get the state machine state */ + { + uint32_t status=cnv->toUnicodeStatus; + inDirectMode=(UBool)((status>>24)&1); + base64Counter=(int8_t)(status>>16); + bits=(uint16_t)status; + } + bytes=cnv->toUBytes; + byteIndex=cnv->toULength; + + /* sourceIndex=-1 if the current character began in the previous buffer */ + sourceIndex=byteIndex==0 ? 0 : -1; + nextSourceIndex=0; + + if(inDirectMode) { +directMode: + /* + * In Direct Mode, US-ASCII characters are encoded directly, i.e., + * with their US-ASCII byte values. + * An ampersand starts Unicode (or "escape") Mode. + * + * In Direct Mode, only the sourceIndex is used. + */ + byteIndex=0; + length=(int32_t)(sourceLimit-source); + targetCapacity=(int32_t)(targetLimit-target); + if(length>targetCapacity) { + length=targetCapacity; + } + while(length>0) { + b=*source++; + if(!isLegalIMAP(b)) { + /* illegal */ + bytes[0]=b; + byteIndex=1; + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + break; + } else if(b!=AMPERSAND) { + /* write directly encoded character */ + *target++=b; + if(offsets!=NULL) { + *offsets++=sourceIndex++; + } + } else /* AMPERSAND */ { + /* switch to Unicode mode */ + nextSourceIndex=++sourceIndex; + inDirectMode=FALSE; + byteIndex=0; + bits=0; + base64Counter=-1; + goto unicodeMode; + } + --length; + } + if(source=targetLimit) { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + } else { +unicodeMode: + /* + * In Unicode (or "escape") Mode, UTF-16BE is base64-encoded. + * The base64 sequence ends with any character that is not in the base64 alphabet. + * A terminating minus sign is consumed. + * US-ASCII must not be base64-ed. + * + * In Unicode Mode, the sourceIndex has the index to the start of the current + * base64 bytes, while nextSourceIndex is precisely parallel to source, + * keeping the index to the following byte. + * Note that in 2 out of 3 cases, UChars overlap within a base64 byte. + */ + while(source0x7e) { + /* illegal - test other illegal US-ASCII values by base64Value==-3 */ + inDirectMode=TRUE; + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + break; + } else if((base64Value=FROM_BASE64_IMAP(b))>=0) { + /* collect base64 bytes into UChars */ + switch(base64Counter) { + case -1: /* -1 is immediately after the & */ + case 0: + bits=base64Value; + base64Counter=1; + break; + case 1: + case 3: + case 4: + case 6: + bits=(uint16_t)((bits<<6)|base64Value); + ++base64Counter; + break; + case 2: + c=(UChar)((bits<<4)|(base64Value>>2)); + if(isLegalIMAP(c)) { + /* illegal */ + inDirectMode=TRUE; + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + goto endloop; + } + *target++=c; + if(offsets!=NULL) { + *offsets++=sourceIndex; + sourceIndex=nextSourceIndex-1; + } + bytes[0]=b; /* keep this byte in case an error occurs */ + byteIndex=1; + bits=(uint16_t)(base64Value&3); + base64Counter=3; + break; + case 5: + c=(UChar)((bits<<2)|(base64Value>>4)); + if(isLegalIMAP(c)) { + /* illegal */ + inDirectMode=TRUE; + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + goto endloop; + } + *target++=c; + if(offsets!=NULL) { + *offsets++=sourceIndex; + sourceIndex=nextSourceIndex-1; + } + bytes[0]=b; /* keep this byte in case an error occurs */ + byteIndex=1; + bits=(uint16_t)(base64Value&15); + base64Counter=6; + break; + case 7: + c=(UChar)((bits<<6)|base64Value); + if(isLegalIMAP(c)) { + /* illegal */ + inDirectMode=TRUE; + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + goto endloop; + } + *target++=c; + if(offsets!=NULL) { + *offsets++=sourceIndex; + sourceIndex=nextSourceIndex; + } + byteIndex=0; + bits=0; + base64Counter=0; + break; + default: + /* will never occur */ + break; + } + } else if(base64Value==-2) { + /* minus sign terminates the base64 sequence */ + inDirectMode=TRUE; + if(base64Counter==-1) { + /* &- i.e. a minus immediately following an ampersand */ + *target++=AMPERSAND; + if(offsets!=NULL) { + *offsets++=sourceIndex-1; + } + } else { + /* absorb the minus and leave the Unicode Mode */ + if(bits!=0 || (base64Counter!=0 && base64Counter!=3 && base64Counter!=6)) { + /* bits are illegally left over, a UChar is incomplete */ + /* base64Counter other than 0, 3, 6 means non-minimal zero-padding, also illegal */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + break; + } + } + sourceIndex=nextSourceIndex; + goto directMode; + } else { + if(base64Counter==-1) { + /* illegal: & immediately followed by something other than base64 or minus sign */ + /* include the ampersand in the reported sequence */ + --sourceIndex; + bytes[0]=AMPERSAND; + bytes[1]=b; + byteIndex=2; + } + /* base64Value==-1 for characters that are illegal only in Unicode mode */ + /* base64Value==-3 for illegal characters */ + /* illegal */ + inDirectMode=TRUE; + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + break; + } + } else { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + } + } +endloop: + + /* + * the end of the input stream and detection of truncated input + * are handled by the framework, but here we must check if we are in Unicode + * mode and byteIndex==0 because we must end in direct mode + * + * conditions: + * successful + * in Unicode mode and byteIndex==0 + * end of input and no truncated input + */ + if( U_SUCCESS(*pErrorCode) && + !inDirectMode && byteIndex==0 && + pArgs->flush && source>=sourceLimit + ) { + if(base64Counter==-1) { + /* & at the very end of the input */ + /* make the ampersand the reported sequence */ + bytes[0]=AMPERSAND; + byteIndex=1; + } + /* else if(base64Counter!=-1) byteIndex remains 0 because there is no particular byte sequence */ + + inDirectMode=TRUE; /* avoid looping */ + *pErrorCode=U_TRUNCATED_CHAR_FOUND; + } + + /* set the converter state back into UConverter */ + cnv->toUnicodeStatus=((uint32_t)inDirectMode<<24)|((uint32_t)((uint8_t)base64Counter)<<16)|(uint32_t)bits; + cnv->toULength=byteIndex; + + /* write back the updated pointers */ + pArgs->source=(const char *)source; + pArgs->target=target; + pArgs->offsets=offsets; + return; +} + +static void +_IMAPFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv; + const UChar *source, *sourceLimit; + uint8_t *target, *targetLimit; + int32_t *offsets; + + int32_t length, targetCapacity, sourceIndex; + UChar c; + uint8_t b; + + /* UTF-7 state */ + uint8_t bits; + int8_t base64Counter; + UBool inDirectMode; + + /* set up the local pointers */ + cnv=pArgs->converter; + + /* set up the local pointers */ + source=pArgs->source; + sourceLimit=pArgs->sourceLimit; + target=(uint8_t *)pArgs->target; + targetLimit=(uint8_t *)pArgs->targetLimit; + offsets=pArgs->offsets; + + /* get the state machine state */ + { + uint32_t status=cnv->fromUnicodeStatus; + inDirectMode=(UBool)((status>>24)&1); + base64Counter=(int8_t)(status>>16); + bits=(uint8_t)status; + } + + /* UTF-7 always encodes UTF-16 code units, therefore we need only a simple sourceIndex */ + sourceIndex=0; + + if(inDirectMode) { +directMode: + length=(int32_t)(sourceLimit-source); + targetCapacity=(int32_t)(targetLimit-target); + if(length>targetCapacity) { + length=targetCapacity; + } + while(length>0) { + c=*source++; + /* encode 0x20..0x7e except '&' directly */ + if(inSetDIMAP(c)) { + /* encode directly */ + *target++=(uint8_t)c; + if(offsets!=NULL) { + *offsets++=sourceIndex++; + } + } else if(c==AMPERSAND) { + /* output &- for & */ + *target++=AMPERSAND; + if(targetcharErrorBuffer[0]=MINUS; + cnv->charErrorBufferLength=1; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + } else { + /* un-read this character and switch to Unicode Mode */ + --source; + *target++=AMPERSAND; + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + inDirectMode=FALSE; + base64Counter=0; + goto unicodeMode; + } + --length; + } + if(source=targetLimit) { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + } else { +unicodeMode: + while(sourcecharErrorBuffer[0]=MINUS; + cnv->charErrorBufferLength=1; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + goto directMode; + } else { + /* + * base64 this character: + * Output 2 or 3 base64 bytes for the remaining bits of the previous character + * and the bits of this character, each implicitly in UTF-16BE. + * + * Here, bits is an 8-bit variable because only 6 bits need to be kept from one + * character to the next. The actual 2 or 4 bits are shifted to the left edge + * of the 6-bits field 5..0 to make the termination of the base64 sequence easier. + */ + switch(base64Counter) { + case 0: + b=(uint8_t)(c>>10); + *target++=TO_BASE64_IMAP(b); + if(target>4)&0x3f); + *target++=TO_BASE64_IMAP(b); + if(offsets!=NULL) { + *offsets++=sourceIndex; + *offsets++=sourceIndex++; + } + } else { + if(offsets!=NULL) { + *offsets++=sourceIndex++; + } + b=(uint8_t)((c>>4)&0x3f); + cnv->charErrorBuffer[0]=TO_BASE64_IMAP(b); + cnv->charErrorBufferLength=1; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + bits=(uint8_t)((c&15)<<2); + base64Counter=1; + break; + case 1: + b=(uint8_t)(bits|(c>>14)); + *target++=TO_BASE64_IMAP(b); + if(target>8)&0x3f); + *target++=TO_BASE64_IMAP(b); + if(target>2)&0x3f); + *target++=TO_BASE64_IMAP(b); + if(offsets!=NULL) { + *offsets++=sourceIndex; + *offsets++=sourceIndex; + *offsets++=sourceIndex++; + } + } else { + if(offsets!=NULL) { + *offsets++=sourceIndex; + *offsets++=sourceIndex++; + } + b=(uint8_t)((c>>2)&0x3f); + cnv->charErrorBuffer[0]=TO_BASE64_IMAP(b); + cnv->charErrorBufferLength=1; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + } else { + if(offsets!=NULL) { + *offsets++=sourceIndex++; + } + b=(uint8_t)((c>>8)&0x3f); + cnv->charErrorBuffer[0]=TO_BASE64_IMAP(b); + b=(uint8_t)((c>>2)&0x3f); + cnv->charErrorBuffer[1]=TO_BASE64_IMAP(b); + cnv->charErrorBufferLength=2; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + bits=(uint8_t)((c&3)<<4); + base64Counter=2; + break; + case 2: + b=(uint8_t)(bits|(c>>12)); + *target++=TO_BASE64_IMAP(b); + if(target>6)&0x3f); + *target++=TO_BASE64_IMAP(b); + if(targetcharErrorBuffer[0]=TO_BASE64_IMAP(b); + cnv->charErrorBufferLength=1; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + } else { + if(offsets!=NULL) { + *offsets++=sourceIndex++; + } + b=(uint8_t)((c>>6)&0x3f); + cnv->charErrorBuffer[0]=TO_BASE64_IMAP(b); + b=(uint8_t)(c&0x3f); + cnv->charErrorBuffer[1]=TO_BASE64_IMAP(b); + cnv->charErrorBufferLength=2; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + bits=0; + base64Counter=0; + break; + default: + /* will never occur */ + break; + } + } + } else { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + } + } + + if(pArgs->flush && source>=sourceLimit) { + /* flush remaining bits to the target */ + if(!inDirectMode) { + if(base64Counter!=0) { + if(targetcharErrorBuffer[cnv->charErrorBufferLength++]=TO_BASE64_IMAP(bits); + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + } + /* need to terminate with a minus */ + if(targetcharErrorBuffer[cnv->charErrorBufferLength++]=MINUS; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + } + /* reset the state for the next conversion */ + cnv->fromUnicodeStatus=(cnv->fromUnicodeStatus&0xf0000000)|0x1000000; /* keep version, inDirectMode=TRUE */ + } else { + /* set the converter state back into UConverter */ + cnv->fromUnicodeStatus= + (cnv->fromUnicodeStatus&0xf0000000)| /* keep version*/ + ((uint32_t)inDirectMode<<24)|((uint32_t)base64Counter<<16)|(uint32_t)bits; + } + + /* write back the updated pointers */ + pArgs->source=source; + pArgs->target=(char *)target; + pArgs->offsets=offsets; + return; +} + +static const UConverterImpl _IMAPImpl={ + UCNV_IMAP_MAILBOX, + + NULL, + NULL, + + _UTF7Open, + NULL, + _UTF7Reset, + + _IMAPToUnicodeWithOffsets, + _IMAPToUnicodeWithOffsets, + _IMAPFromUnicodeWithOffsets, + _IMAPFromUnicodeWithOffsets, + NULL, + + NULL, + NULL, + NULL, /* we don't need writeSub() because we never call a callback at fromUnicode() */ + NULL, + ucnv_getCompleteUnicodeSet +}; + +static const UConverterStaticData _IMAPStaticData={ + sizeof(UConverterStaticData), + "IMAP-mailbox-name", + 0, /* TODO CCSID for IMAP-mailbox-name */ + UCNV_IBM, UCNV_IMAP_MAILBOX, + 1, 4, + { 0x3f, 0, 0, 0 }, 1, /* the subchar is not used */ + FALSE, FALSE, + 0, + 0, + { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */ +}; + +const UConverterSharedData _IMAPData={ + sizeof(UConverterSharedData), ~((uint32_t)0), + NULL, NULL, &_IMAPStaticData, FALSE, &_IMAPImpl, + 0 +}; + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/ucnv_u8.c b/Build/source/libs/icu/icu-xetex/common/ucnv_u8.c new file mode 100644 index 00000000000..75f554c427b --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnv_u8.c @@ -0,0 +1,1084 @@ +/* +********************************************************************** +* Copyright (C) 2002-2007, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* file name: ucnv_u8.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2002jul01 +* created by: Markus W. Scherer +* +* UTF-8 converter implementation. Used to be in ucnv_utf.c. +* +* Also, CESU-8 implementation, see UTR 26. +* The CESU-8 converter uses all the same functions as the +* UTF-8 converter, with a branch for converting supplementary code points. +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "unicode/ucnv.h" +#include "ucnv_bld.h" +#include "ucnv_cnv.h" +#include "cmemory.h" + +/* Prototypes --------------------------------------------------------------- */ + +/* Keep these here to make finicky compilers happy */ + +U_CFUNC void ucnv_fromUnicode_UTF8(UConverterFromUnicodeArgs *args, + UErrorCode *err); +U_CFUNC void ucnv_fromUnicode_UTF8_OFFSETS_LOGIC(UConverterFromUnicodeArgs *args, + UErrorCode *err); + + +/* UTF-8 -------------------------------------------------------------------- */ + +/* UTF-8 Conversion DATA + * for more information see Unicode Standard 2.0, Transformation Formats Appendix A-9 + */ +/*static const uint32_t REPLACEMENT_CHARACTER = 0x0000FFFD;*/ +#define MAXIMUM_UCS2 0x0000FFFF +#define MAXIMUM_UTF 0x0010FFFF +#define MAXIMUM_UCS4 0x7FFFFFFF +#define HALF_SHIFT 10 +#define HALF_BASE 0x0010000 +#define HALF_MASK 0x3FF +#define SURROGATE_HIGH_START 0xD800 +#define SURROGATE_HIGH_END 0xDBFF +#define SURROGATE_LOW_START 0xDC00 +#define SURROGATE_LOW_END 0xDFFF + +/* -SURROGATE_LOW_START + HALF_BASE */ +#define SURROGATE_LOW_BASE 9216 + +static const uint32_t offsetsFromUTF8[7] = {0, + (uint32_t) 0x00000000, (uint32_t) 0x00003080, (uint32_t) 0x000E2080, + (uint32_t) 0x03C82080, (uint32_t) 0xFA082080, (uint32_t) 0x82082080 +}; + +/* END OF UTF-8 Conversion DATA */ + +static const int8_t bytesFromUTF8[256] = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0 +}; + +/* + * Starting with Unicode 3.0.1: + * UTF-8 byte sequences of length N _must_ encode code points of or above utf8_minChar32[N]; + * byte sequences with more than 4 bytes are illegal in UTF-8, + * which is tested with impossible values for them + */ +static const uint32_t +utf8_minChar32[7]={ 0, 0, 0x80, 0x800, 0x10000, 0xffffffff, 0xffffffff }; + +static void ucnv_toUnicode_UTF8 (UConverterToUnicodeArgs * args, + UErrorCode * err) +{ + UConverter *cnv = args->converter; + const unsigned char *mySource = (unsigned char *) args->source; + UChar *myTarget = args->target; + const unsigned char *sourceLimit = (unsigned char *) args->sourceLimit; + const UChar *targetLimit = args->targetLimit; + unsigned char *toUBytes = cnv->toUBytes; + UBool isCESU8 = (UBool)(cnv->sharedData == &_CESU8Data); + uint32_t ch, ch2 = 0; + int32_t i, inBytes; + + /* Restore size of current sequence */ + if (cnv->toUnicodeStatus && myTarget < targetLimit) + { + inBytes = cnv->mode; /* restore # of bytes to consume */ + i = cnv->toULength; /* restore # of bytes consumed */ + cnv->toULength = 0; + + ch = cnv->toUnicodeStatus;/*Stores the previously calculated ch from a previous call*/ + cnv->toUnicodeStatus = 0; + goto morebytes; + } + + + while (mySource < sourceLimit && myTarget < targetLimit) + { + ch = *(mySource++); + if (ch < 0x80) /* Simple case */ + { + *(myTarget++) = (UChar) ch; + } + else + { + /* store the first char */ + toUBytes[0] = (char)ch; + inBytes = bytesFromUTF8[ch]; /* lookup current sequence length */ + i = 1; + +morebytes: + while (i < inBytes) + { + if (mySource < sourceLimit) + { + toUBytes[i] = (char) (ch2 = *mySource); + if (!UTF8_IS_TRAIL(ch2)) + { + break; /* i < inBytes */ + } + ch = (ch << 6) + ch2; + ++mySource; + i++; + } + else + { + /* stores a partially calculated target*/ + cnv->toUnicodeStatus = ch; + cnv->mode = inBytes; + cnv->toULength = (int8_t) i; + goto donefornow; + } + } + + /* Remove the accumulated high bits */ + ch -= offsetsFromUTF8[inBytes]; + + /* + * Legal UTF-8 byte sequences in Unicode 3.0.1 and up: + * - use only trail bytes after a lead byte (checked above) + * - use the right number of trail bytes for a given lead byte + * - encode a code point <= U+10ffff + * - use the fewest possible number of bytes for their code points + * - use at most 4 bytes (for i>=5 it is 0x10ffff= utf8_minChar32[i] && + (isCESU8 ? i <= 3 : !UTF_IS_SURROGATE(ch))) + { + /* Normal valid byte when the loop has not prematurely terminated (i < inBytes) */ + if (ch <= MAXIMUM_UCS2) + { + /* fits in 16 bits */ + *(myTarget++) = (UChar) ch; + } + else + { + /* write out the surrogates */ + ch -= HALF_BASE; + *(myTarget++) = (UChar) ((ch >> HALF_SHIFT) + SURROGATE_HIGH_START); + ch = (ch & HALF_MASK) + SURROGATE_LOW_START; + if (myTarget < targetLimit) + { + *(myTarget++) = (UChar)ch; + } + else + { + /* Put in overflow buffer (not handled here) */ + cnv->UCharErrorBuffer[0] = (UChar) ch; + cnv->UCharErrorBufferLength = 1; + *err = U_BUFFER_OVERFLOW_ERROR; + break; + } + } + } + else + { + cnv->toULength = (int8_t)i; + *err = U_ILLEGAL_CHAR_FOUND; + break; + } + } + } + +donefornow: + if (mySource < sourceLimit && myTarget >= targetLimit && U_SUCCESS(*err)) + { + /* End of target buffer */ + *err = U_BUFFER_OVERFLOW_ERROR; + } + + args->target = myTarget; + args->source = (const char *) mySource; +} + +static void ucnv_toUnicode_UTF8_OFFSETS_LOGIC (UConverterToUnicodeArgs * args, + UErrorCode * err) +{ + UConverter *cnv = args->converter; + const unsigned char *mySource = (unsigned char *) args->source; + UChar *myTarget = args->target; + int32_t *myOffsets = args->offsets; + int32_t offsetNum = 0; + const unsigned char *sourceLimit = (unsigned char *) args->sourceLimit; + const UChar *targetLimit = args->targetLimit; + unsigned char *toUBytes = cnv->toUBytes; + UBool isCESU8 = (UBool)(cnv->sharedData == &_CESU8Data); + uint32_t ch, ch2 = 0; + int32_t i, inBytes; + + /* Restore size of current sequence */ + if (cnv->toUnicodeStatus && myTarget < targetLimit) + { + inBytes = cnv->mode; /* restore # of bytes to consume */ + i = cnv->toULength; /* restore # of bytes consumed */ + cnv->toULength = 0; + + ch = cnv->toUnicodeStatus;/*Stores the previously calculated ch from a previous call*/ + cnv->toUnicodeStatus = 0; + goto morebytes; + } + + while (mySource < sourceLimit && myTarget < targetLimit) + { + ch = *(mySource++); + if (ch < 0x80) /* Simple case */ + { + *(myTarget++) = (UChar) ch; + *(myOffsets++) = offsetNum++; + } + else + { + toUBytes[0] = (char)ch; + inBytes = bytesFromUTF8[ch]; + i = 1; + +morebytes: + while (i < inBytes) + { + if (mySource < sourceLimit) + { + toUBytes[i] = (char) (ch2 = *mySource); + if (!UTF8_IS_TRAIL(ch2)) + { + break; /* i < inBytes */ + } + ch = (ch << 6) + ch2; + ++mySource; + i++; + } + else + { + cnv->toUnicodeStatus = ch; + cnv->mode = inBytes; + cnv->toULength = (int8_t)i; + goto donefornow; + } + } + + /* Remove the accumulated high bits */ + ch -= offsetsFromUTF8[inBytes]; + + /* + * Legal UTF-8 byte sequences in Unicode 3.0.1 and up: + * - use only trail bytes after a lead byte (checked above) + * - use the right number of trail bytes for a given lead byte + * - encode a code point <= U+10ffff + * - use the fewest possible number of bytes for their code points + * - use at most 4 bytes (for i>=5 it is 0x10ffff= utf8_minChar32[i] && + (isCESU8 ? i <= 3 : !UTF_IS_SURROGATE(ch))) + { + /* Normal valid byte when the loop has not prematurely terminated (i < inBytes) */ + if (ch <= MAXIMUM_UCS2) + { + /* fits in 16 bits */ + *(myTarget++) = (UChar) ch; + *(myOffsets++) = offsetNum; + } + else + { + /* write out the surrogates */ + ch -= HALF_BASE; + *(myTarget++) = (UChar) ((ch >> HALF_SHIFT) + SURROGATE_HIGH_START); + *(myOffsets++) = offsetNum; + ch = (ch & HALF_MASK) + SURROGATE_LOW_START; + if (myTarget < targetLimit) + { + *(myTarget++) = (UChar)ch; + *(myOffsets++) = offsetNum; + } + else + { + cnv->UCharErrorBuffer[0] = (UChar) ch; + cnv->UCharErrorBufferLength = 1; + *err = U_BUFFER_OVERFLOW_ERROR; + } + } + offsetNum += i; + } + else + { + cnv->toULength = (int8_t)i; + *err = U_ILLEGAL_CHAR_FOUND; + break; + } + } + } + +donefornow: + if (mySource < sourceLimit && myTarget >= targetLimit && U_SUCCESS(*err)) + { /* End of target buffer */ + *err = U_BUFFER_OVERFLOW_ERROR; + } + + args->target = myTarget; + args->source = (const char *) mySource; + args->offsets = myOffsets; +} + +U_CFUNC void ucnv_fromUnicode_UTF8 (UConverterFromUnicodeArgs * args, + UErrorCode * err) +{ + UConverter *cnv = args->converter; + const UChar *mySource = args->source; + const UChar *sourceLimit = args->sourceLimit; + uint8_t *myTarget = (uint8_t *) args->target; + const uint8_t *targetLimit = (uint8_t *) args->targetLimit; + uint8_t *tempPtr; + UChar32 ch; + uint8_t tempBuf[4]; + int32_t indexToWrite; + UBool isNotCESU8 = (UBool)(cnv->sharedData != &_CESU8Data); + + if (cnv->fromUChar32 && myTarget < targetLimit) + { + ch = cnv->fromUChar32; + cnv->fromUChar32 = 0; + goto lowsurrogate; + } + + while (mySource < sourceLimit && myTarget < targetLimit) + { + ch = *(mySource++); + + if (ch < 0x80) /* Single byte */ + { + *(myTarget++) = (uint8_t) ch; + } + else if (ch < 0x800) /* Double byte */ + { + *(myTarget++) = (uint8_t) ((ch >> 6) | 0xc0); + if (myTarget < targetLimit) + { + *(myTarget++) = (uint8_t) ((ch & 0x3f) | 0x80); + } + else + { + cnv->charErrorBuffer[0] = (uint8_t) ((ch & 0x3f) | 0x80); + cnv->charErrorBufferLength = 1; + *err = U_BUFFER_OVERFLOW_ERROR; + } + } + else { + /* Check for surrogates */ + if(UTF_IS_SURROGATE(ch) && isNotCESU8) { +lowsurrogate: + if (mySource < sourceLimit) { + /* test both code units */ + if(UTF_IS_SURROGATE_FIRST(ch) && UTF_IS_SECOND_SURROGATE(*mySource)) { + /* convert and consume this supplementary code point */ + ch=UTF16_GET_PAIR_VALUE(ch, *mySource); + ++mySource; + /* exit this condition tree */ + } + else { + /* this is an unpaired trail or lead code unit */ + /* callback(illegal) */ + cnv->fromUChar32 = ch; + *err = U_ILLEGAL_CHAR_FOUND; + break; + } + } + else { + /* no more input */ + cnv->fromUChar32 = ch; + break; + } + } + + /* Do we write the buffer directly for speed, + or do we have to be careful about target buffer space? */ + tempPtr = (((targetLimit - myTarget) >= 4) ? myTarget : tempBuf); + + if (ch <= MAXIMUM_UCS2) { + indexToWrite = 2; + tempPtr[0] = (uint8_t) ((ch >> 12) | 0xe0); + } + else { + indexToWrite = 3; + tempPtr[0] = (uint8_t) ((ch >> 18) | 0xf0); + tempPtr[1] = (uint8_t) (((ch >> 12) & 0x3f) | 0x80); + } + tempPtr[indexToWrite-1] = (uint8_t) (((ch >> 6) & 0x3f) | 0x80); + tempPtr[indexToWrite] = (uint8_t) ((ch & 0x3f) | 0x80); + + if (tempPtr == myTarget) { + /* There was enough space to write the codepoint directly. */ + myTarget += (indexToWrite + 1); + } + else { + /* We might run out of room soon. Write it slowly. */ + for (; tempPtr <= (tempBuf + indexToWrite); tempPtr++) { + if (myTarget < targetLimit) { + *(myTarget++) = *tempPtr; + } + else { + cnv->charErrorBuffer[cnv->charErrorBufferLength++] = *tempPtr; + *err = U_BUFFER_OVERFLOW_ERROR; + } + } + } + } + } + + if (mySource < sourceLimit && myTarget >= targetLimit && U_SUCCESS(*err)) + { + *err = U_BUFFER_OVERFLOW_ERROR; + } + + args->target = (char *) myTarget; + args->source = mySource; +} + +U_CFUNC void ucnv_fromUnicode_UTF8_OFFSETS_LOGIC (UConverterFromUnicodeArgs * args, + UErrorCode * err) +{ + UConverter *cnv = args->converter; + const UChar *mySource = args->source; + int32_t *myOffsets = args->offsets; + const UChar *sourceLimit = args->sourceLimit; + uint8_t *myTarget = (uint8_t *) args->target; + const uint8_t *targetLimit = (uint8_t *) args->targetLimit; + uint8_t *tempPtr; + UChar32 ch; + int32_t offsetNum, nextSourceIndex; + int32_t indexToWrite; + uint8_t tempBuf[4]; + UBool isNotCESU8 = (UBool)(cnv->sharedData != &_CESU8Data); + + if (cnv->fromUChar32 && myTarget < targetLimit) + { + ch = cnv->fromUChar32; + cnv->fromUChar32 = 0; + offsetNum = -1; + nextSourceIndex = 0; + goto lowsurrogate; + } else { + offsetNum = 0; + } + + while (mySource < sourceLimit && myTarget < targetLimit) + { + ch = *(mySource++); + + if (ch < 0x80) /* Single byte */ + { + *(myOffsets++) = offsetNum++; + *(myTarget++) = (char) ch; + } + else if (ch < 0x800) /* Double byte */ + { + *(myOffsets++) = offsetNum; + *(myTarget++) = (uint8_t) ((ch >> 6) | 0xc0); + if (myTarget < targetLimit) + { + *(myOffsets++) = offsetNum++; + *(myTarget++) = (uint8_t) ((ch & 0x3f) | 0x80); + } + else + { + cnv->charErrorBuffer[0] = (uint8_t) ((ch & 0x3f) | 0x80); + cnv->charErrorBufferLength = 1; + *err = U_BUFFER_OVERFLOW_ERROR; + } + } + else + /* Check for surrogates */ + { + nextSourceIndex = offsetNum + 1; + + if(UTF_IS_SURROGATE(ch) && isNotCESU8) { +lowsurrogate: + if (mySource < sourceLimit) { + /* test both code units */ + if(UTF_IS_SURROGATE_FIRST(ch) && UTF_IS_SECOND_SURROGATE(*mySource)) { + /* convert and consume this supplementary code point */ + ch=UTF16_GET_PAIR_VALUE(ch, *mySource); + ++mySource; + ++nextSourceIndex; + /* exit this condition tree */ + } + else { + /* this is an unpaired trail or lead code unit */ + /* callback(illegal) */ + cnv->fromUChar32 = ch; + *err = U_ILLEGAL_CHAR_FOUND; + break; + } + } + else { + /* no more input */ + cnv->fromUChar32 = ch; + break; + } + } + + /* Do we write the buffer directly for speed, + or do we have to be careful about target buffer space? */ + tempPtr = (((targetLimit - myTarget) >= 4) ? myTarget : tempBuf); + + if (ch <= MAXIMUM_UCS2) { + indexToWrite = 2; + tempPtr[0] = (uint8_t) ((ch >> 12) | 0xe0); + } + else { + indexToWrite = 3; + tempPtr[0] = (uint8_t) ((ch >> 18) | 0xf0); + tempPtr[1] = (uint8_t) (((ch >> 12) & 0x3f) | 0x80); + } + tempPtr[indexToWrite-1] = (uint8_t) (((ch >> 6) & 0x3f) | 0x80); + tempPtr[indexToWrite] = (uint8_t) ((ch & 0x3f) | 0x80); + + if (tempPtr == myTarget) { + /* There was enough space to write the codepoint directly. */ + myTarget += (indexToWrite + 1); + myOffsets[0] = offsetNum; + myOffsets[1] = offsetNum; + myOffsets[2] = offsetNum; + if (indexToWrite >= 3) { + myOffsets[3] = offsetNum; + } + myOffsets += (indexToWrite + 1); + } + else { + /* We might run out of room soon. Write it slowly. */ + for (; tempPtr <= (tempBuf + indexToWrite); tempPtr++) { + if (myTarget < targetLimit) + { + *(myOffsets++) = offsetNum; + *(myTarget++) = *tempPtr; + } + else + { + cnv->charErrorBuffer[cnv->charErrorBufferLength++] = *tempPtr; + *err = U_BUFFER_OVERFLOW_ERROR; + } + } + } + offsetNum = nextSourceIndex; + } + } + + if (mySource < sourceLimit && myTarget >= targetLimit && U_SUCCESS(*err)) + { + *err = U_BUFFER_OVERFLOW_ERROR; + } + + args->target = (char *) myTarget; + args->source = mySource; + args->offsets = myOffsets; +} + +static UChar32 ucnv_getNextUChar_UTF8(UConverterToUnicodeArgs *args, + UErrorCode *err) { + UConverter *cnv; + const uint8_t *sourceInitial; + const uint8_t *source; + uint16_t extraBytesToWrite; + uint8_t myByte; + UChar32 ch; + int8_t i, isLegalSequence; + + /* UTF-8 only here, the framework handles CESU-8 to combine surrogate pairs */ + + cnv = args->converter; + sourceInitial = source = (const uint8_t *)args->source; + if (source >= (const uint8_t *)args->sourceLimit) + { + /* no input */ + *err = U_INDEX_OUTOFBOUNDS_ERROR; + return 0xffff; + } + + myByte = (uint8_t)*(source++); + if (myByte < 0x80) + { + args->source = (const char *)source; + return (UChar32)myByte; + } + + extraBytesToWrite = (uint16_t)bytesFromUTF8[myByte]; + if (extraBytesToWrite == 0) { + cnv->toUBytes[0] = myByte; + cnv->toULength = 1; + *err = U_ILLEGAL_CHAR_FOUND; + args->source = (const char *)source; + return 0xffff; + } + + /*The byte sequence is longer than the buffer area passed*/ + if (((const char *)source + extraBytesToWrite - 1) > args->sourceLimit) + { + /* check if all of the remaining bytes are trail bytes */ + cnv->toUBytes[0] = myByte; + i = 1; + *err = U_TRUNCATED_CHAR_FOUND; + while(source < (const uint8_t *)args->sourceLimit) { + if(U8_IS_TRAIL(myByte = *source)) { + cnv->toUBytes[i++] = myByte; + ++source; + } else { + /* error even before we run out of input */ + *err = U_ILLEGAL_CHAR_FOUND; + break; + } + } + cnv->toULength = i; + args->source = (const char *)source; + return 0xffff; + } + + isLegalSequence = 1; + ch = myByte << 6; + switch(extraBytesToWrite) + { + /* note: code falls through cases! (sic)*/ + case 6: + ch += (myByte = *source); + ch <<= 6; + if (!UTF8_IS_TRAIL(myByte)) + { + isLegalSequence = 0; + break; + } + ++source; + case 5: + ch += (myByte = *source); + ch <<= 6; + if (!UTF8_IS_TRAIL(myByte)) + { + isLegalSequence = 0; + break; + } + ++source; + case 4: + ch += (myByte = *source); + ch <<= 6; + if (!UTF8_IS_TRAIL(myByte)) + { + isLegalSequence = 0; + break; + } + ++source; + case 3: + ch += (myByte = *source); + ch <<= 6; + if (!UTF8_IS_TRAIL(myByte)) + { + isLegalSequence = 0; + break; + } + ++source; + case 2: + ch += (myByte = *source); + if (!UTF8_IS_TRAIL(myByte)) + { + isLegalSequence = 0; + break; + } + ++source; + }; + ch -= offsetsFromUTF8[extraBytesToWrite]; + args->source = (const char *)source; + + /* + * Legal UTF-8 byte sequences in Unicode 3.0.1 and up: + * - use only trail bytes after a lead byte (checked above) + * - use the right number of trail bytes for a given lead byte + * - encode a code point <= U+10ffff + * - use the fewest possible number of bytes for their code points + * - use at most 4 bytes (for i>=5 it is 0x10ffff= utf8_minChar32[extraBytesToWrite] && + !U_IS_SURROGATE(ch) + ) { + return ch; /* return the code point */ + } + + for(i = 0; sourceInitial < source; ++i) { + cnv->toUBytes[i] = *sourceInitial++; + } + cnv->toULength = i; + *err = U_ILLEGAL_CHAR_FOUND; + return 0xffff; +} + +/* UTF-8-from-UTF-8 conversion functions ------------------------------------ */ + +/* minimum code point values for n-byte UTF-8 sequences, n=0..4 */ +static const UChar32 +utf8_minLegal[5]={ 0, 0, 0x80, 0x800, 0x10000 }; + +/* offsets for n-byte UTF-8 sequences that were calculated with ((lead<<6)+trail)<<6+trail... */ +static const UChar32 +utf8_offsets[7]={ 0, 0, 0x3080, 0xE2080, 0x3C82080 }; + +/* "Convert" UTF-8 to UTF-8: Validate and copy. Modified from ucnv_DBCSFromUTF8(). */ +static void +ucnv_UTF8FromUTF8(UConverterFromUnicodeArgs *pFromUArgs, + UConverterToUnicodeArgs *pToUArgs, + UErrorCode *pErrorCode) { + UConverter *utf8, *cnv; + const uint8_t *source, *sourceLimit; + uint8_t *target; + int32_t targetCapacity; + int32_t count; + + int8_t oldToULength, toULength, toULimit; + + UChar32 c; + uint8_t b, t1, t2; + + /* set up the local pointers */ + utf8=pToUArgs->converter; + cnv=pFromUArgs->converter; + source=(uint8_t *)pToUArgs->source; + sourceLimit=(uint8_t *)pToUArgs->sourceLimit; + target=(uint8_t *)pFromUArgs->target; + targetCapacity=(int32_t)(pFromUArgs->targetLimit-pFromUArgs->target); + + /* get the converter state from the UTF-8 UConverter */ + c=(UChar32)utf8->toUnicodeStatus; + if(c!=0) { + toULength=oldToULength=utf8->toULength; + toULimit=(int8_t)utf8->mode; + } else { + toULength=oldToULength=toULimit=0; + } + + count=(int32_t)(sourceLimit-source)+oldToULength; + if(counttargetCapacity) { + count=targetCapacity; + } + + i=0; + while(i<3 && i<(count-toULimit)) { + b=source[count-oldToULength-i-1]; + if(U8_IS_TRAIL(b)) { + ++i; + } else { + if(itoUnicodeStatus=0; + utf8->toULength=0; + goto moreBytes; + /* See note in ucnv_SBCSFromUTF8() about this goto. */ + } + + /* conversion loop */ + while(count>0) { + b=*source++; + if((int8_t)b>=0) { + /* convert ASCII */ + *target++=b; + --count; + continue; + } else { + if(b>0xe0) { + if( /* handle U+1000..U+D7FF inline */ + (t1=source[0]) >= 0x80 && ((b<0xed && (t1 <= 0xbf)) || + (b==0xed && (t1 <= 0x9f))) && + (t2=source[1]) >= 0x80 && t2 <= 0xbf + ) { + source+=2; + *target++=b; + *target++=t1; + *target++=t2; + count-=3; + continue; + } + } else if(b<0xe0) { + if( /* handle U+0080..U+07FF inline */ + b>=0xc2 && + (t1=*source) >= 0x80 && t1 <= 0xbf + ) { + ++source; + *target++=b; + *target++=t1; + count-=2; + continue; + } + } else if(b==0xe0) { + if( /* handle U+0800..U+0FFF inline */ + (t1=source[0]) >= 0xa0 && t1 <= 0xbf && + (t2=source[1]) >= 0x80 && t2 <= 0xbf + ) { + source+=2; + *target++=b; + *target++=t1; + *target++=t2; + count-=3; + continue; + } + } + + /* handle "complicated" and error cases, and continuing partial characters */ + oldToULength=0; + toULength=1; + toULimit=utf8_countTrailBytes[b]+1; + c=b; +moreBytes: + while(toULengthtoUBytes[oldToULength++]=*source++; + } + utf8->toUnicodeStatus=c; + utf8->toULength=toULength; + utf8->mode=toULimit; + pToUArgs->source=(char *)source; + pFromUArgs->target=(char *)target; + return; + } + } + + if( toULength==toULimit && /* consumed all trail bytes */ + (toULength==3 || toULength==2) && /* BMP */ + (c-=utf8_offsets[toULength])>=utf8_minLegal[toULength] && + (c<=0xd7ff || 0xe000<=c) /* not a surrogate */ + ) { + /* legal byte sequence for BMP code point */ + } else if( + toULength==toULimit && toULength==4 && + (0x10000<=(c-=utf8_offsets[4]) && c<=0x10ffff) + ) { + /* legal byte sequence for supplementary code point */ + } else { + /* error handling: illegal UTF-8 byte sequence */ + source-=(toULength-oldToULength); + while(oldToULengthtoUBytes[oldToULength++]=*source++; + } + utf8->toULength=toULength; + pToUArgs->source=(char *)source; + pFromUArgs->target=(char *)target; + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + return; + } + + /* copy the legal byte sequence to the target */ + { + int8_t i; + + for(i=0; itoUBytes[i]; + } + source-=(toULength-oldToULength); + for(; itargetLimit) { + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } else { + b=*source; + toULimit=utf8_countTrailBytes[b]+1; + if(toULimit>(sourceLimit-source)) { + /* collect a truncated byte sequence */ + toULength=0; + c=b; + for(;;) { + utf8->toUBytes[toULength++]=b; + if(++source==sourceLimit) { + /* partial byte sequence at end of source */ + utf8->toUnicodeStatus=c; + utf8->toULength=toULength; + utf8->mode=toULimit; + break; + } else if(!U8_IS_TRAIL(b=*source)) { + /* lead byte in trail byte position */ + utf8->toULength=toULength; + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + break; + } + c=(c<<6)+b; + } + } else { + /* partial-sequence target overflow: fall back to the pivoting implementation */ + *pErrorCode=U_USING_DEFAULT_WARNING; + } + } + } + + /* write back the updated pointers */ + pToUArgs->source=(char *)source; + pFromUArgs->target=(char *)target; +} + +/* UTF-8 converter data ----------------------------------------------------- */ + +static const UConverterImpl _UTF8Impl={ + UCNV_UTF8, + + NULL, + NULL, + + NULL, + NULL, + NULL, + + ucnv_toUnicode_UTF8, + ucnv_toUnicode_UTF8_OFFSETS_LOGIC, + ucnv_fromUnicode_UTF8, + ucnv_fromUnicode_UTF8_OFFSETS_LOGIC, + ucnv_getNextUChar_UTF8, + + NULL, + NULL, + NULL, + NULL, + ucnv_getNonSurrogateUnicodeSet, + + ucnv_UTF8FromUTF8, + ucnv_UTF8FromUTF8 +}; + +/* The 1208 CCSID refers to any version of Unicode of UTF-8 */ +static const UConverterStaticData _UTF8StaticData={ + sizeof(UConverterStaticData), + "UTF-8", + 1208, UCNV_IBM, UCNV_UTF8, + 1, 3, /* max 3 bytes per UChar from UTF-8 (4 bytes from surrogate _pair_) */ + { 0xef, 0xbf, 0xbd, 0 },3,FALSE,FALSE, + 0, + 0, + { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */ +}; + + +const UConverterSharedData _UTF8Data={ + sizeof(UConverterSharedData), ~((uint32_t) 0), + NULL, NULL, &_UTF8StaticData, FALSE, &_UTF8Impl, + 0 +}; + +/* CESU-8 converter data ---------------------------------------------------- */ + +static const UConverterImpl _CESU8Impl={ + UCNV_CESU8, + + NULL, + NULL, + + NULL, + NULL, + NULL, + + ucnv_toUnicode_UTF8, + ucnv_toUnicode_UTF8_OFFSETS_LOGIC, + ucnv_fromUnicode_UTF8, + ucnv_fromUnicode_UTF8_OFFSETS_LOGIC, + NULL, + + NULL, + NULL, + NULL, + NULL, + ucnv_getCompleteUnicodeSet +}; + +static const UConverterStaticData _CESU8StaticData={ + sizeof(UConverterStaticData), + "CESU-8", + 9400, /* CCSID for CESU-8 */ + UCNV_UNKNOWN, UCNV_CESU8, 1, 3, + { 0xef, 0xbf, 0xbd, 0 },3,FALSE,FALSE, + 0, + 0, + { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */ +}; + + +const UConverterSharedData _CESU8Data={ + sizeof(UConverterSharedData), ~((uint32_t) 0), + NULL, NULL, &_CESU8StaticData, FALSE, &_CESU8Impl, + 0 +}; + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/ucnvbocu.c b/Build/source/libs/icu/icu-xetex/common/ucnvbocu.c new file mode 100644 index 00000000000..8623b114208 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnvbocu.c @@ -0,0 +1,1393 @@ +/* +****************************************************************************** +* +* Copyright (C) 2002-2005, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: ucnvbocu.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2002mar27 +* created by: Markus W. Scherer +* +* This is an implementation of the Binary Ordered Compression for Unicode, +* in its MIME-friendly form as defined in http://www.unicode.org/notes/tn6/ +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "unicode/ucnv.h" +#include "unicode/ucnv_cb.h" +#include "ucnv_bld.h" +#include "ucnv_cnv.h" + +/* BOCU-1 constants and macros ---------------------------------------------- */ + +/* + * BOCU-1 encodes the code points of a Unicode string as + * a sequence of byte-encoded differences (slope detection), + * preserving lexical order. + * + * Optimize the difference-taking for runs of Unicode text within + * small scripts: + * + * Most small scripts are allocated within aligned 128-blocks of Unicode + * code points. Lexical order is preserved if the "previous code point" state + * is always moved into the middle of such a block. + * + * Additionally, "prev" is moved from anywhere in the Unihan and Hangul + * areas into the middle of those areas. + * + * C0 control codes and space are encoded with their US-ASCII bytes. + * "prev" is reset for C0 controls but not for space. + */ + +/* initial value for "prev": middle of the ASCII range */ +#define BOCU1_ASCII_PREV 0x40 + +/* bounding byte values for differences */ +#define BOCU1_MIN 0x21 +#define BOCU1_MIDDLE 0x90 +#define BOCU1_MAX_LEAD 0xfe +#define BOCU1_MAX_TRAIL 0xff +#define BOCU1_RESET 0xff + +/* number of lead bytes */ +#define BOCU1_COUNT (BOCU1_MAX_LEAD-BOCU1_MIN+1) + +/* adjust trail byte counts for the use of some C0 control byte values */ +#define BOCU1_TRAIL_CONTROLS_COUNT 20 +#define BOCU1_TRAIL_BYTE_OFFSET (BOCU1_MIN-BOCU1_TRAIL_CONTROLS_COUNT) + +/* number of trail bytes */ +#define BOCU1_TRAIL_COUNT ((BOCU1_MAX_TRAIL-BOCU1_MIN+1)+BOCU1_TRAIL_CONTROLS_COUNT) + +/* + * number of positive and negative single-byte codes + * (counting 0==BOCU1_MIDDLE among the positive ones) + */ +#define BOCU1_SINGLE 64 + +/* number of lead bytes for positive and negative 2/3/4-byte sequences */ +#define BOCU1_LEAD_2 43 +#define BOCU1_LEAD_3 3 +#define BOCU1_LEAD_4 1 + +/* The difference value range for single-byters. */ +#define BOCU1_REACH_POS_1 (BOCU1_SINGLE-1) +#define BOCU1_REACH_NEG_1 (-BOCU1_SINGLE) + +/* The difference value range for double-byters. */ +#define BOCU1_REACH_POS_2 (BOCU1_REACH_POS_1+BOCU1_LEAD_2*BOCU1_TRAIL_COUNT) +#define BOCU1_REACH_NEG_2 (BOCU1_REACH_NEG_1-BOCU1_LEAD_2*BOCU1_TRAIL_COUNT) + +/* The difference value range for 3-byters. */ +#define BOCU1_REACH_POS_3 \ + (BOCU1_REACH_POS_2+BOCU1_LEAD_3*BOCU1_TRAIL_COUNT*BOCU1_TRAIL_COUNT) + +#define BOCU1_REACH_NEG_3 (BOCU1_REACH_NEG_2-BOCU1_LEAD_3*BOCU1_TRAIL_COUNT*BOCU1_TRAIL_COUNT) + +/* The lead byte start values. */ +#define BOCU1_START_POS_2 (BOCU1_MIDDLE+BOCU1_REACH_POS_1+1) +#define BOCU1_START_POS_3 (BOCU1_START_POS_2+BOCU1_LEAD_2) +#define BOCU1_START_POS_4 (BOCU1_START_POS_3+BOCU1_LEAD_3) + /* ==BOCU1_MAX_LEAD */ + +#define BOCU1_START_NEG_2 (BOCU1_MIDDLE+BOCU1_REACH_NEG_1) +#define BOCU1_START_NEG_3 (BOCU1_START_NEG_2-BOCU1_LEAD_2) +#define BOCU1_START_NEG_4 (BOCU1_START_NEG_3-BOCU1_LEAD_3) + /* ==BOCU1_MIN+1 */ + +/* The length of a byte sequence, according to the lead byte (!=BOCU1_RESET). */ +#define BOCU1_LENGTH_FROM_LEAD(lead) \ + ((BOCU1_START_NEG_2<=(lead) && (lead)>24 : 4) + +/* + * 12 commonly used C0 control codes (and space) are only used to encode + * themselves directly, + * which makes BOCU-1 MIME-usable and reasonably safe for + * ASCII-oriented software. + * + * These controls are + * 0 NUL + * + * 7 BEL + * 8 BS + * + * 9 TAB + * a LF + * b VT + * c FF + * d CR + * + * e SO + * f SI + * + * 1a SUB + * 1b ESC + * + * The other 20 C0 controls are also encoded directly (to preserve order) + * but are also used as trail bytes in difference encoding + * (for better compression). + */ +#define BOCU1_TRAIL_TO_BYTE(t) ((t)>=BOCU1_TRAIL_CONTROLS_COUNT ? (t)+BOCU1_TRAIL_BYTE_OFFSET : bocu1TrailToByte[t]) + +/* + * Byte value map for control codes, + * from external byte values 0x00..0x20 + * to trail byte values 0..19 (0..0x13) as used in the difference calculation. + * External byte values that are illegal as trail bytes are mapped to -1. + */ +static const int8_t +bocu1ByteToTrail[BOCU1_MIN]={ +/* 0 1 2 3 4 5 6 7 */ + -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, -1, + +/* 8 9 a b c d e f */ + -1, -1, -1, -1, -1, -1, -1, -1, + +/* 10 11 12 13 14 15 16 17 */ + 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, + +/* 18 19 1a 1b 1c 1d 1e 1f */ + 0x0e, 0x0f, -1, -1, 0x10, 0x11, 0x12, 0x13, + +/* 20 */ + -1 +}; + +/* + * Byte value map for control codes, + * from trail byte values 0..19 (0..0x13) as used in the difference calculation + * to external byte values 0x00..0x20. + */ +static const int8_t +bocu1TrailToByte[BOCU1_TRAIL_CONTROLS_COUNT]={ +/* 0 1 2 3 4 5 6 7 */ + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x10, 0x11, + +/* 8 9 a b c d e f */ + 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, + +/* 10 11 12 13 */ + 0x1c, 0x1d, 0x1e, 0x1f +}; + +/** + * Integer division and modulo with negative numerators + * yields negative modulo results and quotients that are one more than + * what we need here. + * This macro adjust the results so that the modulo-value m is always >=0. + * + * For positive n, the if() condition is always FALSE. + * + * @param n Number to be split into quotient and rest. + * Will be modified to contain the quotient. + * @param d Divisor. + * @param m Output variable for the rest (modulo result). + */ +#define NEGDIVMOD(n, d, m) { \ + (m)=(n)%(d); \ + (n)/=(d); \ + if((m)<0) { \ + --(n); \ + (m)+=(d); \ + } \ +} + +/* BOCU-1 implementation functions ------------------------------------------ */ + +#define BOCU1_SIMPLE_PREV(c) (((c)&~0x7f)+BOCU1_ASCII_PREV) + +/** + * Compute the next "previous" value for differencing + * from the current code point. + * + * @param c current code point, 0x3040..0xd7a3 (rest handled by macro below) + * @return "previous code point" state value + */ +static U_INLINE int32_t +bocu1Prev(int32_t c) { + /* compute new prev */ + if(/* 0x3040<=c && */ c<=0x309f) { + /* Hiragana is not 128-aligned */ + return 0x3070; + } else if(0x4e00<=c && c<=0x9fa5) { + /* CJK Unihan */ + return 0x4e00-BOCU1_REACH_NEG_2; + } else if(0xac00<=c /* && c<=0xd7a3 */) { + /* Korean Hangul */ + return (0xd7a3+0xac00)/2; + } else { + /* mostly small scripts */ + return BOCU1_SIMPLE_PREV(c); + } +} + +/** Fast version of bocu1Prev() for most scripts. */ +#define BOCU1_PREV(c) ((c)<0x3040 || (c)>0xd7a3 ? BOCU1_SIMPLE_PREV(c) : bocu1Prev(c)) + +/* + * The BOCU-1 converter uses the standard setup code in ucnv.c/ucnv_bld.c. + * The UConverter fields are used as follows: + * + * fromUnicodeStatus encoder's prev (0 will be interpreted as BOCU1_ASCII_PREV) + * + * toUnicodeStatus decoder's prev (0 will be interpreted as BOCU1_ASCII_PREV) + * mode decoder's incomplete (diff<<2)|count (ignored when toULength==0) + */ + +/* BOCU-1-from-Unicode conversion functions --------------------------------- */ + +/** + * Encode a difference -0x10ffff..0x10ffff in 1..4 bytes + * and return a packed integer with them. + * + * The encoding favors small absolut differences with short encodings + * to compress runs of same-script characters. + * + * Optimized version with unrolled loops and fewer floating-point operations + * than the standard packDiff(). + * + * @param diff difference value -0x10ffff..0x10ffff + * @return + * 0x010000zz for 1-byte sequence zz + * 0x0200yyzz for 2-byte sequence yy zz + * 0x03xxyyzz for 3-byte sequence xx yy zz + * 0xwwxxyyzz for 4-byte sequence ww xx yy zz (ww>0x03) + */ +static int32_t +packDiff(int32_t diff) { + int32_t result, m; + + if(diff>=BOCU1_REACH_NEG_1) { + /* mostly positive differences, and single-byte negative ones */ +#if 0 /* single-byte case handled in macros, see below */ + if(diff<=BOCU1_REACH_POS_1) { + /* single byte */ + return 0x01000000|(BOCU1_MIDDLE+diff); + } else +#endif + if(diff<=BOCU1_REACH_POS_2) { + /* two bytes */ + diff-=BOCU1_REACH_POS_1+1; + result=0x02000000; + + m=diff%BOCU1_TRAIL_COUNT; + diff/=BOCU1_TRAIL_COUNT; + result|=BOCU1_TRAIL_TO_BYTE(m); + + result|=(BOCU1_START_POS_2+diff)<<8; + } else if(diff<=BOCU1_REACH_POS_3) { + /* three bytes */ + diff-=BOCU1_REACH_POS_2+1; + result=0x03000000; + + m=diff%BOCU1_TRAIL_COUNT; + diff/=BOCU1_TRAIL_COUNT; + result|=BOCU1_TRAIL_TO_BYTE(m); + + m=diff%BOCU1_TRAIL_COUNT; + diff/=BOCU1_TRAIL_COUNT; + result|=BOCU1_TRAIL_TO_BYTE(m)<<8; + + result|=(BOCU1_START_POS_3+diff)<<16; + } else { + /* four bytes */ + diff-=BOCU1_REACH_POS_3+1; + + m=diff%BOCU1_TRAIL_COUNT; + diff/=BOCU1_TRAIL_COUNT; + result=BOCU1_TRAIL_TO_BYTE(m); + + m=diff%BOCU1_TRAIL_COUNT; + diff/=BOCU1_TRAIL_COUNT; + result|=BOCU1_TRAIL_TO_BYTE(m)<<8; + + /* + * We know that / and % would deliver quotient 0 and rest=diff. + * Avoid division and modulo for performance. + */ + result|=BOCU1_TRAIL_TO_BYTE(diff)<<16; + + result|=((uint32_t)BOCU1_START_POS_4)<<24; + } + } else { + /* two- to four-byte negative differences */ + if(diff>=BOCU1_REACH_NEG_2) { + /* two bytes */ + diff-=BOCU1_REACH_NEG_1; + result=0x02000000; + + NEGDIVMOD(diff, BOCU1_TRAIL_COUNT, m); + result|=BOCU1_TRAIL_TO_BYTE(m); + + result|=(BOCU1_START_NEG_2+diff)<<8; + } else if(diff>=BOCU1_REACH_NEG_3) { + /* three bytes */ + diff-=BOCU1_REACH_NEG_2; + result=0x03000000; + + NEGDIVMOD(diff, BOCU1_TRAIL_COUNT, m); + result|=BOCU1_TRAIL_TO_BYTE(m); + + NEGDIVMOD(diff, BOCU1_TRAIL_COUNT, m); + result|=BOCU1_TRAIL_TO_BYTE(m)<<8; + + result|=(BOCU1_START_NEG_3+diff)<<16; + } else { + /* four bytes */ + diff-=BOCU1_REACH_NEG_3; + + NEGDIVMOD(diff, BOCU1_TRAIL_COUNT, m); + result=BOCU1_TRAIL_TO_BYTE(m); + + NEGDIVMOD(diff, BOCU1_TRAIL_COUNT, m); + result|=BOCU1_TRAIL_TO_BYTE(m)<<8; + + /* + * We know that NEGDIVMOD would deliver + * quotient -1 and rest=diff+BOCU1_TRAIL_COUNT. + * Avoid division and modulo for performance. + */ + m=diff+BOCU1_TRAIL_COUNT; + result|=BOCU1_TRAIL_TO_BYTE(m)<<16; + + result|=BOCU1_MIN<<24; + } + } + return result; +} + +/* Faster versions of packDiff() for single-byte-encoded diff values. */ + +/** Is a diff value encodable in a single byte? */ +#define DIFF_IS_SINGLE(diff) (BOCU1_REACH_NEG_1<=(diff) && (diff)<=BOCU1_REACH_POS_1) + +/** Encode a diff value in a single byte. */ +#define PACK_SINGLE_DIFF(diff) (BOCU1_MIDDLE+(diff)) + +/** Is a diff value encodable in two bytes? */ +#define DIFF_IS_DOUBLE(diff) (BOCU1_REACH_NEG_2<=(diff) && (diff)<=BOCU1_REACH_POS_2) + +static void +_Bocu1FromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv; + const UChar *source, *sourceLimit; + uint8_t *target; + int32_t targetCapacity; + int32_t *offsets; + + int32_t prev, c, diff; + + int32_t sourceIndex, nextSourceIndex; + +U_ALIGN_CODE(16) + + /* set up the local pointers */ + cnv=pArgs->converter; + source=pArgs->source; + sourceLimit=pArgs->sourceLimit; + target=(uint8_t *)pArgs->target; + targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); + offsets=pArgs->offsets; + + /* get the converter state from UConverter */ + c=cnv->fromUChar32; + prev=(int32_t)cnv->fromUnicodeStatus; + if(prev==0) { + prev=BOCU1_ASCII_PREV; + } + + /* sourceIndex=-1 if the current character began in the previous buffer */ + sourceIndex= c==0 ? 0 : -1; + nextSourceIndex=0; + + /* conversion loop */ + if(c!=0 && targetCapacity>0) { + goto getTrail; + } + +fastSingle: + /* fast loop for single-byte differences */ + /* use only one loop counter variable, targetCapacity, not also source */ + diff=(int32_t)(sourceLimit-source); + if(targetCapacity>diff) { + targetCapacity=diff; + } + while(targetCapacity>0 && (c=*source)<0x3000) { + if(c<=0x20) { + if(c!=0x20) { + prev=BOCU1_ASCII_PREV; + } + *target++=(uint8_t)c; + *offsets++=nextSourceIndex++; + ++source; + --targetCapacity; + } else { + diff=c-prev; + if(DIFF_IS_SINGLE(diff)) { + prev=BOCU1_SIMPLE_PREV(c); + *target++=(uint8_t)PACK_SINGLE_DIFF(diff); + *offsets++=nextSourceIndex++; + ++source; + --targetCapacity; + } else { + break; + } + } + } + /* restore real values */ + targetCapacity=(int32_t)((const uint8_t *)pArgs->targetLimit-target); + sourceIndex=nextSourceIndex; /* wrong if offsets==NULL but does not matter */ + + /* regular loop for all cases */ + while(source0) { + c=*source++; + ++nextSourceIndex; + + if(c<=0x20) { + /* + * ISO C0 control & space: + * Encode directly for MIME compatibility, + * and reset state except for space, to not disrupt compression. + */ + if(c!=0x20) { + prev=BOCU1_ASCII_PREV; + } + *target++=(uint8_t)c; + *offsets++=sourceIndex; + --targetCapacity; + + sourceIndex=nextSourceIndex; + continue; + } + + if(UTF_IS_LEAD(c)) { +getTrail: + if(source=0) { + diff-=BOCU1_REACH_POS_1+1; + m=diff%BOCU1_TRAIL_COUNT; + diff/=BOCU1_TRAIL_COUNT; + diff+=BOCU1_START_POS_2; + } else { + diff-=BOCU1_REACH_NEG_1; + NEGDIVMOD(diff, BOCU1_TRAIL_COUNT, m); + diff+=BOCU1_START_NEG_2; + } + *target++=(uint8_t)diff; + *target++=(uint8_t)BOCU1_TRAIL_TO_BYTE(m); + *offsets++=sourceIndex; + *offsets++=sourceIndex; + targetCapacity-=2; + sourceIndex=nextSourceIndex; + } else { + int32_t length; /* will be 2..4 */ + + diff=packDiff(diff); + length=BOCU1_LENGTH_FROM_PACKED(diff); + + /* write the output character bytes from diff and length */ + /* from the first if in the loop we know that targetCapacity>0 */ + if(length<=targetCapacity) { + switch(length) { + /* each branch falls through to the next one */ + case 4: + *target++=(uint8_t)(diff>>24); + *offsets++=sourceIndex; + case 3: + *target++=(uint8_t)(diff>>16); + *offsets++=sourceIndex; + case 2: + *target++=(uint8_t)(diff>>8); + *offsets++=sourceIndex; + /* case 1: handled above */ + *target++=(uint8_t)diff; + *offsets++=sourceIndex; + default: + /* will never occur */ + break; + } + targetCapacity-=length; + sourceIndex=nextSourceIndex; + } else { + uint8_t *charErrorBuffer; + + /* + * We actually do this backwards here: + * In order to save an intermediate variable, we output + * first to the overflow buffer what does not fit into the + * regular target. + */ + /* we know that 1<=targetCapacitycharErrorBuffer; + switch(length) { + /* each branch falls through to the next one */ + case 3: + *charErrorBuffer++=(uint8_t)(diff>>16); + case 2: + *charErrorBuffer++=(uint8_t)(diff>>8); + case 1: + *charErrorBuffer=(uint8_t)diff; + default: + /* will never occur */ + break; + } + cnv->charErrorBufferLength=(int8_t)length; + + /* now output what fits into the regular target */ + diff>>=8*length; /* length was reduced by targetCapacity */ + switch(targetCapacity) { + /* each branch falls through to the next one */ + case 3: + *target++=(uint8_t)(diff>>16); + *offsets++=sourceIndex; + case 2: + *target++=(uint8_t)(diff>>8); + *offsets++=sourceIndex; + case 1: + *target++=(uint8_t)diff; + *offsets++=sourceIndex; + default: + /* will never occur */ + break; + } + + /* target overflow */ + targetCapacity=0; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + } + } else { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + } + + /* set the converter state back into UConverter */ + cnv->fromUChar32= c<0 ? -c : 0; + cnv->fromUnicodeStatus=(uint32_t)prev; + + /* write back the updated pointers */ + pArgs->source=source; + pArgs->target=(char *)target; + pArgs->offsets=offsets; +} + +/* + * Identical to _Bocu1FromUnicodeWithOffsets but without offset handling. + * If a change is made in the original function, then either + * change this function the same way or + * re-copy the original function and remove the variables + * offsets, sourceIndex, and nextSourceIndex. + */ +static void +_Bocu1FromUnicode(UConverterFromUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv; + const UChar *source, *sourceLimit; + uint8_t *target; + int32_t targetCapacity; + + int32_t prev, c, diff; + + /* set up the local pointers */ + cnv=pArgs->converter; + source=pArgs->source; + sourceLimit=pArgs->sourceLimit; + target=(uint8_t *)pArgs->target; + targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); + + /* get the converter state from UConverter */ + c=cnv->fromUChar32; + prev=(int32_t)cnv->fromUnicodeStatus; + if(prev==0) { + prev=BOCU1_ASCII_PREV; + } + + /* conversion loop */ + if(c!=0 && targetCapacity>0) { + goto getTrail; + } + +fastSingle: + /* fast loop for single-byte differences */ + /* use only one loop counter variable, targetCapacity, not also source */ + diff=(int32_t)(sourceLimit-source); + if(targetCapacity>diff) { + targetCapacity=diff; + } + while(targetCapacity>0 && (c=*source)<0x3000) { + if(c<=0x20) { + if(c!=0x20) { + prev=BOCU1_ASCII_PREV; + } + *target++=(uint8_t)c; + } else { + diff=c-prev; + if(DIFF_IS_SINGLE(diff)) { + prev=BOCU1_SIMPLE_PREV(c); + *target++=(uint8_t)PACK_SINGLE_DIFF(diff); + } else { + break; + } + } + ++source; + --targetCapacity; + } + /* restore real values */ + targetCapacity=(int32_t)((const uint8_t *)pArgs->targetLimit-target); + + /* regular loop for all cases */ + while(source0) { + c=*source++; + + if(c<=0x20) { + /* + * ISO C0 control & space: + * Encode directly for MIME compatibility, + * and reset state except for space, to not disrupt compression. + */ + if(c!=0x20) { + prev=BOCU1_ASCII_PREV; + } + *target++=(uint8_t)c; + --targetCapacity; + continue; + } + + if(UTF_IS_LEAD(c)) { +getTrail: + if(source=0) { + diff-=BOCU1_REACH_POS_1+1; + m=diff%BOCU1_TRAIL_COUNT; + diff/=BOCU1_TRAIL_COUNT; + diff+=BOCU1_START_POS_2; + } else { + diff-=BOCU1_REACH_NEG_1; + NEGDIVMOD(diff, BOCU1_TRAIL_COUNT, m); + diff+=BOCU1_START_NEG_2; + } + *target++=(uint8_t)diff; + *target++=(uint8_t)BOCU1_TRAIL_TO_BYTE(m); + targetCapacity-=2; + } else { + int32_t length; /* will be 2..4 */ + + diff=packDiff(diff); + length=BOCU1_LENGTH_FROM_PACKED(diff); + + /* write the output character bytes from diff and length */ + /* from the first if in the loop we know that targetCapacity>0 */ + if(length<=targetCapacity) { + switch(length) { + /* each branch falls through to the next one */ + case 4: + *target++=(uint8_t)(diff>>24); + case 3: + *target++=(uint8_t)(diff>>16); + /* case 2: handled above */ + *target++=(uint8_t)(diff>>8); + /* case 1: handled above */ + *target++=(uint8_t)diff; + default: + /* will never occur */ + break; + } + targetCapacity-=length; + } else { + uint8_t *charErrorBuffer; + + /* + * We actually do this backwards here: + * In order to save an intermediate variable, we output + * first to the overflow buffer what does not fit into the + * regular target. + */ + /* we know that 1<=targetCapacitycharErrorBuffer; + switch(length) { + /* each branch falls through to the next one */ + case 3: + *charErrorBuffer++=(uint8_t)(diff>>16); + case 2: + *charErrorBuffer++=(uint8_t)(diff>>8); + case 1: + *charErrorBuffer=(uint8_t)diff; + default: + /* will never occur */ + break; + } + cnv->charErrorBufferLength=(int8_t)length; + + /* now output what fits into the regular target */ + diff>>=8*length; /* length was reduced by targetCapacity */ + switch(targetCapacity) { + /* each branch falls through to the next one */ + case 3: + *target++=(uint8_t)(diff>>16); + case 2: + *target++=(uint8_t)(diff>>8); + case 1: + *target++=(uint8_t)diff; + default: + /* will never occur */ + break; + } + + /* target overflow */ + targetCapacity=0; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + } + } else { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + } + + /* set the converter state back into UConverter */ + cnv->fromUChar32= c<0 ? -c : 0; + cnv->fromUnicodeStatus=(uint32_t)prev; + + /* write back the updated pointers */ + pArgs->source=source; + pArgs->target=(char *)target; +} + +/* BOCU-1-to-Unicode conversion functions ----------------------------------- */ + +/** + * Function for BOCU-1 decoder; handles multi-byte lead bytes. + * + * @param b lead byte; + * BOCU1_MIN<=b=BOCU1_START_NEG_2) { + /* positive difference */ + if(b=BOCU1_START_NEG_3) { + /* two bytes */ + diff=((int32_t)b-BOCU1_START_NEG_2)*BOCU1_TRAIL_COUNT+BOCU1_REACH_NEG_1; + count=1; + } else if(b>BOCU1_MIN) { + /* three bytes */ + diff=((int32_t)b-BOCU1_START_NEG_3)*BOCU1_TRAIL_COUNT*BOCU1_TRAIL_COUNT+BOCU1_REACH_NEG_2; + count=2; + } else { + /* four bytes */ + diff=-BOCU1_TRAIL_COUNT*BOCU1_TRAIL_COUNT*BOCU1_TRAIL_COUNT+BOCU1_REACH_NEG_3; + count=3; + } + } + + /* return the state for decoding the trail byte(s) */ + return (diff<<2)|count; +} + +/** + * Function for BOCU-1 decoder; handles multi-byte trail bytes. + * + * @param count number of remaining trail bytes including this one + * @param b trail byte + * @return new delta for diff including b - <0 indicates an error + * + * @see decodeBocu1 + */ +static U_INLINE int32_t +decodeBocu1TrailByte(int32_t count, int32_t b) { + if(b<=0x20) { + /* skip some C0 controls and make the trail byte range contiguous */ + b=bocu1ByteToTrail[b]; + /* b<0 for an illegal trail byte value will result in return<0 below */ +#if BOCU1_MAX_TRAIL<0xff + } else if(b>BOCU1_MAX_TRAIL) { + return -99; +#endif + } else { + b-=BOCU1_TRAIL_BYTE_OFFSET; + } + + /* add trail byte into difference and decrement count */ + if(count==1) { + return b; + } else if(count==2) { + return b*BOCU1_TRAIL_COUNT; + } else /* count==3 */ { + return b*(BOCU1_TRAIL_COUNT*BOCU1_TRAIL_COUNT); + } +} + +static void +_Bocu1ToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv; + const uint8_t *source, *sourceLimit; + UChar *target; + const UChar *targetLimit; + int32_t *offsets; + + int32_t prev, count, diff, c; + + int8_t byteIndex; + uint8_t *bytes; + + int32_t sourceIndex, nextSourceIndex; + + /* set up the local pointers */ + cnv=pArgs->converter; + source=(const uint8_t *)pArgs->source; + sourceLimit=(const uint8_t *)pArgs->sourceLimit; + target=pArgs->target; + targetLimit=pArgs->targetLimit; + offsets=pArgs->offsets; + + /* get the converter state from UConverter */ + prev=(int32_t)cnv->toUnicodeStatus; + if(prev==0) { + prev=BOCU1_ASCII_PREV; + } + diff=cnv->mode; /* mode may be set to UCNV_SI by ucnv_bld.c but then toULength==0 */ + count=diff&3; + diff>>=2; + + byteIndex=cnv->toULength; + bytes=cnv->toUBytes; + + /* sourceIndex=-1 if the current character began in the previous buffer */ + sourceIndex=byteIndex==0 ? 0 : -1; + nextSourceIndex=0; + + /* conversion "loop" similar to _SCSUToUnicodeWithOffsets() */ + if(count>0 && byteIndex>0 && targettargetLimit-target); + if(count>diff) { + count=diff; + } + while(count>0) { + if(BOCU1_START_NEG_2<=(c=*source) && c=targetLimit) { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + + ++nextSourceIndex; + c=*source++; + if(BOCU1_START_NEG_2<=c && c=BOCU1_MIDDLE) { + diff=((int32_t)c-BOCU1_START_POS_2)*BOCU1_TRAIL_COUNT+BOCU1_REACH_POS_1+1; + } else { + diff=((int32_t)c-BOCU1_START_NEG_2)*BOCU1_TRAIL_COUNT+BOCU1_REACH_NEG_1; + } + + /* trail byte */ + ++nextSourceIndex; + c=decodeBocu1TrailByte(1, *source++); + if(c<0 || (uint32_t)(c=prev+diff+c)>0x10ffff) { + bytes[0]=source[-2]; + bytes[1]=source[-1]; + byteIndex=2; + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + break; + } + } else if(c==BOCU1_RESET) { + /* only reset the state, no code point */ + prev=BOCU1_ASCII_PREV; + sourceIndex=nextSourceIndex; + continue; + } else { + /* + * For multi-byte difference lead bytes, set the decoder state + * with the partial difference value from the lead byte and + * with the number of trail bytes. + */ + bytes[0]=(uint8_t)c; + byteIndex=1; + + diff=decodeBocu1LeadByte(c); + count=diff&3; + diff>>=2; +getTrail: + for(;;) { + if(source>=sourceLimit) { + goto endloop; + } + ++nextSourceIndex; + c=bytes[byteIndex++]=*source++; + + /* trail byte in any position */ + c=decodeBocu1TrailByte(count, c); + if(c<0) { + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + goto endloop; + } + + diff+=c; + if(--count==0) { + /* final trail byte, deliver a code point */ + byteIndex=0; + c=prev+diff; + if((uint32_t)c>0x10ffff) { + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + goto endloop; + } + break; + } + } + } + + /* calculate the next prev and output c */ + prev=BOCU1_PREV(c); + if(c<=0xffff) { + *target++=(UChar)c; + *offsets++=sourceIndex; + } else { + /* output surrogate pair */ + *target++=UTF16_LEAD(c); + if(targetUCharErrorBuffer[0]=UTF16_TRAIL(c); + cnv->UCharErrorBufferLength=1; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + } + sourceIndex=nextSourceIndex; + } +endloop: + + if(*pErrorCode==U_ILLEGAL_CHAR_FOUND) { + /* set the converter state in UConverter to deal with the next character */ + cnv->toUnicodeStatus=BOCU1_ASCII_PREV; + cnv->mode=0; + } else { + /* set the converter state back into UConverter */ + cnv->toUnicodeStatus=(uint32_t)prev; + cnv->mode=(diff<<2)|count; + } + cnv->toULength=byteIndex; + + /* write back the updated pointers */ + pArgs->source=(const char *)source; + pArgs->target=target; + pArgs->offsets=offsets; + return; +} + +/* + * Identical to _Bocu1ToUnicodeWithOffsets but without offset handling. + * If a change is made in the original function, then either + * change this function the same way or + * re-copy the original function and remove the variables + * offsets, sourceIndex, and nextSourceIndex. + */ +static void +_Bocu1ToUnicode(UConverterToUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv; + const uint8_t *source, *sourceLimit; + UChar *target; + const UChar *targetLimit; + + int32_t prev, count, diff, c; + + int8_t byteIndex; + uint8_t *bytes; + +U_ALIGN_CODE(16) + + /* set up the local pointers */ + cnv=pArgs->converter; + source=(const uint8_t *)pArgs->source; + sourceLimit=(const uint8_t *)pArgs->sourceLimit; + target=pArgs->target; + targetLimit=pArgs->targetLimit; + + /* get the converter state from UConverter */ + prev=(int32_t)cnv->toUnicodeStatus; + if(prev==0) { + prev=BOCU1_ASCII_PREV; + } + diff=cnv->mode; /* mode may be set to UCNV_SI by ucnv_bld.c but then toULength==0 */ + count=diff&3; + diff>>=2; + + byteIndex=cnv->toULength; + bytes=cnv->toUBytes; + + /* conversion "loop" similar to _SCSUToUnicodeWithOffsets() */ + if(count>0 && byteIndex>0 && targettargetLimit-target); + if(count>diff) { + count=diff; + } + while(count>0) { + if(BOCU1_START_NEG_2<=(c=*source) && c=targetLimit) { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + + c=*source++; + if(BOCU1_START_NEG_2<=c && c=BOCU1_MIDDLE) { + diff=((int32_t)c-BOCU1_START_POS_2)*BOCU1_TRAIL_COUNT+BOCU1_REACH_POS_1+1; + } else { + diff=((int32_t)c-BOCU1_START_NEG_2)*BOCU1_TRAIL_COUNT+BOCU1_REACH_NEG_1; + } + + /* trail byte */ + c=decodeBocu1TrailByte(1, *source++); + if(c<0 || (uint32_t)(c=prev+diff+c)>0x10ffff) { + bytes[0]=source[-2]; + bytes[1]=source[-1]; + byteIndex=2; + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + break; + } + } else if(c==BOCU1_RESET) { + /* only reset the state, no code point */ + prev=BOCU1_ASCII_PREV; + continue; + } else { + /* + * For multi-byte difference lead bytes, set the decoder state + * with the partial difference value from the lead byte and + * with the number of trail bytes. + */ + bytes[0]=(uint8_t)c; + byteIndex=1; + + diff=decodeBocu1LeadByte(c); + count=diff&3; + diff>>=2; +getTrail: + for(;;) { + if(source>=sourceLimit) { + goto endloop; + } + c=bytes[byteIndex++]=*source++; + + /* trail byte in any position */ + c=decodeBocu1TrailByte(count, c); + if(c<0) { + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + goto endloop; + } + + diff+=c; + if(--count==0) { + /* final trail byte, deliver a code point */ + byteIndex=0; + c=prev+diff; + if((uint32_t)c>0x10ffff) { + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + goto endloop; + } + break; + } + } + } + + /* calculate the next prev and output c */ + prev=BOCU1_PREV(c); + if(c<=0xffff) { + *target++=(UChar)c; + } else { + /* output surrogate pair */ + *target++=UTF16_LEAD(c); + if(targetUCharErrorBuffer[0]=UTF16_TRAIL(c); + cnv->UCharErrorBufferLength=1; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + } + } +endloop: + + if(*pErrorCode==U_ILLEGAL_CHAR_FOUND) { + /* set the converter state in UConverter to deal with the next character */ + cnv->toUnicodeStatus=BOCU1_ASCII_PREV; + cnv->mode=0; + } else { + /* set the converter state back into UConverter */ + cnv->toUnicodeStatus=(uint32_t)prev; + cnv->mode=(diff<<2)|count; + } + cnv->toULength=byteIndex; + + /* write back the updated pointers */ + pArgs->source=(const char *)source; + pArgs->target=target; + return; +} + +/* miscellaneous ------------------------------------------------------------ */ + +static const UConverterImpl _Bocu1Impl={ + UCNV_BOCU1, + + NULL, + NULL, + + NULL, + NULL, + NULL, + + _Bocu1ToUnicode, + _Bocu1ToUnicodeWithOffsets, + _Bocu1FromUnicode, + _Bocu1FromUnicodeWithOffsets, + NULL, + + NULL, + NULL, + NULL, + NULL, + ucnv_getCompleteUnicodeSet +}; + +static const UConverterStaticData _Bocu1StaticData={ + sizeof(UConverterStaticData), + "BOCU-1", + 1214, /* CCSID for BOCU-1 */ + UCNV_IBM, UCNV_BOCU1, + 1, 4, /* one UChar generates at least 1 byte and at most 4 bytes */ + { 0x1a, 0, 0, 0 }, 1, /* BOCU-1 never needs to write a subchar */ + FALSE, FALSE, + 0, + 0, + { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */ +}; + +const UConverterSharedData _Bocu1Data={ + sizeof(UConverterSharedData), ~((uint32_t)0), + NULL, NULL, &_Bocu1StaticData, FALSE, &_Bocu1Impl, + 0 +}; + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/ucnvdisp.c b/Build/source/libs/icu/icu-xetex/common/ucnvdisp.c new file mode 100644 index 00000000000..4075be66da7 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnvdisp.c @@ -0,0 +1,86 @@ +/* +****************************************************************************** +* +* Copyright (C) 1998-2004, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* +* ucnvdisp.c: +* Implements APIs for the ICU's codeset conversion library display names. +* +* Modification History: +* +* Date Name Description +* 04/04/99 helena Fixed internal header inclusion. +* 05/09/00 helena Added implementation to handle fallback mappings. +* 06/20/2000 helena OS/400 port changes; mostly typecast. +* 09/08/2004 grhoten split from ucnv.c +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "unicode/ustring.h" +#include "unicode/ures.h" +#include "unicode/ucnv.h" +#include "cstring.h" +#include "ustr_imp.h" +#include "ucnv_imp.h" +#include "putilimp.h" + +U_CAPI int32_t U_EXPORT2 +ucnv_getDisplayName(const UConverter *cnv, + const char *displayLocale, + UChar *displayName, int32_t displayNameCapacity, + UErrorCode *pErrorCode) { + UResourceBundle *rb; + const UChar *name; + int32_t length; + UErrorCode localStatus = U_ZERO_ERROR; + + /* check arguments */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + if(cnv==NULL || displayNameCapacity<0 || (displayNameCapacity>0 && displayName==NULL)) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* open the resource bundle and get the display name string */ + rb=ures_open(NULL, displayLocale, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return 0; + } + + /* use the internal name as the key */ + name=ures_getStringByKey(rb, cnv->sharedData->staticData->name, &length, &localStatus); + ures_close(rb); + + if(U_SUCCESS(localStatus)) { + /* copy the string */ + if (*pErrorCode == U_ZERO_ERROR) { + *pErrorCode = localStatus; + } + u_memcpy(displayName, name, uprv_min(length, displayNameCapacity)*U_SIZEOF_UCHAR); + } else { + /* convert the internal name into a Unicode string */ + length=(int32_t)uprv_strlen(cnv->sharedData->staticData->name); + u_charsToUChars(cnv->sharedData->staticData->name, displayName, uprv_min(length, displayNameCapacity)); + } + return u_terminateUChars(displayName, displayNameCapacity, length, pErrorCode); +} + +#endif + +/* + * Hey, Emacs, please set the following: + * + * Local Variables: + * indent-tabs-mode: nil + * End: + * + */ diff --git a/Build/source/libs/icu/icu-xetex/common/ucnvhz.c b/Build/source/libs/icu/icu-xetex/common/ucnvhz.c new file mode 100644 index 00000000000..b94811b8122 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnvhz.c @@ -0,0 +1,591 @@ +/* +********************************************************************** +* Copyright (C) 2000-2006, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* file name: ucnvhz.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2000oct16 +* created by: Ram Viswanadha +* 10/31/2000 Ram Implemented offsets logic function +* +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION && !UCONFIG_NO_LEGACY_CONVERSION + +#include "cmemory.h" +#include "unicode/ucnv.h" +#include "unicode/ucnv_cb.h" +#include "unicode/uset.h" +#include "ucnv_bld.h" +#include "ucnv_cnv.h" + +#define UCNV_TILDE 0x7E /* ~ */ +#define UCNV_OPEN_BRACE 0x7B /* { */ +#define UCNV_CLOSE_BRACE 0x7D /* } */ +#define SB_ESCAPE "\x7E\x7D" +#define DB_ESCAPE "\x7E\x7B" +#define TILDE_ESCAPE "\x7E\x7E" +#define ESC_LEN 2 + + +#define CONCAT_ESCAPE_MACRO( args, targetIndex,targetLength,strToAppend, err, len,sourceIndex){ \ + while(len-->0){ \ + if(targetIndex < targetLength){ \ + args->target[targetIndex] = (unsigned char) *strToAppend; \ + if(args->offsets!=NULL){ \ + *(offsets++) = sourceIndex-1; \ + } \ + targetIndex++; \ + } \ + else{ \ + args->converter->charErrorBuffer[(int)args->converter->charErrorBufferLength++] = (unsigned char) *strToAppend; \ + *err =U_BUFFER_OVERFLOW_ERROR; \ + } \ + strToAppend++; \ + } \ +} + + +typedef struct{ + UConverter* gbConverter; + int32_t targetIndex; + int32_t sourceIndex; + UBool isEscapeAppended; + UBool isStateDBCS; + UBool isTargetUCharDBCS; +}UConverterDataHZ; + + + +static void +_HZOpen(UConverter *cnv, const char *name,const char *locale,uint32_t options, UErrorCode *errorCode){ + cnv->toUnicodeStatus = 0; + cnv->fromUnicodeStatus= 0; + cnv->mode=0; + cnv->fromUChar32=0x0000; + cnv->extraInfo = uprv_malloc(sizeof(UConverterDataHZ)); + if(cnv->extraInfo != NULL){ + uprv_memset(cnv->extraInfo, 0, sizeof(UConverterDataHZ)); + ((UConverterDataHZ*)cnv->extraInfo)->gbConverter = ucnv_open("ibm-1386",errorCode); + } + else { + *errorCode = U_MEMORY_ALLOCATION_ERROR; + return; + } +} + +static void +_HZClose(UConverter *cnv){ + if(cnv->extraInfo != NULL) { + ucnv_close (((UConverterDataHZ *) (cnv->extraInfo))->gbConverter); + if(!cnv->isExtraLocal) { + uprv_free(cnv->extraInfo); + } + cnv->extraInfo = NULL; + } +} + +static void +_HZReset(UConverter *cnv, UConverterResetChoice choice){ + if(choice<=UCNV_RESET_TO_UNICODE) { + cnv->toUnicodeStatus = 0; + cnv->mode=0; + if(cnv->extraInfo != NULL){ + ((UConverterDataHZ*)cnv->extraInfo)->isStateDBCS = FALSE; + } + } + if(choice!=UCNV_RESET_TO_UNICODE) { + cnv->fromUnicodeStatus= 0; + cnv->fromUChar32=0x0000; + if(cnv->extraInfo != NULL){ + ((UConverterDataHZ*)cnv->extraInfo)->isEscapeAppended = FALSE; + ((UConverterDataHZ*)cnv->extraInfo)->targetIndex = 0; + ((UConverterDataHZ*)cnv->extraInfo)->sourceIndex = 0; + ((UConverterDataHZ*)cnv->extraInfo)->isTargetUCharDBCS = FALSE; + } + } +} + +/**************************************HZ Encoding************************************************* +* Rules for HZ encoding +* +* In ASCII mode, a byte is interpreted as an ASCII character, unless a +* '~' is encountered. The character '~' is an escape character. By +* convention, it must be immediately followed ONLY by '~', '{' or '\n' +* (), with the following special meaning. + +* 1. The escape sequence '~~' is interpreted as a '~'. +* 2. The escape-to-GB sequence '~{' switches the mode from ASCII to GB. +* 3. The escape sequence '~\n' is a line-continuation marker to be +* consumed with no output produced. +* In GB mode, characters are interpreted two bytes at a time as (pure) +* GB codes until the escape-from-GB code '~}' is read. This code +* switches the mode from GB back to ASCII. (Note that the escape- +* from-GB code '~}' ($7E7D) is outside the defined GB range.) +* +* Source: RFC 1842 +*/ + + +static void +UConverter_toUnicode_HZ_OFFSETS_LOGIC(UConverterToUnicodeArgs *args, + UErrorCode* err){ + char tempBuf[2]; + const char *mySource = ( char *) args->source; + UChar *myTarget = args->target; + const char *mySourceLimit = args->sourceLimit; + UChar32 targetUniChar = 0x0000; + UChar mySourceChar = 0x0000; + UConverterDataHZ* myData=(UConverterDataHZ*)(args->converter->extraInfo); + tempBuf[0]=0; + tempBuf[1]=0; + if ((args->converter == NULL) || (args->targetLimit < args->target) || (mySourceLimit < args->source)){ + *err = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + while(mySource< mySourceLimit){ + + if(myTarget < args->targetLimit){ + + mySourceChar= (unsigned char) *mySource++; + + switch(mySourceChar){ + case 0x0A: + if(args->converter->mode ==UCNV_TILDE){ + args->converter->mode=0; + + } + *(myTarget++)=(UChar)mySourceChar; + continue; + + case UCNV_TILDE: + if(args->converter->mode ==UCNV_TILDE){ + *(myTarget++)=(UChar)mySourceChar; + args->converter->mode=0; + continue; + + } + else if(args->converter->toUnicodeStatus !=0){ + args->converter->mode=0; + break; + } + else{ + args->converter->mode = UCNV_TILDE; + continue; + } + + + case UCNV_OPEN_BRACE: + if(args->converter->mode == UCNV_TILDE){ + args->converter->mode=0; + myData->isStateDBCS = TRUE; + continue; + } + else{ + break; + } + + + case UCNV_CLOSE_BRACE: + if(args->converter->mode == UCNV_TILDE){ + args->converter->mode=0; + myData->isStateDBCS = FALSE; + continue; + } + else{ + break; + } + + default: + /* if the first byte is equal to TILDE and the trail byte + * is not a valid byte then it is an error condition + */ + if(args->converter->mode == UCNV_TILDE){ + args->converter->mode=0; + mySourceChar= (UChar)(((UCNV_TILDE+0x80) << 8) | ((mySourceChar & 0x00ff)+0x80)); + goto SAVE_STATE; + } + + break; + + } + + if(myData->isStateDBCS){ + if(args->converter->toUnicodeStatus == 0x00){ + args->converter->toUnicodeStatus = (UChar) mySourceChar; + continue; + } + else{ + tempBuf[0] = (char) (args->converter->toUnicodeStatus+0x80) ; + tempBuf[1] = (char) (mySourceChar+0x80); + mySourceChar= (UChar)(((args->converter->toUnicodeStatus+0x80) << 8) | ((mySourceChar & 0x00ff)+0x80)); + args->converter->toUnicodeStatus =0x00; + targetUniChar = ucnv_MBCSSimpleGetNextUChar(myData->gbConverter->sharedData, + tempBuf, 2, args->converter->useFallback); + } + } + else{ + if(args->converter->fromUnicodeStatus == 0x00){ + targetUniChar = ucnv_MBCSSimpleGetNextUChar(myData->gbConverter->sharedData, + mySource - 1, 1, args->converter->useFallback); + } + else{ + goto SAVE_STATE; + } + + } + if(targetUniChar < 0xfffe){ + if(args->offsets) { + args->offsets[myTarget - args->target]=(int32_t)(mySource - args->source - 1-(myData->isStateDBCS)); + } + + *(myTarget++)=(UChar)targetUniChar; + } + else if(targetUniChar>=0xfffe){ +SAVE_STATE: + if(targetUniChar == 0xfffe){ + *err = U_INVALID_CHAR_FOUND; + } + else{ + *err = U_ILLEGAL_CHAR_FOUND; + } + if(myData->isStateDBCS){ + /* this should never occur since isStateDBCS is set to true + * only after tempBuf[0] and tempBuf[1] + * are set to the input .. just to please BEAM + */ + if(tempBuf[0]==0 || tempBuf[1]==0){ + *err = U_INTERNAL_PROGRAM_ERROR; + }else{ + args->converter->toUBytes[0] = (uint8_t)(tempBuf[0]-0x80); + args->converter->toUBytes[1] = (uint8_t)(tempBuf[1]-0x80); + args->converter->toULength=2; + } + } + else{ + args->converter->toUBytes[0] = (uint8_t)mySourceChar; + args->converter->toULength=1; + } + break; + } + } + else{ + *err =U_BUFFER_OVERFLOW_ERROR; + break; + } + } + + args->target = myTarget; + args->source = mySource; +} + + +static void +UConverter_fromUnicode_HZ_OFFSETS_LOGIC (UConverterFromUnicodeArgs * args, + UErrorCode * err){ + const UChar *mySource = args->source; + char *myTarget = args->target; + int32_t* offsets = args->offsets; + int32_t mySourceIndex = 0; + int32_t myTargetIndex = 0; + int32_t targetLength = (int32_t)(args->targetLimit - myTarget); + int32_t mySourceLength = (int32_t)(args->sourceLimit - args->source); + int32_t length=0; + uint32_t targetUniChar = 0x0000; + UChar32 mySourceChar = 0x0000; + UConverterDataHZ *myConverterData=(UConverterDataHZ*)args->converter->extraInfo; + UBool isTargetUCharDBCS = (UBool) myConverterData->isTargetUCharDBCS; + UBool oldIsTargetUCharDBCS = isTargetUCharDBCS; + int len =0; + const char* escSeq=NULL; + + if ((args->converter == NULL) || (args->targetLimit < myTarget) || (args->sourceLimit < args->source)){ + *err = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + if(args->converter->fromUChar32!=0 && myTargetIndex < targetLength) { + goto getTrail; + } + /*writing the char to the output stream */ + while (mySourceIndex < mySourceLength){ + targetUniChar = missingCharMarker; + if (myTargetIndex < targetLength){ + + mySourceChar = (UChar) mySource[mySourceIndex++]; + + + oldIsTargetUCharDBCS = isTargetUCharDBCS; + if(mySourceChar ==UCNV_TILDE){ + /*concatEscape(args, &myTargetIndex, &targetLength,"\x7E\x7E",err,2,&mySourceIndex);*/ + len = ESC_LEN; + escSeq = TILDE_ESCAPE; + CONCAT_ESCAPE_MACRO(args, myTargetIndex, targetLength, escSeq,err,len,mySourceIndex); + continue; + } + else{ + length= ucnv_MBCSFromUChar32(myConverterData->gbConverter->sharedData, + mySourceChar,&targetUniChar,args->converter->useFallback); + + } + /* only DBCS or SBCS characters are expected*/ + /* DB haracters with high bit set to 1 are expected */ + if(length > 2 || length==0 ||(((targetUniChar & 0x8080) != 0x8080)&& length==2)){ + targetUniChar= missingCharMarker; + } + if (targetUniChar != missingCharMarker){ + myConverterData->isTargetUCharDBCS = isTargetUCharDBCS = (UBool)(targetUniChar>0x00FF); + if(oldIsTargetUCharDBCS != isTargetUCharDBCS || !myConverterData->isEscapeAppended ){ + /*Shifting from a double byte to single byte mode*/ + if(!isTargetUCharDBCS){ + len =ESC_LEN; + escSeq = SB_ESCAPE; + CONCAT_ESCAPE_MACRO(args, myTargetIndex, targetLength, escSeq,err,len,mySourceIndex); + myConverterData->isEscapeAppended = TRUE; + } + else{ /* Shifting from a single byte to double byte mode*/ + len =ESC_LEN; + escSeq = DB_ESCAPE; + CONCAT_ESCAPE_MACRO(args, myTargetIndex, targetLength, escSeq,err,len,mySourceIndex); + myConverterData->isEscapeAppended = TRUE; + + } + } + + if(isTargetUCharDBCS){ + if( myTargetIndex > 8) -0x80); + if(offsets){ + *(offsets++) = mySourceIndex-1; + } + if(myTargetIndex < targetLength){ + myTarget[myTargetIndex++] =(char) ((targetUniChar & 0x00FF) -0x80); + if(offsets){ + *(offsets++) = mySourceIndex-1; + } + }else{ + args->converter->charErrorBuffer[args->converter->charErrorBufferLength++] = (char) ((targetUniChar & 0x00FF) -0x80); + *err = U_BUFFER_OVERFLOW_ERROR; + } + }else{ + args->converter->charErrorBuffer[args->converter->charErrorBufferLength++] =(char) ((targetUniChar >> 8) -0x80); + args->converter->charErrorBuffer[args->converter->charErrorBufferLength++] = (char) ((targetUniChar & 0x00FF) -0x80); + *err = U_BUFFER_OVERFLOW_ERROR; + } + + }else{ + if( myTargetIndex converter->charErrorBuffer[args->converter->charErrorBufferLength++] = (char) targetUniChar; + *err = U_BUFFER_OVERFLOW_ERROR; + } + } + + } + else{ + /* oops.. the code point is unassigned */ + /*Handle surrogates */ + /*check if the char is a First surrogate*/ + if(UTF_IS_SURROGATE(mySourceChar)) { + if(UTF_IS_SURROGATE_FIRST(mySourceChar)) { + args->converter->fromUChar32=mySourceChar; +getTrail: + /*look ahead to find the trail surrogate*/ + if(mySourceIndex < mySourceLength) { + /* test the following code unit */ + UChar trail=(UChar) args->source[mySourceIndex]; + if(UTF_IS_SECOND_SURROGATE(trail)) { + ++mySourceIndex; + mySourceChar=UTF16_GET_PAIR_VALUE(args->converter->fromUChar32, trail); + args->converter->fromUChar32=0x00; + /* there are no surrogates in GB2312*/ + *err = U_INVALID_CHAR_FOUND; + /* exit this condition tree */ + } else { + /* this is an unmatched lead code unit (1st surrogate) */ + /* callback(illegal) */ + *err=U_ILLEGAL_CHAR_FOUND; + } + } else { + /* no more input */ + *err = U_ZERO_ERROR; + } + } else { + /* this is an unmatched trail code unit (2nd surrogate) */ + /* callback(illegal) */ + *err=U_ILLEGAL_CHAR_FOUND; + } + } else { + /* callback(unassigned) for a BMP code point */ + *err = U_INVALID_CHAR_FOUND; + } + + args->converter->fromUChar32=mySourceChar; + break; + } + } + else{ + *err = U_BUFFER_OVERFLOW_ERROR; + break; + } + targetUniChar=missingCharMarker; + } + + args->target += myTargetIndex; + args->source += mySourceIndex; + myConverterData->isTargetUCharDBCS = isTargetUCharDBCS; +} + +static void +_HZ_WriteSub(UConverterFromUnicodeArgs *args, int32_t offsetIndex, UErrorCode *err) { + UConverter *cnv = args->converter; + UConverterDataHZ *convData=(UConverterDataHZ *) cnv->extraInfo; + char *p; + char buffer[4]; + p = buffer; + + if( convData->isTargetUCharDBCS){ + *p++= UCNV_TILDE; + *p++= UCNV_CLOSE_BRACE; + convData->isTargetUCharDBCS=FALSE; + } + *p++= (char)cnv->subChars[0]; + + ucnv_cbFromUWriteBytes(args, + buffer, (int32_t)(p - buffer), + offsetIndex, err); +} + +/* + * Structure for cloning an HZ converter into a single memory block. + * ucnv_safeClone() of the HZ converter will align the entire cloneHZStruct, + * and then ucnv_safeClone() of the sub-converter may additionally align + * subCnv inside the cloneHZStruct, for which we need the deadSpace after + * subCnv. This is because UAlignedMemory may be larger than the actually + * necessary alignment size for the platform. + * The other cloneHZStruct fields will not be moved around, + * and are aligned properly with cloneHZStruct's alignment. + */ +struct cloneHZStruct +{ + UConverter cnv; + UConverter subCnv; + UAlignedMemory deadSpace; + UConverterDataHZ mydata; +}; + + +static UConverter * +_HZ_SafeClone(const UConverter *cnv, + void *stackBuffer, + int32_t *pBufferSize, + UErrorCode *status) +{ + struct cloneHZStruct * localClone; + int32_t size, bufferSizeNeeded = sizeof(struct cloneHZStruct); + + if (U_FAILURE(*status)){ + return 0; + } + + if (*pBufferSize == 0){ /* 'preflighting' request - set needed size into *pBufferSize */ + *pBufferSize = bufferSizeNeeded; + return 0; + } + + localClone = (struct cloneHZStruct *)stackBuffer; + /* ucnv.c/ucnv_safeClone() copied the main UConverter already */ + + uprv_memcpy(&localClone->mydata, cnv->extraInfo, sizeof(UConverterDataHZ)); + localClone->cnv.extraInfo = &localClone->mydata; + localClone->cnv.isExtraLocal = TRUE; + + /* deep-clone the sub-converter */ + size = (int32_t)(sizeof(UConverter) + sizeof(UAlignedMemory)); /* include size of padding */ + ((UConverterDataHZ*)localClone->cnv.extraInfo)->gbConverter = + ucnv_safeClone(((UConverterDataHZ*)cnv->extraInfo)->gbConverter, &localClone->subCnv, &size, status); + + return &localClone->cnv; +} + +static void +_HZ_GetUnicodeSet(const UConverter *cnv, + const USetAdder *sa, + UConverterUnicodeSet which, + UErrorCode *pErrorCode) { + /* the tilde '~' is hardcoded in the converter */ + sa->add(sa->set, 0x7e); + + /* add all of the code points that the sub-converter handles */ + ((UConverterDataHZ*)cnv->extraInfo)-> + gbConverter->sharedData->impl-> + getUnicodeSet(((UConverterDataHZ*)cnv->extraInfo)->gbConverter, + sa, which, pErrorCode); +} + +static const UConverterImpl _HZImpl={ + + UCNV_HZ, + + NULL, + NULL, + + _HZOpen, + _HZClose, + _HZReset, + + UConverter_toUnicode_HZ_OFFSETS_LOGIC, + UConverter_toUnicode_HZ_OFFSETS_LOGIC, + UConverter_fromUnicode_HZ_OFFSETS_LOGIC, + UConverter_fromUnicode_HZ_OFFSETS_LOGIC, + NULL, + + NULL, + NULL, + _HZ_WriteSub, + _HZ_SafeClone, + _HZ_GetUnicodeSet +}; + +static const UConverterStaticData _HZStaticData={ + sizeof(UConverterStaticData), + "HZ", + 0, + UCNV_IBM, + UCNV_HZ, + 1, + 4, + { 0x1a, 0, 0, 0 }, + 1, + FALSE, + FALSE, + 0, + 0, + { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 }, /* reserved */ + +}; + + +const UConverterSharedData _HZData={ + sizeof(UConverterSharedData), + ~((uint32_t) 0), + NULL, + NULL, + &_HZStaticData, + FALSE, + &_HZImpl, + 0 +}; + +#endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */ diff --git a/Build/source/libs/icu/icu-xetex/common/ucnvisci.c b/Build/source/libs/icu/icu-xetex/common/ucnvisci.c new file mode 100644 index 00000000000..34f82c0b0fb --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnvisci.c @@ -0,0 +1,1469 @@ +/* +********************************************************************** +* Copyright (C) 2000-2007, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* file name: ucnvisci.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2001JUN26 +* created by: Ram Viswanadha +* +* Date Name Description +* 24/7/2001 Ram Added support for EXT character handling +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION && !UCONFIG_NO_LEGACY_CONVERSION + +#include "cmemory.h" +#include "ucnv_bld.h" +#include "unicode/ucnv.h" +#include "ucnv_cnv.h" +#include "unicode/ucnv_cb.h" +#include "unicode/uset.h" +#include "cstring.h" + +#define UCNV_OPTIONS_VERSION_MASK 0xf +#define NUKTA 0x093c +#define HALANT 0x094d +#define ZWNJ 0x200c /* Zero Width Non Joiner */ +#define ZWJ 0x200d /* Zero width Joiner */ +#define INVALID_CHAR 0xffff +#define ATR 0xEF /* Attribute code */ +#define EXT 0xF0 /* Extension code */ +#define DANDA 0x0964 +#define DOUBLE_DANDA 0x0965 +#define ISCII_NUKTA 0xE9 +#define ISCII_HALANT 0xE8 +#define ISCII_DANDA 0xEA +#define ISCII_INV 0xD9 +#define ISCII_VOWEL_SIGN_E 0xE0 +#define INDIC_BLOCK_BEGIN 0x0900 +#define INDIC_BLOCK_END 0x0D7F +#define INDIC_RANGE (INDIC_BLOCK_END - INDIC_BLOCK_BEGIN) +#define VOCALLIC_RR 0x0931 +#define LF 0x0A +#define ASCII_END 0xA0 +#define NO_CHAR_MARKER 0xFFFE +#define TELUGU_DELTA DELTA * TELUGU +#define DEV_ABBR_SIGN 0x0970 +#define DEV_ANUDATTA 0x0952 +#define EXT_RANGE_BEGIN 0xA1 +#define EXT_RANGE_END 0xEE + + +typedef enum { + DEVANAGARI =0, + BENGALI, + GURMUKHI, + GUJARATI, + ORIYA, + TAMIL, + TELUGU, + KANNADA, + MALAYALAM, + DELTA=0x80 +}UniLang; + + +/** + * Enumeration for switching code pages if + + * is encountered + */ +typedef enum { + DEF = 0x40, + RMN = 0x41, + DEV = 0x42, + BNG = 0x43, + TML = 0x44, + TLG = 0x45, + ASM = 0x46, + ORI = 0x47, + KND = 0x48, + MLM = 0x49, + GJR = 0x4A, + PNJ = 0x4B, + ARB = 0x71, + PES = 0x72, + URD = 0x73, + SND = 0x74, + KSM = 0x75, + PST = 0x76 +}ISCIILang; + +typedef enum{ + DEV_MASK =0x80, + PNJ_MASK =0x40, + GJR_MASK =0x20, + ORI_MASK =0x10, + BNG_MASK =0x08, + KND_MASK =0x04, + MLM_MASK =0x02, + TML_MASK =0x01, + ZERO =0x00 +}MaskEnum; + +#define ISCII_CNV_PREFIX "ISCII,version=" + +typedef struct{ + UChar contextCharToUnicode; /* previous Unicode codepoint for contextual analysis */ + UChar contextCharFromUnicode; /* previous Unicode codepoint for contextual analysis */ + uint16_t defDeltaToUnicode; /* delta for switching to default state when DEF is encountered */ + uint16_t currentDeltaFromUnicode;/* current delta in Indic block */ + uint16_t currentDeltaToUnicode; /* current delta in Indic block */ + MaskEnum currentMaskFromUnicode; /* mask for current state in toUnicode */ + MaskEnum currentMaskToUnicode; /* mask for current state in toUnicode */ + MaskEnum defMaskToUnicode; /* mask for default state in toUnicode */ + UBool isFirstBuffer; /* boolean for fromUnicode to see if we need to announce the first script */ + UBool resetToDefaultToUnicode; /* boolean for reseting to default delta and mask when a newline is encountered*/ + char name[sizeof(ISCII_CNV_PREFIX) + 1]; +}UConverterDataISCII; + +typedef struct LookupDataStruct +{ + UniLang uniLang; + MaskEnum maskEnum; + ISCIILang isciiLang; +} LookupDataStruct; + +static const LookupDataStruct lookupInitialData[]={ + { DEVANAGARI, DEV_MASK, DEV }, + { BENGALI, BNG_MASK, BNG }, + { GURMUKHI, PNJ_MASK, PNJ }, + { GUJARATI, GJR_MASK, GJR }, + { ORIYA, ORI_MASK, ORI }, + { TAMIL, TML_MASK, TML }, + { TELUGU, KND_MASK, TLG }, + { KANNADA, KND_MASK, KND }, + { MALAYALAM, MLM_MASK, MLM } +}; + +static void +_ISCIIOpen(UConverter *cnv, const char *name,const char *locale,uint32_t options, UErrorCode *errorCode){ + cnv->extraInfo = uprv_malloc (sizeof (UConverterDataISCII)); + + if(cnv->extraInfo != NULL) { + int32_t len=0; + UConverterDataISCII *converterData=(UConverterDataISCII *) cnv->extraInfo; + converterData->contextCharToUnicode=NO_CHAR_MARKER; + cnv->toUnicodeStatus = missingCharMarker; + converterData->contextCharFromUnicode=0x0000; + converterData->resetToDefaultToUnicode=FALSE; + /* check if the version requested is supported */ + if((options & UCNV_OPTIONS_VERSION_MASK) < 9){ + /* initialize state variables */ + converterData->currentDeltaFromUnicode=converterData->currentDeltaToUnicode= + converterData->defDeltaToUnicode= + (uint16_t)(lookupInitialData[options & UCNV_OPTIONS_VERSION_MASK].uniLang * DELTA); + + converterData->currentMaskFromUnicode = converterData->currentMaskToUnicode = + converterData->defMaskToUnicode=lookupInitialData[options & UCNV_OPTIONS_VERSION_MASK].maskEnum; + + converterData->isFirstBuffer=TRUE; + (void)uprv_strcpy(converterData->name, ISCII_CNV_PREFIX); + len = (int32_t)uprv_strlen(converterData->name); + converterData->name[len]= (char)((options & UCNV_OPTIONS_VERSION_MASK) + '0'); + converterData->name[len+1]=0; + }else{ + uprv_free(cnv->extraInfo); + cnv->extraInfo = NULL; + *errorCode = U_ILLEGAL_ARGUMENT_ERROR; + } + + }else{ + *errorCode =U_MEMORY_ALLOCATION_ERROR; + } +} +static void +_ISCIIClose(UConverter *cnv){ + if(cnv->extraInfo!=NULL) { + if(!cnv->isExtraLocal) { + uprv_free(cnv->extraInfo); + } + cnv->extraInfo=NULL; + } +} + +static const char* +_ISCIIgetName(const UConverter* cnv){ + if(cnv->extraInfo){ + UConverterDataISCII* myData= (UConverterDataISCII*)cnv->extraInfo; + return myData->name; + } + return NULL; +} + +static void +_ISCIIReset(UConverter *cnv, UConverterResetChoice choice){ + UConverterDataISCII* data =(UConverterDataISCII *) (cnv->extraInfo); + if(choice<=UCNV_RESET_TO_UNICODE) { + cnv->toUnicodeStatus = missingCharMarker; + cnv->mode=0; + data->currentDeltaToUnicode=data->defDeltaToUnicode; + data->currentMaskToUnicode = data->defMaskToUnicode; + data->contextCharToUnicode=NO_CHAR_MARKER; + } + if(choice!=UCNV_RESET_TO_UNICODE) { + cnv->fromUChar32=0x0000; + data->contextCharFromUnicode=0x00; + data->currentMaskFromUnicode=data->defMaskToUnicode; + data->currentDeltaFromUnicode=data->defDeltaToUnicode; + data->isFirstBuffer=TRUE; + data->resetToDefaultToUnicode=FALSE; + } +} + +/** + * The values in validity table are indexed by the lower bits of Unicode + * range 0x0900 - 0x09ff. The values have a structure like: + * --------------------------------------------------------------- + * | DEV | PNJ | GJR | ORI | BNG | TLG | MLM | TML | + * | | | | | ASM | KND | | | + * --------------------------------------------------------------- + * If a code point is valid in a particular script + * then that bit is turned on + * + * Unicode does not distinguish between Bengali and Assamese so we use 1 bit for + * to represent these languages + * + * Telugu and Kannada have same codepoints except for Vocallic_RR which we special case + * and combine and use 1 bit to represent these languages. + * + * TODO: It is probably easier to understand and maintain to change this + * to use uint16_t and give each of the 9 Unicode/script blocks its own bit. + */ + +static const uint8_t validityTable[128] = { +/* This state table is tool generated please do not edit unless you know exactly what you are doing */ +/* Note: This table was edited to mirror the Windows XP implementation */ +/*ISCII:Valid:Unicode */ +/*0xa0 : 0x00: 0x900 */ ZERO + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO , +/*0xa1 : 0xb8: 0x901 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + ZERO + ZERO + ZERO , +/*0xa2 : 0xfe: 0x902 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xa3 : 0xbf: 0x903 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0x00 : 0x00: 0x904 */ DEV_MASK + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO , +/*0xa4 : 0xff: 0x905 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xa5 : 0xff: 0x906 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xa6 : 0xff: 0x907 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xa7 : 0xff: 0x908 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xa8 : 0xff: 0x909 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xa9 : 0xff: 0x90a */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xaa : 0xfe: 0x90b */ DEV_MASK + ZERO + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + ZERO , +/*0x00 : 0x00: 0x90c */ DEV_MASK + ZERO + ZERO + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + ZERO , +/*0xae : 0x80: 0x90d */ DEV_MASK + ZERO + GJR_MASK + ZERO + ZERO + ZERO + ZERO + ZERO , +/*0xab : 0x87: 0x90e */ DEV_MASK + ZERO + ZERO + ZERO + ZERO + KND_MASK + MLM_MASK + TML_MASK , +/*0xac : 0xff: 0x90f */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xad : 0xff: 0x910 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xb2 : 0x80: 0x911 */ DEV_MASK + ZERO + GJR_MASK + ZERO + ZERO + ZERO + ZERO + ZERO , +/*0xaf : 0x87: 0x912 */ DEV_MASK + ZERO + ZERO + ZERO + ZERO + KND_MASK + MLM_MASK + TML_MASK , +/*0xb0 : 0xff: 0x913 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xb1 : 0xff: 0x914 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xb3 : 0xff: 0x915 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xb4 : 0xfe: 0x916 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + ZERO , +/*0xb5 : 0xfe: 0x917 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + ZERO , +/*0xb6 : 0xfe: 0x918 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + ZERO , +/*0xb7 : 0xff: 0x919 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xb8 : 0xff: 0x91a */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xb9 : 0xfe: 0x91b */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + ZERO , +/*0xba : 0xff: 0x91c */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xbb : 0xfe: 0x91d */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + ZERO , +/*0xbc : 0xff: 0x91e */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xbd : 0xff: 0x91f */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xbe : 0xfe: 0x920 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + ZERO , +/*0xbf : 0xfe: 0x921 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + ZERO , +/*0xc0 : 0xfe: 0x922 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + ZERO , +/*0xc1 : 0xff: 0x923 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xc2 : 0xff: 0x924 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xc3 : 0xfe: 0x925 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + ZERO , +/*0xc4 : 0xfe: 0x926 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + ZERO , +/*0xc5 : 0xfe: 0x927 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + ZERO , +/*0xc6 : 0xff: 0x928 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xc7 : 0x81: 0x929 */ DEV_MASK + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO + TML_MASK , +/*0xc8 : 0xff: 0x92a */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xc9 : 0xfe: 0x92b */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + ZERO , +/*0xca : 0xfe: 0x92c */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + ZERO , +/*0xcb : 0xfe: 0x92d */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + ZERO , +/*0xcc : 0xfe: 0x92e */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xcd : 0xff: 0x92f */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xcf : 0xff: 0x930 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xd0 : 0x87: 0x931 */ DEV_MASK + ZERO + ZERO + ZERO + ZERO + KND_MASK + MLM_MASK + TML_MASK , +/*0xd1 : 0xff: 0x932 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xd2 : 0xb7: 0x933 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + ZERO + KND_MASK + MLM_MASK + TML_MASK , +/*0xd3 : 0x83: 0x934 */ DEV_MASK + ZERO + ZERO + ZERO + ZERO + ZERO + MLM_MASK + TML_MASK , +/*0xd4 : 0xff: 0x935 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + ZERO + KND_MASK + MLM_MASK + TML_MASK , +/*0xd5 : 0xfe: 0x936 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + ZERO , +/*0xd6 : 0xbf: 0x937 */ DEV_MASK + ZERO + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xd7 : 0xff: 0x938 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xd8 : 0xff: 0x939 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0x00 : 0x00: 0x93A */ ZERO + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO , +/*0x00 : 0x00: 0x93B */ ZERO + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO , +/*0xe9 : 0xda: 0x93c */ DEV_MASK + PNJ_MASK + ZERO + ORI_MASK + BNG_MASK + ZERO + ZERO + ZERO , +/*0x00 : 0x00: 0x93d */ DEV_MASK + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO , +/*0xda : 0xff: 0x93e */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xdb : 0xff: 0x93f */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xdc : 0xff: 0x940 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xdd : 0xff: 0x941 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xde : 0xff: 0x942 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xdf : 0xbe: 0x943 */ DEV_MASK + ZERO + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + ZERO , +/*0x00 : 0x00: 0x944 */ DEV_MASK + ZERO + GJR_MASK + ZERO + BNG_MASK + KND_MASK + ZERO + ZERO , +/*0xe3 : 0x80: 0x945 */ DEV_MASK + ZERO + GJR_MASK + ZERO + ZERO + ZERO + ZERO + ZERO , +/*0xe0 : 0x87: 0x946 */ DEV_MASK + ZERO + ZERO + ZERO + ZERO + KND_MASK + MLM_MASK + TML_MASK , +/*0xe1 : 0xff: 0x947 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xe2 : 0xff: 0x948 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xe7 : 0x80: 0x949 */ DEV_MASK + ZERO + GJR_MASK + ZERO + ZERO + ZERO + ZERO + ZERO , +/*0xe4 : 0x87: 0x94a */ DEV_MASK + ZERO + ZERO + ZERO + ZERO + KND_MASK + MLM_MASK + TML_MASK , +/*0xe5 : 0xff: 0x94b */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xe6 : 0xff: 0x94c */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xe8 : 0xff: 0x94d */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xec : 0x00: 0x94e */ ZERO + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO , +/*0xed : 0x00: 0x94f */ ZERO + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO , +/*0x00 : 0x00: 0x950 */ DEV_MASK + ZERO + GJR_MASK + ZERO + ZERO + ZERO + ZERO + ZERO , +/*0x00 : 0x00: 0x951 */ DEV_MASK + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO , +/*0x00 : 0x00: 0x952 */ DEV_MASK + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO , +/*0x00 : 0x00: 0x953 */ DEV_MASK + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO , +/*0x00 : 0x00: 0x954 */ DEV_MASK + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO , +/*0x00 : 0x00: 0x955 */ ZERO + ZERO + ZERO + ZERO + ZERO + KND_MASK + ZERO + ZERO , +/*0x00 : 0x00: 0x956 */ ZERO + ZERO + ZERO + ORI_MASK + ZERO + KND_MASK + ZERO + ZERO , +/*0x00 : 0x00: 0x957 */ ZERO + ZERO + ZERO + ORI_MASK + BNG_MASK + ZERO + MLM_MASK + ZERO , +/*0x00 : 0x00: 0x958 */ DEV_MASK + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO , +/*0x00 : 0x00: 0x959 */ DEV_MASK + PNJ_MASK + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO , +/*0x00 : 0x00: 0x95a */ DEV_MASK + PNJ_MASK + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO , +/*0x00 : 0x00: 0x95b */ DEV_MASK + PNJ_MASK + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO , +/*0x00 : 0x00: 0x95c */ DEV_MASK + PNJ_MASK + ZERO + ZERO + BNG_MASK + ZERO + ZERO + ZERO , +/*0x00 : 0x00: 0x95d */ DEV_MASK + ZERO + ZERO + ORI_MASK + BNG_MASK + ZERO + ZERO + ZERO , +/*0x00 : 0x00: 0x95e */ DEV_MASK + PNJ_MASK + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO , +/*0xce : 0x98: 0x95f */ DEV_MASK + ZERO + ZERO + ORI_MASK + BNG_MASK + ZERO + ZERO + ZERO , +/*0x00 : 0x00: 0x960 */ DEV_MASK + ZERO + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + ZERO , +/*0x00 : 0x00: 0x961 */ DEV_MASK + ZERO + ZERO + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + ZERO , +/*0x00 : 0x00: 0x962 */ DEV_MASK + ZERO + ZERO + ZERO + BNG_MASK + ZERO + ZERO + ZERO , +/*0x00 : 0x00: 0x963 */ DEV_MASK + ZERO + ZERO + ZERO + BNG_MASK + ZERO + ZERO + ZERO , +/*0xea : 0xf8: 0x964 */ DEV_MASK + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO , +/*0xeaea : 0x00: 0x965*/ DEV_MASK + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO , +/*0xf1 : 0xff: 0x966 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xf2 : 0xff: 0x967 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xf3 : 0xff: 0x968 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xf4 : 0xff: 0x969 */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xf5 : 0xff: 0x96a */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xf6 : 0xff: 0x96b */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xf7 : 0xff: 0x96c */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xf8 : 0xff: 0x96d */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xf9 : 0xff: 0x96e */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0xfa : 0xff: 0x96f */ DEV_MASK + PNJ_MASK + GJR_MASK + ORI_MASK + BNG_MASK + KND_MASK + MLM_MASK + TML_MASK , +/*0x00 : 0x80: 0x970 */ DEV_MASK + PNJ_MASK + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO , + +/* + * The length of the array is 128 to provide values for 0x900..0x97f. + * The last 15 entries for 0x971..0x97f of the validity table are all zero + * because no Indic script uses such Unicode code points. + */ +/*0x00 : 0x00: 0x9yz */ ZERO + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO + ZERO +}; + +static const uint16_t fromUnicodeTable[128]={ + 0x00a0 ,/* 0x0900 */ + 0x00a1 ,/* 0x0901 */ + 0x00a2 ,/* 0x0902 */ + 0x00a3 ,/* 0x0903 */ + 0xa4e0 ,/* 0x0904 */ + 0x00a4 ,/* 0x0905 */ + 0x00a5 ,/* 0x0906 */ + 0x00a6 ,/* 0x0907 */ + 0x00a7 ,/* 0x0908 */ + 0x00a8 ,/* 0x0909 */ + 0x00a9 ,/* 0x090a */ + 0x00aa ,/* 0x090b */ + 0xA6E9 ,/* 0x090c */ + 0x00ae ,/* 0x090d */ + 0x00ab ,/* 0x090e */ + 0x00ac ,/* 0x090f */ + 0x00ad ,/* 0x0910 */ + 0x00b2 ,/* 0x0911 */ + 0x00af ,/* 0x0912 */ + 0x00b0 ,/* 0x0913 */ + 0x00b1 ,/* 0x0914 */ + 0x00b3 ,/* 0x0915 */ + 0x00b4 ,/* 0x0916 */ + 0x00b5 ,/* 0x0917 */ + 0x00b6 ,/* 0x0918 */ + 0x00b7 ,/* 0x0919 */ + 0x00b8 ,/* 0x091a */ + 0x00b9 ,/* 0x091b */ + 0x00ba ,/* 0x091c */ + 0x00bb ,/* 0x091d */ + 0x00bc ,/* 0x091e */ + 0x00bd ,/* 0x091f */ + 0x00be ,/* 0x0920 */ + 0x00bf ,/* 0x0921 */ + 0x00c0 ,/* 0x0922 */ + 0x00c1 ,/* 0x0923 */ + 0x00c2 ,/* 0x0924 */ + 0x00c3 ,/* 0x0925 */ + 0x00c4 ,/* 0x0926 */ + 0x00c5 ,/* 0x0927 */ + 0x00c6 ,/* 0x0928 */ + 0x00c7 ,/* 0x0929 */ + 0x00c8 ,/* 0x092a */ + 0x00c9 ,/* 0x092b */ + 0x00ca ,/* 0x092c */ + 0x00cb ,/* 0x092d */ + 0x00cc ,/* 0x092e */ + 0x00cd ,/* 0x092f */ + 0x00cf ,/* 0x0930 */ + 0x00d0 ,/* 0x0931 */ + 0x00d1 ,/* 0x0932 */ + 0x00d2 ,/* 0x0933 */ + 0x00d3 ,/* 0x0934 */ + 0x00d4 ,/* 0x0935 */ + 0x00d5 ,/* 0x0936 */ + 0x00d6 ,/* 0x0937 */ + 0x00d7 ,/* 0x0938 */ + 0x00d8 ,/* 0x0939 */ + 0xFFFF ,/* 0x093A */ + 0xFFFF ,/* 0x093B */ + 0x00e9 ,/* 0x093c */ + 0xEAE9 ,/* 0x093d */ + 0x00da ,/* 0x093e */ + 0x00db ,/* 0x093f */ + 0x00dc ,/* 0x0940 */ + 0x00dd ,/* 0x0941 */ + 0x00de ,/* 0x0942 */ + 0x00df ,/* 0x0943 */ + 0xDFE9 ,/* 0x0944 */ + 0x00e3 ,/* 0x0945 */ + 0x00e0 ,/* 0x0946 */ + 0x00e1 ,/* 0x0947 */ + 0x00e2 ,/* 0x0948 */ + 0x00e7 ,/* 0x0949 */ + 0x00e4 ,/* 0x094a */ + 0x00e5 ,/* 0x094b */ + 0x00e6 ,/* 0x094c */ + 0x00e8 ,/* 0x094d */ + 0x00ec ,/* 0x094e */ + 0x00ed ,/* 0x094f */ + 0xA1E9 ,/* 0x0950 */ /* OM Symbol */ + 0xFFFF ,/* 0x0951 */ + 0xF0B8 ,/* 0x0952 */ + 0xFFFF ,/* 0x0953 */ + 0xFFFF ,/* 0x0954 */ + 0xFFFF ,/* 0x0955 */ + 0xFFFF ,/* 0x0956 */ + 0xFFFF ,/* 0x0957 */ + 0xb3e9 ,/* 0x0958 */ + 0xb4e9 ,/* 0x0959 */ + 0xb5e9 ,/* 0x095a */ + 0xbae9 ,/* 0x095b */ + 0xbfe9 ,/* 0x095c */ + 0xC0E9 ,/* 0x095d */ + 0xc9e9 ,/* 0x095e */ + 0x00ce ,/* 0x095f */ + 0xAAe9 ,/* 0x0960 */ + 0xA7E9 ,/* 0x0961 */ + 0xDBE9 ,/* 0x0962 */ + 0xDCE9 ,/* 0x0963 */ + 0x00ea ,/* 0x0964 */ + 0xeaea ,/* 0x0965 */ + 0x00f1 ,/* 0x0966 */ + 0x00f2 ,/* 0x0967 */ + 0x00f3 ,/* 0x0968 */ + 0x00f4 ,/* 0x0969 */ + 0x00f5 ,/* 0x096a */ + 0x00f6 ,/* 0x096b */ + 0x00f7 ,/* 0x096c */ + 0x00f8 ,/* 0x096d */ + 0x00f9 ,/* 0x096e */ + 0x00fa ,/* 0x096f */ + 0xF0BF ,/* 0x0970 */ + 0xFFFF ,/* 0x0971 */ + 0xFFFF ,/* 0x0972 */ + 0xFFFF ,/* 0x0973 */ + 0xFFFF ,/* 0x0974 */ + 0xFFFF ,/* 0x0975 */ + 0xFFFF ,/* 0x0976 */ + 0xFFFF ,/* 0x0977 */ + 0xFFFF ,/* 0x0978 */ + 0xFFFF ,/* 0x0979 */ + 0xFFFF ,/* 0x097a */ + 0xFFFF ,/* 0x097b */ + 0xFFFF ,/* 0x097c */ + 0xFFFF ,/* 0x097d */ + 0xFFFF ,/* 0x097e */ + 0xFFFF ,/* 0x097f */ +}; +static const uint16_t toUnicodeTable[256]={ + 0x0000,/* 0x00 */ + 0x0001,/* 0x01 */ + 0x0002,/* 0x02 */ + 0x0003,/* 0x03 */ + 0x0004,/* 0x04 */ + 0x0005,/* 0x05 */ + 0x0006,/* 0x06 */ + 0x0007,/* 0x07 */ + 0x0008,/* 0x08 */ + 0x0009,/* 0x09 */ + 0x000a,/* 0x0a */ + 0x000b,/* 0x0b */ + 0x000c,/* 0x0c */ + 0x000d,/* 0x0d */ + 0x000e,/* 0x0e */ + 0x000f,/* 0x0f */ + 0x0010,/* 0x10 */ + 0x0011,/* 0x11 */ + 0x0012,/* 0x12 */ + 0x0013,/* 0x13 */ + 0x0014,/* 0x14 */ + 0x0015,/* 0x15 */ + 0x0016,/* 0x16 */ + 0x0017,/* 0x17 */ + 0x0018,/* 0x18 */ + 0x0019,/* 0x19 */ + 0x001a,/* 0x1a */ + 0x001b,/* 0x1b */ + 0x001c,/* 0x1c */ + 0x001d,/* 0x1d */ + 0x001e,/* 0x1e */ + 0x001f,/* 0x1f */ + 0x0020,/* 0x20 */ + 0x0021,/* 0x21 */ + 0x0022,/* 0x22 */ + 0x0023,/* 0x23 */ + 0x0024,/* 0x24 */ + 0x0025,/* 0x25 */ + 0x0026,/* 0x26 */ + 0x0027,/* 0x27 */ + 0x0028,/* 0x28 */ + 0x0029,/* 0x29 */ + 0x002a,/* 0x2a */ + 0x002b,/* 0x2b */ + 0x002c,/* 0x2c */ + 0x002d,/* 0x2d */ + 0x002e,/* 0x2e */ + 0x002f,/* 0x2f */ + 0x0030,/* 0x30 */ + 0x0031,/* 0x31 */ + 0x0032,/* 0x32 */ + 0x0033,/* 0x33 */ + 0x0034,/* 0x34 */ + 0x0035,/* 0x35 */ + 0x0036,/* 0x36 */ + 0x0037,/* 0x37 */ + 0x0038,/* 0x38 */ + 0x0039,/* 0x39 */ + 0x003A,/* 0x3A */ + 0x003B,/* 0x3B */ + 0x003c,/* 0x3c */ + 0x003d,/* 0x3d */ + 0x003e,/* 0x3e */ + 0x003f,/* 0x3f */ + 0x0040,/* 0x40 */ + 0x0041,/* 0x41 */ + 0x0042,/* 0x42 */ + 0x0043,/* 0x43 */ + 0x0044,/* 0x44 */ + 0x0045,/* 0x45 */ + 0x0046,/* 0x46 */ + 0x0047,/* 0x47 */ + 0x0048,/* 0x48 */ + 0x0049,/* 0x49 */ + 0x004a,/* 0x4a */ + 0x004b,/* 0x4b */ + 0x004c,/* 0x4c */ + 0x004d,/* 0x4d */ + 0x004e,/* 0x4e */ + 0x004f,/* 0x4f */ + 0x0050,/* 0x50 */ + 0x0051,/* 0x51 */ + 0x0052,/* 0x52 */ + 0x0053,/* 0x53 */ + 0x0054,/* 0x54 */ + 0x0055,/* 0x55 */ + 0x0056,/* 0x56 */ + 0x0057,/* 0x57 */ + 0x0058,/* 0x58 */ + 0x0059,/* 0x59 */ + 0x005a,/* 0x5a */ + 0x005b,/* 0x5b */ + 0x005c,/* 0x5c */ + 0x005d,/* 0x5d */ + 0x005e,/* 0x5e */ + 0x005f,/* 0x5f */ + 0x0060,/* 0x60 */ + 0x0061,/* 0x61 */ + 0x0062,/* 0x62 */ + 0x0063,/* 0x63 */ + 0x0064,/* 0x64 */ + 0x0065,/* 0x65 */ + 0x0066,/* 0x66 */ + 0x0067,/* 0x67 */ + 0x0068,/* 0x68 */ + 0x0069,/* 0x69 */ + 0x006a,/* 0x6a */ + 0x006b,/* 0x6b */ + 0x006c,/* 0x6c */ + 0x006d,/* 0x6d */ + 0x006e,/* 0x6e */ + 0x006f,/* 0x6f */ + 0x0070,/* 0x70 */ + 0x0071,/* 0x71 */ + 0x0072,/* 0x72 */ + 0x0073,/* 0x73 */ + 0x0074,/* 0x74 */ + 0x0075,/* 0x75 */ + 0x0076,/* 0x76 */ + 0x0077,/* 0x77 */ + 0x0078,/* 0x78 */ + 0x0079,/* 0x79 */ + 0x007a,/* 0x7a */ + 0x007b,/* 0x7b */ + 0x007c,/* 0x7c */ + 0x007d,/* 0x7d */ + 0x007e,/* 0x7e */ + 0x007f,/* 0x7f */ + 0x0080,/* 0x80 */ + 0x0081,/* 0x81 */ + 0x0082,/* 0x82 */ + 0x0083,/* 0x83 */ + 0x0084,/* 0x84 */ + 0x0085,/* 0x85 */ + 0x0086,/* 0x86 */ + 0x0087,/* 0x87 */ + 0x0088,/* 0x88 */ + 0x0089,/* 0x89 */ + 0x008a,/* 0x8a */ + 0x008b,/* 0x8b */ + 0x008c,/* 0x8c */ + 0x008d,/* 0x8d */ + 0x008e,/* 0x8e */ + 0x008f,/* 0x8f */ + 0x0090,/* 0x90 */ + 0x0091,/* 0x91 */ + 0x0092,/* 0x92 */ + 0x0093,/* 0x93 */ + 0x0094,/* 0x94 */ + 0x0095,/* 0x95 */ + 0x0096,/* 0x96 */ + 0x0097,/* 0x97 */ + 0x0098,/* 0x98 */ + 0x0099,/* 0x99 */ + 0x009a,/* 0x9a */ + 0x009b,/* 0x9b */ + 0x009c,/* 0x9c */ + 0x009d,/* 0x9d */ + 0x009e,/* 0x9e */ + 0x009f,/* 0x9f */ + 0x00A0,/* 0xa0 */ + 0x0901,/* 0xa1 */ + 0x0902,/* 0xa2 */ + 0x0903,/* 0xa3 */ + 0x0905,/* 0xa4 */ + 0x0906,/* 0xa5 */ + 0x0907,/* 0xa6 */ + 0x0908,/* 0xa7 */ + 0x0909,/* 0xa8 */ + 0x090a,/* 0xa9 */ + 0x090b,/* 0xaa */ + 0x090e,/* 0xab */ + 0x090f,/* 0xac */ + 0x0910,/* 0xad */ + 0x090d,/* 0xae */ + 0x0912,/* 0xaf */ + 0x0913,/* 0xb0 */ + 0x0914,/* 0xb1 */ + 0x0911,/* 0xb2 */ + 0x0915,/* 0xb3 */ + 0x0916,/* 0xb4 */ + 0x0917,/* 0xb5 */ + 0x0918,/* 0xb6 */ + 0x0919,/* 0xb7 */ + 0x091a,/* 0xb8 */ + 0x091b,/* 0xb9 */ + 0x091c,/* 0xba */ + 0x091d,/* 0xbb */ + 0x091e,/* 0xbc */ + 0x091f,/* 0xbd */ + 0x0920,/* 0xbe */ + 0x0921,/* 0xbf */ + 0x0922,/* 0xc0 */ + 0x0923,/* 0xc1 */ + 0x0924,/* 0xc2 */ + 0x0925,/* 0xc3 */ + 0x0926,/* 0xc4 */ + 0x0927,/* 0xc5 */ + 0x0928,/* 0xc6 */ + 0x0929,/* 0xc7 */ + 0x092a,/* 0xc8 */ + 0x092b,/* 0xc9 */ + 0x092c,/* 0xca */ + 0x092d,/* 0xcb */ + 0x092e,/* 0xcc */ + 0x092f,/* 0xcd */ + 0x095f,/* 0xce */ + 0x0930,/* 0xcf */ + 0x0931,/* 0xd0 */ + 0x0932,/* 0xd1 */ + 0x0933,/* 0xd2 */ + 0x0934,/* 0xd3 */ + 0x0935,/* 0xd4 */ + 0x0936,/* 0xd5 */ + 0x0937,/* 0xd6 */ + 0x0938,/* 0xd7 */ + 0x0939,/* 0xd8 */ + 0x200D,/* 0xd9 */ + 0x093e,/* 0xda */ + 0x093f,/* 0xdb */ + 0x0940,/* 0xdc */ + 0x0941,/* 0xdd */ + 0x0942,/* 0xde */ + 0x0943,/* 0xdf */ + 0x0946,/* 0xe0 */ + 0x0947,/* 0xe1 */ + 0x0948,/* 0xe2 */ + 0x0945,/* 0xe3 */ + 0x094a,/* 0xe4 */ + 0x094b,/* 0xe5 */ + 0x094c,/* 0xe6 */ + 0x0949,/* 0xe7 */ + 0x094d,/* 0xe8 */ + 0x093c,/* 0xe9 */ + 0x0964,/* 0xea */ + 0xFFFF,/* 0xeb */ + 0xFFFF,/* 0xec */ + 0xFFFF,/* 0xed */ + 0xFFFF,/* 0xee */ + 0xFFFF,/* 0xef */ + 0xFFFF,/* 0xf0 */ + 0x0966,/* 0xf1 */ + 0x0967,/* 0xf2 */ + 0x0968,/* 0xf3 */ + 0x0969,/* 0xf4 */ + 0x096a,/* 0xf5 */ + 0x096b,/* 0xf6 */ + 0x096c,/* 0xf7 */ + 0x096d,/* 0xf8 */ + 0x096e,/* 0xf9 */ + 0x096f,/* 0xfa */ + 0xFFFF,/* 0xfb */ + 0xFFFF,/* 0xfc */ + 0xFFFF,/* 0xfd */ + 0xFFFF,/* 0xfe */ + 0xFFFF /* 0xff */ +}; + +static const uint16_t vowelSignESpecialCases[][2]={ + { 2 /*length of array*/ , 0 }, + { 0xA4 , 0x0904 }, +}; + +static const uint16_t nuktaSpecialCases[][2]={ + { 16 /*length of array*/ , 0 }, + { 0xA6 , 0x090c }, + { 0xEA , 0x093D }, + { 0xDF , 0x0944 }, + { 0xA1 , 0x0950 }, + { 0xb3 , 0x0958 }, + { 0xb4 , 0x0959 }, + { 0xb5 , 0x095a }, + { 0xba , 0x095b }, + { 0xbf , 0x095c }, + { 0xC0 , 0x095d }, + { 0xc9 , 0x095e }, + { 0xAA , 0x0960 }, + { 0xA7 , 0x0961 }, + { 0xDB , 0x0962 }, + { 0xDC , 0x0963 }, +}; + +#define WRITE_TO_TARGET_FROM_U(args,offsets,source,target,targetLimit,targetByteUnit,err){ \ + /* write the targetUniChar to target */ \ + if(target source-1); \ + } \ + }else{ \ + *(target)++ = (uint8_t)(targetByteUnit>>8); \ + if(offsets){ \ + *(offsets++) = (int32_t)(source - args->source-1); \ + } \ + if(target < targetLimit){ \ + *(target)++ = (uint8_t) targetByteUnit; \ + if(offsets){ \ + *(offsets++) = (int32_t)(source - args->source-1); \ + } \ + }else{ \ + args->converter->charErrorBuffer[args->converter->charErrorBufferLength++] = \ + (uint8_t) (targetByteUnit); \ + *err = U_BUFFER_OVERFLOW_ERROR; \ + } \ + } \ + }else{ \ + if(targetByteUnit & 0xFF00){ \ + args->converter->charErrorBuffer[args->converter->charErrorBufferLength++] = \ + (uint8_t) (targetByteUnit >>8); \ + } \ + args->converter->charErrorBuffer[args->converter->charErrorBufferLength++] = \ + (uint8_t) (targetByteUnit); \ + *err = U_BUFFER_OVERFLOW_ERROR; \ + } \ +} + +/* Rules: + * Explicit Halant : + * + + * Soft Halant : + * + + */ + +static void +UConverter_fromUnicode_ISCII_OFFSETS_LOGIC (UConverterFromUnicodeArgs * args, + UErrorCode * err){ + const UChar *source = args->source; + const UChar *sourceLimit = args->sourceLimit; + unsigned char *target = (unsigned char *) args->target; + unsigned char *targetLimit = (unsigned char *) args->targetLimit; + int32_t* offsets = args->offsets; + uint32_t targetByteUnit = 0x0000; + UChar32 sourceChar = 0x0000; + UConverterDataISCII *converterData; + uint16_t newDelta=0; + uint16_t range = 0; + UBool deltaChanged = FALSE; + + if ((args->converter == NULL) || (args->targetLimit < args->target) || (args->sourceLimit < args->source)){ + *err = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + /* initialize data */ + converterData=(UConverterDataISCII*)args->converter->extraInfo; + newDelta=converterData->currentDeltaFromUnicode; + range = (uint16_t)(newDelta/DELTA); + + if((sourceChar = args->converter->fromUChar32)!=0) { + goto getTrail; + } + + /*writing the char to the output stream */ + while(source < sourceLimit){ + + targetByteUnit = missingCharMarker; + + sourceChar = *source++; + + /*check if input is in ASCII and C0 control codes range*/ + if (sourceChar <= ASCII_END) { + WRITE_TO_TARGET_FROM_U(args,offsets,source,target,targetLimit,sourceChar,err); + if(U_FAILURE(*err)){ + break; + } + if(sourceChar == LF){ + targetByteUnit = ATR<<8; + targetByteUnit += (uint8_t) lookupInitialData[range].isciiLang; + args->converter->fromUnicodeStatus=sourceChar; + /* now append ATR and language code */ + WRITE_TO_TARGET_FROM_U(args,offsets,source,target,targetLimit,targetByteUnit,err); + if(U_FAILURE(*err)){ + break; + } + } + continue; + } + switch(sourceChar){ + case ZWNJ: + /* contextChar has HALANT */ + if(converterData->contextCharFromUnicode){ + converterData->contextCharFromUnicode = 0x00; + targetByteUnit = ISCII_HALANT; + }else{ + /* consume ZWNJ and continue */ + converterData->contextCharFromUnicode = 0x00; + continue; + } + break; + case ZWJ: + /* contextChar has HALANT */ + if(converterData->contextCharFromUnicode){ + targetByteUnit = ISCII_NUKTA; + }else{ + targetByteUnit =ISCII_INV; + } + converterData->contextCharFromUnicode = 0x00; + break; + default: + /* is the sourceChar in the INDIC_RANGE? */ + if((uint16_t)(INDIC_BLOCK_END-sourceChar) <= INDIC_RANGE){ + /* Danda and Double Danda are valid in Northern scripts.. since Unicode + * does not include these codepoints in all Northern scrips we need to + * filter them out + */ + if(sourceChar!= DANDA && sourceChar != DOUBLE_DANDA){ + /* find out to which block the souceChar belongs*/ + range =(uint16_t)((sourceChar-INDIC_BLOCK_BEGIN)/DELTA); + newDelta =(uint16_t)(range*DELTA); + + /* Now are we in the same block as the previous? */ + if(newDelta!= converterData->currentDeltaFromUnicode || converterData->isFirstBuffer){ + converterData->currentDeltaFromUnicode = newDelta; + converterData->currentMaskFromUnicode = lookupInitialData[range].maskEnum; + deltaChanged =TRUE; + converterData->isFirstBuffer=FALSE; + } + /* Normalize all Indic codepoints to Devanagari and map them to ISCII */ + /* now subtract the new delta from sourceChar*/ + sourceChar -= converterData->currentDeltaFromUnicode ; + } + + /* get the target byte unit */ + targetByteUnit=fromUnicodeTable[(uint8_t)sourceChar]; + + /* is the code point valid in current script? */ + if((validityTable[(uint8_t)sourceChar] & converterData->currentMaskFromUnicode)==0){ + /* Vocallic RR is assigne in ISCII Telugu and Unicode */ + if(converterData->currentDeltaFromUnicode!=(TELUGU_DELTA) && sourceChar!=VOCALLIC_RR){ + targetByteUnit=missingCharMarker; + } + } + + if(deltaChanged){ + /* we are in a script block which is different than + * previous sourceChar's script block write ATR and language codes + */ + uint16_t temp=0; + temp =(uint16_t)(ATR<<8); + temp += (uint16_t)((uint8_t) lookupInitialData[range].isciiLang); + /* reset */ + deltaChanged=FALSE; + /* now append ATR and language code */ + WRITE_TO_TARGET_FROM_U(args,offsets,source,target,targetLimit,temp,err); + if(U_FAILURE(*err)){ + break; + } + } + } + /* reset context char */ + converterData->contextCharFromUnicode = 0x00; + break; + } + + + if(targetByteUnit != missingCharMarker){ + if(targetByteUnit==ISCII_HALANT){ + converterData->contextCharFromUnicode = (UChar)targetByteUnit; + } + /* write targetByteUnit to target*/ + WRITE_TO_TARGET_FROM_U(args,offsets,source,target,targetLimit,targetByteUnit,err); + if(U_FAILURE(*err)){ + break; + } + } + else{ + /* oops.. the code point is unassigned */ + /*check if the char is a First surrogate*/ + if(UTF_IS_SURROGATE(sourceChar)) { + if(UTF_IS_SURROGATE_FIRST(sourceChar)) { +getTrail: + /*look ahead to find the trail surrogate*/ + if(source < sourceLimit) { + /* test the following code unit */ + UChar trail= (*source); + if(UTF_IS_SECOND_SURROGATE(trail)) { + source++; + sourceChar=UTF16_GET_PAIR_VALUE(sourceChar, trail); + *err =U_INVALID_CHAR_FOUND; + /* convert this surrogate code point */ + /* exit this condition tree */ + } else { + /* this is an unmatched lead code unit (1st surrogate) */ + /* callback(illegal) */ + *err=U_ILLEGAL_CHAR_FOUND; + } + } else { + /* no more input */ + *err = U_ZERO_ERROR; + } + } else { + /* this is an unmatched trail code unit (2nd surrogate) */ + /* callback(illegal) */ + *err=U_ILLEGAL_CHAR_FOUND; + } + } else { + /* callback(unassigned) for a BMP code point */ + *err = U_INVALID_CHAR_FOUND; + } + + args->converter->fromUChar32=sourceChar; + break; + } + }/* end while(mySourceIndexsource = source; + args->target = (char*)target; +} + +static const int32_t lookupTable[][2]={ + { ZERO, ZERO }, /*DEFALT*/ + { ZERO, ZERO }, /*ROMAN*/ + { DEVANAGARI, DEV_MASK }, + { BENGALI, BNG_MASK }, + { TAMIL, TML_MASK }, + { TELUGU, KND_MASK }, + { BENGALI, BNG_MASK }, + { ORIYA, ORI_MASK }, + { KANNADA, KND_MASK }, + { MALAYALAM, MLM_MASK }, + { GUJARATI, GJR_MASK }, + { GURMUKHI, PNJ_MASK } + +}; + +#define WRITE_TO_TARGET_TO_U(args,source,target,offsets,offset,targetUniChar,delta, err){\ + /* add offset to current Indic Block */ \ + if(targetUniChar>ASCII_END && \ + targetUniChar != ZWJ && \ + targetUniChar != ZWNJ && \ + targetUniChar != DANDA && \ + targetUniChar != DOUBLE_DANDA){ \ + \ + targetUniChar+=(uint16_t)(delta); \ + } \ + /* now write the targetUniChar */ \ + if(targettargetLimit){ \ + *(target)++ = (UChar)targetUniChar; \ + if(offsets){ \ + *(offsets)++ = (int32_t)(offset); \ + } \ + }else{ \ + args->converter->UCharErrorBuffer[args->converter->UCharErrorBufferLength++] = \ + (UChar)targetUniChar; \ + *err = U_BUFFER_OVERFLOW_ERROR; \ + } \ +} + +#define GET_MAPPING(sourceChar,targetUniChar,data){ \ + targetUniChar = toUnicodeTable[(sourceChar)] ; \ + /* is the code point valid in current script? */ \ + if(sourceChar> ASCII_END && \ + (validityTable[(uint8_t)targetUniChar] & data->currentMaskToUnicode)==0){ \ + /* Vocallic RR is assigne in ISCII Telugu and Unicode */ \ + if(data->currentDeltaToUnicode!=(TELUGU_DELTA) && \ + targetUniChar!=VOCALLIC_RR){ \ + targetUniChar=missingCharMarker; \ + } \ + } \ +} + +/*********** + * Rules for ISCII to Unicode converter + * ISCII is stateful encoding. To convert ISCII bytes to Unicode, + * which has both precomposed and decomposed forms characters + * pre-context and post-context need to be considered. + * + * Post context + * i) ATR : Attribute code is used to declare the font and script switching. + * Currently we only switch scripts and font codes consumed without generating an error + * ii) EXT : Extention code is used to declare switching to Sanskrit and for obscure, + * obsolete characters + * Pre context + * i) Halant: if preceeded by a halant then it is a explicit halant + * ii) Nukta : + * a) if preceeded by a halant then it is a soft halant + * b) if preceeded by specific consonants and the ligatures have pre-composed + * characters in Unicode then convert to pre-composed characters + * iii) Danda: If Danda is preceeded by a Danda then convert to Double Danda + * + */ + +static void +UConverter_toUnicode_ISCII_OFFSETS_LOGIC(UConverterToUnicodeArgs *args, + UErrorCode* err){ + const char *source = ( char *) args->source; + UChar *target = args->target; + const char *sourceLimit = args->sourceLimit; + const UChar* targetLimit = args->targetLimit; + uint32_t targetUniChar = 0x0000; + uint8_t sourceChar = 0x0000; + UConverterDataISCII* data; + UChar32* toUnicodeStatus=NULL; + UChar* contextCharToUnicode = NULL; + UBool found; + int i; + + if ((args->converter == NULL) || (target < args->target) || (source < args->source)){ + *err = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + data = (UConverterDataISCII*)(args->converter->extraInfo); + contextCharToUnicode = &data->contextCharToUnicode; /* contains previous ISCII codepoint visited */ + toUnicodeStatus = (UChar32*)&args->converter->toUnicodeStatus;/* contains the mapping to Unicode of the above codepoint*/ + + while(sourcecurrentDeltaToUnicode = + (uint16_t)(lookupTable[sourceChar & 0x0F][0] * DELTA); + data->currentMaskToUnicode = + (MaskEnum)lookupTable[sourceChar & 0x0F][1] ; + } + else if(sourceChar==DEF){ + /* switch back to default */ + data->currentDeltaToUnicode = data->defDeltaToUnicode; + data->currentMaskToUnicode = data->defMaskToUnicode; + }else{ + if((sourceChar >= 0x21 && sourceChar <= 0x3F)){ + /* these are display codes consume and continue */ + }else{ + *err =U_ILLEGAL_CHAR_FOUND; + /* reset */ + *contextCharToUnicode=NO_CHAR_MARKER; + goto CALLBACK; + } + } + + /* reset */ + *contextCharToUnicode=NO_CHAR_MARKER; + + continue; + + }else if(*contextCharToUnicode==EXT){ + /* check if sourceChar is in 0xA1-0xEE range */ + if((uint8_t) (EXT_RANGE_END - sourceChar) <= (EXT_RANGE_END - EXT_RANGE_BEGIN)){ + /* We currently support only Anudatta and Devanagari abbreviation sign */ + if(sourceChar==0xBF || sourceChar == 0xB8){ + targetUniChar = (sourceChar==0xBF) ? DEV_ABBR_SIGN : DEV_ANUDATTA; + + /* find out if the mapping is valid in this state */ + if(validityTable[(uint8_t)targetUniChar] & data->currentMaskToUnicode){ + + *contextCharToUnicode= NO_CHAR_MARKER; + + /* write to target */ + WRITE_TO_TARGET_TO_U(args,source,target,args->offsets,(source-args->source -2), + targetUniChar,data->currentDeltaToUnicode,err); + + continue; + } + } + /* byte unit is unassigned */ + targetUniChar = missingCharMarker; + *err= U_INVALID_CHAR_FOUND; + }else{ + /* only 0xA1 - 0xEE are legal after EXT char */ + *contextCharToUnicode= NO_CHAR_MARKER; + *err = U_ILLEGAL_CHAR_FOUND; + } + goto CALLBACK; + }else if(*contextCharToUnicode==ISCII_INV){ + if(sourceChar==ISCII_HALANT){ + targetUniChar = 0x0020; /* replace with space accoding to Indic FAQ */ + }else{ + targetUniChar = ZWJ; + } + + /* write to target */ + WRITE_TO_TARGET_TO_U(args,source,target,args->offsets,(source-args->source -2), + targetUniChar,data->currentDeltaToUnicode,err); + /* reset */ + *contextCharToUnicode=NO_CHAR_MARKER; + } + + /* look at the pre-context and perform special processing */ + switch(sourceChar){ + case ISCII_INV: + case EXT: /*falls through*/ + case ATR: + *contextCharToUnicode = (UChar)sourceChar; + + if(*toUnicodeStatus != missingCharMarker){ + + WRITE_TO_TARGET_TO_U(args,source,target,args->offsets,(source-args->source -2), + *toUnicodeStatus,data->currentDeltaToUnicode,err); + *toUnicodeStatus = missingCharMarker; + } + continue; + case ISCII_DANDA: + /* handle double danda*/ + if(*contextCharToUnicode== ISCII_DANDA){ + targetUniChar = DOUBLE_DANDA; + /* clear the context */ + *contextCharToUnicode = NO_CHAR_MARKER; + *toUnicodeStatus = missingCharMarker; + }else{ + GET_MAPPING(sourceChar,targetUniChar,data); + *contextCharToUnicode = sourceChar; + } + break; + case ISCII_HALANT: + /* handle explicit halant */ + if(*contextCharToUnicode == ISCII_HALANT){ + targetUniChar = ZWNJ; + /* clear the context */ + *contextCharToUnicode = NO_CHAR_MARKER; + }else{ + GET_MAPPING(sourceChar,targetUniChar,data); + *contextCharToUnicode = sourceChar; + } + break; + case 0x0A: + /* fall through */ + case 0x0D: + data->resetToDefaultToUnicode = TRUE; + GET_MAPPING(sourceChar,targetUniChar,data); + *contextCharToUnicode = sourceChar; + break; + + case ISCII_VOWEL_SIGN_E: + i=1; + found=FALSE; + for( ;icurrentMaskToUnicode){ + /*targetUniChar += data->currentDeltaToUnicode ;*/ + *contextCharToUnicode= NO_CHAR_MARKER; + *toUnicodeStatus = missingCharMarker; + break; + } + } + GET_MAPPING(sourceChar,targetUniChar,data); + *contextCharToUnicode = sourceChar; + break; + + case ISCII_NUKTA: + /* handle soft halant */ + if(*contextCharToUnicode == ISCII_HALANT){ + targetUniChar = ZWJ; + /* clear the context */ + *contextCharToUnicode = NO_CHAR_MARKER; + break; + }else{ + /* try to handle + ISCII_NUKTA special mappings */ + i=1; + found =FALSE; + for( ;icurrentMaskToUnicode){ + /*targetUniChar += data->currentDeltaToUnicode ;*/ + *contextCharToUnicode= NO_CHAR_MARKER; + *toUnicodeStatus = missingCharMarker; + break; + } + /* else fall through to default */ + } + /* else fall through to default */ + } + default: + GET_MAPPING(sourceChar,targetUniChar,data); + *contextCharToUnicode = sourceChar; + break; + } + + + if(*toUnicodeStatus != missingCharMarker){ + /* write the previously mapped codepoint */ + WRITE_TO_TARGET_TO_U(args,source,target,args->offsets,(source-args->source -2), + *toUnicodeStatus,data->currentDeltaToUnicode,err); + *toUnicodeStatus = missingCharMarker; + } + + + if(targetUniChar != missingCharMarker ){ + /* now save the targetUniChar for delayed write */ + *toUnicodeStatus = (UChar) targetUniChar; + if(data->resetToDefaultToUnicode==TRUE){ + data->currentDeltaToUnicode = data->defDeltaToUnicode; + data->currentMaskToUnicode = data->defMaskToUnicode; + data->resetToDefaultToUnicode=FALSE; + } + }else{ + + /* we reach here only if targetUniChar == missingCharMarker + * so assign codes to reason and err + */ + *err = U_INVALID_CHAR_FOUND; +CALLBACK: + args->converter->toUBytes[0] = (uint8_t) sourceChar; + args->converter->toULength = 1; + break; + } + + } + else{ + *err =U_BUFFER_OVERFLOW_ERROR; + break; + } + } + + if(U_SUCCESS(*err) && args->flush && source == sourceLimit) { + /* end of the input stream */ + UConverter *cnv = args->converter; + + if(*contextCharToUnicode==ATR || *contextCharToUnicode==EXT || *contextCharToUnicode==ISCII_INV){ + /* set toUBytes[] */ + cnv->toUBytes[0] = (uint8_t)*contextCharToUnicode; + cnv->toULength = 1; + + /* avoid looping on truncated sequences */ + *contextCharToUnicode = NO_CHAR_MARKER; + }else{ + cnv->toULength = 0; + } + + if(*toUnicodeStatus != missingCharMarker) { + /* output a remaining target character */ + WRITE_TO_TARGET_TO_U(args,source,target,args->offsets,(source - args->source -1), + *toUnicodeStatus,data->currentDeltaToUnicode,err); + *toUnicodeStatus = missingCharMarker; + } + } + + args->target = target; + args->source = source; +} + +/* structure for SafeClone calculations */ +struct cloneISCIIStruct +{ + UConverter cnv; + UConverterDataISCII mydata; +}; + + +static UConverter * +_ISCII_SafeClone(const UConverter *cnv, + void *stackBuffer, + int32_t *pBufferSize, + UErrorCode *status) +{ + struct cloneISCIIStruct * localClone; + int32_t bufferSizeNeeded = sizeof(struct cloneISCIIStruct); + + if (U_FAILURE(*status)){ + return 0; + } + + if (*pBufferSize == 0){ /* 'preflighting' request - set needed size into *pBufferSize */ + *pBufferSize = bufferSizeNeeded; + return 0; + } + + localClone = (struct cloneISCIIStruct *)stackBuffer; + /* ucnv.c/ucnv_safeClone() copied the main UConverter already */ + + uprv_memcpy(&localClone->mydata, cnv->extraInfo, sizeof(UConverterDataISCII)); + localClone->cnv.extraInfo = &localClone->mydata; + localClone->cnv.isExtraLocal = TRUE; + + return &localClone->cnv; +} + +static void +_ISCIIGetUnicodeSet(const UConverter *cnv, + const USetAdder *sa, + UConverterUnicodeSet which, + UErrorCode *pErrorCode) +{ + int32_t idx, script; + uint8_t mask; + + /* Since all ISCII versions allow switching to other ISCII + scripts, we add all roundtrippable characters to this set. */ + sa->addRange(sa->set, 0, ASCII_END); + for (script = DEVANAGARI; script <= MALAYALAM; script++) { + mask = (uint8_t)(lookupInitialData[script].maskEnum); + for (idx = 0; idx < DELTA; idx++) { + if (validityTable[idx] & mask) { + sa->add(sa->set, idx + (script * DELTA) + INDIC_BLOCK_BEGIN); + } + } + } + sa->add(sa->set, DANDA); + sa->add(sa->set, DOUBLE_DANDA); + sa->add(sa->set, ZWNJ); + sa->add(sa->set, ZWJ); +} + +static const UConverterImpl _ISCIIImpl={ + + UCNV_ISCII, + + NULL, + NULL, + + _ISCIIOpen, + _ISCIIClose, + _ISCIIReset, + + UConverter_toUnicode_ISCII_OFFSETS_LOGIC, + UConverter_toUnicode_ISCII_OFFSETS_LOGIC, + UConverter_fromUnicode_ISCII_OFFSETS_LOGIC, + UConverter_fromUnicode_ISCII_OFFSETS_LOGIC, + NULL, + + NULL, + _ISCIIgetName, + NULL, + _ISCII_SafeClone, + _ISCIIGetUnicodeSet +}; + +static const UConverterStaticData _ISCIIStaticData={ + sizeof(UConverterStaticData), + "ISCII", + 0, + UCNV_IBM, + UCNV_ISCII, + 1, + 4, + { 0x1a, 0, 0, 0 }, + 0x1, + FALSE, + FALSE, + 0x0, + 0x0, + { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 }, /* reserved */ + +}; + +const UConverterSharedData _ISCIIData={ + sizeof(UConverterSharedData), + ~((uint32_t) 0), + NULL, + NULL, + &_ISCIIStaticData, + FALSE, + &_ISCIIImpl, + 0 +}; + +#endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */ diff --git a/Build/source/libs/icu/icu-xetex/common/ucnvlat1.c b/Build/source/libs/icu/icu-xetex/common/ucnvlat1.c new file mode 100644 index 00000000000..4e682dfd1c3 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnvlat1.c @@ -0,0 +1,743 @@ +/* +********************************************************************** +* Copyright (C) 2000-2007, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* file name: ucnvlat1.cpp +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2000feb07 +* created by: Markus W. Scherer +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "unicode/ucnv.h" +#include "unicode/uset.h" +#include "ucnv_bld.h" +#include "ucnv_cnv.h" + +/* control optimizations according to the platform */ +#define LATIN1_UNROLL_FROM_UNICODE 1 + +/* ISO 8859-1 --------------------------------------------------------------- */ + +/* This is a table-less and callback-less version of ucnv_MBCSSingleToBMPWithOffsets(). */ +static void +_Latin1ToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + const uint8_t *source; + UChar *target; + int32_t targetCapacity, length; + int32_t *offsets; + + int32_t sourceIndex; + + /* set up the local pointers */ + source=(const uint8_t *)pArgs->source; + target=pArgs->target; + targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); + offsets=pArgs->offsets; + + sourceIndex=0; + + /* + * since the conversion here is 1:1 UChar:uint8_t, we need only one counter + * for the minimum of the sourceLength and targetCapacity + */ + length=(int32_t)((const uint8_t *)pArgs->sourceLimit-source); + if(length<=targetCapacity) { + targetCapacity=length; + } else { + /* target will be full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + length=targetCapacity; + } + + if(targetCapacity>=8) { + /* This loop is unrolled for speed and improved pipelining. */ + int32_t count, loops; + + loops=count=targetCapacity>>3; + length=targetCapacity&=0x7; + do { + target[0]=source[0]; + target[1]=source[1]; + target[2]=source[2]; + target[3]=source[3]; + target[4]=source[4]; + target[5]=source[5]; + target[6]=source[6]; + target[7]=source[7]; + target+=8; + source+=8; + } while(--count>0); + + if(offsets!=NULL) { + do { + offsets[0]=sourceIndex++; + offsets[1]=sourceIndex++; + offsets[2]=sourceIndex++; + offsets[3]=sourceIndex++; + offsets[4]=sourceIndex++; + offsets[5]=sourceIndex++; + offsets[6]=sourceIndex++; + offsets[7]=sourceIndex++; + offsets+=8; + } while(--loops>0); + } + } + + /* conversion loop */ + while(targetCapacity>0) { + *target++=*source++; + --targetCapacity; + } + + /* write back the updated pointers */ + pArgs->source=(const char *)source; + pArgs->target=target; + + /* set offsets */ + if(offsets!=NULL) { + while(length>0) { + *offsets++=sourceIndex++; + --length; + } + pArgs->offsets=offsets; + } +} + +/* This is a table-less and callback-less version of ucnv_MBCSSingleGetNextUChar(). */ +static UChar32 +_Latin1GetNextUChar(UConverterToUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + const uint8_t *source=(const uint8_t *)pArgs->source; + if(source<(const uint8_t *)pArgs->sourceLimit) { + pArgs->source=(const char *)(source+1); + return *source; + } + + /* no output because of empty input */ + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0xffff; +} + +/* This is a table-less version of ucnv_MBCSSingleFromBMPWithOffsets(). */ +static void +_Latin1FromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv; + const UChar *source, *sourceLimit; + uint8_t *target, *oldTarget; + int32_t targetCapacity, length; + int32_t *offsets; + + UChar32 cp; + UChar c, max; + + int32_t sourceIndex; + + /* set up the local pointers */ + cnv=pArgs->converter; + source=pArgs->source; + sourceLimit=pArgs->sourceLimit; + target=oldTarget=(uint8_t *)pArgs->target; + targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); + offsets=pArgs->offsets; + + if(cnv->sharedData==&_Latin1Data) { + max=0xff; /* Latin-1 */ + } else { + max=0x7f; /* US-ASCII */ + } + + /* get the converter state from UConverter */ + cp=cnv->fromUChar32; + + /* sourceIndex=-1 if the current character began in the previous buffer */ + sourceIndex= cp==0 ? 0 : -1; + + /* + * since the conversion here is 1:1 UChar:uint8_t, we need only one counter + * for the minimum of the sourceLength and targetCapacity + */ + length=(int32_t)(sourceLimit-source); + if(length0) { + goto getTrail; + } + +#if LATIN1_UNROLL_FROM_UNICODE + /* unroll the loop with the most common case */ + if(targetCapacity>=16) { + int32_t count, loops; + UChar u, oredChars; + + loops=count=targetCapacity>>4; + do { + oredChars=u=*source++; + *target++=(uint8_t)u; + oredChars|=u=*source++; + *target++=(uint8_t)u; + oredChars|=u=*source++; + *target++=(uint8_t)u; + oredChars|=u=*source++; + *target++=(uint8_t)u; + oredChars|=u=*source++; + *target++=(uint8_t)u; + oredChars|=u=*source++; + *target++=(uint8_t)u; + oredChars|=u=*source++; + *target++=(uint8_t)u; + oredChars|=u=*source++; + *target++=(uint8_t)u; + oredChars|=u=*source++; + *target++=(uint8_t)u; + oredChars|=u=*source++; + *target++=(uint8_t)u; + oredChars|=u=*source++; + *target++=(uint8_t)u; + oredChars|=u=*source++; + *target++=(uint8_t)u; + oredChars|=u=*source++; + *target++=(uint8_t)u; + oredChars|=u=*source++; + *target++=(uint8_t)u; + oredChars|=u=*source++; + *target++=(uint8_t)u; + oredChars|=u=*source++; + *target++=(uint8_t)u; + + /* were all 16 entries really valid? */ + if(oredChars>max) { + /* no, return to the first of these 16 */ + source-=16; + target-=16; + break; + } + } while(--count>0); + count=loops-count; + targetCapacity-=16*count; + + if(offsets!=NULL) { + oldTarget+=16*count; + while(count>0) { + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + --count; + } + } + } +#endif + + /* conversion loop */ + c=0; + while(targetCapacity>0 && (c=*source++)<=max) { + /* convert the Unicode code point */ + *target++=(uint8_t)c; + --targetCapacity; + } + + if(c>max) { + cp=c; + if(!U_IS_SURROGATE(cp)) { + /* callback(unassigned) */ + } else if(U_IS_SURROGATE_LEAD(cp)) { +getTrail: + if(sourcefromUChar32=cp; + goto noMoreInput; + } + } else { + /* this is an unmatched trail code unit (2nd surrogate) */ + /* callback(illegal) */ + } + + *pErrorCode= U_IS_SURROGATE(cp) ? U_ILLEGAL_CHAR_FOUND : U_INVALID_CHAR_FOUND; + cnv->fromUChar32=cp; + } +noMoreInput: + + /* set offsets since the start */ + if(offsets!=NULL) { + size_t count=target-oldTarget; + while(count>0) { + *offsets++=sourceIndex++; + --count; + } + } + + if(U_SUCCESS(*pErrorCode) && source=(uint8_t *)pArgs->targetLimit) { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + + /* write back the updated pointers */ + pArgs->source=source; + pArgs->target=(char *)target; + pArgs->offsets=offsets; +} + +/* Convert UTF-8 to Latin-1. Adapted from ucnv_SBCSFromUTF8(). */ +static void +ucnv_Latin1FromUTF8(UConverterFromUnicodeArgs *pFromUArgs, + UConverterToUnicodeArgs *pToUArgs, + UErrorCode *pErrorCode) { + UConverter *utf8; + const uint8_t *source, *sourceLimit; + uint8_t *target; + int32_t targetCapacity; + + UChar32 c; + uint8_t b, t1; + + /* set up the local pointers */ + utf8=pToUArgs->converter; + source=(uint8_t *)pToUArgs->source; + sourceLimit=(uint8_t *)pToUArgs->sourceLimit; + target=(uint8_t *)pFromUArgs->target; + targetCapacity=(int32_t)(pFromUArgs->targetLimit-pFromUArgs->target); + + /* get the converter state from the UTF-8 UConverter */ + c=(UChar32)utf8->toUnicodeStatus; + if(c!=0 && source=0xc2 && c<=0xc3 && (t1=(uint8_t)(*source-0x80)) <= 0x3f) { + ++source; + *target++=(uint8_t)(((c&3)<<6)|t1); + --targetCapacity; + + utf8->toUnicodeStatus=0; + utf8->toULength=0; + } else { + /* complicated, illegal or unmappable input: fall back to the pivoting implementation */ + *pErrorCode=U_USING_DEFAULT_WARNING; + return; + } + } + + /* + * Make sure that the last byte sequence before sourceLimit is complete + * or runs into a lead byte. + * In the conversion loop compare source with sourceLimit only once + * per multi-byte character. + * For Latin-1, adjust sourceLimit only for 1 trail byte because + * the conversion loop handles at most 2-byte sequences. + */ + if(source0) { + b=*source++; + if((int8_t)b>=0) { + /* convert ASCII */ + *target++=(uint8_t)b; + --targetCapacity; + } else if( /* handle U+0080..U+00FF inline */ + b>=0xc2 && b<=0xc3 && + (t1=(uint8_t)(*source-0x80)) <= 0x3f + ) { + ++source; + *target++=(uint8_t)(((b&3)<<6)|t1); + --targetCapacity; + } else { + /* complicated, illegal or unmappable input: fall back to the pivoting implementation */ + pToUArgs->source=(char *)(source-1); + pFromUArgs->target=(char *)target; + *pErrorCode=U_USING_DEFAULT_WARNING; + return; + } + } else { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + } + + /* + * The sourceLimit may have been adjusted before the conversion loop + * to stop before a truncated sequence. + * If so, then collect the truncated sequence now. + * For Latin-1, there is at most exactly one lead byte because of the + * smaller sourceLimit adjustment logic. + */ + if(U_SUCCESS(*pErrorCode) && source<(sourceLimit=(uint8_t *)pToUArgs->sourceLimit)) { + utf8->toUnicodeStatus=utf8->toUBytes[0]=b=*source++; + utf8->toULength=1; + utf8->mode=utf8_countTrailBytes[b]+1; + } + + /* write back the updated pointers */ + pToUArgs->source=(char *)source; + pFromUArgs->target=(char *)target; +} + +static void +_Latin1GetUnicodeSet(const UConverter *cnv, + const USetAdder *sa, + UConverterUnicodeSet which, + UErrorCode *pErrorCode) { + sa->addRange(sa->set, 0, 0xff); +} + +static const UConverterImpl _Latin1Impl={ + UCNV_LATIN_1, + + NULL, + NULL, + + NULL, + NULL, + NULL, + + _Latin1ToUnicodeWithOffsets, + _Latin1ToUnicodeWithOffsets, + _Latin1FromUnicodeWithOffsets, + _Latin1FromUnicodeWithOffsets, + _Latin1GetNextUChar, + + NULL, + NULL, + NULL, + NULL, + _Latin1GetUnicodeSet, + + NULL, + ucnv_Latin1FromUTF8 +}; + +static const UConverterStaticData _Latin1StaticData={ + sizeof(UConverterStaticData), + "ISO-8859-1", + 819, UCNV_IBM, UCNV_LATIN_1, 1, 1, + { 0x1a, 0, 0, 0 }, 1, FALSE, FALSE, + 0, + 0, + { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */ +}; + +const UConverterSharedData _Latin1Data={ + sizeof(UConverterSharedData), ~((uint32_t) 0), + NULL, NULL, &_Latin1StaticData, FALSE, &_Latin1Impl, + 0 +}; + +/* US-ASCII ----------------------------------------------------------------- */ + +/* This is a table-less version of ucnv_MBCSSingleToBMPWithOffsets(). */ +static void +_ASCIIToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + const uint8_t *source, *sourceLimit; + UChar *target, *oldTarget; + int32_t targetCapacity, length; + int32_t *offsets; + + int32_t sourceIndex; + + uint8_t c; + + /* set up the local pointers */ + source=(const uint8_t *)pArgs->source; + sourceLimit=(const uint8_t *)pArgs->sourceLimit; + target=oldTarget=pArgs->target; + targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); + offsets=pArgs->offsets; + + /* sourceIndex=-1 if the current character began in the previous buffer */ + sourceIndex=0; + + /* + * since the conversion here is 1:1 UChar:uint8_t, we need only one counter + * for the minimum of the sourceLength and targetCapacity + */ + length=(int32_t)(sourceLimit-source); + if(length=8) { + /* This loop is unrolled for speed and improved pipelining. */ + int32_t count, loops; + UChar oredChars; + + loops=count=targetCapacity>>3; + do { + oredChars=target[0]=source[0]; + oredChars|=target[1]=source[1]; + oredChars|=target[2]=source[2]; + oredChars|=target[3]=source[3]; + oredChars|=target[4]=source[4]; + oredChars|=target[5]=source[5]; + oredChars|=target[6]=source[6]; + oredChars|=target[7]=source[7]; + + /* were all 16 entries really valid? */ + if(oredChars>0x7f) { + /* no, return to the first of these 16 */ + break; + } + source+=8; + target+=8; + } while(--count>0); + count=loops-count; + targetCapacity-=count*8; + + if(offsets!=NULL) { + oldTarget+=count*8; + while(count>0) { + offsets[0]=sourceIndex++; + offsets[1]=sourceIndex++; + offsets[2]=sourceIndex++; + offsets[3]=sourceIndex++; + offsets[4]=sourceIndex++; + offsets[5]=sourceIndex++; + offsets[6]=sourceIndex++; + offsets[7]=sourceIndex++; + offsets+=8; + --count; + } + } + } + + /* conversion loop */ + c=0; + while(targetCapacity>0 && (c=*source++)<=0x7f) { + *target++=c; + --targetCapacity; + } + + if(c>0x7f) { + /* callback(illegal); copy the current bytes to toUBytes[] */ + UConverter *cnv=pArgs->converter; + cnv->toUBytes[0]=c; + cnv->toULength=1; + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + } else if(source=pArgs->targetLimit) { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + + /* set offsets since the start */ + if(offsets!=NULL) { + size_t count=target-oldTarget; + while(count>0) { + *offsets++=sourceIndex++; + --count; + } + } + + /* write back the updated pointers */ + pArgs->source=(const char *)source; + pArgs->target=target; + pArgs->offsets=offsets; +} + +/* This is a table-less version of ucnv_MBCSSingleGetNextUChar(). */ +static UChar32 +_ASCIIGetNextUChar(UConverterToUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + const uint8_t *source; + uint8_t b; + + source=(const uint8_t *)pArgs->source; + if(source<(const uint8_t *)pArgs->sourceLimit) { + b=*source++; + pArgs->source=(const char *)source; + if(b<=0x7f) { + return b; + } else { + UConverter *cnv=pArgs->converter; + cnv->toUBytes[0]=b; + cnv->toULength=1; + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + return 0xffff; + } + } + + /* no output because of empty input */ + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0xffff; +} + +/* "Convert" UTF-8 to US-ASCII: Validate and copy. */ +static void +ucnv_ASCIIFromUTF8(UConverterFromUnicodeArgs *pFromUArgs, + UConverterToUnicodeArgs *pToUArgs, + UErrorCode *pErrorCode) { + const uint8_t *source, *sourceLimit; + uint8_t *target; + int32_t targetCapacity, length; + + uint8_t c; + + if(pToUArgs->converter->toUnicodeStatus!=0) { + /* no handling of partial UTF-8 characters here, fall back to pivoting */ + *pErrorCode=U_USING_DEFAULT_WARNING; + return; + } + + /* set up the local pointers */ + source=(const uint8_t *)pToUArgs->source; + sourceLimit=(const uint8_t *)pToUArgs->sourceLimit; + target=(uint8_t *)pFromUArgs->target; + targetCapacity=(int32_t)(pFromUArgs->targetLimit-pFromUArgs->target); + + /* + * since the conversion here is 1:1 uint8_t:uint8_t, we need only one counter + * for the minimum of the sourceLength and targetCapacity + */ + length=(int32_t)(sourceLimit-source); + if(length=16) { + int32_t count, loops; + uint8_t oredChars; + + loops=count=targetCapacity>>4; + do { + oredChars=*target++=*source++; + oredChars|=*target++=*source++; + oredChars|=*target++=*source++; + oredChars|=*target++=*source++; + oredChars|=*target++=*source++; + oredChars|=*target++=*source++; + oredChars|=*target++=*source++; + oredChars|=*target++=*source++; + oredChars|=*target++=*source++; + oredChars|=*target++=*source++; + oredChars|=*target++=*source++; + oredChars|=*target++=*source++; + oredChars|=*target++=*source++; + oredChars|=*target++=*source++; + oredChars|=*target++=*source++; + oredChars|=*target++=*source++; + + /* were all 16 entries really valid? */ + if(oredChars>0x7f) { + /* no, return to the first of these 16 */ + source-=16; + target-=16; + break; + } + } while(--count>0); + count=loops-count; + targetCapacity-=16*count; + } + + /* conversion loop */ + c=0; + while(targetCapacity>0 && (c=*source)<=0x7f) { + ++source; + *target++=c; + --targetCapacity; + } + + if(c>0x7f) { + /* non-ASCII character, handle in standard converter */ + *pErrorCode=U_USING_DEFAULT_WARNING; + } else if(source=(const uint8_t *)pFromUArgs->targetLimit) { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + + /* write back the updated pointers */ + pToUArgs->source=(const char *)source; + pFromUArgs->target=(char *)target; +} + +static void +_ASCIIGetUnicodeSet(const UConverter *cnv, + const USetAdder *sa, + UConverterUnicodeSet which, + UErrorCode *pErrorCode) { + sa->addRange(sa->set, 0, 0x7f); +} + +static const UConverterImpl _ASCIIImpl={ + UCNV_US_ASCII, + + NULL, + NULL, + + NULL, + NULL, + NULL, + + _ASCIIToUnicodeWithOffsets, + _ASCIIToUnicodeWithOffsets, + _Latin1FromUnicodeWithOffsets, + _Latin1FromUnicodeWithOffsets, + _ASCIIGetNextUChar, + + NULL, + NULL, + NULL, + NULL, + _ASCIIGetUnicodeSet, + + NULL, + ucnv_ASCIIFromUTF8 +}; + +static const UConverterStaticData _ASCIIStaticData={ + sizeof(UConverterStaticData), + "US-ASCII", + 367, UCNV_IBM, UCNV_US_ASCII, 1, 1, + { 0x1a, 0, 0, 0 }, 1, FALSE, FALSE, + 0, + 0, + { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */ +}; + +const UConverterSharedData _ASCIIData={ + sizeof(UConverterSharedData), ~((uint32_t) 0), + NULL, NULL, &_ASCIIStaticData, FALSE, &_ASCIIImpl, + 0 +}; + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/ucnvmbcs.c b/Build/source/libs/icu/icu-xetex/common/ucnvmbcs.c new file mode 100644 index 00000000000..37a7d4b5c50 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnvmbcs.c @@ -0,0 +1,4946 @@ +/* +****************************************************************************** +* +* Copyright (C) 2000-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: ucnvmbcs.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2000jul03 +* created by: Markus W. Scherer +* +* The current code in this file replaces the previous implementation +* of conversion code from multi-byte codepages to Unicode and back. +* This implementation supports the following: +* - legacy variable-length codepages with up to 4 bytes per character +* - all Unicode code points (up to 0x10ffff) +* - efficient distinction of unassigned vs. illegal byte sequences +* - it is possible in fromUnicode() to directly deal with simple +* stateful encodings (used for EBCDIC_STATEFUL) +* - it is possible to convert Unicode code points +* to a single zero byte (but not as a fallback except for SBCS) +* +* Remaining limitations in fromUnicode: +* - byte sequences must not have leading zero bytes +* - except for SBCS codepages: no fallback mapping from Unicode to a zero byte +* - limitation to up to 4 bytes per character +* +* ICU 2.8 (late 2003) adds a secondary data structure which lifts some of these +* limitations and adds m:n character mappings and other features. +* See ucnv_ext.h for details. +* +* Change history: +* +* 5/6/2001 Ram Moved MBCS_SINGLE_RESULT_FROM_U,MBCS_STAGE_2_FROM_U, +* MBCS_VALUE_2_FROM_STAGE_2, MBCS_VALUE_4_FROM_STAGE_2 +* macros to ucnvmbcs.h file +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION && !UCONFIG_NO_LEGACY_CONVERSION + +#include "unicode/ucnv.h" +#include "unicode/ucnv_cb.h" +#include "unicode/udata.h" +#include "unicode/uset.h" +#include "ucnv_bld.h" +#include "ucnvmbcs.h" +#include "ucnv_ext.h" +#include "ucnv_cnv.h" +#include "umutex.h" +#include "cmemory.h" +#include "cstring.h" + +/* control optimizations according to the platform */ +#define MBCS_UNROLL_SINGLE_TO_BMP 1 +#define MBCS_UNROLL_SINGLE_FROM_BMP 0 + +/* + * _MBCSHeader version 4.3 + * (Note that the _MBCSHeader version is in addition to the converter formatVersion.) + * + * Change from version 4.2: + * - Optional utf8Friendly data structures, with 64-entry stage 3 block + * allocation for parts of the BMP, and an additional mbcsIndex in non-SBCS + * files which can be used instead of stages 1 & 2. + * Faster lookups for roundtrips from most commonly used characters, + * and lookups from UTF-8 byte sequences with a natural bit distribution. + * See ucnvmbcs.h for more details. + * + * Change from version 4.1: + * - Added an optional extension table structure at the end of the .cnv file. + * It is present if the upper bits of the header flags field contains a non-zero + * byte offset to it. + * Files that contain only a conversion table and no base table + * use the special outputType MBCS_OUTPUT_EXT_ONLY. + * These contain the base table name between the MBCS header and the extension + * data. + * + * Change from version 4.0: + * - Replace header.reserved with header.fromUBytesLength so that all + * fields in the data have length. + * + * Changes from version 3 (for performance improvements): + * - new bit distribution for state table entries + * - reordered action codes + * - new data structure for single-byte fromUnicode + * + stage 2 only contains indexes + * + stage 3 stores 16 bits per character with classification bits 15..8 + * - no multiplier for stage 1 entries + * - stage 2 for non-single-byte codepages contains the index and the flags in + * one 32-bit value + * - 2-byte and 4-byte fromUnicode results are stored directly as 16/32-bit integers + * + * For more details about old versions of the MBCS data structure, see + * the corresponding versions of this file. + * + * Converting stateless codepage data ---------------------------------------*** + * (or codepage data with simple states) to Unicode. + * + * Data structure and algorithm for converting from complex legacy codepages + * to Unicode. (Designed before 2000-may-22.) + * + * The basic idea is that the structure of legacy codepages can be described + * with state tables. + * When reading a byte stream, each input byte causes a state transition. + * Some transitions result in the output of a code point, some result in + * "unassigned" or "illegal" output. + * This is used here for character conversion. + * + * The data structure begins with a state table consisting of a row + * per state, with 256 entries (columns) per row for each possible input + * byte value. + * Each entry is 32 bits wide, with two formats distinguished by + * the sign bit (bit 31): + * + * One format for transitional entries (bit 31 not set) for non-final bytes, and + * one format for final entries (bit 31 set). + * Both formats contain the number of the next state in the same bit + * positions. + * State 0 is the initial state. + * + * Most of the time, the offset values of subsequent states are added + * up to a scalar value. This value will eventually be the index of + * the Unicode code point in a table that follows the state table. + * The effect is that the code points for final state table rows + * are contiguous. The code points of final state rows follow each other + * in the order of the references to those final states by previous + * states, etc. + * + * For some terminal states, the offset is itself the output Unicode + * code point (16 bits for a BMP code point or 20 bits for a supplementary + * code point (stored as code point minus 0x10000 so that 20 bits are enough). + * For others, the code point in the Unicode table is stored with either + * one or two code units: one for BMP code points, two for a pair of + * surrogates. + * All code points for a final state entry take up the same number of code + * units, regardless of whether they all actually _use_ the same number + * of code units. This is necessary for simple array access. + * + * An additional feature comes in with what in ICU is called "fallback" + * mappings: + * + * In addition to round-trippable, precise, 1:1 mappings, there are often + * mappings defined between similar, though not the same, characters. + * Typically, such mappings occur only in fromUnicode mapping tables because + * Unicode has a superset repertoire of most other codepages. However, it + * is possible to provide such mappings in the toUnicode tables, too. + * In this case, the fallback mappings are partly integrated into the + * general state tables because the structure of the encoding includes their + * byte sequences. + * For final entries in an initial state, fallback mappings are stored in + * the entry itself like with roundtrip mappings. + * For other final entries, they are stored in the code units table if + * the entry is for a pair of code units. + * For single-unit results in the code units table, there is no space to + * alternatively hold a fallback mapping; in this case, the code unit + * is stored as U+fffe (unassigned), and the fallback mapping needs to + * be looked up by the scalar offset value in a separate table. + * + * "Unassigned" state entries really mean "structurally unassigned", + * i.e., such a byte sequence will never have a mapping result. + * + * The interpretation of the bits in each entry is as follows: + * + * Bit 31 not set, not a terminal entry ("transitional"): + * 30..24 next state + * 23..0 offset delta, to be added up + * + * Bit 31 set, terminal ("final") entry: + * 30..24 next state (regardless of action code) + * 23..20 action code: + * action codes 0 and 1 result in precise-mapping Unicode code points + * 0 valid byte sequence + * 19..16 not used, 0 + * 15..0 16-bit Unicode BMP code point + * never U+fffe or U+ffff + * 1 valid byte sequence + * 19..0 20-bit Unicode supplementary code point + * never U+fffe or U+ffff + * + * action codes 2 and 3 result in fallback (unidirectional-mapping) Unicode code points + * 2 valid byte sequence (fallback) + * 19..16 not used, 0 + * 15..0 16-bit Unicode BMP code point as fallback result + * 3 valid byte sequence (fallback) + * 19..0 20-bit Unicode supplementary code point as fallback result + * + * action codes 4 and 5 may result in roundtrip/fallback/unassigned/illegal results + * depending on the code units they result in + * 4 valid byte sequence + * 19..9 not used, 0 + * 8..0 final offset delta + * pointing to one 16-bit code unit which may be + * fffe unassigned -- look for a fallback for this offset + * ffff illegal + * 5 valid byte sequence + * 19..9 not used, 0 + * 8..0 final offset delta + * pointing to two 16-bit code units + * (typically UTF-16 surrogates) + * the result depends on the first code unit as follows: + * 0000..d7ff roundtrip BMP code point (1st alone) + * d800..dbff roundtrip surrogate pair (1st, 2nd) + * dc00..dfff fallback surrogate pair (1st-400, 2nd) + * e000 roundtrip BMP code point (2nd alone) + * e001 fallback BMP code point (2nd alone) + * fffe unassigned + * ffff illegal + * (the final offset deltas are at most 255 * 2, + * times 2 because of storing code unit pairs) + * + * 6 unassigned byte sequence + * 19..16 not used, 0 + * 15..0 16-bit Unicode BMP code point U+fffe (new with version 2) + * this does not contain a final offset delta because the main + * purpose of this action code is to save scalar offset values; + * therefore, fallback values cannot be assigned to byte + * sequences that result in this action code + * 7 illegal byte sequence + * 19..16 not used, 0 + * 15..0 16-bit Unicode BMP code point U+ffff (new with version 2) + * 8 state change only + * 19..0 not used, 0 + * useful for state changes in simple stateful encodings, + * at Shift-In/Shift-Out codes + * + * + * 9..15 reserved for future use + * current implementations will only perform a state change + * and ignore bits 19..0 + * + * An encoding with contiguous ranges of unassigned byte sequences, like + * Shift-JIS and especially EUC-TW, can be stored efficiently by having + * at least two states for the trail bytes: + * One trail byte state that results in code points, and one that only + * has "unassigned" and "illegal" terminal states. + * + * Note: partly by accident, this data structure supports simple stateful + * encodings without any additional logic. + * Currently, only simple Shift-In/Shift-Out schemes are handled with + * appropriate state tables (especially EBCDIC_STATEFUL!). + * + * MBCS version 2 added: + * unassigned and illegal action codes have U+fffe and U+ffff + * instead of unused bits; this is useful for _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP() + * + * Converting from Unicode to codepage bytes --------------------------------*** + * + * The conversion data structure for fromUnicode is designed for the known + * structure of Unicode. It maps from 21-bit code points (0..0x10ffff) to + * a sequence of 1..4 bytes, in addition to a flag that indicates if there is + * a roundtrip mapping. + * + * The lookup is done with a 3-stage trie, using 11/6/4 bits for stage 1/2/3 + * like in the character properties table. + * The beginning of the trie is at offsetFromUTable, the beginning of stage 3 + * with the resulting bytes is at offsetFromUBytes. + * + * Beginning with version 4, single-byte codepages have a significantly different + * trie compared to other codepages. + * In all cases, the entry in stage 1 is directly the index of the block of + * 64 entries in stage 2. + * + * Single-byte lookup: + * + * Stage 2 only contains 16-bit indexes directly to the 16-blocks in stage 3. + * Stage 3 contains one 16-bit word per result: + * Bits 15..8 indicate the kind of result: + * f roundtrip result + * c fallback result from private-use code point + * 8 fallback result from other code points + * 0 unassigned + * Bits 7..0 contain the codepage byte. A zero byte is always possible. + * + * In version 4.3, the runtime code can build an sbcsIndex for a utf8Friendly + * file. For 2-byte UTF-8 byte sequences and some 3-byte sequences the lookup + * becomes a 2-stage (single-index) trie lookup with 6 bits for stage 3. + * ASCII code points can be looked up with a linear array access into stage 3. + * See maxFastUChar and other details in ucnvmbcs.h. + * + * Multi-byte lookup: + * + * Stage 2 contains a 32-bit word for each 16-block in stage 3: + * Bits 31..16 contain flags for which stage 3 entries contain roundtrip results + * test: MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) + * If this test is false, then a non-zero result will be interpreted as + * a fallback mapping. + * Bits 15..0 contain the index to stage 3, which must be multiplied by 16*(bytes per char) + * + * Stage 3 contains 2, 3, or 4 bytes per result. + * 2 or 4 bytes are stored as uint16_t/uint32_t in platform endianness, + * while 3 bytes are stored as bytes in big-endian order. + * Leading zero bytes are ignored, and the number of bytes is counted. + * A zero byte mapping result is possible as a roundtrip result. + * For some output types, the actual result is processed from this; + * see ucnv_MBCSFromUnicodeWithOffsets(). + * + * Note that stage 1 always contains 0x440=1088 entries (0x440==0x110000>>10), + * or (version 3 and up) for BMP-only codepages, it contains 64 entries. + * + * In version 4.3, a utf8Friendly file contains an mbcsIndex table. + * For 2-byte UTF-8 byte sequences and most 3-byte sequences the lookup + * becomes a 2-stage (single-index) trie lookup with 6 bits for stage 3. + * ASCII code points can be looked up with a linear array access into stage 3. + * See maxFastUChar, mbcsIndex and other details in ucnvmbcs.h. + * + * In version 3, stage 2 blocks may overlap by multiples of the multiplier + * for compaction. + * In version 4, stage 2 blocks (and for single-byte codepages, stage 3 blocks) + * may overlap by any number of entries. + * + * MBCS version 2 added: + * the converter checks for known output types, which allows + * adding new ones without crashing an unaware converter + */ + +static const UConverterImpl _SBCSUTF8Impl; +static const UConverterImpl _DBCSUTF8Impl; + +/* GB 18030 data ------------------------------------------------------------ */ + +/* helper macros for linear values for GB 18030 four-byte sequences */ +#define LINEAR_18030(a, b, c, d) ((((a)*10+(b))*126L+(c))*10L+(d)) + +#define LINEAR_18030_BASE LINEAR_18030(0x81, 0x30, 0x81, 0x30) + +#define LINEAR(x) LINEAR_18030(x>>24, (x>>16)&0xff, (x>>8)&0xff, x&0xff) + +/* + * Some ranges of GB 18030 where both the Unicode code points and the + * GB four-byte sequences are contiguous and are handled algorithmically by + * the special callback functions below. + * The values are start & end of Unicode & GB codes. + * + * Note that single surrogates are not mapped by GB 18030 + * as of the re-released mapping tables from 2000-nov-30. + */ +static const uint32_t +gb18030Ranges[13][4]={ + {0x10000, 0x10FFFF, LINEAR(0x90308130), LINEAR(0xE3329A35)}, + {0x9FA6, 0xD7FF, LINEAR(0x82358F33), LINEAR(0x8336C738)}, + {0x0452, 0x200F, LINEAR(0x8130D330), LINEAR(0x8136A531)}, + {0xE865, 0xF92B, LINEAR(0x8336D030), LINEAR(0x84308534)}, + {0x2643, 0x2E80, LINEAR(0x8137A839), LINEAR(0x8138FD38)}, + {0xFA2A, 0xFE2F, LINEAR(0x84309C38), LINEAR(0x84318537)}, + {0x3CE1, 0x4055, LINEAR(0x8231D438), LINEAR(0x8232AF32)}, + {0x361B, 0x3917, LINEAR(0x8230A633), LINEAR(0x8230F237)}, + {0x49B8, 0x4C76, LINEAR(0x8234A131), LINEAR(0x8234E733)}, + {0x4160, 0x4336, LINEAR(0x8232C937), LINEAR(0x8232F837)}, + {0x478E, 0x4946, LINEAR(0x8233E838), LINEAR(0x82349638)}, + {0x44D7, 0x464B, LINEAR(0x8233A339), LINEAR(0x8233C931)}, + {0xFFE6, 0xFFFF, LINEAR(0x8431A234), LINEAR(0x8431A439)} +}; + +/* bit flag for UConverter.options indicating GB 18030 special handling */ +#define _MBCS_OPTION_GB18030 0x8000 + +/* Miscellaneous ------------------------------------------------------------ */ + +/* similar to ucnv_MBCSGetNextUChar() but recursive */ +static void +_getUnicodeSetForBytes(const UConverterSharedData *sharedData, + const int32_t (*stateTable)[256], const uint16_t *unicodeCodeUnits, + const USetAdder *sa, + UConverterUnicodeSet which, + uint8_t state, uint32_t offset, int32_t lowByte, int32_t highByte, + + UErrorCode *pErrorCode) { + int32_t b, entry; + + for(b=lowByte; b<=highByte; ++b) { + entry=stateTable[state][b]; + if(MBCS_ENTRY_IS_TRANSITION(entry)) { + _getUnicodeSetForBytes( + sharedData, stateTable, unicodeCodeUnits, + sa, which, + (uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry), + offset+MBCS_ENTRY_TRANSITION_OFFSET(entry), + 0, 0xff, + pErrorCode); + } else { + UChar32 c; + int32_t rowOffset=offset; + uint8_t action; + + c=U_SENTINEL; + + /* + * An if-else-if chain provides more reliable performance for + * the most common cases compared to a switch. + */ + action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); + if(action==MBCS_STATE_VALID_DIRECT_16) { + /* output BMP code point */ + c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + } else if(action==MBCS_STATE_VALID_16) { + offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); + c=unicodeCodeUnits[offset]; + if(c<0xfffe) { + /* output BMP code point */ + } else { + c=U_SENTINEL; + } + } else if(action==MBCS_STATE_VALID_16_PAIR) { + offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); + c=unicodeCodeUnits[offset++]; + if(c<0xd800) { + /* output BMP code point below 0xd800 */ + } else if(c<=0xdbff) { + /* output roundtrip or fallback supplementary code point */ + c=((c&0x3ff)<<10)+unicodeCodeUnits[offset]+(0x10000-0xdc00); + } else if(c==0xe000) { + /* output roundtrip BMP code point above 0xd800 or fallback BMP code point */ + c=unicodeCodeUnits[offset]; + } else { + c=U_SENTINEL; + } + } else if(action==MBCS_STATE_VALID_DIRECT_20) { + /* output supplementary code point */ + c=(UChar32)(MBCS_ENTRY_FINAL_VALUE(entry)+0x10000); + } + + if(c>=0) { + sa->add(sa->set, c); + } + offset=rowOffset; + } + } +} + +/* + * Internal function returning a UnicodeSet for toUnicode() conversion. + * Currently only used for ISO-2022-CN, and only handles roundtrip mappings. + * In the future, if we add support for reverse-fallback sets, this function + * needs to be updated, and called for each initial state. + * Does not currently handle extensions. + * Does not empty the set first. + */ +U_CFUNC void +ucnv_MBCSGetUnicodeSetForBytes(const UConverterSharedData *sharedData, + const USetAdder *sa, + UConverterUnicodeSet which, + uint8_t state, int32_t lowByte, int32_t highByte, + UErrorCode *pErrorCode) { + _getUnicodeSetForBytes( + sharedData, sharedData->mbcs.stateTable, sharedData->mbcs.unicodeCodeUnits, + sa, which, + state, 0, lowByte, highByte, + pErrorCode); +} + +U_CFUNC void +ucnv_MBCSGetUnicodeSetForUnicode(const UConverterSharedData *sharedData, + const USetAdder *sa, + UConverterUnicodeSet which, + UErrorCode *pErrorCode) { + const UConverterMBCSTable *mbcsTable; + const uint16_t *table; + + uint32_t st3; + uint16_t st1, maxStage1, st2; + + UChar32 c; + + /* enumerate the from-Unicode trie table */ + mbcsTable=&sharedData->mbcs; + table=mbcsTable->fromUnicodeTable; + if(mbcsTable->unicodeMask&UCNV_HAS_SUPPLEMENTARY) { + maxStage1=0x440; + } else { + maxStage1=0x40; + } + + c=0; /* keep track of the current code point while enumerating */ + + if(mbcsTable->outputType==MBCS_OUTPUT_1) { + const uint16_t *stage2, *stage3, *results; + + results=(const uint16_t *)mbcsTable->fromUnicodeBytes; + + for(st1=0; st1maxStage1) { + stage2=table+st2; + for(st2=0; st2<64; ++st2) { + if((st3=stage2[st2])!=0) { + /* read the stage 3 block */ + stage3=results+st3; + + /* + * Add code points for which the roundtrip flag is set. + * Once we get a set for fallback mappings, we have to use + * a threshold variable with a value of 0x800. + * See ucnv_MBCSSingleFromBMPWithOffsets() and + * MBCS_SINGLE_RESULT_FROM_U() for details. + */ + do { + if(*stage3++>=0xf00) { + sa->add(sa->set, c); + } + } while((++c&0xf)!=0); + } else { + c+=16; /* empty stage 3 block */ + } + } + } else { + c+=1024; /* empty stage 2 block */ + } + } + } else if(mbcsTable->outputType==MBCS_OUTPUT_DBCS_ONLY) { + /* ignore single-byte results */ + const uint32_t *stage2; + const uint16_t *stage3, *results; + + results=(const uint16_t *)mbcsTable->fromUnicodeBytes; + + for(st1=0; st1(maxStage1>>1)) { + stage2=(const uint32_t *)table+st2; + for(st2=0; st2<64; ++st2) { + if((st3=stage2[st2])!=0) { + /* read the stage 3 block */ + stage3=results+16*(uint32_t)(uint16_t)st3; + + /* get the roundtrip flags for the stage 3 block */ + st3>>=16; + + /* + * Add code points for which the roundtrip flag is set. + * Once we get a set for fallback mappings, we have to check + * non-roundtrip stage 3 results for whether they are 0. + * See ucnv_MBCSFromUnicodeWithOffsets() for details. + * + * Ignore single-byte results (<0x100). + */ + do { + if((st3&1)!=0 && *stage3>=0x100) { + sa->add(sa->set, c); + } + st3>>=1; + ++stage3; + } while((++c&0xf)!=0); + } else { + c+=16; /* empty stage 3 block */ + } + } + } else { + c+=1024; /* empty stage 2 block */ + } + } + } else { + const uint32_t *stage2; + + for(st1=0; st1(maxStage1>>1)) { + stage2=(const uint32_t *)table+st2; + for(st2=0; st2<64; ++st2) { + if((st3=stage2[st2])!=0) { + /* get the roundtrip flags for the stage 3 block */ + st3>>=16; + + /* + * Add code points for which the roundtrip flag is set. + * Once we get a set for fallback mappings, we have to check + * non-roundtrip stage 3 results for whether they are 0. + * See ucnv_MBCSFromUnicodeWithOffsets() for details. + */ + do { + if(st3&1) { + sa->add(sa->set, c); + } + st3>>=1; + } while((++c&0xf)!=0); + } else { + c+=16; /* empty stage 3 block */ + } + } + } else { + c+=1024; /* empty stage 2 block */ + } + } + } + + ucnv_extGetUnicodeSet(sharedData, sa, which, pErrorCode); +} + +static void +ucnv_MBCSGetUnicodeSet(const UConverter *cnv, + const USetAdder *sa, + UConverterUnicodeSet which, + UErrorCode *pErrorCode) { + if(cnv->options&_MBCS_OPTION_GB18030) { + sa->addRange(sa->set, 0, 0xd7ff); + sa->addRange(sa->set, 0xe000, 0x10ffff); + } else { + ucnv_MBCSGetUnicodeSetForUnicode(cnv->sharedData, sa, which, pErrorCode); + } +} + +/* conversion extensions for input not in the main table -------------------- */ + +/* + * Hardcoded extension handling for GB 18030. + * Definition of LINEAR macros and gb18030Ranges see near the beginning of the file. + * + * In the future, conversion extensions may handle m:n mappings and delta tables, + * see http://source.icu-project.org/repos/icu/icuhtml/trunk/design/conversion/conversion_extensions.html + * + * If an input character cannot be mapped, then these functions set an error + * code. The framework will then call the callback function. + */ + +/* + * @return if(U_FAILURE) return the code point for cnv->fromUChar32 + * else return 0 after output has been written to the target + */ +static UChar32 +_extFromU(UConverter *cnv, const UConverterSharedData *sharedData, + UChar32 cp, + const UChar **source, const UChar *sourceLimit, + uint8_t **target, const uint8_t *targetLimit, + int32_t **offsets, int32_t sourceIndex, + UBool flush, + UErrorCode *pErrorCode) { + const int32_t *cx; + + cnv->useSubChar1=FALSE; + + if( (cx=sharedData->mbcs.extIndexes)!=NULL && + ucnv_extInitialMatchFromU( + cnv, cx, + cp, source, sourceLimit, + (char **)target, (char *)targetLimit, + offsets, sourceIndex, + flush, + pErrorCode) + ) { + return 0; /* an extension mapping handled the input */ + } + + /* GB 18030 */ + if((cnv->options&_MBCS_OPTION_GB18030)!=0) { + const uint32_t *range; + int32_t i; + + range=gb18030Ranges[0]; + for(i=0; itoUBytes[0..length[ + * @return if(U_FAILURE) return the length (toULength, byteIndex) for the input + * else return 0 after output has been written to the target + */ +static int8_t +_extToU(UConverter *cnv, const UConverterSharedData *sharedData, + int8_t length, + const uint8_t **source, const uint8_t *sourceLimit, + UChar **target, const UChar *targetLimit, + int32_t **offsets, int32_t sourceIndex, + UBool flush, + UErrorCode *pErrorCode) { + const int32_t *cx; + + if( (cx=sharedData->mbcs.extIndexes)!=NULL && + ucnv_extInitialMatchToU( + cnv, cx, + length, (const char **)source, (const char *)sourceLimit, + target, targetLimit, + offsets, sourceIndex, + flush, + pErrorCode) + ) { + return 0; /* an extension mapping handled the input */ + } + + /* GB 18030 */ + if(length==4 && (cnv->options&_MBCS_OPTION_GB18030)!=0) { + const uint32_t *range; + uint32_t linear; + int32_t i; + + linear=LINEAR_18030(cnv->toUBytes[0], cnv->toUBytes[1], cnv->toUBytes[2], cnv->toUBytes[3]); + range=gb18030Ranges[0]; + for(i=0; iNL ------------------------------------------------------ */ + +/* + * This code modifies a standard EBCDIC<->Unicode mapping table for + * OS/390 (z/OS) Unix System Services (Open Edition). + * The difference is in the mapping of Line Feed and New Line control codes: + * Standard EBCDIC maps + * + * \x25 |0 + * \x15 |0 + * + * but OS/390 USS EBCDIC swaps the control codes for LF and NL, + * mapping + * + * \x15 |0 + * \x25 |0 + * + * This code modifies a loaded standard EBCDIC<->Unicode mapping table + * by copying it into allocated memory and swapping the LF and NL values. + * It allows to support the same EBCDIC charset in both versions without + * duplicating the entire installed table. + */ + +/* standard EBCDIC codes */ +#define EBCDIC_LF 0x25 +#define EBCDIC_NL 0x15 + +/* standard EBCDIC codes with roundtrip flag as stored in Unicode-to-single-byte tables */ +#define EBCDIC_RT_LF 0xf25 +#define EBCDIC_RT_NL 0xf15 + +/* Unicode code points */ +#define U_LF 0x0a +#define U_NL 0x85 + +static UBool +_EBCDICSwapLFNL(UConverterSharedData *sharedData, UErrorCode *pErrorCode) { + UConverterMBCSTable *mbcsTable; + + const uint16_t *table, *results; + const uint8_t *bytes; + + int32_t (*newStateTable)[256]; + uint16_t *newResults; + uint8_t *p; + char *name; + + uint32_t stage2Entry; + uint32_t size, sizeofFromUBytes; + + mbcsTable=&sharedData->mbcs; + + table=mbcsTable->fromUnicodeTable; + bytes=mbcsTable->fromUnicodeBytes; + results=(const uint16_t *)bytes; + + /* + * Check that this is an EBCDIC table with SBCS portion - + * SBCS or EBCDIC_STATEFUL with standard EBCDIC LF and NL mappings. + * + * If not, ignore the option. Options are always ignored if they do not apply. + */ + if(!( + (mbcsTable->outputType==MBCS_OUTPUT_1 || mbcsTable->outputType==MBCS_OUTPUT_2_SISO) && + mbcsTable->stateTable[0][EBCDIC_LF]==MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, U_LF) && + mbcsTable->stateTable[0][EBCDIC_NL]==MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, U_NL) + )) { + return FALSE; + } + + if(mbcsTable->outputType==MBCS_OUTPUT_1) { + if(!( + EBCDIC_RT_LF==MBCS_SINGLE_RESULT_FROM_U(table, results, U_LF) && + EBCDIC_RT_NL==MBCS_SINGLE_RESULT_FROM_U(table, results, U_NL) + )) { + return FALSE; + } + } else /* MBCS_OUTPUT_2_SISO */ { + stage2Entry=MBCS_STAGE_2_FROM_U(table, U_LF); + if(!( + MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, U_LF)!=0 && + EBCDIC_LF==MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, U_LF) + )) { + return FALSE; + } + + stage2Entry=MBCS_STAGE_2_FROM_U(table, U_NL); + if(!( + MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, U_NL)!=0 && + EBCDIC_NL==MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, U_NL) + )) { + return FALSE; + } + } + + if(mbcsTable->fromUBytesLength>0) { + /* + * We _know_ the number of bytes in the fromUnicodeBytes array + * starting with header.version 4.1. + */ + sizeofFromUBytes=mbcsTable->fromUBytesLength; + } else { + /* + * Otherwise: + * There used to be code to enumerate the fromUnicode + * trie and find the highest entry, but it was removed in ICU 3.2 + * because it was not tested and caused a low code coverage number. + * See Jitterbug 3674. + * This affects only some .cnv file formats with a header.version + * below 4.1, and only when swaplfnl is requested. + * + * ucnvmbcs.c revision 1.99 is the last one with the + * ucnv_MBCSSizeofFromUBytes() function. + */ + *pErrorCode=U_INVALID_FORMAT_ERROR; + return FALSE; + } + + /* + * The table has an appropriate format. + * Allocate and build + * - a modified to-Unicode state table + * - a modified from-Unicode output array + * - a converter name string with the swap option appended + */ + size= + mbcsTable->countStates*1024+ + sizeofFromUBytes+ + UCNV_MAX_CONVERTER_NAME_LENGTH+20; + p=(uint8_t *)uprv_malloc(size); + if(p==NULL) { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return FALSE; + } + + /* copy and modify the to-Unicode state table */ + newStateTable=(int32_t (*)[256])p; + uprv_memcpy(newStateTable, mbcsTable->stateTable, mbcsTable->countStates*1024); + + newStateTable[0][EBCDIC_LF]=MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, U_NL); + newStateTable[0][EBCDIC_NL]=MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, U_LF); + + /* copy and modify the from-Unicode result table */ + newResults=(uint16_t *)newStateTable[mbcsTable->countStates]; + uprv_memcpy(newResults, bytes, sizeofFromUBytes); + + /* conveniently, the table access macros work on the left side of expressions */ + if(mbcsTable->outputType==MBCS_OUTPUT_1) { + MBCS_SINGLE_RESULT_FROM_U(table, newResults, U_LF)=EBCDIC_RT_NL; + MBCS_SINGLE_RESULT_FROM_U(table, newResults, U_NL)=EBCDIC_RT_LF; + } else /* MBCS_OUTPUT_2_SISO */ { + stage2Entry=MBCS_STAGE_2_FROM_U(table, U_LF); + MBCS_VALUE_2_FROM_STAGE_2(newResults, stage2Entry, U_LF)=EBCDIC_NL; + + stage2Entry=MBCS_STAGE_2_FROM_U(table, U_NL); + MBCS_VALUE_2_FROM_STAGE_2(newResults, stage2Entry, U_NL)=EBCDIC_LF; + } + + /* set the canonical converter name */ + name=(char *)newResults+sizeofFromUBytes; + uprv_strcpy(name, sharedData->staticData->name); + uprv_strcat(name, UCNV_SWAP_LFNL_OPTION_STRING); + + /* set the pointers */ + umtx_lock(NULL); + if(mbcsTable->swapLFNLStateTable==NULL) { + mbcsTable->swapLFNLStateTable=newStateTable; + mbcsTable->swapLFNLFromUnicodeBytes=(uint8_t *)newResults; + mbcsTable->swapLFNLName=name; + + newStateTable=NULL; + } + umtx_unlock(NULL); + + /* release the allocated memory if another thread beat us to it */ + if(newStateTable!=NULL) { + uprv_free(newStateTable); + } + return TRUE; +} + +/* MBCS setup functions ----------------------------------------------------- */ + +static void +ucnv_MBCSLoad(UConverterSharedData *sharedData, + UConverterLoadArgs *pArgs, + const uint8_t *raw, + UErrorCode *pErrorCode) { + UDataInfo info; + UConverterMBCSTable *mbcsTable=&sharedData->mbcs; + _MBCSHeader *header=(_MBCSHeader *)raw; + uint32_t offset; + + if(header->version[0]!=4) { + *pErrorCode=U_INVALID_TABLE_FORMAT; + return; + } + + mbcsTable->outputType=(uint8_t)header->flags; + + /* extension data, header version 4.2 and higher */ + offset=header->flags>>8; + if(offset!=0) { + mbcsTable->extIndexes=(const int32_t *)(raw+offset); + } + + if(mbcsTable->outputType==MBCS_OUTPUT_EXT_ONLY) { + UConverterLoadArgs args={ 0 }; + UConverterSharedData *baseSharedData; + const int32_t *extIndexes; + const char *baseName; + + /* extension-only file, load the base table and set values appropriately */ + if((extIndexes=mbcsTable->extIndexes)==NULL) { + /* extension-only file without extension */ + *pErrorCode=U_INVALID_TABLE_FORMAT; + return; + } + + if(pArgs->nestedLoads!=1) { + /* an extension table must not be loaded as a base table */ + *pErrorCode=U_INVALID_TABLE_FILE; + return; + } + + /* load the base table */ + baseName=(const char *)(header+1); + if(0==uprv_strcmp(baseName, sharedData->staticData->name)) { + /* forbid loading this same extension-only file */ + *pErrorCode=U_INVALID_TABLE_FORMAT; + return; + } + + /* TODO parse package name out of the prefix of the base name in the extension .cnv file? */ + args.size=sizeof(UConverterLoadArgs); + args.nestedLoads=2; + args.reserved=pArgs->reserved; + args.options=pArgs->options; + args.pkg=pArgs->pkg; + args.name=baseName; + baseSharedData=ucnv_load(&args, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return; + } + if( baseSharedData->staticData->conversionType!=UCNV_MBCS || + baseSharedData->mbcs.baseSharedData!=NULL + ) { + ucnv_unload(baseSharedData); + *pErrorCode=U_INVALID_TABLE_FORMAT; + return; + } + + /* copy the base table data */ + uprv_memcpy(mbcsTable, &baseSharedData->mbcs, sizeof(UConverterMBCSTable)); + + /* overwrite values with relevant ones for the extension converter */ + mbcsTable->baseSharedData=baseSharedData; + mbcsTable->extIndexes=extIndexes; + + /* + * It would be possible to share the swapLFNL data with a base converter, + * but the generated name would have to be different, and the memory + * would have to be free'd only once. + * It is easier to just create the data for the extension converter + * separately when it is requested. + */ + mbcsTable->swapLFNLStateTable=NULL; + mbcsTable->swapLFNLFromUnicodeBytes=NULL; + mbcsTable->swapLFNLName=NULL; + + /* + * Set a special, runtime-only outputType if the extension converter + * is a DBCS version of a base converter that also maps single bytes. + */ + if( sharedData->staticData->conversionType==UCNV_DBCS || + (sharedData->staticData->conversionType==UCNV_MBCS && + sharedData->staticData->minBytesPerChar>=2) + ) { + if(baseSharedData->mbcs.outputType==MBCS_OUTPUT_2_SISO) { + /* the base converter is SI/SO-stateful */ + int32_t entry; + + /* get the dbcs state from the state table entry for SO=0x0e */ + entry=mbcsTable->stateTable[0][0xe]; + if( MBCS_ENTRY_IS_FINAL(entry) && + MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_CHANGE_ONLY && + MBCS_ENTRY_FINAL_STATE(entry)!=0 + ) { + mbcsTable->dbcsOnlyState=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); + + mbcsTable->outputType=MBCS_OUTPUT_DBCS_ONLY; + } + } else if( + baseSharedData->staticData->conversionType==UCNV_MBCS && + baseSharedData->staticData->minBytesPerChar==1 && + baseSharedData->staticData->maxBytesPerChar==2 && + mbcsTable->countStates<=127 + ) { + /* non-stateful base converter, need to modify the state table */ + int32_t (*newStateTable)[256]; + int32_t *state; + int32_t i, count; + + /* allocate a new state table and copy the base state table contents */ + count=mbcsTable->countStates; + newStateTable=(int32_t (*)[256])uprv_malloc((count+1)*1024); + if(newStateTable==NULL) { + ucnv_unload(baseSharedData); + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return; + } + + uprv_memcpy(newStateTable, mbcsTable->stateTable, count*1024); + + /* change all final single-byte entries to go to a new all-illegal state */ + state=newStateTable[0]; + for(i=0; i<256; ++i) { + if(MBCS_ENTRY_IS_FINAL(state[i])) { + state[i]=MBCS_ENTRY_TRANSITION(count, 0); + } + } + + /* build the new all-illegal state */ + state=newStateTable[count]; + for(i=0; i<256; ++i) { + state[i]=MBCS_ENTRY_FINAL(0, MBCS_STATE_ILLEGAL, 0); + } + mbcsTable->stateTable=(const int32_t (*)[256])newStateTable; + mbcsTable->countStates=(uint8_t)(count+1); + mbcsTable->stateTableOwned=TRUE; + + mbcsTable->outputType=MBCS_OUTPUT_DBCS_ONLY; + } + } + + /* + * unlike below for files with base tables, do not get the unicodeMask + * from the sharedData; instead, use the base table's unicodeMask, + * which we copied in the memcpy above; + * this is necessary because the static data unicodeMask, especially + * the UCNV_HAS_SUPPLEMENTARY flag, is part of the base table data + */ + } else { + /* conversion file with a base table; an additional extension table is optional */ + /* make sure that the output type is known */ + switch(mbcsTable->outputType) { + case MBCS_OUTPUT_1: + case MBCS_OUTPUT_2: + case MBCS_OUTPUT_3: + case MBCS_OUTPUT_4: + case MBCS_OUTPUT_3_EUC: + case MBCS_OUTPUT_4_EUC: + case MBCS_OUTPUT_2_SISO: + /* OK */ + break; + default: + *pErrorCode=U_INVALID_TABLE_FORMAT; + return; + } + + mbcsTable->countStates=(uint8_t)header->countStates; + mbcsTable->countToUFallbacks=header->countToUFallbacks; + mbcsTable->stateTable=(const int32_t (*)[256])(raw+sizeof(_MBCSHeader)); + mbcsTable->toUFallbacks=(const _MBCSToUFallback *)(mbcsTable->stateTable+header->countStates); + mbcsTable->unicodeCodeUnits=(const uint16_t *)(raw+header->offsetToUCodeUnits); + + mbcsTable->fromUnicodeTable=(const uint16_t *)(raw+header->offsetFromUTable); + mbcsTable->fromUnicodeBytes=(const uint8_t *)(raw+header->offsetFromUBytes); + mbcsTable->fromUBytesLength=header->fromUBytesLength; + + /* + * converter versions 6.1 and up contain a unicodeMask that is + * used here to select the most efficient function implementations + */ + info.size=sizeof(UDataInfo); + udata_getInfo((UDataMemory *)sharedData->dataMemory, &info); + if(info.formatVersion[0]>6 || (info.formatVersion[0]==6 && info.formatVersion[1]>=1)) { + /* mask off possible future extensions to be safe */ + mbcsTable->unicodeMask=(uint8_t)(sharedData->staticData->unicodeMask&3); + } else { + /* for older versions, assume worst case: contains anything possible (prevent over-optimizations) */ + mbcsTable->unicodeMask=UCNV_HAS_SUPPLEMENTARY|UCNV_HAS_SURROGATES; + } + + /* + * _MBCSHeader.version 4.3 adds utf8Friendly data structures. + * Check for the header version, SBCS vs. MBCS, and for whether the + * data structures are optimized for code points as high as what the + * runtime code is designed for. + * The implementation does not handle mapping tables with entries for + * unpaired surrogates. + */ + if( header->version[1]>=3 && + (mbcsTable->unicodeMask&UCNV_HAS_SURROGATES)==0 && + (mbcsTable->countStates==1 ? + (header->version[2]>=(SBCS_FAST_MAX>>8)) : + (header->version[2]>=(MBCS_FAST_MAX>>8)) + ) + ) { + mbcsTable->utf8Friendly=TRUE; + + if(mbcsTable->countStates==1) { + /* + * SBCS: Stage 3 is allocated in 64-entry blocks for U+0000..SBCS_FAST_MAX or higher. + * Build a table with indexes to each block, to be used instead of + * the regular stage 1/2 table. + */ + int32_t i; + for(i=0; i<(SBCS_FAST_LIMIT>>6); ++i) { + mbcsTable->sbcsIndex[i]=mbcsTable->fromUnicodeTable[mbcsTable->fromUnicodeTable[i>>4]+((i<<2)&0x3c)]; + } + /* set SBCS_FAST_MAX to reflect the reach of sbcsIndex[] even if header->version[2]>(SBCS_FAST_MAX>>8) */ + mbcsTable->maxFastUChar=SBCS_FAST_MAX; + } else { + /* + * MBCS: Stage 3 is allocated in 64-entry blocks for U+0000..MBCS_FAST_MAX or higher. + * The .cnv file is prebuilt with an additional stage table with indexes + * to each block. + */ + mbcsTable->mbcsIndex=(const uint16_t *)(mbcsTable->fromUnicodeBytes+mbcsTable->fromUBytesLength); + mbcsTable->maxFastUChar=(((UChar)header->version[2])<<8)|0xff; + } + } + + /* calculate a bit set of 4 ASCII characters per bit that round-trip to ASCII bytes */ + { + uint32_t asciiRoundtrips=0xffffffff; + int32_t i; + + for(i=0; i<0x80; ++i) { + if(mbcsTable->stateTable[0][i]!=MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, i)) { + asciiRoundtrips&=~((uint32_t)1<<(i>>2)); + } + } + mbcsTable->asciiRoundtrips=asciiRoundtrips; + } + } + + /* Set the impl pointer here so that it is set for both extension-only and base tables. */ + if(mbcsTable->utf8Friendly) { + if(mbcsTable->countStates==1) { + sharedData->impl=&_SBCSUTF8Impl; + } else { + if(mbcsTable->outputType==MBCS_OUTPUT_2) { + sharedData->impl=&_DBCSUTF8Impl; + } + } + } + + if(mbcsTable->outputType==MBCS_OUTPUT_DBCS_ONLY || mbcsTable->outputType==MBCS_OUTPUT_2_SISO) { + /* + * MBCS_OUTPUT_DBCS_ONLY: No SBCS mappings, therefore ASCII does not roundtrip. + * MBCS_OUTPUT_2_SISO: Bypass the ASCII fastpath to handle prevLength correctly. + */ + mbcsTable->asciiRoundtrips=0; + } +} + +static void +ucnv_MBCSUnload(UConverterSharedData *sharedData) { + UConverterMBCSTable *mbcsTable=&sharedData->mbcs; + + if(mbcsTable->swapLFNLStateTable!=NULL) { + uprv_free(mbcsTable->swapLFNLStateTable); + } + if(mbcsTable->stateTableOwned) { + uprv_free((void *)mbcsTable->stateTable); + } + if(mbcsTable->baseSharedData!=NULL) { + ucnv_unload(mbcsTable->baseSharedData); + } +} + +static void +ucnv_MBCSOpen(UConverter *cnv, + const char *name, + const char *locale, + uint32_t options, + UErrorCode *pErrorCode) { + UConverterMBCSTable *mbcsTable; + const int32_t *extIndexes; + uint8_t outputType; + int8_t maxBytesPerUChar; + + mbcsTable=&cnv->sharedData->mbcs; + outputType=mbcsTable->outputType; + + if(outputType==MBCS_OUTPUT_DBCS_ONLY) { + /* the swaplfnl option does not apply, remove it */ + cnv->options=options&=~UCNV_OPTION_SWAP_LFNL; + } + + if((options&UCNV_OPTION_SWAP_LFNL)!=0) { + /* do this because double-checked locking is broken */ + UBool isCached; + + umtx_lock(NULL); + isCached=mbcsTable->swapLFNLStateTable!=NULL; + umtx_unlock(NULL); + + if(!isCached) { + if(!_EBCDICSwapLFNL(cnv->sharedData, pErrorCode)) { + if(U_FAILURE(*pErrorCode)) { + return; /* something went wrong */ + } + + /* the option does not apply, remove it */ + cnv->options=options&=~UCNV_OPTION_SWAP_LFNL; + } + } + } + + if(uprv_strstr(name, "18030")!=NULL) { + if(uprv_strstr(name, "gb18030")!=NULL || uprv_strstr(name, "GB18030")!=NULL) { + /* set a flag for GB 18030 mode, which changes the callback behavior */ + cnv->options|=_MBCS_OPTION_GB18030; + } + } + + /* fix maxBytesPerUChar depending on outputType and options etc. */ + if(outputType==MBCS_OUTPUT_2_SISO) { + cnv->maxBytesPerUChar=3; /* SO+DBCS */ + } + + extIndexes=mbcsTable->extIndexes; + if(extIndexes!=NULL) { + maxBytesPerUChar=(int8_t)UCNV_GET_MAX_BYTES_PER_UCHAR(extIndexes); + if(outputType==MBCS_OUTPUT_2_SISO) { + ++maxBytesPerUChar; /* SO + multiple DBCS */ + } + + if(maxBytesPerUChar>cnv->maxBytesPerUChar) { + cnv->maxBytesPerUChar=maxBytesPerUChar; + } + } + +#if 0 + /* + * documentation of UConverter fields used for status + * all of these fields are (re)set to 0 by ucnv_bld.c and ucnv_reset() + */ + + /* toUnicode */ + cnv->toUnicodeStatus=0; /* offset */ + cnv->mode=0; /* state */ + cnv->toULength=0; /* byteIndex */ + + /* fromUnicode */ + cnv->fromUChar32=0; + cnv->fromUnicodeStatus=1; /* prevLength */ +#endif +} + +static const char * +ucnv_MBCSGetName(const UConverter *cnv) { + if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0 && cnv->sharedData->mbcs.swapLFNLName!=NULL) { + return cnv->sharedData->mbcs.swapLFNLName; + } else { + return cnv->sharedData->staticData->name; + } +} + +/* MBCS-to-Unicode conversion functions ------------------------------------- */ + +static UChar32 +ucnv_MBCSGetFallback(UConverterMBCSTable *mbcsTable, uint32_t offset) { + const _MBCSToUFallback *toUFallbacks; + uint32_t i, start, limit; + + limit=mbcsTable->countToUFallbacks; + if(limit>0) { + /* do a binary search for the fallback mapping */ + toUFallbacks=mbcsTable->toUFallbacks; + start=0; + while(startconverter; + source=(const uint8_t *)pArgs->source; + sourceLimit=(const uint8_t *)pArgs->sourceLimit; + target=pArgs->target; + targetLimit=pArgs->targetLimit; + offsets=pArgs->offsets; + + if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { + stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable; + } else { + stateTable=cnv->sharedData->mbcs.stateTable; + } + + /* sourceIndex=-1 if the current character began in the previous buffer */ + sourceIndex=0; + + /* conversion loop */ + while(source=targetLimit) { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + + entry=stateTable[0][*source++]; + /* MBCS_ENTRY_IS_FINAL(entry) */ + + /* test the most common case first */ + if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) { + /* output BMP code point */ + *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + + /* normal end of action codes: prepare for a new character */ + ++sourceIndex; + continue; + } + + /* + * An if-else-if chain provides more reliable performance for + * the most common cases compared to a switch. + */ + action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); + if(action==MBCS_STATE_VALID_DIRECT_20 || + (action==MBCS_STATE_FALLBACK_DIRECT_20 && UCNV_TO_U_USE_FALLBACK(cnv)) + ) { + entry=MBCS_ENTRY_FINAL_VALUE(entry); + /* output surrogate pair */ + *target++=(UChar)(0xd800|(UChar)(entry>>10)); + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + c=(UChar)(0xdc00|(UChar)(entry&0x3ff)); + if(targetUCharErrorBuffer[0]=c; + cnv->UCharErrorBufferLength=1; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + + ++sourceIndex; + continue; + } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) { + if(UCNV_TO_U_USE_FALLBACK(cnv)) { + /* output BMP code point */ + *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + + ++sourceIndex; + continue; + } + } else if(action==MBCS_STATE_UNASSIGNED) { + /* just fall through */ + } else if(action==MBCS_STATE_ILLEGAL) { + /* callback(illegal) */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + } else { + /* reserved, must never occur */ + ++sourceIndex; + continue; + } + + if(U_FAILURE(*pErrorCode)) { + /* callback(illegal) */ + break; + } else /* unassigned sequences indicated with byteIndex>0 */ { + /* try an extension mapping */ + pArgs->source=(const char *)source; + cnv->toUBytes[0]=*(source-1); + cnv->toULength=_extToU(cnv, cnv->sharedData, + 1, &source, sourceLimit, + &target, targetLimit, + &offsets, sourceIndex, + pArgs->flush, + pErrorCode); + sourceIndex+=1+(int32_t)(source-(const uint8_t *)pArgs->source); + + if(U_FAILURE(*pErrorCode)) { + /* not mappable or buffer overflow */ + break; + } + } + } + + /* write back the updated pointers */ + pArgs->source=(const char *)source; + pArgs->target=target; + pArgs->offsets=offsets; +} + +/* + * This version of ucnv_MBCSSingleToUnicodeWithOffsets() is optimized for single-byte, single-state codepages + * that only map to and from the BMP. + * In addition to single-byte optimizations, the offset calculations + * become much easier. + */ +static void +ucnv_MBCSSingleToBMPWithOffsets(UConverterToUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv; + const uint8_t *source, *sourceLimit, *lastSource; + UChar *target; + int32_t targetCapacity, length; + int32_t *offsets; + + const int32_t (*stateTable)[256]; + + int32_t sourceIndex; + + int32_t entry; + uint8_t action; + + /* set up the local pointers */ + cnv=pArgs->converter; + source=(const uint8_t *)pArgs->source; + sourceLimit=(const uint8_t *)pArgs->sourceLimit; + target=pArgs->target; + targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); + offsets=pArgs->offsets; + + if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { + stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable; + } else { + stateTable=cnv->sharedData->mbcs.stateTable; + } + + /* sourceIndex=-1 if the current character began in the previous buffer */ + sourceIndex=0; + lastSource=source; + + /* + * since the conversion here is 1:1 UChar:uint8_t, we need only one counter + * for the minimum of the sourceLength and targetCapacity + */ + length=(int32_t)(sourceLimit-source); + if(length=16) { + int32_t count, loops, oredEntries; + + loops=count=targetCapacity>>4; + do { + oredEntries=entry=stateTable[0][*source++]; + *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + oredEntries|=entry=stateTable[0][*source++]; + *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + oredEntries|=entry=stateTable[0][*source++]; + *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + oredEntries|=entry=stateTable[0][*source++]; + *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + oredEntries|=entry=stateTable[0][*source++]; + *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + oredEntries|=entry=stateTable[0][*source++]; + *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + oredEntries|=entry=stateTable[0][*source++]; + *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + oredEntries|=entry=stateTable[0][*source++]; + *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + oredEntries|=entry=stateTable[0][*source++]; + *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + oredEntries|=entry=stateTable[0][*source++]; + *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + oredEntries|=entry=stateTable[0][*source++]; + *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + oredEntries|=entry=stateTable[0][*source++]; + *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + oredEntries|=entry=stateTable[0][*source++]; + *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + oredEntries|=entry=stateTable[0][*source++]; + *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + oredEntries|=entry=stateTable[0][*source++]; + *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + oredEntries|=entry=stateTable[0][*source++]; + *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + + /* were all 16 entries really valid? */ + if(!MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(oredEntries)) { + /* no, return to the first of these 16 */ + source-=16; + target-=16; + break; + } + } while(--count>0); + count=loops-count; + targetCapacity-=16*count; + + if(offsets!=NULL) { + lastSource+=16*count; + while(count>0) { + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + --count; + } + } + } +#endif + + /* conversion loop */ + while(targetCapacity>0) { + entry=stateTable[0][*source++]; + /* MBCS_ENTRY_IS_FINAL(entry) */ + + /* test the most common case first */ + if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) { + /* output BMP code point */ + *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + --targetCapacity; + continue; + } + + /* + * An if-else-if chain provides more reliable performance for + * the most common cases compared to a switch. + */ + action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); + if(action==MBCS_STATE_FALLBACK_DIRECT_16) { + if(UCNV_TO_U_USE_FALLBACK(cnv)) { + /* output BMP code point */ + *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + --targetCapacity; + continue; + } + } else if(action==MBCS_STATE_UNASSIGNED) { + /* just fall through */ + } else if(action==MBCS_STATE_ILLEGAL) { + /* callback(illegal) */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + } else { + /* reserved, must never occur */ + continue; + } + + /* set offsets since the start or the last extension */ + if(offsets!=NULL) { + int32_t count=(int32_t)(source-lastSource); + + /* predecrement: do not set the offset for the callback-causing character */ + while(--count>0) { + *offsets++=sourceIndex++; + } + /* offset and sourceIndex are now set for the current character */ + } + + if(U_FAILURE(*pErrorCode)) { + /* callback(illegal) */ + break; + } else /* unassigned sequences indicated with byteIndex>0 */ { + /* try an extension mapping */ + lastSource=source; + cnv->toUBytes[0]=*(source-1); + cnv->toULength=_extToU(cnv, cnv->sharedData, + 1, &source, sourceLimit, + &target, target+targetCapacity, + &offsets, sourceIndex, + pArgs->flush, + pErrorCode); + sourceIndex+=1+(int32_t)(source-lastSource); + + if(U_FAILURE(*pErrorCode)) { + /* not mappable or buffer overflow */ + break; + } + + /* recalculate the targetCapacity after an extension mapping */ + targetCapacity=(int32_t)(pArgs->targetLimit-target); + length=(int32_t)(sourceLimit-source); + if(length=pArgs->targetLimit) { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + + /* set offsets since the start or the last callback */ + if(offsets!=NULL) { + size_t count=source-lastSource; + while(count>0) { + *offsets++=sourceIndex++; + --count; + } + } + + /* write back the updated pointers */ + pArgs->source=(const char *)source; + pArgs->target=target; + pArgs->offsets=offsets; +} + +U_CFUNC void +ucnv_MBCSToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv; + const uint8_t *source, *sourceLimit; + UChar *target; + const UChar *targetLimit; + int32_t *offsets; + + const int32_t (*stateTable)[256]; + const uint16_t *unicodeCodeUnits; + + uint32_t offset; + uint8_t state; + int8_t byteIndex; + uint8_t *bytes; + + int32_t sourceIndex, nextSourceIndex; + + int32_t entry; + UChar c; + uint8_t action; + + /* use optimized function if possible */ + cnv=pArgs->converter; + + if(cnv->preToULength>0) { + /* + * pass sourceIndex=-1 because we continue from an earlier buffer + * in the future, this may change with continuous offsets + */ + ucnv_extContinueMatchToU(cnv, pArgs, -1, pErrorCode); + + if(U_FAILURE(*pErrorCode) || cnv->preToULength<0) { + return; + } + } + + if(cnv->sharedData->mbcs.countStates==1) { + if(!(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY)) { + ucnv_MBCSSingleToBMPWithOffsets(pArgs, pErrorCode); + } else { + ucnv_MBCSSingleToUnicodeWithOffsets(pArgs, pErrorCode); + } + return; + } + + /* set up the local pointers */ + source=(const uint8_t *)pArgs->source; + sourceLimit=(const uint8_t *)pArgs->sourceLimit; + target=pArgs->target; + targetLimit=pArgs->targetLimit; + offsets=pArgs->offsets; + + if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { + stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable; + } else { + stateTable=cnv->sharedData->mbcs.stateTable; + } + unicodeCodeUnits=cnv->sharedData->mbcs.unicodeCodeUnits; + + /* get the converter state from UConverter */ + offset=cnv->toUnicodeStatus; + byteIndex=cnv->toULength; + bytes=cnv->toUBytes; + + /* + * if we are in the SBCS state for a DBCS-only converter, + * then load the DBCS state from the MBCS data + * (dbcsOnlyState==0 if it is not a DBCS-only converter) + */ + if((state=(uint8_t)(cnv->mode))==0) { + state=cnv->sharedData->mbcs.dbcsOnlyState; + } + + /* sourceIndex=-1 if the current character began in the previous buffer */ + sourceIndex=byteIndex==0 ? 0 : -1; + nextSourceIndex=0; + + /* conversion loop */ + while(source=targetLimit) { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + + if(byteIndex==0) { + /* optimized loop for 1/2-byte input and BMP output */ + if(offsets==NULL) { + do { + entry=stateTable[state][*source]; + if(MBCS_ENTRY_IS_TRANSITION(entry)) { + state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry); + offset=MBCS_ENTRY_TRANSITION_OFFSET(entry); + + ++source; + if( source=sourceLimit) { + break; + } + if(target>=targetLimit) { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + + ++nextSourceIndex; + bytes[byteIndex++]=*source++; + } else /* byteIndex>0 */ { + ++nextSourceIndex; + entry=stateTable[state][bytes[byteIndex++]=*source++]; + } + + if(MBCS_ENTRY_IS_TRANSITION(entry)) { + state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry); + offset+=MBCS_ENTRY_TRANSITION_OFFSET(entry); + continue; + } + + /* save the previous state for proper extension mapping with SI/SO-stateful converters */ + cnv->mode=state; + + /* set the next state early so that we can reuse the entry variable */ + state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */ + + /* + * An if-else-if chain provides more reliable performance for + * the most common cases compared to a switch. + */ + action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); + if(action==MBCS_STATE_VALID_16) { + offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); + c=unicodeCodeUnits[offset]; + if(c<0xfffe) { + /* output BMP code point */ + *target++=c; + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + byteIndex=0; + } else if(c==0xfffe) { + if(UCNV_TO_U_USE_FALLBACK(cnv) && (entry=(int32_t)ucnv_MBCSGetFallback(&cnv->sharedData->mbcs, offset))!=0xfffe) { + /* output fallback BMP code point */ + *target++=(UChar)entry; + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + byteIndex=0; + } + } else { + /* callback(illegal) */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + } + } else if(action==MBCS_STATE_VALID_DIRECT_16) { + /* output BMP code point */ + *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + byteIndex=0; + } else if(action==MBCS_STATE_VALID_16_PAIR) { + offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); + c=unicodeCodeUnits[offset++]; + if(c<0xd800) { + /* output BMP code point below 0xd800 */ + *target++=c; + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + byteIndex=0; + } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? c<=0xdfff : c<=0xdbff) { + /* output roundtrip or fallback surrogate pair */ + *target++=(UChar)(c&0xdbff); + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + byteIndex=0; + if(targetUCharErrorBuffer[0]=unicodeCodeUnits[offset]; + cnv->UCharErrorBufferLength=1; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + + offset=0; + break; + } + } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? (c&0xfffe)==0xe000 : c==0xe000) { + /* output roundtrip BMP code point above 0xd800 or fallback BMP code point */ + *target++=unicodeCodeUnits[offset]; + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + byteIndex=0; + } else if(c==0xffff) { + /* callback(illegal) */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + } + } else if(action==MBCS_STATE_VALID_DIRECT_20 || + (action==MBCS_STATE_FALLBACK_DIRECT_20 && UCNV_TO_U_USE_FALLBACK(cnv)) + ) { + entry=MBCS_ENTRY_FINAL_VALUE(entry); + /* output surrogate pair */ + *target++=(UChar)(0xd800|(UChar)(entry>>10)); + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + byteIndex=0; + c=(UChar)(0xdc00|(UChar)(entry&0x3ff)); + if(targetUCharErrorBuffer[0]=c; + cnv->UCharErrorBufferLength=1; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + + offset=0; + break; + } + } else if(action==MBCS_STATE_CHANGE_ONLY) { + /* + * This serves as a state change without any output. + * It is useful for reading simple stateful encodings, + * for example using just Shift-In/Shift-Out codes. + * The 21 unused bits may later be used for more sophisticated + * state transitions. + */ + if(cnv->sharedData->mbcs.dbcsOnlyState==0) { + byteIndex=0; + } else { + /* SI/SO are illegal for DBCS-only conversion */ + state=(uint8_t)(cnv->mode); /* restore the previous state */ + + /* callback(illegal) */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + } + } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) { + if(UCNV_TO_U_USE_FALLBACK(cnv)) { + /* output BMP code point */ + *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + byteIndex=0; + } + } else if(action==MBCS_STATE_UNASSIGNED) { + /* just fall through */ + } else if(action==MBCS_STATE_ILLEGAL) { + /* callback(illegal) */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + } else { + /* reserved, must never occur */ + byteIndex=0; + } + + /* end of action codes: prepare for a new character */ + offset=0; + + if(byteIndex==0) { + sourceIndex=nextSourceIndex; + } else if(U_FAILURE(*pErrorCode)) { + /* callback(illegal) */ + break; + } else /* unassigned sequences indicated with byteIndex>0 */ { + /* try an extension mapping */ + pArgs->source=(const char *)source; + byteIndex=_extToU(cnv, cnv->sharedData, + byteIndex, &source, sourceLimit, + &target, targetLimit, + &offsets, sourceIndex, + pArgs->flush, + pErrorCode); + sourceIndex=nextSourceIndex+(int32_t)(source-(const uint8_t *)pArgs->source); + + if(U_FAILURE(*pErrorCode)) { + /* not mappable or buffer overflow */ + break; + } + } + } + + /* set the converter state back into UConverter */ + cnv->toUnicodeStatus=offset; + cnv->mode=state; + cnv->toULength=byteIndex; + + /* write back the updated pointers */ + pArgs->source=(const char *)source; + pArgs->target=target; + pArgs->offsets=offsets; +} + +/* + * This version of ucnv_MBCSGetNextUChar() is optimized for single-byte, single-state codepages. + * We still need a conversion loop in case we find reserved action codes, which are to be ignored. + */ +static UChar32 +ucnv_MBCSSingleGetNextUChar(UConverterToUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv; + const int32_t (*stateTable)[256]; + const uint8_t *source, *sourceLimit; + + int32_t entry; + uint8_t action; + + /* set up the local pointers */ + cnv=pArgs->converter; + source=(const uint8_t *)pArgs->source; + sourceLimit=(const uint8_t *)pArgs->sourceLimit; + if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { + stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable; + } else { + stateTable=cnv->sharedData->mbcs.stateTable; + } + + /* conversion loop */ + while(sourcesource=(const char *)source; + + if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) { + /* output BMP code point */ + return (UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + } + + /* + * An if-else-if chain provides more reliable performance for + * the most common cases compared to a switch. + */ + action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); + if( action==MBCS_STATE_VALID_DIRECT_20 || + (action==MBCS_STATE_FALLBACK_DIRECT_20 && UCNV_TO_U_USE_FALLBACK(cnv)) + ) { + /* output supplementary code point */ + return (UChar32)(MBCS_ENTRY_FINAL_VALUE(entry)+0x10000); + } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) { + if(UCNV_TO_U_USE_FALLBACK(cnv)) { + /* output BMP code point */ + return (UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + } + } else if(action==MBCS_STATE_UNASSIGNED) { + /* just fall through */ + } else if(action==MBCS_STATE_ILLEGAL) { + /* callback(illegal) */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + } else { + /* reserved, must never occur */ + continue; + } + + if(U_FAILURE(*pErrorCode)) { + /* callback(illegal) */ + break; + } else /* unassigned sequence */ { + /* defer to the generic implementation */ + pArgs->source=(const char *)source-1; + return UCNV_GET_NEXT_UCHAR_USE_TO_U; + } + } + + /* no output because of empty input or only state changes */ + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0xffff; +} + +/* + * Version of _MBCSToUnicodeWithOffsets() optimized for single-character + * conversion without offset handling. + * + * When a character does not have a mapping to Unicode, then we return to the + * generic ucnv_getNextUChar() code for extension/GB 18030 and error/callback + * handling. + * We also defer to the generic code in other complicated cases and have them + * ultimately handled by _MBCSToUnicodeWithOffsets() itself. + * + * All normal mappings and errors are handled here. + */ +static UChar32 +ucnv_MBCSGetNextUChar(UConverterToUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv; + const uint8_t *source, *sourceLimit, *lastSource; + + const int32_t (*stateTable)[256]; + const uint16_t *unicodeCodeUnits; + + uint32_t offset; + uint8_t state; + + int32_t entry; + UChar32 c; + uint8_t action; + + /* use optimized function if possible */ + cnv=pArgs->converter; + + if(cnv->preToULength>0) { + /* use the generic code in ucnv_getNextUChar() to continue with a partial match */ + return UCNV_GET_NEXT_UCHAR_USE_TO_U; + } + + if(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SURROGATES) { + /* + * Using the generic ucnv_getNextUChar() code lets us deal correctly + * with the rare case of a codepage that maps single surrogates + * without adding the complexity to this already complicated function here. + */ + return UCNV_GET_NEXT_UCHAR_USE_TO_U; + } else if(cnv->sharedData->mbcs.countStates==1) { + return ucnv_MBCSSingleGetNextUChar(pArgs, pErrorCode); + } + + /* set up the local pointers */ + source=lastSource=(const uint8_t *)pArgs->source; + sourceLimit=(const uint8_t *)pArgs->sourceLimit; + + if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { + stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable; + } else { + stateTable=cnv->sharedData->mbcs.stateTable; + } + unicodeCodeUnits=cnv->sharedData->mbcs.unicodeCodeUnits; + + /* get the converter state from UConverter */ + offset=cnv->toUnicodeStatus; + + /* + * if we are in the SBCS state for a DBCS-only converter, + * then load the DBCS state from the MBCS data + * (dbcsOnlyState==0 if it is not a DBCS-only converter) + */ + if((state=(uint8_t)(cnv->mode))==0) { + state=cnv->sharedData->mbcs.dbcsOnlyState; + } + + /* conversion loop */ + c=U_SENTINEL; + while(sourcemode=state; + + /* set the next state early so that we can reuse the entry variable */ + state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */ + + /* + * An if-else-if chain provides more reliable performance for + * the most common cases compared to a switch. + */ + action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); + if(action==MBCS_STATE_VALID_DIRECT_16) { + /* output BMP code point */ + c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + break; + } else if(action==MBCS_STATE_VALID_16) { + offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); + c=unicodeCodeUnits[offset]; + if(c<0xfffe) { + /* output BMP code point */ + break; + } else if(c==0xfffe) { + if(UCNV_TO_U_USE_FALLBACK(cnv) && (c=ucnv_MBCSGetFallback(&cnv->sharedData->mbcs, offset))!=0xfffe) { + break; + } + } else { + /* callback(illegal) */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + } + } else if(action==MBCS_STATE_VALID_16_PAIR) { + offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); + c=unicodeCodeUnits[offset++]; + if(c<0xd800) { + /* output BMP code point below 0xd800 */ + break; + } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? c<=0xdfff : c<=0xdbff) { + /* output roundtrip or fallback supplementary code point */ + c=((c&0x3ff)<<10)+unicodeCodeUnits[offset]+(0x10000-0xdc00); + break; + } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? (c&0xfffe)==0xe000 : c==0xe000) { + /* output roundtrip BMP code point above 0xd800 or fallback BMP code point */ + c=unicodeCodeUnits[offset]; + break; + } else if(c==0xffff) { + /* callback(illegal) */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + } + } else if(action==MBCS_STATE_VALID_DIRECT_20 || + (action==MBCS_STATE_FALLBACK_DIRECT_20 && UCNV_TO_U_USE_FALLBACK(cnv)) + ) { + /* output supplementary code point */ + c=(UChar32)(MBCS_ENTRY_FINAL_VALUE(entry)+0x10000); + break; + } else if(action==MBCS_STATE_CHANGE_ONLY) { + /* + * This serves as a state change without any output. + * It is useful for reading simple stateful encodings, + * for example using just Shift-In/Shift-Out codes. + * The 21 unused bits may later be used for more sophisticated + * state transitions. + */ + if(cnv->sharedData->mbcs.dbcsOnlyState!=0) { + /* SI/SO are illegal for DBCS-only conversion */ + state=(uint8_t)(cnv->mode); /* restore the previous state */ + + /* callback(illegal) */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + } + } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) { + if(UCNV_TO_U_USE_FALLBACK(cnv)) { + /* output BMP code point */ + c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + break; + } + } else if(action==MBCS_STATE_UNASSIGNED) { + /* just fall through */ + } else if(action==MBCS_STATE_ILLEGAL) { + /* callback(illegal) */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + } else { + /* reserved (must never occur), or only state change */ + offset=0; + lastSource=source; + continue; + } + + /* end of action codes: prepare for a new character */ + offset=0; + + if(U_FAILURE(*pErrorCode)) { + /* callback(illegal) */ + break; + } else /* unassigned sequence */ { + /* defer to the generic implementation */ + cnv->toUnicodeStatus=0; + cnv->mode=state; + pArgs->source=(const char *)lastSource; + return UCNV_GET_NEXT_UCHAR_USE_TO_U; + } + } + } + + if(c<0) { + if(U_SUCCESS(*pErrorCode) && source==sourceLimit && lastSourcetoUBytes; + cnv->toULength=(int8_t)(source-lastSource); + do { + *bytes++=*lastSource++; + } while(lastSourcetoUnicodeStatus=0; + cnv->mode=state; + + /* write back the updated pointer */ + pArgs->source=(const char *)source; + return c; +} + +#if 0 +/* + * Code disabled 2002dec09 (ICU 2.4) because it is not currently used in ICU. markus + * Removal improves code coverage. + */ +/** + * This version of ucnv_MBCSSimpleGetNextUChar() is optimized for single-byte, single-state codepages. + * It does not handle the EBCDIC swaplfnl option (set in UConverter). + * It does not handle conversion extensions (_extToU()). + */ +U_CFUNC UChar32 +ucnv_MBCSSingleSimpleGetNextUChar(UConverterSharedData *sharedData, + uint8_t b, UBool useFallback) { + int32_t entry; + uint8_t action; + + entry=sharedData->mbcs.stateTable[0][b]; + /* MBCS_ENTRY_IS_FINAL(entry) */ + + if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) { + /* output BMP code point */ + return (UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + } + + /* + * An if-else-if chain provides more reliable performance for + * the most common cases compared to a switch. + */ + action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); + if(action==MBCS_STATE_VALID_DIRECT_20) { + /* output supplementary code point */ + return 0x10000+MBCS_ENTRY_FINAL_VALUE(entry); + } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) { + if(!TO_U_USE_FALLBACK(useFallback)) { + return 0xfffe; + } + /* output BMP code point */ + return (UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + } else if(action==MBCS_STATE_FALLBACK_DIRECT_20) { + if(!TO_U_USE_FALLBACK(useFallback)) { + return 0xfffe; + } + /* output supplementary code point */ + return 0x10000+MBCS_ENTRY_FINAL_VALUE(entry); + } else if(action==MBCS_STATE_UNASSIGNED) { + return 0xfffe; + } else if(action==MBCS_STATE_ILLEGAL) { + return 0xffff; + } else { + /* reserved, must never occur */ + return 0xffff; + } +} +#endif + +/* + * This is a simple version of _MBCSGetNextUChar() that is used + * by other converter implementations. + * It only returns an "assigned" result if it consumes the entire input. + * It does not use state from the converter, nor error codes. + * It does not handle the EBCDIC swaplfnl option (set in UConverter). + * It handles conversion extensions but not GB 18030. + * + * Return value: + * U+fffe unassigned + * U+ffff illegal + * otherwise the Unicode code point + */ +U_CFUNC UChar32 +ucnv_MBCSSimpleGetNextUChar(UConverterSharedData *sharedData, + const char *source, int32_t length, + UBool useFallback) { + const int32_t (*stateTable)[256]; + const uint16_t *unicodeCodeUnits; + + uint32_t offset; + uint8_t state, action; + + UChar32 c; + int32_t i, entry; + + if(length<=0) { + /* no input at all: "illegal" */ + return 0xffff; + } + +#if 0 +/* + * Code disabled 2002dec09 (ICU 2.4) because it is not currently used in ICU. markus + * TODO In future releases, verify that this function is never called for SBCS + * conversions, i.e., that sharedData->mbcs.countStates==1 is still true. + * Removal improves code coverage. + */ + /* use optimized function if possible */ + if(sharedData->mbcs.countStates==1) { + if(length==1) { + return ucnv_MBCSSingleSimpleGetNextUChar(sharedData, (uint8_t)*source, useFallback); + } else { + return 0xffff; /* illegal: more than a single byte for an SBCS converter */ + } + } +#endif + + /* set up the local pointers */ + stateTable=sharedData->mbcs.stateTable; + unicodeCodeUnits=sharedData->mbcs.unicodeCodeUnits; + + /* converter state */ + offset=0; + state=sharedData->mbcs.dbcsOnlyState; + + /* conversion loop */ + for(i=0;;) { + entry=stateTable[state][(uint8_t)source[i++]]; + if(MBCS_ENTRY_IS_TRANSITION(entry)) { + state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry); + offset+=MBCS_ENTRY_TRANSITION_OFFSET(entry); + + if(i==length) { + return 0xffff; /* truncated character */ + } + } else { + /* + * An if-else-if chain provides more reliable performance for + * the most common cases compared to a switch. + */ + action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); + if(action==MBCS_STATE_VALID_16) { + offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); + c=unicodeCodeUnits[offset]; + if(c!=0xfffe) { + /* done */ + } else if(UCNV_TO_U_USE_FALLBACK(cnv)) { + c=ucnv_MBCSGetFallback(&sharedData->mbcs, offset); + /* else done with 0xfffe */ + } + break; + } else if(action==MBCS_STATE_VALID_DIRECT_16) { + /* output BMP code point */ + c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + break; + } else if(action==MBCS_STATE_VALID_16_PAIR) { + offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); + c=unicodeCodeUnits[offset++]; + if(c<0xd800) { + /* output BMP code point below 0xd800 */ + } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? c<=0xdfff : c<=0xdbff) { + /* output roundtrip or fallback supplementary code point */ + c=(UChar32)(((c&0x3ff)<<10)+unicodeCodeUnits[offset]+(0x10000-0xdc00)); + } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? (c&0xfffe)==0xe000 : c==0xe000) { + /* output roundtrip BMP code point above 0xd800 or fallback BMP code point */ + c=unicodeCodeUnits[offset]; + } else if(c==0xffff) { + return 0xffff; + } else { + c=0xfffe; + } + break; + } else if(action==MBCS_STATE_VALID_DIRECT_20) { + /* output supplementary code point */ + c=0x10000+MBCS_ENTRY_FINAL_VALUE(entry); + break; + } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) { + if(!TO_U_USE_FALLBACK(useFallback)) { + c=0xfffe; + break; + } + /* output BMP code point */ + c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); + break; + } else if(action==MBCS_STATE_FALLBACK_DIRECT_20) { + if(!TO_U_USE_FALLBACK(useFallback)) { + c=0xfffe; + break; + } + /* output supplementary code point */ + c=0x10000+MBCS_ENTRY_FINAL_VALUE(entry); + break; + } else if(action==MBCS_STATE_UNASSIGNED) { + c=0xfffe; + break; + } + + /* + * forbid MBCS_STATE_CHANGE_ONLY for this function, + * and MBCS_STATE_ILLEGAL and reserved action codes + */ + return 0xffff; + } + } + + if(i!=length) { + /* illegal for this function: not all input consumed */ + return 0xffff; + } + + if(c==0xfffe) { + /* try an extension mapping */ + const int32_t *cx=sharedData->mbcs.extIndexes; + if(cx!=NULL) { + return ucnv_extSimpleMatchToU(cx, source, length, useFallback); + } + } + + return c; +} + +/* MBCS-from-Unicode conversion functions ----------------------------------- */ + +/* This version of ucnv_MBCSFromUnicodeWithOffsets() is optimized for double-byte codepages. */ +static void +ucnv_MBCSDoubleFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv; + const UChar *source, *sourceLimit; + uint8_t *target; + int32_t targetCapacity; + int32_t *offsets; + + const uint16_t *table; + const uint16_t *mbcsIndex; + const uint8_t *bytes; + + UChar32 c; + + int32_t sourceIndex, nextSourceIndex; + + uint32_t stage2Entry; + uint32_t asciiRoundtrips; + uint32_t value; + uint8_t unicodeMask; + + /* use optimized function if possible */ + cnv=pArgs->converter; + unicodeMask=cnv->sharedData->mbcs.unicodeMask; + + /* set up the local pointers */ + source=pArgs->source; + sourceLimit=pArgs->sourceLimit; + target=(uint8_t *)pArgs->target; + targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); + offsets=pArgs->offsets; + + table=cnv->sharedData->mbcs.fromUnicodeTable; + mbcsIndex=cnv->sharedData->mbcs.mbcsIndex; + if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { + bytes=cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes; + } else { + bytes=cnv->sharedData->mbcs.fromUnicodeBytes; + } + asciiRoundtrips=cnv->sharedData->mbcs.asciiRoundtrips; + + /* get the converter state from UConverter */ + c=cnv->fromUChar32; + + /* sourceIndex=-1 if the current character began in the previous buffer */ + sourceIndex= c==0 ? 0 : -1; + nextSourceIndex=0; + + /* conversion loop */ + if(c!=0 && targetCapacity>0) { + goto getTrail; + } + + while(source0) { + /* + * Get a correct Unicode code point: + * a single UChar for a BMP code point or + * a matched surrogate pair for a "supplementary code point". + */ + c=*source++; + ++nextSourceIndex; + if(c<=0x7f && IS_ASCII_ROUNDTRIP(c, asciiRoundtrips)) { + *target++=(uint8_t)c; + if(offsets!=NULL) { + *offsets++=sourceIndex; + sourceIndex=nextSourceIndex; + } + --targetCapacity; + c=0; + continue; + } + /* + * utf8Friendly table: Test for <=0xd7ff rather than <=MBCS_FAST_MAX + * to avoid dealing with surrogates. + * MBCS_FAST_MAX must be >=0xd7ff. + */ + if(c<=0xd7ff) { + value=DBCS_RESULT_FROM_MOST_BMP(mbcsIndex, (const uint16_t *)bytes, c); + /* There are only roundtrips (!=0) and no-mapping (==0) entries. */ + if(value==0) { + goto unassigned; + } + /* output the value */ + } else { + /* + * This also tests if the codepage maps single surrogates. + * If it does, then surrogates are not paired but mapped separately. + * Note that in this case unmatched surrogates are not detected. + */ + if(UTF_IS_SURROGATE(c) && !(unicodeMask&UCNV_HAS_SURROGATES)) { + if(UTF_IS_SURROGATE_FIRST(c)) { +getTrail: + if(sourcesource=source; + c=_extFromU(cnv, cnv->sharedData, + c, &source, sourceLimit, + &target, target+targetCapacity, + &offsets, sourceIndex, + pArgs->flush, + pErrorCode); + nextSourceIndex+=(int32_t)(source-pArgs->source); + + if(U_FAILURE(*pErrorCode)) { + /* not mappable or buffer overflow */ + break; + } else { + /* a mapping was written to the target, continue */ + + /* recalculate the targetCapacity after an extension mapping */ + targetCapacity=(int32_t)(pArgs->targetLimit-(char *)target); + + /* normal end of conversion: prepare for a new character */ + sourceIndex=nextSourceIndex; + continue; + } + } + } + + /* write the output character bytes from value and length */ + /* from the first if in the loop we know that targetCapacity>0 */ + if(value<=0xff) { + /* this is easy because we know that there is enough space */ + *target++=(uint8_t)value; + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + --targetCapacity; + } else /* length==2 */ { + *target++=(uint8_t)(value>>8); + if(2<=targetCapacity) { + *target++=(uint8_t)value; + if(offsets!=NULL) { + *offsets++=sourceIndex; + *offsets++=sourceIndex; + } + targetCapacity-=2; + } else { + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + cnv->charErrorBuffer[0]=(char)value; + cnv->charErrorBufferLength=1; + + /* target overflow */ + targetCapacity=0; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + c=0; + break; + } + } + + /* normal end of conversion: prepare for a new character */ + c=0; + sourceIndex=nextSourceIndex; + continue; + } else { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + } + + /* set the converter state back into UConverter */ + cnv->fromUChar32=c; + + /* write back the updated pointers */ + pArgs->source=source; + pArgs->target=(char *)target; + pArgs->offsets=offsets; +} + +/* This version of ucnv_MBCSFromUnicodeWithOffsets() is optimized for single-byte codepages. */ +static void +ucnv_MBCSSingleFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv; + const UChar *source, *sourceLimit; + uint8_t *target; + int32_t targetCapacity; + int32_t *offsets; + + const uint16_t *table; + const uint16_t *results; + + UChar32 c; + + int32_t sourceIndex, nextSourceIndex; + + uint16_t value, minValue; + UBool hasSupplementary; + + /* set up the local pointers */ + cnv=pArgs->converter; + source=pArgs->source; + sourceLimit=pArgs->sourceLimit; + target=(uint8_t *)pArgs->target; + targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); + offsets=pArgs->offsets; + + table=cnv->sharedData->mbcs.fromUnicodeTable; + if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { + results=(uint16_t *)cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes; + } else { + results=(uint16_t *)cnv->sharedData->mbcs.fromUnicodeBytes; + } + + if(cnv->useFallback) { + /* use all roundtrip and fallback results */ + minValue=0x800; + } else { + /* use only roundtrips and fallbacks from private-use characters */ + minValue=0xc00; + } + hasSupplementary=(UBool)(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY); + + /* get the converter state from UConverter */ + c=cnv->fromUChar32; + + /* sourceIndex=-1 if the current character began in the previous buffer */ + sourceIndex= c==0 ? 0 : -1; + nextSourceIndex=0; + + /* conversion loop */ + if(c!=0 && targetCapacity>0) { + goto getTrail; + } + + while(source0) { + /* + * Get a correct Unicode code point: + * a single UChar for a BMP code point or + * a matched surrogate pair for a "supplementary code point". + */ + c=*source++; + ++nextSourceIndex; + if(UTF_IS_SURROGATE(c)) { + if(UTF_IS_SURROGATE_FIRST(c)) { +getTrail: + if(source=minValue) { + /* assigned, write the output character bytes from value and length */ + /* length==1 */ + /* this is easy because we know that there is enough space */ + *target++=(uint8_t)value; + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + --targetCapacity; + + /* normal end of conversion: prepare for a new character */ + c=0; + sourceIndex=nextSourceIndex; + } else { /* unassigned */ +unassigned: + /* try an extension mapping */ + pArgs->source=source; + c=_extFromU(cnv, cnv->sharedData, + c, &source, sourceLimit, + &target, target+targetCapacity, + &offsets, sourceIndex, + pArgs->flush, + pErrorCode); + nextSourceIndex+=(int32_t)(source-pArgs->source); + + if(U_FAILURE(*pErrorCode)) { + /* not mappable or buffer overflow */ + break; + } else { + /* a mapping was written to the target, continue */ + + /* recalculate the targetCapacity after an extension mapping */ + targetCapacity=(int32_t)(pArgs->targetLimit-(char *)target); + + /* normal end of conversion: prepare for a new character */ + sourceIndex=nextSourceIndex; + } + } + } else { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + } + + /* set the converter state back into UConverter */ + cnv->fromUChar32=c; + + /* write back the updated pointers */ + pArgs->source=source; + pArgs->target=(char *)target; + pArgs->offsets=offsets; +} + +/* + * This version of ucnv_MBCSFromUnicode() is optimized for single-byte codepages + * that map only to and from the BMP. + * In addition to single-byte/state optimizations, the offset calculations + * become much easier. + * It would be possible to use the sbcsIndex for UTF-8-friendly tables, + * but measurements have shown that this diminishes performance + * in more cases than it improves it. + * See SVN revision 21013 (2007-feb-06) for the last version with #if switches + * for various MBCS and SBCS optimizations. + */ +static void +ucnv_MBCSSingleFromBMPWithOffsets(UConverterFromUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv; + const UChar *source, *sourceLimit, *lastSource; + uint8_t *target; + int32_t targetCapacity, length; + int32_t *offsets; + + const uint16_t *table; + const uint16_t *results; + + UChar32 c; + + int32_t sourceIndex; + + uint32_t asciiRoundtrips; + uint16_t value, minValue; + + /* set up the local pointers */ + cnv=pArgs->converter; + source=pArgs->source; + sourceLimit=pArgs->sourceLimit; + target=(uint8_t *)pArgs->target; + targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); + offsets=pArgs->offsets; + + table=cnv->sharedData->mbcs.fromUnicodeTable; + if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { + results=(uint16_t *)cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes; + } else { + results=(uint16_t *)cnv->sharedData->mbcs.fromUnicodeBytes; + } + asciiRoundtrips=cnv->sharedData->mbcs.asciiRoundtrips; + + if(cnv->useFallback) { + /* use all roundtrip and fallback results */ + minValue=0x800; + } else { + /* use only roundtrips and fallbacks from private-use characters */ + minValue=0xc00; + } + + /* get the converter state from UConverter */ + c=cnv->fromUChar32; + + /* sourceIndex=-1 if the current character began in the previous buffer */ + sourceIndex= c==0 ? 0 : -1; + lastSource=source; + + /* + * since the conversion here is 1:1 UChar:uint8_t, we need only one counter + * for the minimum of the sourceLength and targetCapacity + */ + length=(int32_t)(sourceLimit-source); + if(length0) { + goto getTrail; + } + +#if MBCS_UNROLL_SINGLE_FROM_BMP + /* unrolling makes it slower on Pentium III/Windows 2000?! */ + /* unroll the loop with the most common case */ +unrolled: + if(targetCapacity>=4) { + int32_t count, loops; + uint16_t andedValues; + + loops=count=targetCapacity>>2; + do { + c=*source++; + andedValues=value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); + *target++=(uint8_t)value; + c=*source++; + andedValues&=value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); + *target++=(uint8_t)value; + c=*source++; + andedValues&=value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); + *target++=(uint8_t)value; + c=*source++; + andedValues&=value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); + *target++=(uint8_t)value; + + /* were all 4 entries really valid? */ + if(andedValues0); + count=loops-count; + targetCapacity-=4*count; + + if(offsets!=NULL) { + lastSource+=4*count; + while(count>0) { + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + *offsets++=sourceIndex++; + --count; + } + } + + c=0; + } +#endif + + while(targetCapacity>0) { + /* + * Get a correct Unicode code point: + * a single UChar for a BMP code point or + * a matched surrogate pair for a "supplementary code point". + */ + c=*source++; + /* + * Do not immediately check for single surrogates: + * Assume that they are unassigned and check for them in that case. + * This speeds up the conversion of assigned characters. + */ + /* convert the Unicode code point in c into codepage bytes */ + if(c<=0x7f && IS_ASCII_ROUNDTRIP(c, asciiRoundtrips)) { + *target++=(uint8_t)c; + --targetCapacity; + c=0; + continue; + } + value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); + /* is this code point assigned, or do we use fallbacks? */ + if(value>=minValue) { + /* assigned, write the output character bytes from value and length */ + /* length==1 */ + /* this is easy because we know that there is enough space */ + *target++=(uint8_t)value; + --targetCapacity; + + /* normal end of conversion: prepare for a new character */ + c=0; + continue; + } else if(!UTF_IS_SURROGATE(c)) { + /* normal, unassigned BMP character */ + } else if(UTF_IS_SURROGATE_FIRST(c)) { +getTrail: + if(sourceflush) { + *pErrorCode=U_TRUNCATED_CHAR_FOUND; + } + break; + } + } else { + /* this is an unmatched trail code unit (2nd surrogate) */ + /* callback(illegal) */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + break; + } + + /* c does not have a mapping */ + + /* get the number of code units for c to correctly advance sourceIndex */ + length=U16_LENGTH(c); + + /* set offsets since the start or the last extension */ + if(offsets!=NULL) { + int32_t count=(int32_t)(source-lastSource); + + /* do not set the offset for this character */ + count-=length; + + while(count>0) { + *offsets++=sourceIndex++; + --count; + } + /* offsets and sourceIndex are now set for the current character */ + } + + /* try an extension mapping */ + lastSource=source; + c=_extFromU(cnv, cnv->sharedData, + c, &source, sourceLimit, + &target, target+targetCapacity, + &offsets, sourceIndex, + pArgs->flush, + pErrorCode); + sourceIndex+=length+(int32_t)(source-lastSource); + lastSource=source; + + if(U_FAILURE(*pErrorCode)) { + /* not mappable or buffer overflow */ + break; + } else { + /* a mapping was written to the target, continue */ + + /* recalculate the targetCapacity after an extension mapping */ + targetCapacity=(int32_t)(pArgs->targetLimit-(char *)target); + length=(int32_t)(sourceLimit-source); + if(length=(uint8_t *)pArgs->targetLimit) { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + + /* set offsets since the start or the last callback */ + if(offsets!=NULL) { + size_t count=source-lastSource; + if (count > 0 && *pErrorCode == U_TRUNCATED_CHAR_FOUND) { + /* + Caller gave us a partial supplementary character, + which this function couldn't convert in any case. + The callback will handle the offset. + */ + count--; + } + while(count>0) { + *offsets++=sourceIndex++; + --count; + } + } + + /* set the converter state back into UConverter */ + cnv->fromUChar32=c; + + /* write back the updated pointers */ + pArgs->source=source; + pArgs->target=(char *)target; + pArgs->offsets=offsets; +} + +U_CFUNC void +ucnv_MBCSFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv; + const UChar *source, *sourceLimit; + uint8_t *target; + int32_t targetCapacity; + int32_t *offsets; + + const uint16_t *table; + const uint16_t *mbcsIndex; + const uint8_t *p, *bytes; + uint8_t outputType; + + UChar32 c; + + int32_t prevSourceIndex, sourceIndex, nextSourceIndex; + + uint32_t stage2Entry; + uint32_t asciiRoundtrips; + uint32_t value; + int32_t length, prevLength; + uint8_t unicodeMask; + + cnv=pArgs->converter; + + if(cnv->preFromUFirstCP>=0) { + /* + * pass sourceIndex=-1 because we continue from an earlier buffer + * in the future, this may change with continuous offsets + */ + ucnv_extContinueMatchFromU(cnv, pArgs, -1, pErrorCode); + + if(U_FAILURE(*pErrorCode) || cnv->preFromULength<0) { + return; + } + } + + /* use optimized function if possible */ + outputType=cnv->sharedData->mbcs.outputType; + unicodeMask=cnv->sharedData->mbcs.unicodeMask; + if(outputType==MBCS_OUTPUT_1 && !(unicodeMask&UCNV_HAS_SURROGATES)) { + if(!(unicodeMask&UCNV_HAS_SUPPLEMENTARY)) { + ucnv_MBCSSingleFromBMPWithOffsets(pArgs, pErrorCode); + } else { + ucnv_MBCSSingleFromUnicodeWithOffsets(pArgs, pErrorCode); + } + return; + } else if(outputType==MBCS_OUTPUT_2 && cnv->sharedData->mbcs.utf8Friendly) { + ucnv_MBCSDoubleFromUnicodeWithOffsets(pArgs, pErrorCode); + return; + } + + /* set up the local pointers */ + source=pArgs->source; + sourceLimit=pArgs->sourceLimit; + target=(uint8_t *)pArgs->target; + targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); + offsets=pArgs->offsets; + + table=cnv->sharedData->mbcs.fromUnicodeTable; + if(cnv->sharedData->mbcs.utf8Friendly) { + mbcsIndex=cnv->sharedData->mbcs.mbcsIndex; + } else { + mbcsIndex=NULL; + } + if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { + bytes=cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes; + } else { + bytes=cnv->sharedData->mbcs.fromUnicodeBytes; + } + asciiRoundtrips=cnv->sharedData->mbcs.asciiRoundtrips; + + /* get the converter state from UConverter */ + c=cnv->fromUChar32; + + if(outputType==MBCS_OUTPUT_2_SISO) { + prevLength=cnv->fromUnicodeStatus; + if(prevLength==0) { + /* set the real value */ + prevLength=1; + } + } else { + /* prevent fromUnicodeStatus from being set to something non-0 */ + prevLength=0; + } + + /* sourceIndex=-1 if the current character began in the previous buffer */ + prevSourceIndex=-1; + sourceIndex= c==0 ? 0 : -1; + nextSourceIndex=0; + + /* conversion loop */ + /* + * This is another piece of ugly code: + * A goto into the loop if the converter state contains a first surrogate + * from the previous function call. + * It saves me to check in each loop iteration a check of if(c==0) + * and duplicating the trail-surrogate-handling code in the else + * branch of that check. + * I could not find any other way to get around this other than + * using a function call for the conversion and callback, which would + * be even more inefficient. + * + * Markus Scherer 2000-jul-19 + */ + if(c!=0 && targetCapacity>0) { + goto getTrail; + } + + while(source0) { + /* + * Get a correct Unicode code point: + * a single UChar for a BMP code point or + * a matched surrogate pair for a "supplementary code point". + */ + c=*source++; + ++nextSourceIndex; + if(c<=0x7f && IS_ASCII_ROUNDTRIP(c, asciiRoundtrips)) { + *target++=(uint8_t)c; + if(offsets!=NULL) { + *offsets++=sourceIndex; + prevSourceIndex=sourceIndex; + sourceIndex=nextSourceIndex; + } + --targetCapacity; + c=0; + continue; + } + /* + * utf8Friendly table: Test for <=0xd7ff rather than <=MBCS_FAST_MAX + * to avoid dealing with surrogates. + * MBCS_FAST_MAX must be >=0xd7ff. + */ + if(c<=0xd7ff && mbcsIndex!=NULL) { + value=mbcsIndex[c>>6]; + + /* get the bytes and the length for the output (copied from below and adapted for utf8Friendly data) */ + /* There are only roundtrips (!=0) and no-mapping (==0) entries. */ + switch(outputType) { + case MBCS_OUTPUT_2: + value=((const uint16_t *)bytes)[value +(c&0x3f)]; + if(value<=0xff) { + if(value==0) { + goto unassigned; + } else { + length=1; + } + } else { + length=2; + } + break; + case MBCS_OUTPUT_2_SISO: + /* 1/2-byte stateful with Shift-In/Shift-Out */ + /* + * Save the old state in the converter object + * right here, then change the local prevLength state variable if necessary. + * Then, if this character turns out to be unassigned or a fallback that + * is not taken, the callback code must not save the new state in the converter + * because the new state is for a character that is not output. + * However, the callback must still restore the state from the converter + * in case the callback function changed it for its output. + */ + cnv->fromUnicodeStatus=prevLength; /* save the old state */ + value=((const uint16_t *)bytes)[value +(c&0x3f)]; + if(value<=0xff) { + if(value==0) { + goto unassigned; + } else if(prevLength<=1) { + length=1; + } else { + /* change from double-byte mode to single-byte */ + value|=(uint32_t)UCNV_SI<<8; + length=2; + prevLength=1; + } + } else { + if(prevLength==2) { + length=2; + } else { + /* change from single-byte mode to double-byte */ + value|=(uint32_t)UCNV_SO<<16; + length=3; + prevLength=2; + } + } + break; + case MBCS_OUTPUT_DBCS_ONLY: + /* table with single-byte results, but only DBCS mappings used */ + value=((const uint16_t *)bytes)[value +(c&0x3f)]; + if(value<=0xff) { + /* no mapping or SBCS result, not taken for DBCS-only */ + goto unassigned; + } else { + length=2; + } + break; + case MBCS_OUTPUT_3: + p=bytes+(value+(c&0x3f))*3; + value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; + if(value<=0xff) { + if(value==0) { + goto unassigned; + } else { + length=1; + } + } else if(value<=0xffff) { + length=2; + } else { + length=3; + } + break; + case MBCS_OUTPUT_4: + value=((const uint32_t *)bytes)[value +(c&0x3f)]; + if(value<=0xff) { + if(value==0) { + goto unassigned; + } else { + length=1; + } + } else if(value<=0xffff) { + length=2; + } else if(value<=0xffffff) { + length=3; + } else { + length=4; + } + break; + case MBCS_OUTPUT_3_EUC: + value=((const uint16_t *)bytes)[value +(c&0x3f)]; + /* EUC 16-bit fixed-length representation */ + if(value<=0xff) { + if(value==0) { + goto unassigned; + } else { + length=1; + } + } else if((value&0x8000)==0) { + value|=0x8e8000; + length=3; + } else if((value&0x80)==0) { + value|=0x8f0080; + length=3; + } else { + length=2; + } + break; + case MBCS_OUTPUT_4_EUC: + p=bytes+(value+(c&0x3f))*3; + value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; + /* EUC 16-bit fixed-length representation applied to the first two bytes */ + if(value<=0xff) { + if(value==0) { + goto unassigned; + } else { + length=1; + } + } else if(value<=0xffff) { + length=2; + } else if((value&0x800000)==0) { + value|=0x8e800000; + length=4; + } else if((value&0x8000)==0) { + value|=0x8f008000; + length=4; + } else { + length=3; + } + break; + default: + /* must not occur */ + /* + * To avoid compiler warnings that value & length may be + * used without having been initialized, we set them here. + * In reality, this is unreachable code. + * Not having a default branch also causes warnings with + * some compilers. + */ + value=0; + length=0; + break; + } + /* output the value */ + } else { + /* + * This also tests if the codepage maps single surrogates. + * If it does, then surrogates are not paired but mapped separately. + * Note that in this case unmatched surrogates are not detected. + */ + if(UTF_IS_SURROGATE(c) && !(unicodeMask&UCNV_HAS_SURROGATES)) { + if(UTF_IS_SURROGATE_FIRST(c)) { +getTrail: + if(sourcefromUnicodeStatus=prevLength; /* save the old state */ + /* callback(unassigned) */ + goto unassigned; + } + /* convert this supplementary code point */ + /* exit this condition tree */ + } else { + /* this is an unmatched lead code unit (1st surrogate) */ + /* callback(illegal) */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + break; + } + } else { + /* no more input */ + break; + } + } else { + /* this is an unmatched trail code unit (2nd surrogate) */ + /* callback(illegal) */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + break; + } + } + + /* convert the Unicode code point in c into codepage bytes */ + + /* + * The basic lookup is a triple-stage compact array (trie) lookup. + * For details see the beginning of this file. + * + * Single-byte codepages are handled with a different data structure + * by _MBCSSingle... functions. + * + * The result consists of a 32-bit value from stage 2 and + * a pointer to as many bytes as are stored per character. + * The pointer points to the character's bytes in stage 3. + * Bits 15..0 of the stage 2 entry contain the stage 3 index + * for that pointer, while bits 31..16 are flags for which of + * the 16 characters in the block are roundtrip-assigned. + * + * For 2-byte and 4-byte codepages, the bytes are stored as uint16_t + * respectively as uint32_t, in the platform encoding. + * For 3-byte codepages, the bytes are always stored in big-endian order. + * + * For EUC encodings that use only either 0x8e or 0x8f as the first + * byte of their longest byte sequences, the first two bytes in + * this third stage indicate with their 7th bits whether these bytes + * are to be written directly or actually need to be preceeded by + * one of the two Single-Shift codes. With this, the third stage + * stores one byte fewer per character than the actual maximum length of + * EUC byte sequences. + * + * Other than that, leading zero bytes are removed and the other + * bytes output. A single zero byte may be output if the "assigned" + * bit in stage 2 was on. + * The data structure does not support zero byte output as a fallback, + * and also does not allow output of leading zeros. + */ + stage2Entry=MBCS_STAGE_2_FROM_U(table, c); + + /* get the bytes and the length for the output */ + switch(outputType) { + case MBCS_OUTPUT_2: + value=MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c); + if(value<=0xff) { + length=1; + } else { + length=2; + } + break; + case MBCS_OUTPUT_2_SISO: + /* 1/2-byte stateful with Shift-In/Shift-Out */ + /* + * Save the old state in the converter object + * right here, then change the local prevLength state variable if necessary. + * Then, if this character turns out to be unassigned or a fallback that + * is not taken, the callback code must not save the new state in the converter + * because the new state is for a character that is not output. + * However, the callback must still restore the state from the converter + * in case the callback function changed it for its output. + */ + cnv->fromUnicodeStatus=prevLength; /* save the old state */ + value=MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c); + if(value<=0xff) { + if(value==0 && MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c)==0) { + /* no mapping, leave value==0 */ + length=0; + } else if(prevLength<=1) { + length=1; + } else { + /* change from double-byte mode to single-byte */ + value|=(uint32_t)UCNV_SI<<8; + length=2; + prevLength=1; + } + } else { + if(prevLength==2) { + length=2; + } else { + /* change from single-byte mode to double-byte */ + value|=(uint32_t)UCNV_SO<<16; + length=3; + prevLength=2; + } + } + break; + case MBCS_OUTPUT_DBCS_ONLY: + /* table with single-byte results, but only DBCS mappings used */ + value=MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c); + if(value<=0xff) { + /* no mapping or SBCS result, not taken for DBCS-only */ + value=stage2Entry=0; /* stage2Entry=0 to reset roundtrip flags */ + length=0; + } else { + length=2; + } + break; + case MBCS_OUTPUT_3: + p=MBCS_POINTER_3_FROM_STAGE_2(bytes, stage2Entry, c); + value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; + if(value<=0xff) { + length=1; + } else if(value<=0xffff) { + length=2; + } else { + length=3; + } + break; + case MBCS_OUTPUT_4: + value=MBCS_VALUE_4_FROM_STAGE_2(bytes, stage2Entry, c); + if(value<=0xff) { + length=1; + } else if(value<=0xffff) { + length=2; + } else if(value<=0xffffff) { + length=3; + } else { + length=4; + } + break; + case MBCS_OUTPUT_3_EUC: + value=MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c); + /* EUC 16-bit fixed-length representation */ + if(value<=0xff) { + length=1; + } else if((value&0x8000)==0) { + value|=0x8e8000; + length=3; + } else if((value&0x80)==0) { + value|=0x8f0080; + length=3; + } else { + length=2; + } + break; + case MBCS_OUTPUT_4_EUC: + p=MBCS_POINTER_3_FROM_STAGE_2(bytes, stage2Entry, c); + value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; + /* EUC 16-bit fixed-length representation applied to the first two bytes */ + if(value<=0xff) { + length=1; + } else if(value<=0xffff) { + length=2; + } else if((value&0x800000)==0) { + value|=0x8e800000; + length=4; + } else if((value&0x8000)==0) { + value|=0x8f008000; + length=4; + } else { + length=3; + } + break; + default: + /* must not occur */ + /* + * To avoid compiler warnings that value & length may be + * used without having been initialized, we set them here. + * In reality, this is unreachable code. + * Not having a default branch also causes warnings with + * some compilers. + */ + value=stage2Entry=0; /* stage2Entry=0 to reset roundtrip flags */ + length=0; + break; + } + + /* is this code point assigned, or do we use fallbacks? */ + if(!(MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c)!=0 || + (UCNV_FROM_U_USE_FALLBACK(cnv, c) && value!=0)) + ) { + /* + * We allow a 0 byte output if the "assigned" bit is set for this entry. + * There is no way with this data structure for fallback output + * to be a zero byte. + */ + +unassigned: + /* try an extension mapping */ + pArgs->source=source; + c=_extFromU(cnv, cnv->sharedData, + c, &source, sourceLimit, + &target, target+targetCapacity, + &offsets, sourceIndex, + pArgs->flush, + pErrorCode); + nextSourceIndex+=(int32_t)(source-pArgs->source); + prevLength=cnv->fromUnicodeStatus; /* restore SISO state */ + + if(U_FAILURE(*pErrorCode)) { + /* not mappable or buffer overflow */ + break; + } else { + /* a mapping was written to the target, continue */ + + /* recalculate the targetCapacity after an extension mapping */ + targetCapacity=(int32_t)(pArgs->targetLimit-(char *)target); + + /* normal end of conversion: prepare for a new character */ + if(offsets!=NULL) { + prevSourceIndex=sourceIndex; + sourceIndex=nextSourceIndex; + } + continue; + } + } + } + + /* write the output character bytes from value and length */ + /* from the first if in the loop we know that targetCapacity>0 */ + if(length<=targetCapacity) { + if(offsets==NULL) { + switch(length) { + /* each branch falls through to the next one */ + case 4: + *target++=(uint8_t)(value>>24); + case 3: + *target++=(uint8_t)(value>>16); + case 2: + *target++=(uint8_t)(value>>8); + case 1: + *target++=(uint8_t)value; + default: + /* will never occur */ + break; + } + } else { + switch(length) { + /* each branch falls through to the next one */ + case 4: + *target++=(uint8_t)(value>>24); + *offsets++=sourceIndex; + case 3: + *target++=(uint8_t)(value>>16); + *offsets++=sourceIndex; + case 2: + *target++=(uint8_t)(value>>8); + *offsets++=sourceIndex; + case 1: + *target++=(uint8_t)value; + *offsets++=sourceIndex; + default: + /* will never occur */ + break; + } + } + targetCapacity-=length; + } else { + uint8_t *charErrorBuffer; + + /* + * We actually do this backwards here: + * In order to save an intermediate variable, we output + * first to the overflow buffer what does not fit into the + * regular target. + */ + /* we know that 1<=targetCapacitycharErrorBuffer; + switch(length) { + /* each branch falls through to the next one */ + case 3: + *charErrorBuffer++=(uint8_t)(value>>16); + case 2: + *charErrorBuffer++=(uint8_t)(value>>8); + case 1: + *charErrorBuffer=(uint8_t)value; + default: + /* will never occur */ + break; + } + cnv->charErrorBufferLength=(int8_t)length; + + /* now output what fits into the regular target */ + value>>=8*length; /* length was reduced by targetCapacity */ + switch(targetCapacity) { + /* each branch falls through to the next one */ + case 3: + *target++=(uint8_t)(value>>16); + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + case 2: + *target++=(uint8_t)(value>>8); + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + case 1: + *target++=(uint8_t)value; + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + default: + /* will never occur */ + break; + } + + /* target overflow */ + targetCapacity=0; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + c=0; + break; + } + + /* normal end of conversion: prepare for a new character */ + c=0; + if(offsets!=NULL) { + prevSourceIndex=sourceIndex; + sourceIndex=nextSourceIndex; + } + continue; + } else { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + } + + /* + * the end of the input stream and detection of truncated input + * are handled by the framework, but for EBCDIC_STATEFUL conversion + * we need to emit an SI at the very end + * + * conditions: + * successful + * EBCDIC_STATEFUL in DBCS mode + * end of input and no truncated input + */ + if( U_SUCCESS(*pErrorCode) && + outputType==MBCS_OUTPUT_2_SISO && prevLength==2 && + pArgs->flush && source>=sourceLimit && c==0 + ) { + /* EBCDIC_STATEFUL ending with DBCS: emit an SI to return the output stream to SBCS */ + if(targetCapacity>0) { + *target++=(uint8_t)UCNV_SI; + if(offsets!=NULL) { + /* set the last source character's index (sourceIndex points at sourceLimit now) */ + *offsets++=prevSourceIndex; + } + } else { + /* target is full */ + cnv->charErrorBuffer[0]=(char)UCNV_SI; + cnv->charErrorBufferLength=1; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + prevLength=1; /* we switched into SBCS */ + } + + /* set the converter state back into UConverter */ + cnv->fromUChar32=c; + cnv->fromUnicodeStatus=prevLength; + + /* write back the updated pointers */ + pArgs->source=source; + pArgs->target=(char *)target; + pArgs->offsets=offsets; +} + +/* + * This is another simple conversion function for internal use by other + * conversion implementations. + * It does not use the converter state nor call callbacks. + * It does not handle the EBCDIC swaplfnl option (set in UConverter). + * It handles conversion extensions but not GB 18030. + * + * It converts one single Unicode code point into codepage bytes, encoded + * as one 32-bit value. The function returns the number of bytes in *pValue: + * 1..4 the number of bytes in *pValue + * 0 unassigned (*pValue undefined) + * -1 illegal (currently not used, *pValue undefined) + * + * *pValue will contain the resulting bytes with the last byte in bits 7..0, + * the second to last byte in bits 15..8, etc. + * Currently, the function assumes but does not check that 0<=c<=0x10ffff. + */ +U_CFUNC int32_t +ucnv_MBCSFromUChar32(UConverterSharedData *sharedData, + UChar32 c, uint32_t *pValue, + UBool useFallback) { + const int32_t *cx; + const uint16_t *table; +#if 0 +/* #if 0 because this is not currently used in ICU - reduce code, increase code coverage */ + const uint8_t *p; +#endif + uint32_t stage2Entry; + uint32_t value; + int32_t length; + + /* BMP-only codepages are stored without stage 1 entries for supplementary code points */ + if(c<=0xffff || (sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY)) { + table=sharedData->mbcs.fromUnicodeTable; + + /* convert the Unicode code point in c into codepage bytes (same as in _MBCSFromUnicodeWithOffsets) */ + if(sharedData->mbcs.outputType==MBCS_OUTPUT_1) { + value=MBCS_SINGLE_RESULT_FROM_U(table, (uint16_t *)sharedData->mbcs.fromUnicodeBytes, c); + /* is this code point assigned, or do we use fallbacks? */ + if(useFallback ? value>=0x800 : value>=0xc00) { + *pValue=value&0xff; + return 1; + } + } else /* outputType!=MBCS_OUTPUT_1 */ { + stage2Entry=MBCS_STAGE_2_FROM_U(table, c); + + /* get the bytes and the length for the output */ + switch(sharedData->mbcs.outputType) { + case MBCS_OUTPUT_2: + value=MBCS_VALUE_2_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); + if(value<=0xff) { + length=1; + } else { + length=2; + } + break; +#if 0 +/* #if 0 because this is not currently used in ICU - reduce code, increase code coverage */ + case MBCS_OUTPUT_DBCS_ONLY: + /* table with single-byte results, but only DBCS mappings used */ + value=MBCS_VALUE_2_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); + if(value<=0xff) { + /* no mapping or SBCS result, not taken for DBCS-only */ + value=stage2Entry=0; /* stage2Entry=0 to reset roundtrip flags */ + length=0; + } else { + length=2; + } + break; + case MBCS_OUTPUT_3: + p=MBCS_POINTER_3_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); + value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; + if(value<=0xff) { + length=1; + } else if(value<=0xffff) { + length=2; + } else { + length=3; + } + break; + case MBCS_OUTPUT_4: + value=MBCS_VALUE_4_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); + if(value<=0xff) { + length=1; + } else if(value<=0xffff) { + length=2; + } else if(value<=0xffffff) { + length=3; + } else { + length=4; + } + break; + case MBCS_OUTPUT_3_EUC: + value=MBCS_VALUE_2_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); + /* EUC 16-bit fixed-length representation */ + if(value<=0xff) { + length=1; + } else if((value&0x8000)==0) { + value|=0x8e8000; + length=3; + } else if((value&0x80)==0) { + value|=0x8f0080; + length=3; + } else { + length=2; + } + break; + case MBCS_OUTPUT_4_EUC: + p=MBCS_POINTER_3_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); + value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; + /* EUC 16-bit fixed-length representation applied to the first two bytes */ + if(value<=0xff) { + length=1; + } else if(value<=0xffff) { + length=2; + } else if((value&0x800000)==0) { + value|=0x8e800000; + length=4; + } else if((value&0x8000)==0) { + value|=0x8f008000; + length=4; + } else { + length=3; + } + break; +#endif + default: + /* must not occur */ + return -1; + } + + /* is this code point assigned, or do we use fallbacks? */ + if( MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) || + (FROM_U_USE_FALLBACK(useFallback, c) && value!=0) + ) { + /* + * We allow a 0 byte output if the "assigned" bit is set for this entry. + * There is no way with this data structure for fallback output + * to be a zero byte. + */ + /* assigned */ + *pValue=value; + return length; + } + } + } + + cx=sharedData->mbcs.extIndexes; + if(cx!=NULL) { + length=ucnv_extSimpleMatchFromU(cx, c, pValue, useFallback); + return length>=0 ? length : -length; /* return abs(length); */ + } + + /* unassigned */ + return 0; +} + + +#if 0 +/* + * This function has been moved to ucnv2022.c for inlining. + * This implementation is here only for documentation purposes + */ + +/** + * This version of ucnv_MBCSFromUChar32() is optimized for single-byte codepages. + * It does not handle the EBCDIC swaplfnl option (set in UConverter). + * It does not handle conversion extensions (_extFromU()). + * + * It returns the codepage byte for the code point, or -1 if it is unassigned. + */ +U_CFUNC int32_t +ucnv_MBCSSingleFromUChar32(UConverterSharedData *sharedData, + UChar32 c, + UBool useFallback) { + const uint16_t *table; + int32_t value; + + /* BMP-only codepages are stored without stage 1 entries for supplementary code points */ + if(c>=0x10000 && !(sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY)) { + return -1; + } + + /* convert the Unicode code point in c into codepage bytes (same as in _MBCSFromUnicodeWithOffsets) */ + table=sharedData->mbcs.fromUnicodeTable; + + /* get the byte for the output */ + value=MBCS_SINGLE_RESULT_FROM_U(table, (uint16_t *)sharedData->mbcs.fromUnicodeBytes, c); + /* is this code point assigned, or do we use fallbacks? */ + if(useFallback ? value>=0x800 : value>=0xc00) { + return value&0xff; + } else { + return -1; + } +} +#endif + +/* MBCS-from-UTF-8 conversion functions ------------------------------------- */ + +/* minimum code point values for n-byte UTF-8 sequences, n=0..4 */ +static const UChar32 +utf8_minLegal[5]={ 0, 0, 0x80, 0x800, 0x10000 }; + +/* offsets for n-byte UTF-8 sequences that were calculated with ((lead<<6)+trail)<<6+trail... */ +static const UChar32 +utf8_offsets[7]={ 0, 0, 0x3080, 0xE2080, 0x3C82080 }; + +static void +ucnv_SBCSFromUTF8(UConverterFromUnicodeArgs *pFromUArgs, + UConverterToUnicodeArgs *pToUArgs, + UErrorCode *pErrorCode) { + UConverter *utf8, *cnv; + const uint8_t *source, *sourceLimit; + uint8_t *target; + int32_t targetCapacity; + + const uint16_t *table, *sbcsIndex; + const uint16_t *results; + + int8_t oldToULength, toULength, toULimit; + + UChar32 c; + uint8_t b, t1, t2; + + uint32_t asciiRoundtrips; + uint16_t value, minValue; + UBool hasSupplementary; + + /* set up the local pointers */ + utf8=pToUArgs->converter; + cnv=pFromUArgs->converter; + source=(uint8_t *)pToUArgs->source; + sourceLimit=(uint8_t *)pToUArgs->sourceLimit; + target=(uint8_t *)pFromUArgs->target; + targetCapacity=(int32_t)(pFromUArgs->targetLimit-pFromUArgs->target); + + table=cnv->sharedData->mbcs.fromUnicodeTable; + sbcsIndex=cnv->sharedData->mbcs.sbcsIndex; + if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { + results=(uint16_t *)cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes; + } else { + results=(uint16_t *)cnv->sharedData->mbcs.fromUnicodeBytes; + } + asciiRoundtrips=cnv->sharedData->mbcs.asciiRoundtrips; + + if(cnv->useFallback) { + /* use all roundtrip and fallback results */ + minValue=0x800; + } else { + /* use only roundtrips and fallbacks from private-use characters */ + minValue=0xc00; + } + hasSupplementary=(UBool)(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY); + + /* get the converter state from the UTF-8 UConverter */ + c=(UChar32)utf8->toUnicodeStatus; + if(c!=0) { + toULength=oldToULength=utf8->toULength; + toULimit=(int8_t)utf8->mode; + } else { + toULength=oldToULength=toULimit=0; + } + + /* + * Make sure that the last byte sequence before sourceLimit is complete + * or runs into a lead byte. + * Do not go back into the bytes that will be read for finishing a partial + * sequence from the previous buffer. + * In the conversion loop compare source with sourceLimit only once + * per multi-byte character. + */ + { + int32_t i, length; + + length=(int32_t)(sourceLimit-source) - (toULimit-oldToULength); + for(i=0; i<3 && i0) { + utf8->toUnicodeStatus=0; + utf8->toULength=0; + goto moreBytes; + /* + * Note: We could avoid the goto by duplicating some of the moreBytes + * code, but only up to the point of collecting a complete UTF-8 + * sequence; then recurse for the toUBytes[toULength] + * and then continue with normal conversion. + * + * If so, move this code to just after initializing the minimum + * set of local variables for reading the UTF-8 input + * (utf8, source, target, limits but not cnv, table, minValue, etc.). + * + * Potential advantages: + * - avoid the goto + * - oldToULength could become a local variable in just those code blocks + * that deal with buffer boundaries + * - possibly faster if the goto prevents some compiler optimizations + * (this would need measuring to confirm) + * Disadvantage: + * - code duplication + */ + } + + /* conversion loop */ + while(source0) { + b=*source++; + if((int8_t)b>=0) { + /* convert ASCII */ + if(IS_ASCII_ROUNDTRIP(b, asciiRoundtrips)) { + *target++=(uint8_t)b; + --targetCapacity; + continue; + } else { + c=b; + value=SBCS_RESULT_FROM_UTF8(sbcsIndex, results, 0, c); + } + } else { + if(b<0xe0) { + if( /* handle U+0080..U+07FF inline */ + b>=0xc2 && + (t1=(uint8_t)(*source-0x80)) <= 0x3f + ) { + c=b&0x1f; + ++source; + value=SBCS_RESULT_FROM_UTF8(sbcsIndex, results, c, t1); + if(value>=minValue) { + *target++=(uint8_t)value; + --targetCapacity; + continue; + } else { + c=(c<<6)|t1; + } + } else { + c=-1; + } + } else if(b==0xe0) { + if( /* handle U+0800..U+0FFF inline */ + (t1=(uint8_t)(source[0]-0x80)) <= 0x3f && t1 >= 0x20 && + (t2=(uint8_t)(source[1]-0x80)) <= 0x3f + ) { + c=t1; + source+=2; + value=SBCS_RESULT_FROM_UTF8(sbcsIndex, results, c, t2); + if(value>=minValue) { + *target++=(uint8_t)value; + --targetCapacity; + continue; + } else { + c=(c<<6)|t2; + } + } else { + c=-1; + } + } else { + c=-1; + } + + if(c<0) { + /* handle "complicated" and error cases, and continuing partial characters */ + oldToULength=0; + toULength=1; + toULimit=utf8_countTrailBytes[b]+1; + c=b; +moreBytes: + while(toULengthtoUBytes[oldToULength++]=*source++; + } + utf8->toUnicodeStatus=c; + utf8->toULength=toULength; + utf8->mode=toULimit; + pToUArgs->source=(char *)source; + pFromUArgs->target=(char *)target; + return; + } + } + + if( toULength==toULimit && /* consumed all trail bytes */ + (toULength==3 || toULength==2) && /* BMP */ + (c-=utf8_offsets[toULength])>=utf8_minLegal[toULength] && + (c<=0xd7ff || 0xe000<=c) /* not a surrogate */ + ) { + value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); + } else if( + toULength==toULimit && toULength==4 && + (0x10000<=(c-=utf8_offsets[4]) && c<=0x10ffff) + ) { + /* supplementary code point */ + if(!hasSupplementary) { + /* BMP-only codepages are stored without stage 1 entries for supplementary code points */ + value=0; + } else { + value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); + } + } else { + /* error handling: illegal UTF-8 byte sequence */ + source-=(toULength-oldToULength); + while(oldToULengthtoUBytes[oldToULength++]=*source++; + } + utf8->toULength=toULength; + pToUArgs->source=(char *)source; + pFromUArgs->target=(char *)target; + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + return; + } + } + } + + if(value>=minValue) { + /* output the mapping for c */ + *target++=(uint8_t)value; + --targetCapacity; + } else { + /* valueUTF-16->charset conversion. + */ + static const UChar nul=0; + const UChar *noSource=&nul; + c=_extFromU(cnv, cnv->sharedData, + c, &noSource, noSource, + &target, target+targetCapacity, + NULL, -1, + pFromUArgs->flush, + pErrorCode); + + if(U_FAILURE(*pErrorCode)) { + /* not mappable or buffer overflow */ + cnv->fromUChar32=c; + break; + } else if(cnv->preFromUFirstCP>=0) { + /* + * Partial match, return and revert to pivoting. + * In normal from-UTF-16 conversion, we would just continue + * but then exit the loop because the extension match would + * have consumed the source. + */ + break; + } else { + /* a mapping was written to the target, continue */ + + /* recalculate the targetCapacity after an extension mapping */ + targetCapacity=(int32_t)(pFromUArgs->targetLimit-(char *)target); + } + } + } else { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + } + + /* + * The sourceLimit may have been adjusted before the conversion loop + * to stop before a truncated sequence. + * If so, then collect the truncated sequence now. + */ + if(U_SUCCESS(*pErrorCode) && source<(sourceLimit=(uint8_t *)pToUArgs->sourceLimit)) { + c=utf8->toUBytes[0]=b=*source++; + toULength=1; + toULimit=utf8_countTrailBytes[b]+1; + while(sourcetoUBytes[toULength++]=b=*source++; + c=(c<<6)+b; + } + utf8->toUnicodeStatus=c; + utf8->toULength=toULength; + utf8->mode=toULimit; + } + + /* write back the updated pointers */ + pToUArgs->source=(char *)source; + pFromUArgs->target=(char *)target; +} + +static void +ucnv_DBCSFromUTF8(UConverterFromUnicodeArgs *pFromUArgs, + UConverterToUnicodeArgs *pToUArgs, + UErrorCode *pErrorCode) { + UConverter *utf8, *cnv; + const uint8_t *source, *sourceLimit; + uint8_t *target; + int32_t targetCapacity; + + const uint16_t *table, *mbcsIndex; + const uint16_t *results; + + int8_t oldToULength, toULength, toULimit; + + UChar32 c; + uint8_t b, t1, t2; + + uint32_t stage2Entry; + uint32_t asciiRoundtrips; + uint16_t value, minValue; + UBool hasSupplementary; + + /* set up the local pointers */ + utf8=pToUArgs->converter; + cnv=pFromUArgs->converter; + source=(uint8_t *)pToUArgs->source; + sourceLimit=(uint8_t *)pToUArgs->sourceLimit; + target=(uint8_t *)pFromUArgs->target; + targetCapacity=(int32_t)(pFromUArgs->targetLimit-pFromUArgs->target); + + table=cnv->sharedData->mbcs.fromUnicodeTable; + mbcsIndex=cnv->sharedData->mbcs.mbcsIndex; + if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { + results=(uint16_t *)cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes; + } else { + results=(uint16_t *)cnv->sharedData->mbcs.fromUnicodeBytes; + } + asciiRoundtrips=cnv->sharedData->mbcs.asciiRoundtrips; + + if(cnv->useFallback) { + /* use all roundtrip and fallback results */ + minValue=0x800; + } else { + /* use only roundtrips and fallbacks from private-use characters */ + minValue=0xc00; + } + hasSupplementary=(UBool)(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY); + + /* get the converter state from the UTF-8 UConverter */ + c=(UChar32)utf8->toUnicodeStatus; + if(c!=0) { + toULength=oldToULength=utf8->toULength; + toULimit=(int8_t)utf8->mode; + } else { + toULength=oldToULength=toULimit=0; + } + + /* + * Make sure that the last byte sequence before sourceLimit is complete + * or runs into a lead byte. + * Do not go back into the bytes that will be read for finishing a partial + * sequence from the previous buffer. + * In the conversion loop compare source with sourceLimit only once + * per multi-byte character. + */ + { + int32_t i, length; + + length=(int32_t)(sourceLimit-source) - (toULimit-oldToULength); + for(i=0; i<3 && i0) { + utf8->toUnicodeStatus=0; + utf8->toULength=0; + goto moreBytes; + /* See note in ucnv_SBCSFromUTF8() about this goto. */ + } + + /* conversion loop */ + while(source0) { + b=*source++; + if((int8_t)b>=0) { + /* convert ASCII */ + if(IS_ASCII_ROUNDTRIP(b, asciiRoundtrips)) { + *target++=b; + --targetCapacity; + continue; + } else { + value=DBCS_RESULT_FROM_UTF8(mbcsIndex, results, 0, b); + if(value==0) { + c=b; + goto unassigned; + } + } + } else { + if(b>0xe0) { + if( /* handle U+1000..U+D7FF inline */ + (((t1=(uint8_t)(source[0]-0x80), b<0xed) && (t1 <= 0x3f)) || + (b==0xed && (t1 <= 0x1f))) && + (t2=(uint8_t)(source[1]-0x80)) <= 0x3f + ) { + c=((b&0xf)<<6)|t1; + source+=2; + value=DBCS_RESULT_FROM_UTF8(mbcsIndex, results, c, t2); + if(value==0) { + c=(c<<6)|t2; + goto unassigned; + } + } else { + c=-1; + } + } else if(b<0xe0) { + if( /* handle U+0080..U+07FF inline */ + b>=0xc2 && + (t1=(uint8_t)(*source-0x80)) <= 0x3f + ) { + c=b&0x1f; + ++source; + value=DBCS_RESULT_FROM_UTF8(mbcsIndex, results, c, t1); + if(value==0) { + c=(c<<6)|t1; + goto unassigned; + } + } else { + c=-1; + } + } else { + c=-1; + } + + if(c<0) { + /* handle "complicated" and error cases, and continuing partial characters */ + oldToULength=0; + toULength=1; + toULimit=utf8_countTrailBytes[b]+1; + c=b; +moreBytes: + while(toULengthtoUBytes[oldToULength++]=*source++; + } + utf8->toUnicodeStatus=c; + utf8->toULength=toULength; + utf8->mode=toULimit; + pToUArgs->source=(char *)source; + pFromUArgs->target=(char *)target; + return; + } + } + + if( toULength==toULimit && /* consumed all trail bytes */ + (toULength==3 || toULength==2) && /* BMP */ + (c-=utf8_offsets[toULength])>=utf8_minLegal[toULength] && + (c<=0xd7ff || 0xe000<=c) /* not a surrogate */ + ) { + stage2Entry=MBCS_STAGE_2_FROM_U(table, c); + } else if( + toULength==toULimit && toULength==4 && + (0x10000<=(c-=utf8_offsets[4]) && c<=0x10ffff) + ) { + /* supplementary code point */ + if(!hasSupplementary) { + /* BMP-only codepages are stored without stage 1 entries for supplementary code points */ + stage2Entry=0; + } else { + stage2Entry=MBCS_STAGE_2_FROM_U(table, c); + } + } else { + /* error handling: illegal UTF-8 byte sequence */ + source-=(toULength-oldToULength); + while(oldToULengthtoUBytes[oldToULength++]=*source++; + } + utf8->toULength=toULength; + pToUArgs->source=(char *)source; + pFromUArgs->target=(char *)target; + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + return; + } + + /* get the bytes and the length for the output */ + /* MBCS_OUTPUT_2 */ + value=MBCS_VALUE_2_FROM_STAGE_2(results, stage2Entry, c); + + /* is this code point assigned, or do we use fallbacks? */ + if(!(MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) || + (UCNV_FROM_U_USE_FALLBACK(cnv, c) && value!=0)) + ) { + goto unassigned; + } + } + } + + /* write the output character bytes from value and length */ + /* from the first if in the loop we know that targetCapacity>0 */ + if(value<=0xff) { + /* this is easy because we know that there is enough space */ + *target++=(uint8_t)value; + --targetCapacity; + } else /* length==2 */ { + *target++=(uint8_t)(value>>8); + if(2<=targetCapacity) { + *target++=(uint8_t)value; + targetCapacity-=2; + } else { + cnv->charErrorBuffer[0]=(char)value; + cnv->charErrorBufferLength=1; + + /* target overflow */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + } + continue; + +unassigned: + { + /* + * Try an extension mapping. + * Pass in no source because we don't have UTF-16 input. + * If we have a partial match on c, we will return and revert + * to UTF-8->UTF-16->charset conversion. + */ + static const UChar nul=0; + const UChar *noSource=&nul; + c=_extFromU(cnv, cnv->sharedData, + c, &noSource, noSource, + &target, target+targetCapacity, + NULL, -1, + pFromUArgs->flush, + pErrorCode); + + if(U_FAILURE(*pErrorCode)) { + /* not mappable or buffer overflow */ + cnv->fromUChar32=c; + break; + } else if(cnv->preFromUFirstCP>=0) { + /* + * Partial match, return and revert to pivoting. + * In normal from-UTF-16 conversion, we would just continue + * but then exit the loop because the extension match would + * have consumed the source. + */ + break; + } else { + /* a mapping was written to the target, continue */ + + /* recalculate the targetCapacity after an extension mapping */ + targetCapacity=(int32_t)(pFromUArgs->targetLimit-(char *)target); + continue; + } + } + } else { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + } + + /* + * The sourceLimit may have been adjusted before the conversion loop + * to stop before a truncated sequence. + * If so, then collect the truncated sequence now. + */ + if(U_SUCCESS(*pErrorCode) && source<(sourceLimit=(uint8_t *)pToUArgs->sourceLimit)) { + c=utf8->toUBytes[0]=b=*source++; + toULength=1; + toULimit=utf8_countTrailBytes[b]+1; + while(sourcetoUBytes[toULength++]=b=*source++; + c=(c<<6)+b; + } + utf8->toUnicodeStatus=c; + utf8->toULength=toULength; + utf8->mode=toULimit; + } + + /* write back the updated pointers */ + pToUArgs->source=(char *)source; + pFromUArgs->target=(char *)target; +} + +/* miscellaneous ------------------------------------------------------------ */ + +static void +ucnv_MBCSGetStarters(const UConverter* cnv, + UBool starters[256], + UErrorCode *pErrorCode) { + const int32_t *state0; + int i; + + state0=cnv->sharedData->mbcs.stateTable[cnv->sharedData->mbcs.dbcsOnlyState]; + for(i=0; i<256; ++i) { + /* all bytes that cause a state transition from state 0 are lead bytes */ + starters[i]= (UBool)MBCS_ENTRY_IS_TRANSITION(state0[i]); + } +} + +/* + * This is an internal function that allows other converter implementations + * to check whether a byte is a lead byte. + */ +U_CFUNC UBool +ucnv_MBCSIsLeadByte(UConverterSharedData *sharedData, char byte) { + return (UBool)MBCS_ENTRY_IS_TRANSITION(sharedData->mbcs.stateTable[0][(uint8_t)byte]); +} + +static void +ucnv_MBCSWriteSub(UConverterFromUnicodeArgs *pArgs, + int32_t offsetIndex, + UErrorCode *pErrorCode) { + UConverter *cnv=pArgs->converter; + char *p, *subchar; + char buffer[4]; + int32_t length; + + /* first, select between subChar and subChar1 */ + if( cnv->subChar1!=0 && + (cnv->sharedData->mbcs.extIndexes!=NULL ? + cnv->useSubChar1 : + (cnv->invalidUCharBuffer[0]<=0xff)) + ) { + /* select subChar1 if it is set (not 0) and the unmappable Unicode code point is up to U+00ff (IBM MBCS behavior) */ + subchar=(char *)&cnv->subChar1; + length=1; + } else { + /* select subChar in all other cases */ + subchar=(char *)cnv->subChars; + length=cnv->subCharLen; + } + + /* reset the selector for the next code point */ + cnv->useSubChar1=FALSE; + + if (cnv->sharedData->mbcs.outputType == MBCS_OUTPUT_2_SISO) { + p=buffer; + + /* fromUnicodeStatus contains prevLength */ + switch(length) { + case 1: + if(cnv->fromUnicodeStatus==2) { + /* DBCS mode and SBCS sub char: change to SBCS */ + cnv->fromUnicodeStatus=1; + *p++=UCNV_SI; + } + *p++=subchar[0]; + break; + case 2: + if(cnv->fromUnicodeStatus<=1) { + /* SBCS mode and DBCS sub char: change to DBCS */ + cnv->fromUnicodeStatus=2; + *p++=UCNV_SO; + } + *p++=subchar[0]; + *p++=subchar[1]; + break; + default: + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return; + } + subchar=buffer; + length=(int32_t)(p-buffer); + } + + ucnv_cbFromUWriteBytes(pArgs, subchar, length, offsetIndex, pErrorCode); +} + +U_CFUNC UConverterType +ucnv_MBCSGetType(const UConverter* converter) { + /* SBCS, DBCS, and EBCDIC_STATEFUL are replaced by MBCS, but here we cheat a little */ + if(converter->sharedData->mbcs.countStates==1) { + return (UConverterType)UCNV_SBCS; + } else if((converter->sharedData->mbcs.outputType&0xff)==MBCS_OUTPUT_2_SISO) { + return (UConverterType)UCNV_EBCDIC_STATEFUL; + } else if(converter->sharedData->staticData->minBytesPerChar==2 && converter->sharedData->staticData->maxBytesPerChar==2) { + return (UConverterType)UCNV_DBCS; + } + return (UConverterType)UCNV_MBCS; +} + +static const UConverterImpl _SBCSUTF8Impl={ + UCNV_MBCS, + + ucnv_MBCSLoad, + ucnv_MBCSUnload, + + ucnv_MBCSOpen, + NULL, + NULL, + + ucnv_MBCSToUnicodeWithOffsets, + ucnv_MBCSToUnicodeWithOffsets, + ucnv_MBCSFromUnicodeWithOffsets, + ucnv_MBCSFromUnicodeWithOffsets, + ucnv_MBCSGetNextUChar, + + ucnv_MBCSGetStarters, + ucnv_MBCSGetName, + ucnv_MBCSWriteSub, + NULL, + ucnv_MBCSGetUnicodeSet, + + NULL, + ucnv_SBCSFromUTF8 +}; + +static const UConverterImpl _DBCSUTF8Impl={ + UCNV_MBCS, + + ucnv_MBCSLoad, + ucnv_MBCSUnload, + + ucnv_MBCSOpen, + NULL, + NULL, + + ucnv_MBCSToUnicodeWithOffsets, + ucnv_MBCSToUnicodeWithOffsets, + ucnv_MBCSFromUnicodeWithOffsets, + ucnv_MBCSFromUnicodeWithOffsets, + ucnv_MBCSGetNextUChar, + + ucnv_MBCSGetStarters, + ucnv_MBCSGetName, + ucnv_MBCSWriteSub, + NULL, + ucnv_MBCSGetUnicodeSet, + + NULL, + ucnv_DBCSFromUTF8 +}; + +static const UConverterImpl _MBCSImpl={ + UCNV_MBCS, + + ucnv_MBCSLoad, + ucnv_MBCSUnload, + + ucnv_MBCSOpen, + NULL, + NULL, + + ucnv_MBCSToUnicodeWithOffsets, + ucnv_MBCSToUnicodeWithOffsets, + ucnv_MBCSFromUnicodeWithOffsets, + ucnv_MBCSFromUnicodeWithOffsets, + ucnv_MBCSGetNextUChar, + + ucnv_MBCSGetStarters, + ucnv_MBCSGetName, + ucnv_MBCSWriteSub, + NULL, + ucnv_MBCSGetUnicodeSet +}; + + +/* Static data is in tools/makeconv/ucnvstat.c for data-based + * converters. Be sure to update it as well. + */ + +const UConverterSharedData _MBCSData={ + sizeof(UConverterSharedData), 1, + NULL, NULL, NULL, FALSE, &_MBCSImpl, + 0 +}; + +#endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */ diff --git a/Build/source/libs/icu/icu-xetex/common/ucnvmbcs.h b/Build/source/libs/icu/icu-xetex/common/ucnvmbcs.h new file mode 100644 index 00000000000..32439cbbebe --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnvmbcs.h @@ -0,0 +1,490 @@ +/* +****************************************************************************** +* +* Copyright (C) 2000-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: ucnvmbcs.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2000jul07 +* created by: Markus W. Scherer +*/ + +#ifndef __UCNVMBCS_H__ +#define __UCNVMBCS_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "unicode/ucnv.h" +#include "ucnv_cnv.h" + +/** + * ICU conversion (.cnv) data file structure, following the usual UDataInfo + * header. + * + * Format version: 6.2 + * + * struct UConverterStaticData -- struct containing the converter name, IBM CCSID, + * min/max bytes per character, etc. + * see ucnv_bld.h + * + * -------------------- + * + * The static data is followed by conversionType-specific data structures. + * At the moment, there are only variations of MBCS converters. They all have + * the same toUnicode structures, while the fromUnicode structures for SBCS + * differ from those for other MBCS-style converters. + * + * _MBCSHeader.version 4.3 optionally modifies the fromUnicode data structures + * slightly and optionally adds a table for conversion to MBCS (non-SBCS) + * charsets. + * + * The modifications are to make the data utf8Friendly. Not every 4.3 file + * file contains utf8Friendly data. + * It is utf8Friendly if _MBCSHeader.version[2]!=0. + * In this case, the data structures are utf8Friendly up to the code point + * maxFastUChar=((_MBCSHeader.version[2]<<8)|0xff) + * + * A utf8Friendly file has fromUnicode stage 3 entries for code points up to + * maxFastUChar allocated in blocks of 64 for indexing with the 6 bits from + * a UTF-8 trail byte. ASCII is allocated linearly with 128 contiguous entries. + * + * In addition, a utf8Friendly MBCS file contains an additional + * uint16_t mbcsIndex[(maxFastUChar+1)>>6]; + * which replaces the stage 1 and 2 tables for indexing with bits from the + * UTF-8 lead byte and middle trail byte. Unlike the older MBCS stage 2 table, + * the mbcsIndex does not contain roundtrip flags. Therefore, all fallbacks + * from code points up to maxFastUChar (and roundtrips to 0x00) are moved to + * the extension data structure. This also allows for faster roundtrip + * conversion from UTF-16. + * + * SBCS files do not contain an additional sbcsIndex[] array because the + * proportional size increase would be noticeable, but the runtime + * code builds one for the code point range for which the runtime conversion + * code is optimized. + * + * For SBCS, maxFastUChar should be at least U+0FFF. The initial makeconv + * implementation sets it to U+1FFF. Because the sbcsIndex is not stored in + * the file, a larger maxFastUChar only affects stage 3 block allocation size + * and is free in empty blocks. (Larger blocks with sparse contents cause larger + * files.) U+1FFF includes almost all of the small scripts. + * U+0FFF covers UTF-8 two-byte sequences and three-byte sequences starting with + * 0xe0. This includes most scripts with legacy SBCS charsets. + * The initial runtime implementation using 4.3 files only builds an sbcsIndex + * for code points up to U+0FFF. + * + * For MBCS, maxFastUChar should be at least U+D7FF (=initial value). + * This boundary is convenient because practically all of the commonly used + * characters are below it, and because it is the boundary to surrogate + * code points, above which special handling is necessary anyway. + * (Surrogate pair assembly for UTF-16, validity checking for UTF-8.) + * + * maxFastUChar could be up to U+FFFF to cover the whole BMP, which could be + * useful especially for conversion from UTF-8 when the input can be assumed + * to be valid, because the surrogate range would then not have to be + * checked. + * (With maxFastUChar=0xffff, makeconv would have to check for mbcsIndex value + * overflow because with the all-unassigned block 0 and nearly full mappings + * from the BMP it is theoretically possible that an index into stage 3 + * exceeds 16 bits.) + * + * _MBCSHeader.version 4.2 adds an optional conversion extension data structure. + * If it is present, then an ICU version reading header versions 4.0 or 4.1 + * will be able to use the base table and ignore the extension. + * + * The unicodeMask in the static data is part of the base table data structure. + * Especially, the UCNV_HAS_SUPPLEMENTARY flag determines the length of the + * fromUnicode stage 1 array. + * The static data unicodeMask refers only to the base table's properties if + * a base table is included. + * In an extension-only file, the static data unicodeMask is 0. + * The extension data indexes have a separate field with the unicodeMask flags. + * + * MBCS-style data structure following the static data. + * Offsets are counted in bytes from the beginning of the MBCS header structure. + * Details about usage in comments in ucnvmbcs.c. + * + * struct _MBCSHeader (see the definition in this header file below) + * contains 32-bit fields as follows: + * 8 values: + * 0 uint8_t[4] MBCS version in UVersionInfo format (currently 4.3.x.0) + * 1 uint32_t countStates + * 2 uint32_t countToUFallbacks + * 3 uint32_t offsetToUCodeUnits + * 4 uint32_t offsetFromUTable + * 5 uint32_t offsetFromUBytes + * 6 uint32_t flags, bits: + * 31.. 8 offsetExtension -- _MBCSHeader.version 4.2 (ICU 2.8) and higher + * 0 for older versions and if + * there is not extension structure + * 7.. 0 outputType + * 7 uint32_t fromUBytesLength -- _MBCSHeader.version 4.1 (ICU 2.4) and higher + * counts bytes in fromUBytes[] + * + * if(outputType==MBCS_OUTPUT_EXT_ONLY) { + * -- base table name for extension-only table + * char baseTableName[variable]; -- with NUL plus padding for 4-alignment + * + * -- all _MBCSHeader fields except for version and flags are 0 + * } else { + * -- normal base table with optional extension + * + * int32_t stateTable[countStates][256]; + * + * struct _MBCSToUFallback { (fallbacks are sorted by offset) + * uint32_t offset; + * UChar32 codePoint; + * } toUFallbacks[countToUFallbacks]; + * + * uint16_t unicodeCodeUnits[(offsetFromUTable-offsetToUCodeUnits)/2]; + * (padded to an even number of units) + * + * -- stage 1 tables + * if(staticData.unicodeMask&UCNV_HAS_SUPPLEMENTARY) { + * -- stage 1 table for all of Unicode + * uint16_t fromUTable[0x440]; (32-bit-aligned) + * } else { + * -- BMP-only tables have a smaller stage 1 table + * uint16_t fromUTable[0x40]; (32-bit-aligned) + * } + * + * -- stage 2 tables + * length determined by top of stage 1 and bottom of stage 3 tables + * if(outputType==MBCS_OUTPUT_1) { + * -- SBCS: pure indexes + * uint16_t stage 2 indexes[?]; + * } else { + * -- DBCS, MBCS, EBCDIC_STATEFUL, ...: roundtrip flags and indexes + * uint32_t stage 2 flags and indexes[?]; + * } + * + * -- stage 3 tables with byte results + * if(outputType==MBCS_OUTPUT_1) { + * -- SBCS: each 16-bit result contains flags and the result byte, see ucnvmbcs.c + * uint16_t fromUBytes[fromUBytesLength/2]; + * } else { + * -- DBCS, MBCS, EBCDIC_STATEFUL, ... 2/3/4 bytes result, see ucnvmbcs.c + * uint8_t fromUBytes[fromUBytesLength]; or + * uint16_t fromUBytes[fromUBytesLength/2]; or + * uint32_t fromUBytes[fromUBytesLength/4]; + * } + * + * -- optional utf8Friendly mbcsIndex -- _MBCSHeader.version 4.3 (ICU 3.8) and higher + * if(outputType!=MBCS_OUTPUT_1 && + * _MBCSHeader.version[1]>=3 && + * (maxFastUChar=_MBCSHeader.version[2])!=0 + * ) { + * maxFastUChar=(maxFastUChar<<8)|0xff; + * uint16_t mbcsIndex[(maxFastUChar+1)>>6]; + * } + * } + * + * -- extension table, details see ucnv_ext.h + * int32_t indexes[>=32]; ... + */ + +/* MBCS converter data and state -------------------------------------------- */ + +enum { + MBCS_MAX_STATE_COUNT=128 +}; + +/** + * MBCS action codes for conversions to Unicode. + * These values are in bits 23..20 of the state table entries. + */ +enum { + MBCS_STATE_VALID_DIRECT_16, + MBCS_STATE_VALID_DIRECT_20, + + MBCS_STATE_FALLBACK_DIRECT_16, + MBCS_STATE_FALLBACK_DIRECT_20, + + MBCS_STATE_VALID_16, + MBCS_STATE_VALID_16_PAIR, + + MBCS_STATE_UNASSIGNED, + MBCS_STATE_ILLEGAL, + + MBCS_STATE_CHANGE_ONLY +}; + +/* Macros for state table entries */ +#define MBCS_ENTRY_TRANSITION(state, offset) (int32_t)(((int32_t)(state)<<24L)|(offset)) +#define MBCS_ENTRY_TRANSITION_SET_OFFSET(entry, offset) (int32_t)(((entry)&0xff000000)|(offset)) +#define MBCS_ENTRY_TRANSITION_ADD_OFFSET(entry, offset) (int32_t)((entry)+(offset)) + +#define MBCS_ENTRY_FINAL(state, action, value) (int32_t)(0x80000000|((int32_t)(state)<<24L)|((action)<<20L)|(value)) +#define MBCS_ENTRY_SET_FINAL(entry) (int32_t)((entry)|0x80000000) +#define MBCS_ENTRY_FINAL_SET_ACTION(entry, action) (int32_t)(((entry)&0xff0fffff)|((int32_t)(action)<<20L)) +#define MBCS_ENTRY_FINAL_SET_VALUE(entry, value) (int32_t)(((entry)&0xfff00000)|(value)) +#define MBCS_ENTRY_FINAL_SET_ACTION_VALUE(entry, action, value) (int32_t)(((entry)&0xff000000)|((int32_t)(action)<<20L)|(value)) + +#define MBCS_ENTRY_SET_STATE(entry, state) (int32_t)(((entry)&0x80ffffff)|((int32_t)(state)<<24L)) + +#define MBCS_ENTRY_STATE(entry) (((entry)>>24)&0x7f) + +#define MBCS_ENTRY_IS_TRANSITION(entry) ((entry)>=0) +#define MBCS_ENTRY_IS_FINAL(entry) ((entry)<0) + +#define MBCS_ENTRY_TRANSITION_STATE(entry) ((entry)>>24) +#define MBCS_ENTRY_TRANSITION_OFFSET(entry) ((entry)&0xffffff) + +#define MBCS_ENTRY_FINAL_STATE(entry) (((entry)>>24)&0x7f) +#define MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry) ((entry)<(int32_t)0x80100000) +#define MBCS_ENTRY_FINAL_ACTION(entry) (((entry)>>20)&0xf) +#define MBCS_ENTRY_FINAL_VALUE(entry) ((entry)&0xfffff) +#define MBCS_ENTRY_FINAL_VALUE_16(entry) (uint16_t)(entry) + +#define IS_ASCII_ROUNDTRIP(b, asciiRoundtrips) (((asciiRoundtrips) & (1<<((b)>>2)))!=0) + +/* single-byte fromUnicode: get the 16-bit result word */ +#define MBCS_SINGLE_RESULT_FROM_U(table, results, c) (results)[ (table)[ (table)[(c)>>10] +(((c)>>4)&0x3f) ] +((c)&0xf) ] + +/* single-byte fromUnicode using the sbcsIndex */ +#define SBCS_RESULT_FROM_LOW_BMP(table, results, c) (results)[ (table)[(c)>>6] +((c)&0x3f) ] + +/* single-byte fromUTF8 using the sbcsIndex; l and t must be masked externally; can be l=0 and t<=0x7f */ +#define SBCS_RESULT_FROM_UTF8(table, results, l, t) (results)[ (table)[l] +(t) ] + +/* multi-byte fromUnicode: get the 32-bit stage 2 entry */ +#define MBCS_STAGE_2_FROM_U(table, c) ((const uint32_t *)(table))[ (table)[(c)>>10] +(((c)>>4)&0x3f) ] +#define MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) ( ((stage2Entry) & ((uint32_t)1<< (16+((c)&0xf)) )) !=0) + +#define MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c) ((uint16_t *)(bytes))[16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf)] +#define MBCS_VALUE_4_FROM_STAGE_2(bytes, stage2Entry, c) ((uint32_t *)(bytes))[16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf)] + +#define MBCS_POINTER_3_FROM_STAGE_2(bytes, stage2Entry, c) ((bytes)+(16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf))*3) + +/* double-byte fromUnicode using the mbcsIndex */ +#define DBCS_RESULT_FROM_MOST_BMP(table, results, c) (results)[ (table)[(c)>>6] +((c)&0x3f) ] + +/* double-byte fromUTF8 using the mbcsIndex; l and t1 combined into lt1; lt1 and t2 must be masked externally */ +#define DBCS_RESULT_FROM_UTF8(table, results, lt1, t2) (results)[ (table)[lt1] +(t2) ] + + +/** + * MBCS output types for conversions from Unicode. + * These per-converter types determine the storage method in stage 3 of the lookup table, + * mostly how many bytes are stored per entry. + */ +enum { + MBCS_OUTPUT_1, /* 0 */ + MBCS_OUTPUT_2, /* 1 */ + MBCS_OUTPUT_3, /* 2 */ + MBCS_OUTPUT_4, /* 3 */ + + MBCS_OUTPUT_3_EUC=8, /* 8 */ + MBCS_OUTPUT_4_EUC, /* 9 */ + + MBCS_OUTPUT_2_SISO=12, /* c */ + MBCS_OUTPUT_2_HZ, /* d */ + + MBCS_OUTPUT_EXT_ONLY, /* e */ + + MBCS_OUTPUT_COUNT, + + MBCS_OUTPUT_DBCS_ONLY=0xdb /* runtime-only type for DBCS-only handling of SISO tables */ +}; + +/** + * Fallbacks to Unicode are stored outside the normal state table and code point structures + * in a vector of items of this type. They are sorted by offset. + */ +typedef struct { + uint32_t offset; + UChar32 codePoint; +} _MBCSToUFallback; + +/** Constants for fast and UTF-8-friendly conversion. */ +enum { + SBCS_FAST_MAX=0x0fff, /* maximum code point with UTF-8-friendly SBCS runtime code, see makeconv SBCS_UTF8_MAX */ + SBCS_FAST_LIMIT=SBCS_FAST_MAX+1, /* =0x1000 */ + MBCS_FAST_MAX=0xd7ff, /* maximum code point with UTF-8-friendly MBCS runtime code, see makeconv MBCS_UTF8_MAX */ + MBCS_FAST_LIMIT=MBCS_FAST_MAX+1 /* =0xd800 */ +}; + +/** + * This is the MBCS part of the UConverterTable union (a runtime data structure). + * It keeps all the per-converter data and points into the loaded mapping tables. + * + * utf8Friendly data structures added with _MBCSHeader.version 4.3 + */ +typedef struct UConverterMBCSTable { + /* toUnicode */ + uint8_t countStates, dbcsOnlyState, stateTableOwned; + uint32_t countToUFallbacks; + + const int32_t (*stateTable)/*[countStates]*/[256]; + int32_t (*swapLFNLStateTable)/*[countStates]*/[256]; /* for swaplfnl */ + const uint16_t *unicodeCodeUnits/*[countUnicodeResults]*/; + const _MBCSToUFallback *toUFallbacks; + + /* fromUnicode */ + const uint16_t *fromUnicodeTable; + const uint16_t *mbcsIndex; /* for fast conversion from most of BMP to MBCS (utf8Friendly data) */ + uint16_t sbcsIndex[SBCS_FAST_LIMIT>>6]; /* for fast conversion from low BMP to SBCS (utf8Friendly data) */ + const uint8_t *fromUnicodeBytes; + uint8_t *swapLFNLFromUnicodeBytes; /* for swaplfnl */ + uint32_t fromUBytesLength; + uint8_t outputType, unicodeMask; + UBool utf8Friendly; /* for utf8Friendly data */ + UChar maxFastUChar; /* for utf8Friendly data */ + + /* roundtrips */ + uint32_t asciiRoundtrips; + + /* converter name for swaplfnl */ + char *swapLFNLName; + + /* extension data */ + struct UConverterSharedData *baseSharedData; + const int32_t *extIndexes; +} UConverterMBCSTable; + +/** + * MBCS data header. See data format description above. + */ +typedef struct { + UVersionInfo version; + uint32_t countStates, + countToUFallbacks, + offsetToUCodeUnits, + offsetFromUTable, + offsetFromUBytes, + flags, + fromUBytesLength; +} _MBCSHeader; + +/* + * This is a simple version of _MBCSGetNextUChar() that is used + * by other converter implementations. + * It only returns an "assigned" result if it consumes the entire input. + * It does not use state from the converter, nor error codes. + * It does not handle the EBCDIC swaplfnl option (set in UConverter). + * It handles conversion extensions but not GB 18030. + * + * Return value: + * U+fffe unassigned + * U+ffff illegal + * otherwise the Unicode code point + */ +U_CFUNC UChar32 +ucnv_MBCSSimpleGetNextUChar(UConverterSharedData *sharedData, + const char *source, int32_t length, + UBool useFallback); + +/** + * This version of _MBCSSimpleGetNextUChar() is optimized for single-byte, single-state codepages. + * It does not handle the EBCDIC swaplfnl option (set in UConverter). + * It does not handle conversion extensions (_extToU()). + */ +U_CFUNC UChar32 +ucnv_MBCSSingleSimpleGetNextUChar(UConverterSharedData *sharedData, + uint8_t b, UBool useFallback); + +/** + * This macro version of _MBCSSingleSimpleGetNextUChar() gets a code point from a byte. + * It works for single-byte, single-state codepages that only map + * to and from BMP code points, and it always + * returns fallback values. + */ +#define _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP(sharedData, b) \ + (UChar)MBCS_ENTRY_FINAL_VALUE_16((sharedData)->mbcs.stateTable[0][(uint8_t)(b)]) + +/** + * This is an internal function that allows other converter implementations + * to check whether a byte is a lead byte. + */ +U_CFUNC UBool +ucnv_MBCSIsLeadByte(UConverterSharedData *sharedData, char byte); + +/** This is a macro version of _MBCSIsLeadByte(). */ +#define _MBCS_IS_LEAD_BYTE(sharedData, byte) \ + (UBool)MBCS_ENTRY_IS_TRANSITION((sharedData)->mbcs.stateTable[0][(uint8_t)(byte)]) + +/* + * This is another simple conversion function for internal use by other + * conversion implementations. + * It does not use the converter state nor call callbacks. + * It does not handle the EBCDIC swaplfnl option (set in UConverter). + * It handles conversion extensions but not GB 18030. + * + * It converts one single Unicode code point into codepage bytes, encoded + * as one 32-bit value. The function returns the number of bytes in *pValue: + * 1..4 the number of bytes in *pValue + * 0 unassigned (*pValue undefined) + * -1 illegal (currently not used, *pValue undefined) + * + * *pValue will contain the resulting bytes with the last byte in bits 7..0, + * the second to last byte in bits 15..8, etc. + * Currently, the function assumes but does not check that 0<=c<=0x10ffff. + */ +U_CFUNC int32_t +ucnv_MBCSFromUChar32(UConverterSharedData *sharedData, + UChar32 c, uint32_t *pValue, + UBool useFallback); + +/** + * This version of _MBCSFromUChar32() is optimized for single-byte codepages. + * It does not handle the EBCDIC swaplfnl option (set in UConverter). + * + * It returns the codepage byte for the code point, or -1 if it is unassigned. + */ +U_CFUNC int32_t +ucnv_MBCSSingleFromUChar32(UConverterSharedData *sharedData, + UChar32 c, + UBool useFallback); + +/** + * SBCS, DBCS, and EBCDIC_STATEFUL are replaced by MBCS, but + * we cheat a little about the type, returning the old types if appropriate. + */ +U_CFUNC UConverterType +ucnv_MBCSGetType(const UConverter* converter); + +U_CFUNC void +ucnv_MBCSFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, + UErrorCode *pErrorCode); +U_CFUNC void +ucnv_MBCSToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs, + UErrorCode *pErrorCode); + +/* + * Internal function returning a UnicodeSet for toUnicode() conversion. + * Currently only used for ISO-2022-CN, and only handles roundtrip mappings. + * In the future, if we add support for reverse-fallback sets, this function + * needs to be updated, and called for each initial state. + * Does not currently handle extensions. + * Does not empty the set first. + */ +U_CFUNC void +ucnv_MBCSGetUnicodeSetForBytes(const UConverterSharedData *sharedData, + const USetAdder *sa, + UConverterUnicodeSet which, + uint8_t state, int32_t lowByte, int32_t highByte, + UErrorCode *pErrorCode); + +/* + * Internal function returning a UnicodeSet for toUnicode() conversion. + * Currently only used for ISO-2022-CN, and only handles roundtrip mappings. + * In the future, if we add support for fallback sets, this function + * needs to be updated. + * Handles extensions. + * Does not empty the set first. + */ +U_CFUNC void +ucnv_MBCSGetUnicodeSetForUnicode(const UConverterSharedData *sharedData, + const USetAdder *sa, + UConverterUnicodeSet which, + UErrorCode *pErrorCode); + +#endif + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/ucnvscsu.c b/Build/source/libs/icu/icu-xetex/common/ucnvscsu.c new file mode 100644 index 00000000000..910a4920225 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucnvscsu.c @@ -0,0 +1,2015 @@ +/* +****************************************************************************** +* +* Copyright (C) 2000-2006, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: ucnvscsu.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2000nov18 +* created by: Markus W. Scherer +* +* This is an implementation of the Standard Compression Scheme for Unicode +* as defined in http://www.unicode.org/unicode/reports/tr6/ . +* Reserved commands and window settings are treated as illegal sequences and +* will result in callback calls. +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "unicode/ucnv.h" +#include "unicode/ucnv_cb.h" +#include "ucnv_bld.h" +#include "ucnv_cnv.h" +#include "cmemory.h" + +/* SCSU definitions --------------------------------------------------------- */ + +/* SCSU command byte values */ +enum { + SQ0=0x01, /* Quote from window pair 0 */ + SQ7=0x08, /* Quote from window pair 7 */ + SDX=0x0B, /* Define a window as extended */ + Srs=0x0C, /* reserved */ + SQU=0x0E, /* Quote a single Unicode character */ + SCU=0x0F, /* Change to Unicode mode */ + SC0=0x10, /* Select window 0 */ + SC7=0x17, /* Select window 7 */ + SD0=0x18, /* Define and select window 0 */ + SD7=0x1F, /* Define and select window 7 */ + + UC0=0xE0, /* Select window 0 */ + UC7=0xE7, /* Select window 7 */ + UD0=0xE8, /* Define and select window 0 */ + UD7=0xEF, /* Define and select window 7 */ + UQU=0xF0, /* Quote a single Unicode character */ + UDX=0xF1, /* Define a Window as extended */ + Urs=0xF2 /* reserved */ +}; + +enum { + /* + * Unicode code points from 3400 to E000 are not adressible by + * dynamic window, since in these areas no short run alphabets are + * found. Therefore add gapOffset to all values from gapThreshold. + */ + gapThreshold=0x68, + gapOffset=0xAC00, + + /* values between reservedStart and fixedThreshold are reserved */ + reservedStart=0xA8, + + /* use table of predefined fixed offsets for values from fixedThreshold */ + fixedThreshold=0xF9 +}; + +/* constant offsets for the 8 static windows */ +static const uint32_t staticOffsets[8]={ + 0x0000, /* ASCII for quoted tags */ + 0x0080, /* Latin - 1 Supplement (for access to punctuation) */ + 0x0100, /* Latin Extended-A */ + 0x0300, /* Combining Diacritical Marks */ + 0x2000, /* General Punctuation */ + 0x2080, /* Currency Symbols */ + 0x2100, /* Letterlike Symbols and Number Forms */ + 0x3000 /* CJK Symbols and punctuation */ +}; + +/* initial offsets for the 8 dynamic (sliding) windows */ +static const uint32_t initialDynamicOffsets[8]={ + 0x0080, /* Latin-1 */ + 0x00C0, /* Latin Extended A */ + 0x0400, /* Cyrillic */ + 0x0600, /* Arabic */ + 0x0900, /* Devanagari */ + 0x3040, /* Hiragana */ + 0x30A0, /* Katakana */ + 0xFF00 /* Fullwidth ASCII */ +}; + +/* Table of fixed predefined Offsets */ +static const uint32_t fixedOffsets[]={ + /* 0xF9 */ 0x00C0, /* Latin-1 Letters + half of Latin Extended A */ + /* 0xFA */ 0x0250, /* IPA extensions */ + /* 0xFB */ 0x0370, /* Greek */ + /* 0xFC */ 0x0530, /* Armenian */ + /* 0xFD */ 0x3040, /* Hiragana */ + /* 0xFE */ 0x30A0, /* Katakana */ + /* 0xFF */ 0xFF60 /* Halfwidth Katakana */ +}; + +/* state values */ +enum { + readCommand, + quotePairOne, + quotePairTwo, + quoteOne, + definePairOne, + definePairTwo, + defineOne +}; + +typedef struct SCSUData { + /* dynamic window offsets, intitialize to default values from initialDynamicOffsets */ + uint32_t toUDynamicOffsets[8]; + uint32_t fromUDynamicOffsets[8]; + + /* state machine state - toUnicode */ + UBool toUIsSingleByteMode; + uint8_t toUState; + int8_t toUQuoteWindow, toUDynamicWindow; + uint8_t toUByteOne; + uint8_t toUPadding[3]; + + /* state machine state - fromUnicode */ + UBool fromUIsSingleByteMode; + int8_t fromUDynamicWindow; + + /* + * windowUse[] keeps track of the use of the dynamic windows: + * At nextWindowUseIndex there is the least recently used window, + * and the following windows (in a wrapping manner) are more and more + * recently used. + * At nextWindowUseIndex-1 there is the most recently used window. + */ + uint8_t locale; + int8_t nextWindowUseIndex; + int8_t windowUse[8]; +} SCSUData; + +static const int8_t initialWindowUse[8]={ 7, 0, 3, 2, 4, 5, 6, 1 }; +static const int8_t initialWindowUse_ja[8]={ 3, 2, 4, 1, 0, 7, 5, 6 }; + +enum { + lGeneric, l_ja +}; + +/* SCSU setup functions ----------------------------------------------------- */ + +static void +_SCSUReset(UConverter *cnv, UConverterResetChoice choice) { + SCSUData *scsu=(SCSUData *)cnv->extraInfo; + + if(choice<=UCNV_RESET_TO_UNICODE) { + /* reset toUnicode */ + uprv_memcpy(scsu->toUDynamicOffsets, initialDynamicOffsets, 32); + + scsu->toUIsSingleByteMode=TRUE; + scsu->toUState=readCommand; + scsu->toUQuoteWindow=scsu->toUDynamicWindow=0; + scsu->toUByteOne=0; + + cnv->toULength=0; + } + if(choice!=UCNV_RESET_TO_UNICODE) { + /* reset fromUnicode */ + uprv_memcpy(scsu->fromUDynamicOffsets, initialDynamicOffsets, 32); + + scsu->fromUIsSingleByteMode=TRUE; + scsu->fromUDynamicWindow=0; + + scsu->nextWindowUseIndex=0; + switch(scsu->locale) { + case l_ja: + uprv_memcpy(scsu->windowUse, initialWindowUse_ja, 8); + break; + default: + uprv_memcpy(scsu->windowUse, initialWindowUse, 8); + break; + } + + cnv->fromUChar32=0; + } +} + +static void +_SCSUOpen(UConverter *cnv, + const char *name, + const char *locale, + uint32_t options, + UErrorCode *pErrorCode) { + cnv->extraInfo=uprv_malloc(sizeof(SCSUData)); + if(cnv->extraInfo!=NULL) { + if(locale!=NULL && locale[0]=='j' && locale[1]=='a' && (locale[2]==0 || locale[2]=='_')) { + ((SCSUData *)cnv->extraInfo)->locale=l_ja; + } else { + ((SCSUData *)cnv->extraInfo)->locale=lGeneric; + } + _SCSUReset(cnv, UCNV_RESET_BOTH); + } else { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + } + + /* Set the substitution character U+fffd as a Unicode string. */ + cnv->subUChars[0]=0xfffd; + cnv->subCharLen=-1; +} + +static void +_SCSUClose(UConverter *cnv) { + if(cnv->extraInfo!=NULL) { + if(!cnv->isExtraLocal) { + uprv_free(cnv->extraInfo); + } + cnv->extraInfo=NULL; + } +} + +/* SCSU-to-Unicode conversion functions ------------------------------------- */ + +static void +_SCSUToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv; + SCSUData *scsu; + const uint8_t *source, *sourceLimit; + UChar *target; + const UChar *targetLimit; + int32_t *offsets; + UBool isSingleByteMode; + uint8_t state, byteOne; + int8_t quoteWindow, dynamicWindow; + + int32_t sourceIndex, nextSourceIndex; + + uint8_t b; + + /* set up the local pointers */ + cnv=pArgs->converter; + scsu=(SCSUData *)cnv->extraInfo; + + source=(const uint8_t *)pArgs->source; + sourceLimit=(const uint8_t *)pArgs->sourceLimit; + target=pArgs->target; + targetLimit=pArgs->targetLimit; + offsets=pArgs->offsets; + + /* get the state machine state */ + isSingleByteMode=scsu->toUIsSingleByteMode; + state=scsu->toUState; + quoteWindow=scsu->toUQuoteWindow; + dynamicWindow=scsu->toUDynamicWindow; + byteOne=scsu->toUByteOne; + + /* sourceIndex=-1 if the current character began in the previous buffer */ + sourceIndex=state==readCommand ? 0 : -1; + nextSourceIndex=0; + + /* + * conversion "loop" + * + * For performance, this is not a normal C loop. + * Instead, there are two code blocks for the two SCSU modes. + * The function branches to either one, and a change of the mode is done with a goto to + * the other branch. + * + * Each branch has two conventional loops: + * - a fast-path loop for the most common codes in the mode + * - a loop for all other codes in the mode + * When the fast-path runs into a code that it cannot handle, its loop ends and it + * runs into the following loop to handle the other codes. + * The end of the input or output buffer is also handled by the slower loop. + * The slow loop jumps (goto) to the fast-path loop again as soon as possible. + * + * The callback handling is done by returning with an error code. + * The conversion framework actually calls the callback function. + */ + if(isSingleByteMode) { + /* fast path for single-byte mode */ + if(state==readCommand) { +fastSingle: + while(source=0x20) { + ++source; + ++nextSourceIndex; + if(b<=0x7f) { + /* write US-ASCII graphic character or DEL */ + *target++=(UChar)b; + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + } else { + /* write from dynamic window */ + uint32_t c=scsu->toUDynamicOffsets[dynamicWindow]+(b&0x7f); + if(c<=0xffff) { + *target++=(UChar)c; + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + } else { + /* output surrogate pair */ + *target++=(UChar)(0xd7c0+(c>>10)); + if(targetUCharErrorBuffer[0]=(UChar)(0xdc00|(c&0x3ff)); + cnv->UCharErrorBufferLength=1; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + goto endloop; + } + } + } + sourceIndex=nextSourceIndex; + } + } + + /* normal state machine for single-byte mode, minus handling for what fastSingle covers */ +singleByteMode: + while(source=targetLimit) { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + b=*source++; + ++nextSourceIndex; + switch(state) { + case readCommand: + /* redundant conditions are commented out */ + /* here: b<0x20 because otherwise we would be in fastSingle */ + if((1UL<toUBytes[0]=b; + cnv->toULength=1; + goto endloop; + } + + /* store the first byte of a multibyte sequence in toUBytes[] */ + cnv->toUBytes[0]=b; + cnv->toULength=1; + break; + case quotePairOne: + byteOne=b; + cnv->toUBytes[1]=b; + cnv->toULength=2; + state=quotePairTwo; + break; + case quotePairTwo: + *target++=(UChar)((byteOne<<8)|b); + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + sourceIndex=nextSourceIndex; + state=readCommand; + goto fastSingle; + case quoteOne: + if(b<0x80) { + /* all static offsets are in the BMP */ + *target++=(UChar)(staticOffsets[quoteWindow]+b); + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + } else { + /* write from dynamic window */ + uint32_t c=scsu->toUDynamicOffsets[quoteWindow]+(b&0x7f); + if(c<=0xffff) { + *target++=(UChar)c; + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + } else { + /* output surrogate pair */ + *target++=(UChar)(0xd7c0+(c>>10)); + if(targetUCharErrorBuffer[0]=(UChar)(0xdc00|(c&0x3ff)); + cnv->UCharErrorBufferLength=1; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + goto endloop; + } + } + } + sourceIndex=nextSourceIndex; + state=readCommand; + goto fastSingle; + case definePairOne: + dynamicWindow=(int8_t)((b>>5)&7); + byteOne=(uint8_t)(b&0x1f); + cnv->toUBytes[1]=b; + cnv->toULength=2; + state=definePairTwo; + break; + case definePairTwo: + scsu->toUDynamicOffsets[dynamicWindow]=0x10000+(byteOne<<15UL | b<<7UL); + sourceIndex=nextSourceIndex; + state=readCommand; + goto fastSingle; + case defineOne: + if(b==0) { + /* callback(illegal): Reserved window offset value 0 */ + cnv->toUBytes[1]=b; + cnv->toULength=2; + goto endloop; + } else if(btoUDynamicOffsets[dynamicWindow]=b<<7UL; + } else if((uint8_t)(b-gapThreshold)<(reservedStart-gapThreshold)) { + scsu->toUDynamicOffsets[dynamicWindow]=(b<<7UL)+gapOffset; + } else if(b>=fixedThreshold) { + scsu->toUDynamicOffsets[dynamicWindow]=fixedOffsets[b-fixedThreshold]; + } else { + /* callback(illegal): Reserved window offset value 0xa8..0xf8 */ + cnv->toUBytes[1]=b; + cnv->toULength=2; + goto endloop; + } + sourceIndex=nextSourceIndex; + state=readCommand; + goto fastSingle; + } + } + } else { + /* fast path for Unicode mode */ + if(state==readCommand) { +fastUnicode: + while(source+1(Urs-UC0)) { + *target++=(UChar)((b<<8)|source[1]); + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + sourceIndex=nextSourceIndex; + nextSourceIndex+=2; + source+=2; + } + } + + /* normal state machine for Unicode mode */ +/* unicodeByteMode: */ + while(source=targetLimit) { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + b=*source++; + ++nextSourceIndex; + switch(state) { + case readCommand: + if((uint8_t)(b-UC0)>(Urs-UC0)) { + byteOne=b; + cnv->toUBytes[0]=b; + cnv->toULength=1; + state=quotePairTwo; + } else if(/* UC0<=b && */ b<=UC7) { + dynamicWindow=(int8_t)(b-UC0); + sourceIndex=nextSourceIndex; + isSingleByteMode=TRUE; + goto fastSingle; + } else if(/* UD0<=b && */ b<=UD7) { + dynamicWindow=(int8_t)(b-UD0); + isSingleByteMode=TRUE; + cnv->toUBytes[0]=b; + cnv->toULength=1; + state=defineOne; + goto singleByteMode; + } else if(b==UDX) { + isSingleByteMode=TRUE; + cnv->toUBytes[0]=b; + cnv->toULength=1; + state=definePairOne; + goto singleByteMode; + } else if(b==UQU) { + cnv->toUBytes[0]=b; + cnv->toULength=1; + state=quotePairOne; + } else /* Urs */ { + /* callback(illegal) */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + cnv->toUBytes[0]=b; + cnv->toULength=1; + goto endloop; + } + break; + case quotePairOne: + byteOne=b; + cnv->toUBytes[1]=b; + cnv->toULength=2; + state=quotePairTwo; + break; + case quotePairTwo: + *target++=(UChar)((byteOne<<8)|b); + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + sourceIndex=nextSourceIndex; + state=readCommand; + goto fastUnicode; + } + } + } +endloop: + + /* set the converter state back into UConverter */ + if(U_FAILURE(*pErrorCode) && *pErrorCode!=U_BUFFER_OVERFLOW_ERROR) { + /* reset to deal with the next character */ + state=readCommand; + } else if(state==readCommand) { + /* not in a multi-byte sequence, reset toULength */ + cnv->toULength=0; + } + scsu->toUIsSingleByteMode=isSingleByteMode; + scsu->toUState=state; + scsu->toUQuoteWindow=quoteWindow; + scsu->toUDynamicWindow=dynamicWindow; + scsu->toUByteOne=byteOne; + + /* write back the updated pointers */ + pArgs->source=(const char *)source; + pArgs->target=target; + pArgs->offsets=offsets; + return; +} + +/* + * Identical to _SCSUToUnicodeWithOffsets but without offset handling. + * If a change is made in the original function, then either + * change this function the same way or + * re-copy the original function and remove the variables + * offsets, sourceIndex, and nextSourceIndex. + */ +static void +_SCSUToUnicode(UConverterToUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv; + SCSUData *scsu; + const uint8_t *source, *sourceLimit; + UChar *target; + const UChar *targetLimit; + UBool isSingleByteMode; + uint8_t state, byteOne; + int8_t quoteWindow, dynamicWindow; + + uint8_t b; + + /* set up the local pointers */ + cnv=pArgs->converter; + scsu=(SCSUData *)cnv->extraInfo; + + source=(const uint8_t *)pArgs->source; + sourceLimit=(const uint8_t *)pArgs->sourceLimit; + target=pArgs->target; + targetLimit=pArgs->targetLimit; + + /* get the state machine state */ + isSingleByteMode=scsu->toUIsSingleByteMode; + state=scsu->toUState; + quoteWindow=scsu->toUQuoteWindow; + dynamicWindow=scsu->toUDynamicWindow; + byteOne=scsu->toUByteOne; + + /* + * conversion "loop" + * + * For performance, this is not a normal C loop. + * Instead, there are two code blocks for the two SCSU modes. + * The function branches to either one, and a change of the mode is done with a goto to + * the other branch. + * + * Each branch has two conventional loops: + * - a fast-path loop for the most common codes in the mode + * - a loop for all other codes in the mode + * When the fast-path runs into a code that it cannot handle, its loop ends and it + * runs into the following loop to handle the other codes. + * The end of the input or output buffer is also handled by the slower loop. + * The slow loop jumps (goto) to the fast-path loop again as soon as possible. + * + * The callback handling is done by returning with an error code. + * The conversion framework actually calls the callback function. + */ + if(isSingleByteMode) { + /* fast path for single-byte mode */ + if(state==readCommand) { +fastSingle: + while(source=0x20) { + ++source; + if(b<=0x7f) { + /* write US-ASCII graphic character or DEL */ + *target++=(UChar)b; + } else { + /* write from dynamic window */ + uint32_t c=scsu->toUDynamicOffsets[dynamicWindow]+(b&0x7f); + if(c<=0xffff) { + *target++=(UChar)c; + } else { + /* output surrogate pair */ + *target++=(UChar)(0xd7c0+(c>>10)); + if(targetUCharErrorBuffer[0]=(UChar)(0xdc00|(c&0x3ff)); + cnv->UCharErrorBufferLength=1; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + goto endloop; + } + } + } + } + } + + /* normal state machine for single-byte mode, minus handling for what fastSingle covers */ +singleByteMode: + while(source=targetLimit) { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + b=*source++; + switch(state) { + case readCommand: + /* redundant conditions are commented out */ + /* here: b<0x20 because otherwise we would be in fastSingle */ + if((1UL<toUBytes[0]=b; + cnv->toULength=1; + goto endloop; + } + + /* store the first byte of a multibyte sequence in toUBytes[] */ + cnv->toUBytes[0]=b; + cnv->toULength=1; + break; + case quotePairOne: + byteOne=b; + cnv->toUBytes[1]=b; + cnv->toULength=2; + state=quotePairTwo; + break; + case quotePairTwo: + *target++=(UChar)((byteOne<<8)|b); + state=readCommand; + goto fastSingle; + case quoteOne: + if(b<0x80) { + /* all static offsets are in the BMP */ + *target++=(UChar)(staticOffsets[quoteWindow]+b); + } else { + /* write from dynamic window */ + uint32_t c=scsu->toUDynamicOffsets[quoteWindow]+(b&0x7f); + if(c<=0xffff) { + *target++=(UChar)c; + } else { + /* output surrogate pair */ + *target++=(UChar)(0xd7c0+(c>>10)); + if(targetUCharErrorBuffer[0]=(UChar)(0xdc00|(c&0x3ff)); + cnv->UCharErrorBufferLength=1; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + goto endloop; + } + } + } + state=readCommand; + goto fastSingle; + case definePairOne: + dynamicWindow=(int8_t)((b>>5)&7); + byteOne=(uint8_t)(b&0x1f); + cnv->toUBytes[1]=b; + cnv->toULength=2; + state=definePairTwo; + break; + case definePairTwo: + scsu->toUDynamicOffsets[dynamicWindow]=0x10000+(byteOne<<15UL | b<<7UL); + state=readCommand; + goto fastSingle; + case defineOne: + if(b==0) { + /* callback(illegal): Reserved window offset value 0 */ + cnv->toUBytes[1]=b; + cnv->toULength=2; + goto endloop; + } else if(btoUDynamicOffsets[dynamicWindow]=b<<7UL; + } else if((uint8_t)(b-gapThreshold)<(reservedStart-gapThreshold)) { + scsu->toUDynamicOffsets[dynamicWindow]=(b<<7UL)+gapOffset; + } else if(b>=fixedThreshold) { + scsu->toUDynamicOffsets[dynamicWindow]=fixedOffsets[b-fixedThreshold]; + } else { + /* callback(illegal): Reserved window offset value 0xa8..0xf8 */ + cnv->toUBytes[1]=b; + cnv->toULength=2; + goto endloop; + } + state=readCommand; + goto fastSingle; + } + } + } else { + /* fast path for Unicode mode */ + if(state==readCommand) { +fastUnicode: + while(source+1(Urs-UC0)) { + *target++=(UChar)((b<<8)|source[1]); + source+=2; + } + } + + /* normal state machine for Unicode mode */ +/* unicodeByteMode: */ + while(source=targetLimit) { + /* target is full */ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + break; + } + b=*source++; + switch(state) { + case readCommand: + if((uint8_t)(b-UC0)>(Urs-UC0)) { + byteOne=b; + cnv->toUBytes[0]=b; + cnv->toULength=1; + state=quotePairTwo; + } else if(/* UC0<=b && */ b<=UC7) { + dynamicWindow=(int8_t)(b-UC0); + isSingleByteMode=TRUE; + goto fastSingle; + } else if(/* UD0<=b && */ b<=UD7) { + dynamicWindow=(int8_t)(b-UD0); + isSingleByteMode=TRUE; + cnv->toUBytes[0]=b; + cnv->toULength=1; + state=defineOne; + goto singleByteMode; + } else if(b==UDX) { + isSingleByteMode=TRUE; + cnv->toUBytes[0]=b; + cnv->toULength=1; + state=definePairOne; + goto singleByteMode; + } else if(b==UQU) { + cnv->toUBytes[0]=b; + cnv->toULength=1; + state=quotePairOne; + } else /* Urs */ { + /* callback(illegal) */ + *pErrorCode=U_ILLEGAL_CHAR_FOUND; + cnv->toUBytes[0]=b; + cnv->toULength=1; + goto endloop; + } + break; + case quotePairOne: + byteOne=b; + cnv->toUBytes[1]=b; + cnv->toULength=2; + state=quotePairTwo; + break; + case quotePairTwo: + *target++=(UChar)((byteOne<<8)|b); + state=readCommand; + goto fastUnicode; + } + } + } +endloop: + + /* set the converter state back into UConverter */ + if(U_FAILURE(*pErrorCode) && *pErrorCode!=U_BUFFER_OVERFLOW_ERROR) { + /* reset to deal with the next character */ + state=readCommand; + } else if(state==readCommand) { + /* not in a multi-byte sequence, reset toULength */ + cnv->toULength=0; + } + scsu->toUIsSingleByteMode=isSingleByteMode; + scsu->toUState=state; + scsu->toUQuoteWindow=quoteWindow; + scsu->toUDynamicWindow=dynamicWindow; + scsu->toUByteOne=byteOne; + + /* write back the updated pointers */ + pArgs->source=(const char *)source; + pArgs->target=target; + return; +} + +/* SCSU-from-Unicode conversion functions ----------------------------------- */ + +/* + * This SCSU Encoder is fairly simple but uses all SCSU commands to achieve + * reasonable results. The lookahead is minimal. + * Many cases are simple: + * A character fits directly into the current mode, a dynamic or static window, + * or is not compressible. These cases are tested first. + * Real compression heuristics are applied to the rest, in code branches for + * single/Unicode mode and BMP/supplementary code points. + * The heuristics used here are extremely simple. + */ + +/* get the number of the window that this character is in, or -1 */ +static int8_t +getWindow(const uint32_t offsets[8], uint32_t c) { + int i; + for(i=0; i<8; ++i) { + if((uint32_t)(c-offsets[i])<=0x7f) { + return (int8_t)(i); + } + } + return -1; +} + +/* is the character in the dynamic window starting at the offset, or in the direct-encoded range? */ +static UBool +isInOffsetWindowOrDirect(uint32_t offset, uint32_t c) { + return (UBool)(c<=offset+0x7f && + (c>=offset || (c<=0x7f && + (c>=0x20 || (1UL<windowUse[scsu->nextWindowUseIndex]; + if(++scsu->nextWindowUseIndex==8) { + scsu->nextWindowUseIndex=0; + } + return window; +} + +/* + * useDynamicWindow() adjusts + * windowUse[] and nextWindowUseIndex for the algorithm to choose + * the next dynamic window to be defined; + * a subclass may override it and provide its own algorithm. + */ +static void +useDynamicWindow(SCSUData *scsu, int8_t window) { + /* + * move the existing window, which just became the most recently used one, + * up in windowUse[] to nextWindowUseIndex-1 + */ + + /* first, find the index of the window - backwards to favor the more recently used windows */ + int i, j; + + i=scsu->nextWindowUseIndex; + do { + if(--i<0) { + i=7; + } + } while(scsu->windowUse[i]!=window); + + /* now copy each windowUse[i+1] to [i] */ + j=i+1; + if(j==8) { + j=0; + } + while(j!=scsu->nextWindowUseIndex) { + scsu->windowUse[i]=scsu->windowUse[j]; + i=j; + if(++j==8) { j=0; } + } + + /* finally, set the window into the most recently used index */ + scsu->windowUse[i]=window; +} + +/* + * calculate the offset and the code for a dynamic window that contains the character + * takes fixed offsets into account + * the offset of the window is stored in the offset variable, + * the code is returned + * + * return offset code: -1 none <=0xff code for SDn/UDn else code for SDX/UDX, subtract 0x200 to get the true code + */ +static int +getDynamicOffset(uint32_t c, uint32_t *pOffset) { + int i; + + for(i=0; i<7; ++i) { + if((uint32_t)(c-fixedOffsets[i])<=0x7f) { + *pOffset=fixedOffsets[i]; + return 0xf9+i; + } + } + + if(c<0x80) { + /* No dynamic window for US-ASCII. */ + return -1; + } else if(c<0x3400 || + (uint32_t)(c-0x10000)<(0x14000-0x10000) || + (uint32_t)(c-0x1d000)<=(0x1ffff-0x1d000) + ) { + /* This character is in a code range for a "small", i.e., reasonably windowable, script. */ + *pOffset=c&0x7fffff80; + return (int)(c>>7); + } else if(0xe000<=c && c!=0xfeff && c<0xfff0) { + /* For these characters we need to take the gapOffset into account. */ + *pOffset=c&0x7fffff80; + return (int)((c-gapOffset)>>7); + } else { + return -1; + } +} + +/* + * Idea for compression: + * - save SCSUData and other state before really starting work + * - at endloop, see if compression could be better with just unicode mode + * - don't do this if a callback has been called + * - if unicode mode would be smaller, then override the results with it - may need SCU at the beginning + * - different buffer handling! + * + * Drawback or need for corrective handling: + * it is desirable to encode U+feff as SQU fe ff for the SCSU signature, and + * it is desirable to start a document in US-ASCII/Latin-1 for as long as possible + * not only for compression but also for HTML/XML documents with following charset/encoding announcers. + * + * How to achieve both? + * - Only replace the result after an SDX or SCU? + */ + +static void +_SCSUFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv; + SCSUData *scsu; + const UChar *source, *sourceLimit; + uint8_t *target; + int32_t targetCapacity; + int32_t *offsets; + + UBool isSingleByteMode; + uint8_t dynamicWindow; + uint32_t currentOffset; + + uint32_t c, delta; + + int32_t sourceIndex, nextSourceIndex; + + int32_t length; + + /* variables for compression heuristics */ + uint32_t offset; + UChar lead, trail; + int code; + int8_t window; + + /* set up the local pointers */ + cnv=pArgs->converter; + scsu=(SCSUData *)cnv->extraInfo; + + /* set up the local pointers */ + source=pArgs->source; + sourceLimit=pArgs->sourceLimit; + target=(uint8_t *)pArgs->target; + targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); + offsets=pArgs->offsets; + + /* get the state machine state */ + isSingleByteMode=scsu->fromUIsSingleByteMode; + dynamicWindow=scsu->fromUDynamicWindow; + currentOffset=scsu->fromUDynamicOffsets[dynamicWindow]; + + c=cnv->fromUChar32; + + /* sourceIndex=-1 if the current character began in the previous buffer */ + sourceIndex= c==0 ? 0 : -1; + nextSourceIndex=0; + + /* similar conversion "loop" as in toUnicode */ +loop: + if(isSingleByteMode) { + if(c!=0 && targetCapacity>0) { + goto getTrailSingle; + } + + /* state machine for single-byte mode */ +/* singleByteMode: */ + while(sourcefromUDynamicOffsets, c))>=0) { + /* there is a dynamic window that contains this character, change to it */ + dynamicWindow=window; + currentOffset=scsu->fromUDynamicOffsets[dynamicWindow]; + useDynamicWindow(scsu, dynamicWindow); + c=((uint32_t)(SC0+dynamicWindow)<<8)|(c-currentOffset)|0x80; + length=2; + goto outputBytes; + } else if((code=getDynamicOffset(c, &offset))>=0) { + /* might check if there are more characters in this window to come */ + /* define an extended window with this character */ + code-=0x200; + dynamicWindow=getNextDynamicWindow(scsu); + currentOffset=scsu->fromUDynamicOffsets[dynamicWindow]=offset; + useDynamicWindow(scsu, dynamicWindow); + c=((uint32_t)SDX<<24)|((uint32_t)dynamicWindow<<21)|((uint32_t)code<<8)|(c-currentOffset)|0x80; + length=4; + goto outputBytes; + } else { + /* change to Unicode mode and output this (lead, trail) pair */ + isSingleByteMode=FALSE; + *target++=(uint8_t)SCU; + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + --targetCapacity; + c=((uint32_t)lead<<16)|trail; + length=4; + goto outputBytes; + } + } else if(c<0xa0) { + /* quote C1 control character */ + c=(c&0x7f)|(SQ0+1)<<8; /* SQ0+1==SQ1 */ + length=2; + goto outputBytes; + } else if(c==0xfeff || c>=0xfff0) { + /* quote signature character=byte order mark and specials */ + c|=SQU<<16; + length=3; + goto outputBytes; + } else { + /* compress all other BMP characters */ + if((window=getWindow(scsu->fromUDynamicOffsets, c))>=0) { + /* there is a window defined that contains this character - switch to it or quote from it? */ + if(source>=sourceLimit || isInOffsetWindowOrDirect(scsu->fromUDynamicOffsets[window], *source)) { + /* change to dynamic window */ + dynamicWindow=window; + currentOffset=scsu->fromUDynamicOffsets[dynamicWindow]; + useDynamicWindow(scsu, dynamicWindow); + c=((uint32_t)(SC0+dynamicWindow)<<8)|(c-currentOffset)|0x80; + length=2; + goto outputBytes; + } else { + /* quote from dynamic window */ + c=((uint32_t)(SQ0+window)<<8)|(c-scsu->fromUDynamicOffsets[window])|0x80; + length=2; + goto outputBytes; + } + } else if((window=getWindow(staticOffsets, c))>=0) { + /* quote from static window */ + c=((uint32_t)(SQ0+window)<<8)|(c-staticOffsets[window]); + length=2; + goto outputBytes; + } else if((code=getDynamicOffset(c, &offset))>=0) { + /* define a dynamic window with this character */ + dynamicWindow=getNextDynamicWindow(scsu); + currentOffset=scsu->fromUDynamicOffsets[dynamicWindow]=offset; + useDynamicWindow(scsu, dynamicWindow); + c=((uint32_t)(SD0+dynamicWindow)<<16)|((uint32_t)code<<8)|(c-currentOffset)|0x80; + length=3; + goto outputBytes; + } else if((uint32_t)(c-0x3400)<(0xd800-0x3400) && + (source>=sourceLimit || (uint32_t)(*source-0x3400)<(0xd800-0x3400)) + ) { + /* + * this character is not compressible (a BMP ideograph or similar); + * switch to Unicode mode if this is the last character in the block + * or there is at least one more ideograph following immediately + */ + isSingleByteMode=FALSE; + c|=SCU<<16; + length=3; + goto outputBytes; + } else { + /* quote Unicode */ + c|=SQU<<16; + length=3; + goto outputBytes; + } + } + + /* normal end of conversion: prepare for a new character */ + c=0; + sourceIndex=nextSourceIndex; + } + } else { + if(c!=0 && targetCapacity>0) { + goto getTrailUnicode; + } + + /* state machine for Unicode mode */ +/* unicodeByteMode: */ + while(source=2) { + *target++=(uint8_t)(c>>8); + *target++=(uint8_t)c; + if(offsets!=NULL) { + *offsets++=sourceIndex; + *offsets++=sourceIndex; + } + targetCapacity-=2; + } else { + length=2; + goto outputBytes; + } + } else if((uint32_t)(c-0x3400)>=(0xf300-0x3400) /* c<0x3400 || c>=0xf300 */) { + /* compress BMP character if the following one is not an uncompressible ideograph */ + if(!(sourcefromUDynamicOffsets, c))>=0) { + /* there is a dynamic window that contains this character, change to it */ + isSingleByteMode=TRUE; + dynamicWindow=window; + currentOffset=scsu->fromUDynamicOffsets[dynamicWindow]; + useDynamicWindow(scsu, dynamicWindow); + c=((uint32_t)(UC0+dynamicWindow)<<8)|(c-currentOffset)|0x80; + length=2; + goto outputBytes; + } else if((code=getDynamicOffset(c, &offset))>=0) { + /* define a dynamic window with this character */ + isSingleByteMode=TRUE; + dynamicWindow=getNextDynamicWindow(scsu); + currentOffset=scsu->fromUDynamicOffsets[dynamicWindow]=offset; + useDynamicWindow(scsu, dynamicWindow); + c=((uint32_t)(UD0+dynamicWindow)<<16)|((uint32_t)code<<8)|(c-currentOffset)|0x80; + length=3; + goto outputBytes; + } + } + + /* don't know how to compress this character, just write it directly */ + length=2; + goto outputBytes; + } else if(c<0xe000) { + /* c is a surrogate */ + if(UTF_IS_SURROGATE_FIRST(c)) { +getTrailUnicode: + lead=(UChar)c; + if(sourcefromUDynamicOffsets, c))>=0 && + !(sourcefromUDynamicOffsets[dynamicWindow]; + useDynamicWindow(scsu, dynamicWindow); + c=((uint32_t)(UC0+dynamicWindow)<<8)|(c-currentOffset)|0x80; + length=2; + goto outputBytes; + } else if(source=0 + ) { + /* two supplementary characters in (probably) the same window - define an extended one */ + isSingleByteMode=TRUE; + code-=0x200; + dynamicWindow=getNextDynamicWindow(scsu); + currentOffset=scsu->fromUDynamicOffsets[dynamicWindow]=offset; + useDynamicWindow(scsu, dynamicWindow); + c=((uint32_t)UDX<<24)|((uint32_t)dynamicWindow<<21)|((uint32_t)code<<8)|(c-currentOffset)|0x80; + length=4; + goto outputBytes; + } else { + /* don't know how to compress this character, just write it directly */ + c=((uint32_t)lead<<16)|trail; + length=4; + goto outputBytes; + } + } else /* 0xe000<=c<0xf300 */ { + /* quote to avoid SCSU tags */ + c|=UQU<<16; + length=3; + goto outputBytes; + } + + /* normal end of conversion: prepare for a new character */ + c=0; + sourceIndex=nextSourceIndex; + } + } +endloop: + + /* set the converter state back into UConverter */ + scsu->fromUIsSingleByteMode=isSingleByteMode; + scsu->fromUDynamicWindow=dynamicWindow; + + cnv->fromUChar32=c; + + /* write back the updated pointers */ + pArgs->source=source; + pArgs->target=(char *)target; + pArgs->offsets=offsets; + return; + +outputBytes: + /* write the output character bytes from c and length [code copied from ucnvmbcs.c] */ + /* from the first if in the loop we know that targetCapacity>0 */ + if(length<=targetCapacity) { + if(offsets==NULL) { + switch(length) { + /* each branch falls through to the next one */ + case 4: + *target++=(uint8_t)(c>>24); + case 3: + *target++=(uint8_t)(c>>16); + case 2: + *target++=(uint8_t)(c>>8); + case 1: + *target++=(uint8_t)c; + default: + /* will never occur */ + break; + } + } else { + switch(length) { + /* each branch falls through to the next one */ + case 4: + *target++=(uint8_t)(c>>24); + *offsets++=sourceIndex; + case 3: + *target++=(uint8_t)(c>>16); + *offsets++=sourceIndex; + case 2: + *target++=(uint8_t)(c>>8); + *offsets++=sourceIndex; + case 1: + *target++=(uint8_t)c; + *offsets++=sourceIndex; + default: + /* will never occur */ + break; + } + } + targetCapacity-=length; + + /* normal end of conversion: prepare for a new character */ + c=0; + sourceIndex=nextSourceIndex; + goto loop; + } else { + uint8_t *p; + + /* + * We actually do this backwards here: + * In order to save an intermediate variable, we output + * first to the overflow buffer what does not fit into the + * regular target. + */ + /* we know that 0<=targetCapacitycharErrorBuffer; + switch(length) { + /* each branch falls through to the next one */ + case 4: + *p++=(uint8_t)(c>>24); + case 3: + *p++=(uint8_t)(c>>16); + case 2: + *p++=(uint8_t)(c>>8); + case 1: + *p=(uint8_t)c; + default: + /* will never occur */ + break; + } + cnv->charErrorBufferLength=(int8_t)length; + + /* now output what fits into the regular target */ + c>>=8*length; /* length was reduced by targetCapacity */ + switch(targetCapacity) { + /* each branch falls through to the next one */ + case 3: + *target++=(uint8_t)(c>>16); + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + case 2: + *target++=(uint8_t)(c>>8); + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + case 1: + *target++=(uint8_t)c; + if(offsets!=NULL) { + *offsets++=sourceIndex; + } + default: + break; + } + + /* target overflow */ + targetCapacity=0; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + c=0; + goto endloop; + } +} + +/* + * Identical to _SCSUFromUnicodeWithOffsets but without offset handling. + * If a change is made in the original function, then either + * change this function the same way or + * re-copy the original function and remove the variables + * offsets, sourceIndex, and nextSourceIndex. + */ +static void +_SCSUFromUnicode(UConverterFromUnicodeArgs *pArgs, + UErrorCode *pErrorCode) { + UConverter *cnv; + SCSUData *scsu; + const UChar *source, *sourceLimit; + uint8_t *target; + int32_t targetCapacity; + + UBool isSingleByteMode; + uint8_t dynamicWindow; + uint32_t currentOffset; + + uint32_t c, delta; + + int32_t length; + + /* variables for compression heuristics */ + uint32_t offset; + UChar lead, trail; + int code; + int8_t window; + + /* set up the local pointers */ + cnv=pArgs->converter; + scsu=(SCSUData *)cnv->extraInfo; + + /* set up the local pointers */ + source=pArgs->source; + sourceLimit=pArgs->sourceLimit; + target=(uint8_t *)pArgs->target; + targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); + + /* get the state machine state */ + isSingleByteMode=scsu->fromUIsSingleByteMode; + dynamicWindow=scsu->fromUDynamicWindow; + currentOffset=scsu->fromUDynamicOffsets[dynamicWindow]; + + c=cnv->fromUChar32; + + /* similar conversion "loop" as in toUnicode */ +loop: + if(isSingleByteMode) { + if(c!=0 && targetCapacity>0) { + goto getTrailSingle; + } + + /* state machine for single-byte mode */ +/* singleByteMode: */ + while(sourcefromUDynamicOffsets, c))>=0) { + /* there is a dynamic window that contains this character, change to it */ + dynamicWindow=window; + currentOffset=scsu->fromUDynamicOffsets[dynamicWindow]; + useDynamicWindow(scsu, dynamicWindow); + c=((uint32_t)(SC0+dynamicWindow)<<8)|(c-currentOffset)|0x80; + length=2; + goto outputBytes; + } else if((code=getDynamicOffset(c, &offset))>=0) { + /* might check if there are more characters in this window to come */ + /* define an extended window with this character */ + code-=0x200; + dynamicWindow=getNextDynamicWindow(scsu); + currentOffset=scsu->fromUDynamicOffsets[dynamicWindow]=offset; + useDynamicWindow(scsu, dynamicWindow); + c=((uint32_t)SDX<<24)|((uint32_t)dynamicWindow<<21)|((uint32_t)code<<8)|(c-currentOffset)|0x80; + length=4; + goto outputBytes; + } else { + /* change to Unicode mode and output this (lead, trail) pair */ + isSingleByteMode=FALSE; + *target++=(uint8_t)SCU; + --targetCapacity; + c=((uint32_t)lead<<16)|trail; + length=4; + goto outputBytes; + } + } else if(c<0xa0) { + /* quote C1 control character */ + c=(c&0x7f)|(SQ0+1)<<8; /* SQ0+1==SQ1 */ + length=2; + goto outputBytes; + } else if(c==0xfeff || c>=0xfff0) { + /* quote signature character=byte order mark and specials */ + c|=SQU<<16; + length=3; + goto outputBytes; + } else { + /* compress all other BMP characters */ + if((window=getWindow(scsu->fromUDynamicOffsets, c))>=0) { + /* there is a window defined that contains this character - switch to it or quote from it? */ + if(source>=sourceLimit || isInOffsetWindowOrDirect(scsu->fromUDynamicOffsets[window], *source)) { + /* change to dynamic window */ + dynamicWindow=window; + currentOffset=scsu->fromUDynamicOffsets[dynamicWindow]; + useDynamicWindow(scsu, dynamicWindow); + c=((uint32_t)(SC0+dynamicWindow)<<8)|(c-currentOffset)|0x80; + length=2; + goto outputBytes; + } else { + /* quote from dynamic window */ + c=((uint32_t)(SQ0+window)<<8)|(c-scsu->fromUDynamicOffsets[window])|0x80; + length=2; + goto outputBytes; + } + } else if((window=getWindow(staticOffsets, c))>=0) { + /* quote from static window */ + c=((uint32_t)(SQ0+window)<<8)|(c-staticOffsets[window]); + length=2; + goto outputBytes; + } else if((code=getDynamicOffset(c, &offset))>=0) { + /* define a dynamic window with this character */ + dynamicWindow=getNextDynamicWindow(scsu); + currentOffset=scsu->fromUDynamicOffsets[dynamicWindow]=offset; + useDynamicWindow(scsu, dynamicWindow); + c=((uint32_t)(SD0+dynamicWindow)<<16)|((uint32_t)code<<8)|(c-currentOffset)|0x80; + length=3; + goto outputBytes; + } else if((uint32_t)(c-0x3400)<(0xd800-0x3400) && + (source>=sourceLimit || (uint32_t)(*source-0x3400)<(0xd800-0x3400)) + ) { + /* + * this character is not compressible (a BMP ideograph or similar); + * switch to Unicode mode if this is the last character in the block + * or there is at least one more ideograph following immediately + */ + isSingleByteMode=FALSE; + c|=SCU<<16; + length=3; + goto outputBytes; + } else { + /* quote Unicode */ + c|=SQU<<16; + length=3; + goto outputBytes; + } + } + + /* normal end of conversion: prepare for a new character */ + c=0; + } + } else { + if(c!=0 && targetCapacity>0) { + goto getTrailUnicode; + } + + /* state machine for Unicode mode */ +/* unicodeByteMode: */ + while(source=2) { + *target++=(uint8_t)(c>>8); + *target++=(uint8_t)c; + targetCapacity-=2; + } else { + length=2; + goto outputBytes; + } + } else if((uint32_t)(c-0x3400)>=(0xf300-0x3400) /* c<0x3400 || c>=0xf300 */) { + /* compress BMP character if the following one is not an uncompressible ideograph */ + if(!(sourcefromUDynamicOffsets, c))>=0) { + /* there is a dynamic window that contains this character, change to it */ + isSingleByteMode=TRUE; + dynamicWindow=window; + currentOffset=scsu->fromUDynamicOffsets[dynamicWindow]; + useDynamicWindow(scsu, dynamicWindow); + c=((uint32_t)(UC0+dynamicWindow)<<8)|(c-currentOffset)|0x80; + length=2; + goto outputBytes; + } else if((code=getDynamicOffset(c, &offset))>=0) { + /* define a dynamic window with this character */ + isSingleByteMode=TRUE; + dynamicWindow=getNextDynamicWindow(scsu); + currentOffset=scsu->fromUDynamicOffsets[dynamicWindow]=offset; + useDynamicWindow(scsu, dynamicWindow); + c=((uint32_t)(UD0+dynamicWindow)<<16)|((uint32_t)code<<8)|(c-currentOffset)|0x80; + length=3; + goto outputBytes; + } + } + + /* don't know how to compress this character, just write it directly */ + length=2; + goto outputBytes; + } else if(c<0xe000) { + /* c is a surrogate */ + if(UTF_IS_SURROGATE_FIRST(c)) { +getTrailUnicode: + lead=(UChar)c; + if(sourcefromUDynamicOffsets, c))>=0 && + !(sourcefromUDynamicOffsets[dynamicWindow]; + useDynamicWindow(scsu, dynamicWindow); + c=((uint32_t)(UC0+dynamicWindow)<<8)|(c-currentOffset)|0x80; + length=2; + goto outputBytes; + } else if(source=0 + ) { + /* two supplementary characters in (probably) the same window - define an extended one */ + isSingleByteMode=TRUE; + code-=0x200; + dynamicWindow=getNextDynamicWindow(scsu); + currentOffset=scsu->fromUDynamicOffsets[dynamicWindow]=offset; + useDynamicWindow(scsu, dynamicWindow); + c=((uint32_t)UDX<<24)|((uint32_t)dynamicWindow<<21)|((uint32_t)code<<8)|(c-currentOffset)|0x80; + length=4; + goto outputBytes; + } else { + /* don't know how to compress this character, just write it directly */ + c=((uint32_t)lead<<16)|trail; + length=4; + goto outputBytes; + } + } else /* 0xe000<=c<0xf300 */ { + /* quote to avoid SCSU tags */ + c|=UQU<<16; + length=3; + goto outputBytes; + } + + /* normal end of conversion: prepare for a new character */ + c=0; + } + } +endloop: + + /* set the converter state back into UConverter */ + scsu->fromUIsSingleByteMode=isSingleByteMode; + scsu->fromUDynamicWindow=dynamicWindow; + + cnv->fromUChar32=c; + + /* write back the updated pointers */ + pArgs->source=source; + pArgs->target=(char *)target; + return; + +outputBytes: + /* write the output character bytes from c and length [code copied from ucnvmbcs.c] */ + /* from the first if in the loop we know that targetCapacity>0 */ + if(length<=targetCapacity) { + switch(length) { + /* each branch falls through to the next one */ + case 4: + *target++=(uint8_t)(c>>24); + case 3: + *target++=(uint8_t)(c>>16); + case 2: + *target++=(uint8_t)(c>>8); + case 1: + *target++=(uint8_t)c; + default: + /* will never occur */ + break; + } + targetCapacity-=length; + + /* normal end of conversion: prepare for a new character */ + c=0; + goto loop; + } else { + uint8_t *p; + + /* + * We actually do this backwards here: + * In order to save an intermediate variable, we output + * first to the overflow buffer what does not fit into the + * regular target. + */ + /* we know that 0<=targetCapacitycharErrorBuffer; + switch(length) { + /* each branch falls through to the next one */ + case 4: + *p++=(uint8_t)(c>>24); + case 3: + *p++=(uint8_t)(c>>16); + case 2: + *p++=(uint8_t)(c>>8); + case 1: + *p=(uint8_t)c; + default: + /* will never occur */ + break; + } + cnv->charErrorBufferLength=(int8_t)length; + + /* now output what fits into the regular target */ + c>>=8*length; /* length was reduced by targetCapacity */ + switch(targetCapacity) { + /* each branch falls through to the next one */ + case 3: + *target++=(uint8_t)(c>>16); + case 2: + *target++=(uint8_t)(c>>8); + case 1: + *target++=(uint8_t)c; + default: + break; + } + + /* target overflow */ + targetCapacity=0; + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + c=0; + goto endloop; + } +} + +/* miscellaneous ------------------------------------------------------------ */ + +static const char * +_SCSUGetName(const UConverter *cnv) { + SCSUData *scsu=(SCSUData *)cnv->extraInfo; + + switch(scsu->locale) { + case l_ja: + return "SCSU,locale=ja"; + default: + return "SCSU"; + } +} + +/* structure for SafeClone calculations */ +struct cloneSCSUStruct +{ + UConverter cnv; + SCSUData mydata; +}; + +static UConverter * +_SCSUSafeClone(const UConverter *cnv, + void *stackBuffer, + int32_t *pBufferSize, + UErrorCode *status) +{ + struct cloneSCSUStruct * localClone; + int32_t bufferSizeNeeded = sizeof(struct cloneSCSUStruct); + + if (U_FAILURE(*status)){ + return 0; + } + + if (*pBufferSize == 0){ /* 'preflighting' request - set needed size into *pBufferSize */ + *pBufferSize = bufferSizeNeeded; + return 0; + } + + localClone = (struct cloneSCSUStruct *)stackBuffer; + /* ucnv.c/ucnv_safeClone() copied the main UConverter already */ + + uprv_memcpy(&localClone->mydata, cnv->extraInfo, sizeof(SCSUData)); + localClone->cnv.extraInfo = &localClone->mydata; + localClone->cnv.isExtraLocal = TRUE; + + return &localClone->cnv; +} + + +static const UConverterImpl _SCSUImpl={ + UCNV_SCSU, + + NULL, + NULL, + + _SCSUOpen, + _SCSUClose, + _SCSUReset, + + _SCSUToUnicode, + _SCSUToUnicodeWithOffsets, + _SCSUFromUnicode, + _SCSUFromUnicodeWithOffsets, + NULL, + + NULL, + _SCSUGetName, + NULL, + _SCSUSafeClone, + ucnv_getCompleteUnicodeSet +}; + +static const UConverterStaticData _SCSUStaticData={ + sizeof(UConverterStaticData), + "SCSU", + 1212, /* CCSID for SCSU */ + UCNV_IBM, UCNV_SCSU, + 1, 3, /* one UChar generates at least 1 byte and at most 3 bytes */ + /* + * The subchar here is ignored because _SCSUOpen() sets U+fffd as a Unicode + * substitution string. + */ + { 0x0e, 0xff, 0xfd, 0 }, 3, + FALSE, FALSE, + 0, + 0, + { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */ +}; + +const UConverterSharedData _SCSUData={ + sizeof(UConverterSharedData), ~((uint32_t)0), + NULL, NULL, &_SCSUStaticData, FALSE, &_SCSUImpl, + 0 +}; + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/ucol_swp.c b/Build/source/libs/icu/icu-xetex/common/ucol_swp.c new file mode 100644 index 00000000000..3b8ac14c30f --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucol_swp.c @@ -0,0 +1,402 @@ +/* +******************************************************************************* +* +* Copyright (C) 2003-2006, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: ucol_swp.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2003sep10 +* created by: Markus W. Scherer +* +* Swap collation binaries. +*/ + +#include "unicode/udata.h" /* UDataInfo */ +#include "utrie.h" +#include "udataswp.h" +#include "cmemory.h" +#include "ucol_imp.h" +#include "ucol_swp.h" + +/* swapping ----------------------------------------------------------------- */ + +/* + * This performs data swapping for a folded trie (see utrie.c for details). + */ + +U_CAPI int32_t U_EXPORT2 +utrie_swap(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode) { + const UTrieHeader *inTrie; + UTrieHeader trie; + int32_t size; + UBool dataIs32; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + if(ds==NULL || inData==NULL || (length>=0 && outData==NULL)) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* setup and swapping */ + if(length>=0 && lengthreadUInt32(inTrie->signature); + trie.options=ds->readUInt32(inTrie->options); + trie.indexLength=udata_readInt32(ds, inTrie->indexLength); + trie.dataLength=udata_readInt32(ds, inTrie->dataLength); + + if( trie.signature!=0x54726965 || + (trie.options&UTRIE_OPTIONS_SHIFT_MASK)!=UTRIE_SHIFT || + ((trie.options>>UTRIE_OPTIONS_INDEX_SHIFT)&UTRIE_OPTIONS_SHIFT_MASK)!=UTRIE_INDEX_SHIFT || + trie.indexLength=0) { + UTrieHeader *outTrie; + + if(lengthswapArray32(ds, inTrie, sizeof(UTrieHeader), outTrie, pErrorCode); + + /* swap the index and the data */ + if(dataIs32) { + ds->swapArray16(ds, inTrie+1, trie.indexLength*2, outTrie+1, pErrorCode); + ds->swapArray32(ds, (const uint16_t *)(inTrie+1)+trie.indexLength, trie.dataLength*4, + (uint16_t *)(outTrie+1)+trie.indexLength, pErrorCode); + } else { + ds->swapArray16(ds, inTrie+1, (trie.indexLength+trie.dataLength)*2, outTrie+1, pErrorCode); + } + } + + return size; +} + +#if !UCONFIG_NO_COLLATION + +/* swap a header-less collation binary, inside a resource bundle or ucadata.icu */ +U_CAPI int32_t U_EXPORT2 +ucol_swapBinary(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode) { + const uint8_t *inBytes; + uint8_t *outBytes; + + const UCATableHeader *inHeader; + UCATableHeader *outHeader; + UCATableHeader header={ 0 }; + + uint32_t count; + + /* argument checking in case we were not called from ucol_swap() */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + if(ds==NULL || inData==NULL || length<-1 || (length>0 && outData==NULL)) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + inBytes=(const uint8_t *)inData; + outBytes=(uint8_t *)outData; + + inHeader=(const UCATableHeader *)inData; + outHeader=(UCATableHeader *)outData; + + /* + * The collation binary must contain at least the UCATableHeader, + * starting with its size field. + * sizeof(UCATableHeader)==42*4 in ICU 2.8 + * check the length against the header size before reading the size field + */ + if(length<0) { + header.size=udata_readInt32(ds, inHeader->size); + } else if((length<(42*4) || length<(header.size=udata_readInt32(ds, inHeader->size)))) { + udata_printError(ds, "ucol_swapBinary(): too few bytes (%d after header) for collation data\n", + length); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + + header.magic=ds->readUInt32(inHeader->magic); + if(!( + header.magic==UCOL_HEADER_MAGIC && + inHeader->formatVersion[0]==2 && + inHeader->formatVersion[1]>=3 + )) { + udata_printError(ds, "ucol_swapBinary(): magic 0x%08x or format version %02x.%02x is not a collation binary\n", + header.magic, + inHeader->formatVersion[0], inHeader->formatVersion[1]); + *pErrorCode=U_UNSUPPORTED_ERROR; + return 0; + } + + if(inHeader->isBigEndian!=ds->inIsBigEndian || inHeader->charSetFamily!=ds->inCharset) { + udata_printError(ds, "ucol_swapBinary(): endianness %d or charset %d does not match the swapper\n", + inHeader->isBigEndian, inHeader->charSetFamily); + *pErrorCode=U_INVALID_FORMAT_ERROR; + return 0; + } + + if(length>=0) { + /* copy everything, takes care of data that needs no swapping */ + if(inBytes!=outBytes) { + uprv_memcpy(outBytes, inBytes, header.size); + } + + /* swap the necessary pieces in the order of their occurrence in the data */ + + /* read more of the UCATableHeader (the size field was read above) */ + header.options= ds->readUInt32(inHeader->options); + header.UCAConsts= ds->readUInt32(inHeader->UCAConsts); + header.contractionUCACombos= ds->readUInt32(inHeader->contractionUCACombos); + header.mappingPosition= ds->readUInt32(inHeader->mappingPosition); + header.expansion= ds->readUInt32(inHeader->expansion); + header.contractionIndex= ds->readUInt32(inHeader->contractionIndex); + header.contractionCEs= ds->readUInt32(inHeader->contractionCEs); + header.contractionSize= ds->readUInt32(inHeader->contractionSize); + header.endExpansionCE= ds->readUInt32(inHeader->endExpansionCE); + header.expansionCESize= ds->readUInt32(inHeader->expansionCESize); + header.endExpansionCECount= udata_readInt32(ds, inHeader->endExpansionCECount); + header.contractionUCACombosSize=udata_readInt32(ds, inHeader->contractionUCACombosSize); + + /* swap the 32-bit integers in the header */ + ds->swapArray32(ds, inHeader, (int32_t)((const char *)&inHeader->jamoSpecial-(const char *)inHeader), + outHeader, pErrorCode); + + /* set the output platform properties */ + outHeader->isBigEndian=ds->outIsBigEndian; + outHeader->charSetFamily=ds->outCharset; + + /* swap the options */ + if(header.options!=0) { + ds->swapArray32(ds, inBytes+header.options, header.expansion-header.options, + outBytes+header.options, pErrorCode); + } + + /* swap the expansions */ + if(header.mappingPosition!=0 && header.expansion!=0) { + if(header.contractionIndex!=0) { + /* expansions bounded by contractions */ + count=header.contractionIndex-header.expansion; + } else { + /* no contractions: expansions bounded by the main trie */ + count=header.mappingPosition-header.expansion; + } + ds->swapArray32(ds, inBytes+header.expansion, (int32_t)count, + outBytes+header.expansion, pErrorCode); + } + + /* swap the contractions */ + if(header.contractionSize!=0) { + /* contractionIndex: UChar[] */ + ds->swapArray16(ds, inBytes+header.contractionIndex, header.contractionSize*2, + outBytes+header.contractionIndex, pErrorCode); + + /* contractionCEs: CEs[] */ + ds->swapArray32(ds, inBytes+header.contractionCEs, header.contractionSize*4, + outBytes+header.contractionCEs, pErrorCode); + } + + /* swap the main trie */ + if(header.mappingPosition!=0) { + count=header.endExpansionCE-header.mappingPosition; + utrie_swap(ds, inBytes+header.mappingPosition, (int32_t)count, + outBytes+header.mappingPosition, pErrorCode); + } + + /* swap the max expansion table */ + if(header.endExpansionCECount!=0) { + ds->swapArray32(ds, inBytes+header.endExpansionCE, header.endExpansionCECount*4, + outBytes+header.endExpansionCE, pErrorCode); + } + + /* expansionCESize, unsafeCP, contrEndCP: uint8_t[], no need to swap */ + + /* swap UCA constants */ + if(header.UCAConsts!=0) { + /* + * if UCAConsts!=0 then contractionUCACombos because we are swapping + * the UCA data file, and we know that the UCA contains contractions + */ + count=header.contractionUCACombos-header.UCAConsts; + ds->swapArray32(ds, inBytes+header.UCAConsts, header.contractionUCACombos-header.UCAConsts, + outBytes+header.UCAConsts, pErrorCode); + } + + /* swap UCA contractions */ + if(header.contractionUCACombosSize!=0) { + count=header.contractionUCACombosSize*inHeader->contractionUCACombosWidth*U_SIZEOF_UCHAR; + ds->swapArray16(ds, inBytes+header.contractionUCACombos, (int32_t)count, + outBytes+header.contractionUCACombos, pErrorCode); + } + } + + return header.size; +} + +/* swap ICU collation data like ucadata.icu */ +U_CAPI int32_t U_EXPORT2 +ucol_swap(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode) { + const UDataInfo *pInfo; + int32_t headerSize, collationSize; + + /* udata_swapDataHeader checks the arguments */ + headerSize=udata_swapDataHeader(ds, inData, length, outData, pErrorCode); + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + /* check data format and format version */ + pInfo=(const UDataInfo *)((const char *)inData+4); + if(!( + pInfo->dataFormat[0]==0x55 && /* dataFormat="UCol" */ + pInfo->dataFormat[1]==0x43 && + pInfo->dataFormat[2]==0x6f && + pInfo->dataFormat[3]==0x6c && + pInfo->formatVersion[0]==2 && + pInfo->formatVersion[1]>=3 + )) { + udata_printError(ds, "ucol_swap(): data format %02x.%02x.%02x.%02x (format version %02x.%02x) is not a collation file\n", + pInfo->dataFormat[0], pInfo->dataFormat[1], + pInfo->dataFormat[2], pInfo->dataFormat[3], + pInfo->formatVersion[0], pInfo->formatVersion[1]); + *pErrorCode=U_UNSUPPORTED_ERROR; + return 0; + } + + collationSize=ucol_swapBinary(ds, + (const char *)inData+headerSize, + length>=0 ? length-headerSize : -1, + (char *)outData+headerSize, + pErrorCode); + if(U_SUCCESS(*pErrorCode)) { + return headerSize+collationSize; + } else { + return 0; + } +} + +/* swap inverse UCA collation data (invuca.icu) */ +U_CAPI int32_t U_EXPORT2 +ucol_swapInverseUCA(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode) { + const UDataInfo *pInfo; + int32_t headerSize; + + const uint8_t *inBytes; + uint8_t *outBytes; + + const InverseUCATableHeader *inHeader; + InverseUCATableHeader *outHeader; + InverseUCATableHeader header={ 0 }; + + /* udata_swapDataHeader checks the arguments */ + headerSize=udata_swapDataHeader(ds, inData, length, outData, pErrorCode); + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + /* check data format and format version */ + pInfo=(const UDataInfo *)((const char *)inData+4); + if(!( + pInfo->dataFormat[0]==0x49 && /* dataFormat="InvC" */ + pInfo->dataFormat[1]==0x6e && + pInfo->dataFormat[2]==0x76 && + pInfo->dataFormat[3]==0x43 && + pInfo->formatVersion[0]==2 && + pInfo->formatVersion[1]>=1 + )) { + udata_printError(ds, "ucol_swapInverseUCA(): data format %02x.%02x.%02x.%02x (format version %02x.%02x) is not an inverse UCA collation file\n", + pInfo->dataFormat[0], pInfo->dataFormat[1], + pInfo->dataFormat[2], pInfo->dataFormat[3], + pInfo->formatVersion[0], pInfo->formatVersion[1]); + *pErrorCode=U_UNSUPPORTED_ERROR; + return 0; + } + + inBytes=(const uint8_t *)inData+headerSize; + outBytes=(uint8_t *)outData+headerSize; + + inHeader=(const InverseUCATableHeader *)inBytes; + outHeader=(InverseUCATableHeader *)outBytes; + + /* + * The inverse UCA collation binary must contain at least the InverseUCATableHeader, + * starting with its size field. + * sizeof(UCATableHeader)==8*4 in ICU 2.8 + * check the length against the header size before reading the size field + */ + if(length<0) { + header.byteSize=udata_readInt32(ds, inHeader->byteSize); + } else if( + ((length-headerSize)<(8*4) || + (uint32_t)(length-headerSize)<(header.byteSize=udata_readInt32(ds, inHeader->byteSize))) + ) { + udata_printError(ds, "ucol_swapInverseUCA(): too few bytes (%d after header) for inverse UCA collation data\n", + length); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + + if(length>=0) { + /* copy everything, takes care of data that needs no swapping */ + if(inBytes!=outBytes) { + uprv_memcpy(outBytes, inBytes, header.byteSize); + } + + /* swap the necessary pieces in the order of their occurrence in the data */ + + /* read more of the InverseUCATableHeader (the byteSize field was read above) */ + header.tableSize= ds->readUInt32(inHeader->tableSize); + header.contsSize= ds->readUInt32(inHeader->contsSize); + header.table= ds->readUInt32(inHeader->table); + header.conts= ds->readUInt32(inHeader->conts); + + /* swap the 32-bit integers in the header */ + ds->swapArray32(ds, inHeader, 5*4, outHeader, pErrorCode); + + /* swap the inverse table; tableSize counts uint32_t[3] rows */ + ds->swapArray32(ds, inBytes+header.table, header.tableSize*3*4, + outBytes+header.table, pErrorCode); + + /* swap the continuation table; contsSize counts UChars */ + ds->swapArray16(ds, inBytes+header.conts, header.contsSize*U_SIZEOF_UCHAR, + outBytes+header.conts, pErrorCode); + } + + return headerSize+header.byteSize; +} + +#endif /* #if !UCONFIG_NO_COLLATION */ diff --git a/Build/source/libs/icu/icu-xetex/common/ucol_swp.h b/Build/source/libs/icu/icu-xetex/common/ucol_swp.h new file mode 100644 index 00000000000..e0cd6e479bf --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ucol_swp.h @@ -0,0 +1,58 @@ +/* +******************************************************************************* +* +* Copyright (C) 2003, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: ucol_swp.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2003sep10 +* created by: Markus W. Scherer +* +* Swap collation binaries. +*/ + +#ifndef __UCOL_SWP_H__ +#define __UCOL_SWP_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "udataswp.h" + +/** + * Swap a header-less collation binary, inside a resource bundle or ucadata.icu. + * See udataswp.h. + * @internal + */ +U_CAPI int32_t U_EXPORT2 +ucol_swapBinary(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode); + +/** + * Swap ICU collation data like ucadata.icu. See udataswp.h. + * @internal + */ +U_CAPI int32_t U_EXPORT2 +ucol_swap(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode); + +/** + * Swap inverse UCA collation data (invuca.icu). See udataswp.h. + * @internal + */ +U_CAPI int32_t U_EXPORT2 +ucol_swapInverseUCA(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode); + +#endif /* #if !UCONFIG_NO_COLLATION */ + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/udata.c b/Build/source/libs/icu/icu-xetex/common/udata.c new file mode 100644 index 00000000000..d87a23e9dd3 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/udata.c @@ -0,0 +1,1474 @@ +/* +****************************************************************************** +* +* Copyright (C) 1999-2006, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: udata.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 1999oct25 +* created by: Markus W. Scherer +*/ + +#include "unicode/utypes.h" +#include "unicode/putil.h" +#include "umutex.h" +#include "cmemory.h" +#include "cstring.h" +#include "unicode/udata.h" +#include "unicode/uversion.h" +#include "uhash.h" +#include "ucln_cmn.h" +#include "putilimp.h" + +#include "udatamem.h" +#include "umapfile.h" +#include "ucmndata.h" + +/*********************************************************************** +* +* Notes on the organization of the ICU data implementation +* +* All of the public API is defined in udata.h +* +* The implementation is split into several files... +* +* - udata.c (this file) contains higher level code that knows about +* the search paths for locating data, caching opened data, etc. +* +* - umapfile.c contains the low level platform-specific code for actually loading +* (memory mapping, file reading, whatever) data into memory. +* +* - ucmndata.c deals with the tables of contents of ICU data items within +* an ICU common format data file. The implementation includes +* an abstract interface and support for multiple TOC formats. +* All knowledge of any specific TOC format is encapsulated here. +* +* - udatamem.c has code for managing UDataMemory structs. These are little +* descriptor objects for blocks of memory holding ICU data of +* various types. +*/ + +/* configuration ---------------------------------------------------------- */ + +/* If you are excruciatingly bored turn this on .. */ +/* #define UDATA_DEBUG 1 */ + +#if defined(UDATA_DEBUG) +# include +#endif + + +/*********************************************************************** +* +* static (Global) data +* +************************************************************************/ +static UDataMemory *gCommonICUData = NULL; /* Pointer to the common ICU data. */ + /* May be updated once, if we started with */ + /* a stub or subset library. */ + +static UDataMemory *gStubICUData = NULL; /* If gCommonICUData does get updated, remember */ + /* the original one so that it can be cleaned */ + /* up when ICU is shut down. */ + +static UHashtable *gCommonDataCache = NULL; /* Global hash table of opened ICU data files. */ + +static UDataFileAccess gDataFileAccess = UDATA_DEFAULT_ACCESS; + +static UBool U_CALLCONV +udata_cleanup(void) +{ + if (gCommonDataCache) { /* Delete the cache of user data mappings. */ + uhash_close(gCommonDataCache); /* Table owns the contents, and will delete them. */ + gCommonDataCache = NULL; /* Cleanup is not thread safe. */ + } + + if (gCommonICUData != NULL) { + udata_close(gCommonICUData); /* Clean up common ICU Data */ + gCommonICUData = NULL; + } + + if (gStubICUData != NULL) { + udata_close(gStubICUData); /* Clean up the stub ICU Data */ + gStubICUData = NULL; + } + + + return TRUE; /* Everything was cleaned up */ +} + + + + +/* + * setCommonICUData. Set a UDataMemory to be the global ICU Data + */ +static void +setCommonICUData(UDataMemory *pData, /* The new common data. Belongs to caller, we copy it. */ + UDataMemory *oldData, /* Old ICUData ptr. Overwrite of this value is ok, */ + /* of any others is not. */ + UBool warn, /* If true, set USING_DEFAULT warning if ICUData was */ + /* changed by another thread before we got to it. */ + UErrorCode *pErr) +{ + UDataMemory *newCommonData = UDataMemory_createNewInstance(pErr); + if (U_FAILURE(*pErr)) { + return; + } + + /* For the assignment, other threads must cleanly see either the old */ + /* or the new, not some partially initialized new. The old can not be */ + /* deleted - someone may still have a pointer to it lying around in */ + /* their locals. */ + UDatamemory_assign(newCommonData, pData); + umtx_lock(NULL); + if (gCommonICUData==oldData) { + gStubICUData = gCommonICUData; /* remember the old Common Data, so it can be cleaned up. */ + gCommonICUData = newCommonData; + ucln_common_registerCleanup(UCLN_COMMON_UDATA, udata_cleanup); + } + else { + if (warn==TRUE) { + *pErr = U_USING_DEFAULT_WARNING; + } + uprv_free(newCommonData); + } + umtx_unlock(NULL); +} + +static const char * +findBasename(const char *path) { + const char *basename=uprv_strrchr(path, U_FILE_SEP_CHAR); + if(basename==NULL) { + return path; + } else { + return basename+1; + } +} + +#ifdef UDATA_DEBUG +static const char * +packageNameFromPath(const char *path) +{ + if((path == NULL) || (*path == 0)) { + return U_ICUDATA_NAME; + } + + path = findBasename(path); + + if((path == NULL) || (*path == 0)) { + return U_ICUDATA_NAME; + } + + return path; +} +#endif + +/*----------------------------------------------------------------------* + * * + * Cache for common data * + * Functions for looking up or adding entries to a cache of * + * data that has been previously opened. Avoids a potentially * + * expensive operation of re-opening the data for subsequent * + * uses. * + * * + * Data remains cached for the duration of the process. * + * * + *----------------------------------------------------------------------*/ + +typedef struct DataCacheElement { + char *name; + UDataMemory *item; +} DataCacheElement; + + + +/* + * Deleter function for DataCacheElements. + * udata cleanup function closes the hash table; hash table in turn calls back to + * here for each entry. + */ +static void U_CALLCONV DataCacheElement_deleter(void *pDCEl) { + DataCacheElement *p = (DataCacheElement *)pDCEl; + udata_close(p->item); /* unmaps storage */ + uprv_free(p->name); /* delete the hash key string. */ + uprv_free(pDCEl); /* delete 'this' */ +} + + /* udata_getCacheHashTable() + * Get the hash table used to store the data cache entries. + * Lazy create it if it doesn't yet exist. + */ +static UHashtable *udata_getHashTable() { + UErrorCode err = U_ZERO_ERROR; + UBool cacheIsInitialized; + UHashtable *tHT = NULL; + + UMTX_CHECK(NULL, (gCommonDataCache != NULL), cacheIsInitialized); + + if (cacheIsInitialized) { + return gCommonDataCache; + } + + tHT = uhash_open(uhash_hashChars, uhash_compareChars, NULL, &err); + uhash_setValueDeleter(tHT, DataCacheElement_deleter); + + umtx_lock(NULL); + if (gCommonDataCache == NULL) { + gCommonDataCache = tHT; + tHT = NULL; + ucln_common_registerCleanup(UCLN_COMMON_UDATA, udata_cleanup); + } + umtx_unlock(NULL); + if (tHT != NULL) { + uhash_close(tHT); + } + + if (U_FAILURE(err)) { + return NULL; /* TODO: handle this error better. */ + } + return gCommonDataCache; +} + + + +static UDataMemory *udata_findCachedData(const char *path) +{ + UHashtable *htable; + UDataMemory *retVal = NULL; + DataCacheElement *el; + const char *baseName; + + baseName = findBasename(path); /* Cache remembers only the base name, not the full path. */ + htable = udata_getHashTable(); + umtx_lock(NULL); + el = (DataCacheElement *)uhash_get(htable, baseName); + umtx_unlock(NULL); + if (el != NULL) { + retVal = el->item; + } +#ifdef UDATA_DEBUG + fprintf(stderr, "Cache: [%s] -> %p\n", baseName, retVal); +#endif + return retVal; +} + + +static UDataMemory *udata_cacheDataItem(const char *path, UDataMemory *item, UErrorCode *pErr) { + DataCacheElement *newElement; + const char *baseName; + int32_t nameLen; + UHashtable *htable; + UDataMemory *oldValue = NULL; + UErrorCode subErr = U_ZERO_ERROR; + + if (U_FAILURE(*pErr)) { + return NULL; + } + + /* Create a new DataCacheElement - the thingy we store in the hash table - + * and copy the supplied path and UDataMemoryItems into it. + */ + newElement = uprv_malloc(sizeof(DataCacheElement)); + if (newElement == NULL) { + *pErr = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + newElement->item = UDataMemory_createNewInstance(pErr); + if (U_FAILURE(*pErr)) { + uprv_free(newElement); + return NULL; + } + UDatamemory_assign(newElement->item, item); + + baseName = findBasename(path); + nameLen = (int32_t)uprv_strlen(baseName); + newElement->name = uprv_malloc(nameLen+1); + if (newElement->name == NULL) { + *pErr = U_MEMORY_ALLOCATION_ERROR; + uprv_free(newElement->item); + uprv_free(newElement); + return NULL; + } + uprv_strcpy(newElement->name, baseName); + + /* Stick the new DataCacheElement into the hash table. + */ + htable = udata_getHashTable(); + umtx_lock(NULL); + oldValue = uhash_get(htable, path); + if (oldValue != NULL) { + subErr = U_USING_DEFAULT_WARNING; + } + else { + uhash_put( + htable, + newElement->name, /* Key */ + newElement, /* Value */ + &subErr); + } + umtx_unlock(NULL); + +#ifdef UDATA_DEBUG + fprintf(stderr, "Cache: [%s] <<< %p : %s. vFunc=%p\n", newElement->name, + newElement->item, u_errorName(subErr), newElement->item->vFuncs); +#endif + + if (subErr == U_USING_DEFAULT_WARNING || U_FAILURE(subErr)) { + *pErr = subErr; /* copy sub err unto fillin ONLY if something happens. */ + uprv_free(newElement->name); + uprv_free(newElement->item); + uprv_free(newElement); + return oldValue; + } + + return newElement->item; +} + + + +/*------------------------------------------------------------------------------- + * + * TinyString - a small set of really simple string functions, for + * the purpose of consolidating buffer overflow code in one place + * + * Use wherever you would otherwise declare a fixed sized char[xx] buffer. + * Do non-growing ops by accessing fields of struct directly + * Grow using the append function to automatically extend buffer + * as needed. + * + *-------------------------------------------------------------------------------*/ +typedef struct TinyString { + char *s; + int32_t length; + char fStaticBuf[100]; + int32_t fCapacity; +} TinyString; + +static void TinyString_init(TinyString *This) { + This->s = This->fStaticBuf; + *This->s = 0; + This->length = 0; + This->fCapacity = sizeof(This->fStaticBuf)-1; +} + +static void TinyString_append(TinyString *This, const char *what) { + int32_t newLen; + newLen = This->length + (int32_t)uprv_strlen(what); + if (newLen >= This->fCapacity) { + int32_t newCapacity = newLen * 2; + char *newBuf = (char *)uprv_malloc(newCapacity+1); + if (newBuf != NULL) { + uprv_strcpy(newBuf, This->s); + if (This->s != This->fStaticBuf) { + uprv_free(This->s); + } + This->s = newBuf; + This->fCapacity = newCapacity; + } + } + if (newLen < This->fCapacity) { + uprv_strcat(This->s+This->length, what); + This->length = newLen; + } +} + +static void TinyString_appendn(TinyString *This, const char *what, int32_t n) { + int32_t newLen; + newLen = This->length + n; + if (newLen >= This->fCapacity) { + int32_t newCapacity = newLen * 2; + char *newBuf = (char *)uprv_malloc(newCapacity+1); + if (newBuf != NULL) { + uprv_strcpy(newBuf, This->s); + if (This->s != This->fStaticBuf) { + uprv_free(This->s); + } + This->s = newBuf; + This->fCapacity = newCapacity; + } + } + if (newLen < This->fCapacity) { + uprv_strncat(This->s+This->length, what, n); + This->length = newLen; + } +} + +static void TinyString_dt(TinyString *This) { + if (This->s != This->fStaticBuf) { + uprv_free(This->s); + } + TinyString_init(This); +} + + + + +/*----------------------------------------------------------------------*============== + * * + * Path management. Could be shared with other tools/etc if need be * + * later on. * + * * + *----------------------------------------------------------------------*/ + +#define U_DATA_PATHITER_BUFSIZ 128 /* Size of local buffer for paths */ + /* Overflow causes malloc of larger buf */ + +typedef struct +{ + const char *path; /* working path (u_icudata_Dir) */ + const char *nextPath; /* path following this one */ + const char *basename; /* item's basename (icudt22e_mt.res)*/ + const char *suffix; /* item suffix (can be null) */ + + uint32_t basenameLen; /* length of basename */ + + char *itemPath; /* path passed in with item name */ + char itemPathBuf[U_DATA_PATHITER_BUFSIZ]; + + char *pathBuffer; /* output path for this it'ion */ + char pathBufferA[U_DATA_PATHITER_BUFSIZ]; + + char *packageStub; /* example: "/icudt28b". Will ignore that leaf in set paths. */ + char packageStubBuf[U_DATA_PATHITER_BUFSIZ]; + uint32_t packageStubLen; + + UBool checkLastFour; /* if TRUE then allow paths such as '/foo/myapp.dat' + * to match, checks last 4 chars of suffix with + * last 4 of path, then previous chars. */ + +} UDataPathIterator; + +/** + * Initialize (or re-initialize) a user-supplied UDataPathIterator + * Note: UDataPathIterator does not allocate storage, so it doesn't need to be closed. + * + * @param iter The iterator to be initialized. Its current state does not matter. + * @param path The full pathname to be iterated over. If NULL, defaults to U_ICUDATA_NAME + * @param pkg Package which is being searched for, ex "icudt28l". Will ignore leave directories such as /icudt28l + * @param item Item to be searched for. Can include full path, such as /a/b/foo.dat + * @param suffix Optional item suffix, if not-null (ex. ".dat") then 'path' can contain 'item' explicitly. + * Ex: 'stuff.dat' would be found in '/a/foo:/tmp/stuff.dat:/bar/baz' as item #2. + * '/blarg/stuff.dat' would also be found. + */ +static void udata_pathiter_init(UDataPathIterator *iter, const char *path, const char *pkg, + const char *item, const char *suffix, UBool doCheckLastFour) +{ +#ifdef UDATA_DEBUG + fprintf(stderr, "SUFFIX1=%s PATH=%s\n", suffix, path); +#endif + /** Path **/ + if(path == NULL) { + iter->path = u_getDataDirectory(); + } else { + iter->path = path; + } + + /** Package **/ + if(pkg == NULL) { + iter->packageStubLen = 0; + iter->packageStub=iter->packageStubBuf; + iter->packageStub[0] = 0; + } else { + if(uprv_strlen(pkg) + 2 > U_DATA_PATHITER_BUFSIZ) { + iter->packageStub = uprv_malloc(uprv_strlen(pkg)+2); + } else { + iter->packageStub = iter->packageStubBuf; + } + iter->packageStub[0] = U_FILE_SEP_CHAR; + uprv_strcpy(iter->packageStub+1, pkg); + iter->packageStubLen = (int32_t)uprv_strlen(iter->packageStub); + +#ifdef UDATA_DEBUG + fprintf(stderr, "STUB=%s [%d]\n", iter->packageStub, iter->packageStubLen); +#endif + } + + /** Item **/ + iter->basename = findBasename(item); + iter->basenameLen = (int32_t)uprv_strlen(iter->basename); + + /** Item path **/ + iter->itemPath = iter->itemPathBuf; + if(iter->basename == item) { + iter->itemPath[0] = 0; + iter->nextPath = iter->path; + } else { + int32_t itemPathLen = (int32_t)(iter->basename-item); + if (itemPathLen >= U_DATA_PATHITER_BUFSIZ) { + char *t = (char *)uprv_malloc(itemPathLen+1); + if (t != NULL) { + iter->itemPath = t; + } else { + /* Malloc failed. Ignore the itemPath. */ + itemPathLen = 0; + } + } + uprv_strncpy(iter->itemPath, item, itemPathLen); + iter->itemPath[itemPathLen]=0; + iter->nextPath = iter->itemPath; + } +#ifdef UDATA_DEBUG + fprintf(stderr, "SUFFIX=%s [%p]\n", suffix, suffix); +#endif + + /** Suffix **/ + if(suffix != NULL) { + iter->suffix = suffix; + } else { + iter->suffix = ""; + } + + iter->checkLastFour = doCheckLastFour; + + /* pathBuffer will hold the output path strings returned by the this iterator + * Get an upper bound of possible string size, and make sure that the buffer + * is big enough (sum of length of each piece, 2 extra delimiters, + trailing NULL) */ + { + int32_t maxPathLen = (int32_t)(uprv_strlen(iter->path) + uprv_strlen(item) + uprv_strlen(iter->suffix) + iter->packageStubLen + 3); + iter->pathBuffer = iter->pathBufferA; + if (maxPathLen >= U_DATA_PATHITER_BUFSIZ) { + iter->pathBuffer = (char *)uprv_malloc(maxPathLen); + if (iter->pathBuffer == NULL) { + iter->pathBuffer = iter->pathBufferA; + iter->path = ""; + } + } + } + +#ifdef UDATA_DEBUG + fprintf(stderr, "%p: init %s -> [path=%s], [base=%s], [suff=%s], [itempath=%s], [nextpath=%s], [checklast4=%s]\n", + iter, + item, + iter->path, + iter->basename, + iter->suffix, + iter->itemPath, + iter->nextPath, + iter->checkLastFour?"TRUE":"false"); +#endif + +} + +/** + * Get the next path on the list. + * + * @param iter The Iter to be used + * @param len If set, pointer to the length of the returned path, for convenience. + * @return Pointer to the next path segment, or NULL if there are no more. + */ +static const char *udata_pathiter_next(UDataPathIterator *iter) +{ + const char *path = NULL; + uint32_t pathLen = 0; + const char *pathBasename; + + do + { + if( iter->nextPath == NULL ) { + break; + } + + path = iter->nextPath; + + if(iter->nextPath == iter->itemPath) { /* we were processing item's path. */ + iter->nextPath = iter->path; /* start with regular path next tm. */ + pathLen = (int32_t)uprv_strlen(path); + } else { + /* fix up next for next time */ + iter->nextPath = uprv_strchr(path, U_PATH_SEP_CHAR); + if(iter->nextPath == NULL) { + /* segment: entire path */ + pathLen = (int32_t)uprv_strlen(path); + } else { + /* segment: until next segment */ + pathLen = (int32_t)(iter->nextPath - path); + if(*iter->nextPath) { /* skip divider */ + iter->nextPath ++; + } + } + } + + if(pathLen == 0) { + continue; + } + +#ifdef UDATA_DEBUG + fprintf(stderr, "rest of path (IDD) = %s\n", path); + fprintf(stderr, " "); + { + uint32_t qqq; + for(qqq=0;qqqpathBuffer, path, pathLen); + iter->pathBuffer[pathLen] = 0; + + /* check for .dat files */ + pathBasename = findBasename(iter->pathBuffer); + + if(iter->checkLastFour == TRUE && + (pathLen>=4) && + uprv_strncmp(iter->pathBuffer +(pathLen-4),iter->suffix,4)==0 && /* suffix matches */ + uprv_strncmp(findBasename(iter->pathBuffer),iter->basename,iter->basenameLen)==0 && /* base matches */ + uprv_strlen(pathBasename)==(iter->basenameLen+4)) { /* base+suffix = full len */ + +#ifdef UDATA_DEBUG + fprintf(stderr, "Have %s file on the path: %s\n", iter->suffix, iter->pathBuffer); +#endif + /* do nothing */ + } + else + { /* regular dir path */ + if(iter->pathBuffer[pathLen-1] != U_FILE_SEP_CHAR) { + if((pathLen>=4) && + uprv_strncmp(iter->pathBuffer+(pathLen-4), ".dat", 4) == 0) + { +#ifdef UDATA_DEBUG + fprintf(stderr, "skipping non-directory .dat file %s\n", iter->pathBuffer); +#endif + continue; + } + + /* Check if it is a directory with the same name as our package */ + if(iter->packageStubLen && + (pathLen > iter->packageStubLen) && + !uprv_strcmp(iter->pathBuffer + pathLen - iter->packageStubLen, iter->packageStub)) { +#ifdef UDATA_DEBUG + fprintf(stderr, "Found stub %s ( will add package %s of len %d)\n", iter->packageStub, iter->basename, iter->basenameLen); +#endif + pathLen -= iter->packageStubLen; + } + + iter->pathBuffer[pathLen++] = U_FILE_SEP_CHAR; + } + + uprv_strncpy(iter->pathBuffer + pathLen, /* + basename */ + iter->packageStub+1, + iter->packageStubLen-1); + + pathLen += iter->packageStubLen-1; + + if(*iter->suffix) /* tack on suffix */ + { + uprv_strcpy(iter->pathBuffer + pathLen, + iter->suffix); + pathLen += (int32_t)uprv_strlen(iter->suffix); + } + + } + +#ifdef UDATA_DEBUG + fprintf(stderr, " --> %s\n", iter->pathBuffer); +#endif + + return iter->pathBuffer; + + } while(iter->path); + + /* fell way off the end */ + return NULL; +} + + +/* + * Path Iterator Destructor. Clean up any allocated storage + */ +static void udata_pathiter_dt(UDataPathIterator *iter) { + if (iter->itemPath != iter->itemPathBuf) { + uprv_free(iter->itemPath); + iter->itemPath = NULL; + } + if (iter->pathBuffer != iter->pathBufferA) { + uprv_free(iter->pathBuffer); + iter->pathBuffer = NULL; + } + if (iter->packageStub != iter->packageStubBuf) { + uprv_free(iter->packageStub); + iter->packageStub = NULL; + } +} + +/* ==================================================================================*/ + + +/*----------------------------------------------------------------------* + * * + * Add a static reference to the common data library * + * Unless overridden by an explicit udata_setCommonData, this will be * + * our common data. * + * * + *----------------------------------------------------------------------*/ +extern const DataHeader U_DATA_API U_ICUDATA_ENTRY_POINT; + + +/*----------------------------------------------------------------------* + * * + * openCommonData Attempt to open a common format (.dat) file * + * Map it into memory (if it's not there already) * + * and return a UDataMemory object for it. * + * * + * If the requested data is already open and cached * + * just return the cached UDataMem object. * + * * + *----------------------------------------------------------------------*/ +static UDataMemory * +openCommonData(const char *path, /* Path from OpenChoice? */ + UBool isICUData, /* ICU Data true if path == NULL */ + UErrorCode *pErrorCode) +{ + UDataMemory tData; + UDataPathIterator iter; + const char *pathBuffer; + const char *inBasename; + + if (U_FAILURE(*pErrorCode)) { + return NULL; + } + + UDataMemory_init(&tData); + + /* ??????? TODO revisit this */ + if (isICUData) { + /* "mini-cache" for common ICU data */ + if(gCommonICUData != NULL) { + return gCommonICUData; + } + + tData.pHeader = &U_ICUDATA_ENTRY_POINT; + udata_checkCommonData(&tData, pErrorCode); + setCommonICUData(&tData, NULL, FALSE, pErrorCode); + return gCommonICUData; + } + + + /* request is NOT for ICU Data. */ + + /* Find the base name portion of the supplied path. */ + /* inBasename will be left pointing somewhere within the original path string. */ + inBasename = findBasename(path); +#ifdef UDATA_DEBUG + fprintf(stderr, "inBasename = %s\n", inBasename); +#endif + + if(*inBasename==0) { + /* no basename. This will happen if the original path was a directory name, */ + /* like "a/b/c/". (Fallback to separate files will still work.) */ +#ifdef UDATA_DEBUG + fprintf(stderr, "ocd: no basename in %s, bailing.\n", path); +#endif + *pErrorCode=U_FILE_ACCESS_ERROR; + return NULL; + } + + /* Is the requested common data file already open and cached? */ + /* Note that the cache is keyed by the base name only. The rest of the path, */ + /* if any, is not considered. */ + { + UDataMemory *dataToReturn = udata_findCachedData(inBasename); + if (dataToReturn != NULL) { + return dataToReturn; + } + } + + /* Requested item is not in the cache. + * Hunt it down, trying all the path locations + */ + + udata_pathiter_init(&iter, u_getDataDirectory(), inBasename, path, ".dat", TRUE); + + while((UDataMemory_isLoaded(&tData)==FALSE) && + (pathBuffer = udata_pathiter_next(&iter)) != NULL) + { +#ifdef UDATA_DEBUG + fprintf(stderr, "ocd: trying path %s - ", pathBuffer); +#endif + uprv_mapFile(&tData, pathBuffer); +#ifdef UDATA_DEBUG + fprintf(stderr, "%s\n", UDataMemory_isLoaded(&tData)?"LOADED":"not loaded"); +#endif + } + udata_pathiter_dt(&iter); /* Note: this call may invalidate "pathBuffer" */ + +#if defined(OS390_STUBDATA) && defined(OS390BATCH) + if (!UDataMemory_isLoaded(&tData)) { + char ourPathBuffer[1024]; + /* One more chance, for extendCommonData() */ + uprv_strncpy(ourPathBuffer, path, 1019); + ourPathBuffer[1019]=0; + uprv_strcat(ourPathBuffer, ".dat"); + uprv_mapFile(&tData, ourPathBuffer); + } +#endif + + if (!UDataMemory_isLoaded(&tData)) { + /* no common data */ + *pErrorCode=U_FILE_ACCESS_ERROR; + return NULL; + } + + /* we have mapped a file, check its header */ + udata_checkCommonData(&tData, pErrorCode); + + + /* Cache the UDataMemory struct for this .dat file, + * so we won't need to hunt it down and map it again next time + * something is needed from it. */ + return udata_cacheDataItem(inBasename, &tData, pErrorCode); +} + + +#ifdef OS390 +# define MAX_STUB_ENTRIES 8 +#else +# define MAX_STUB_ENTRIES 0 +#endif + + +/*----------------------------------------------------------------------* + * * + * extendICUData If the full set of ICU data was not loaded at * + * program startup, load it now. This function will * + * be called when the lookup of an ICU data item in * + * the common ICU data fails. * + * * + * The parameter is the UDataMemory in which the * + * search for a requested item failed. * + * * + * return true if new data is loaded, false otherwise.* + * * + *----------------------------------------------------------------------*/ +static UBool extendICUData(UDataMemory *failedData, UErrorCode *pErr) +{ + /* If the data library that we are running with turns out to be the + * stub library (or, on the 390, the subset library), we will try to + * load a .dat file instead. The stub library has no entries in its + * TOC, which is how we identify it here. + */ + UDataMemory *pData; + UDataMemory copyPData; + + if (failedData->vFuncs->NumEntries(failedData) > MAX_STUB_ENTRIES) { + /* Not the stub. We can't extend. */ + return FALSE; + } + + /* See if we can explicitly open a .dat file for the ICUData. */ + pData = openCommonData( + U_ICUDATA_NAME, /* "icudt20l" , for example. */ + FALSE, /* Pretend we're not opening ICUData */ + pErr); + + /* How about if there is no pData, eh... */ + + UDataMemory_init(©PData); + if(pData != NULL) { + UDatamemory_assign(©PData, pData); + copyPData.map = 0; /* The mapping for this data is owned by the hash table */ + copyPData.mapAddr = 0; /* which will unmap it when ICU is shut down. */ + /* CommonICUData is also unmapped when ICU is shut down.*/ + /* To avoid unmapping the data twice, zero out the map */ + /* fields in the UDataMemory that we're assigning */ + /* to CommonICUData. */ + + setCommonICUData(©PData, /* The new common data. */ + failedData, /* Old ICUData ptr. Overwrite of this value is ok, */ + FALSE, /* No warnings if write didn't happen */ + pErr); /* setCommonICUData honors errors; NOP if error set */ + } + + + return gCommonICUData != failedData; /* Return true if ICUData pointer was updated. */ + /* (Could potentialy have been done by another thread racing */ + /* us through here, but that's fine, we still return true */ + /* so that current thread will also examine extended data. */ +} + + + + +/*----------------------------------------------------------------------* + * * + * udata_setCommonData * + * * + *----------------------------------------------------------------------*/ +U_CAPI void U_EXPORT2 +udata_setCommonData(const void *data, UErrorCode *pErrorCode) { + UDataMemory dataMemory; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return; + } + + if(data==NULL) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + /* do we already have common ICU data set? */ + if(gCommonICUData != NULL) { + *pErrorCode=U_USING_DEFAULT_WARNING; + return; + } + + /* set the data pointer and test for validity */ + UDataMemory_init(&dataMemory); + UDataMemory_setData(&dataMemory, data); + udata_checkCommonData(&dataMemory, pErrorCode); + if (U_FAILURE(*pErrorCode)) {return;} + + /* we have good data */ + /* Set it up as the ICU Common Data. */ + setCommonICUData(&dataMemory, NULL, TRUE, pErrorCode); +} + + + + +/*--------------------------------------------------------------------------- + * + * udata_setAppData + * + *---------------------------------------------------------------------------- */ +U_CAPI void U_EXPORT2 +udata_setAppData(const char *path, const void *data, UErrorCode *err) +{ + UDataMemory udm; + + if(err==NULL || U_FAILURE(*err)) { + return; + } + if(data==NULL) { + *err=U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + UDataMemory_init(&udm); + udm.pHeader = data; + udata_checkCommonData(&udm, err); + udata_cacheDataItem(path, &udm, err); +} + +/*----------------------------------------------------------------------------* + * * + * checkDataItem Given a freshly located/loaded data item, either * + * an entry in a common file or a separately loaded file, * + * sanity check its header, and see if the data is * + * acceptable to the app. * + * If the data is good, create and return a UDataMemory * + * object that can be returned to the application. * + * Return NULL on any sort of failure. * + * * + *----------------------------------------------------------------------------*/ +static UDataMemory * +checkDataItem +( + const DataHeader *pHeader, /* The data item to be checked. */ + UDataMemoryIsAcceptable *isAcceptable, /* App's call-back function */ + void *context, /* pass-thru param for above. */ + const char *type, /* pass-thru param for above. */ + const char *name, /* pass-thru param for above. */ + UErrorCode *nonFatalErr, /* Error code if this data was not acceptable */ + /* but openChoice should continue with */ + /* trying to get data from fallback path. */ + UErrorCode *fatalErr /* Bad error, caller should return immediately */ + ) +{ + UDataMemory *rDataMem = NULL; /* the new UDataMemory, to be returned. */ + + if (U_FAILURE(*fatalErr)) { + return NULL; + } + + if(pHeader->dataHeader.magic1==0xda && + pHeader->dataHeader.magic2==0x27 && + (isAcceptable==NULL || isAcceptable(context, type, name, &pHeader->info)) + ) { + rDataMem=UDataMemory_createNewInstance(fatalErr); + if (U_FAILURE(*fatalErr)) { + return NULL; + } + rDataMem->pHeader = pHeader; + } else { + /* the data is not acceptable, look further */ + /* If we eventually find something good, this errorcode will be */ + /* cleared out. */ + *nonFatalErr=U_INVALID_FORMAT_ERROR; + } + return rDataMem; +} + +/** + * @return 0 if not loaded, 1 if loaded or err + */ +static UDataMemory *doLoadFromIndividualFiles(const char *pkgName, + const char *dataPath, const char *tocEntryPathSuffix, + /* following arguments are the same as doOpenChoice itself */ + const char *path, const char *type, const char *name, + UDataMemoryIsAcceptable *isAcceptable, void *context, + UErrorCode *subErrorCode, + UErrorCode *pErrorCode) +{ + UDataMemory *retVal = NULL; + const char *pathBuffer; + UDataMemory dataMemory; + UDataMemory *pEntryData; + + UDataPathIterator iter; + /* look in ind. files: package\nam.typ ========================= */ + /* init path iterator for individual files */ + udata_pathiter_init(&iter, dataPath, pkgName, path, tocEntryPathSuffix, FALSE); + + while((pathBuffer = udata_pathiter_next(&iter))) + { +#ifdef UDATA_DEBUG + fprintf(stderr, "UDATA: trying individual file %s\n", pathBuffer); +#endif + if(uprv_mapFile(&dataMemory, pathBuffer)) + { + pEntryData = checkDataItem(dataMemory.pHeader, isAcceptable, context, type, name, subErrorCode, pErrorCode); + if (pEntryData != NULL) { + /* Data is good. + * Hand off ownership of the backing memory to the user's UDataMemory. + * and return it. */ + pEntryData->mapAddr = dataMemory.mapAddr; + pEntryData->map = dataMemory.map; + +#ifdef UDATA_DEBUG + fprintf(stderr, "** Mapped file: %s\n", pathBuffer); +#endif + retVal = pEntryData; + goto commonReturn; + } + + /* the data is not acceptable, or some error occured. Either way, unmap the memory */ + udata_close(&dataMemory); + + /* If we had a nasty error, bail out completely. */ + if (U_FAILURE(*pErrorCode)) { + retVal = NULL; + goto commonReturn; + } + + /* Otherwise remember that we found data but didn't like it for some reason */ + *subErrorCode=U_INVALID_FORMAT_ERROR; + } +#ifdef UDATA_DEBUG + fprintf(stderr, "%s\n", UDataMemory_isLoaded(&dataMemory)?"LOADED":"not loaded"); +#endif + } +commonReturn: + udata_pathiter_dt(&iter); + return retVal; +} + +/** + * @return 0 if not loaded, 1 if loaded or err + */ +static UDataMemory *doLoadFromCommonData(UBool isICUData, const char *pkgName, + const char *dataPath, const char *tocEntryPathSuffix, const char *tocEntryName, + /* following arguments are the same as doOpenChoice itself */ + const char *path, const char *type, const char *name, + UDataMemoryIsAcceptable *isAcceptable, void *context, + UErrorCode *subErrorCode, + UErrorCode *pErrorCode) +{ + UDataMemory *retVal = NULL; + UDataMemory *pEntryData; + const DataHeader *pHeader; + UDataMemory *pCommonData; + /* try to get common data. The loop is for platforms such as the 390 that do + * not initially load the full set of ICU data. If the lookup of an ICU data item + * fails, the full (but slower to load) set is loaded, the and the loop repeats, + * trying the lookup again. Once the full set of ICU data is loaded, the loop wont + * repeat because the full set will be checked the first time through. + * + * The loop also handles the fallback to a .dat file if the application linked + * to the stub data library rather than a real library. + */ + for (;;) { + pCommonData=openCommonData(path, isICUData, subErrorCode); /** search for pkg **/ + + if(U_SUCCESS(*subErrorCode)) { + int32_t length; + + /* look up the data piece in the common data */ + pHeader=pCommonData->vFuncs->Lookup(pCommonData, tocEntryName, &length, subErrorCode); +#ifdef UDATA_DEBUG + fprintf(stderr, "%s: pHeader=%p - %s\n", tocEntryName, pHeader, u_errorName(*subErrorCode)); +#endif + + if(pHeader!=NULL) { + pEntryData = checkDataItem(pHeader, isAcceptable, context, type, name, subErrorCode, pErrorCode); +#ifdef UDATA_DEBUG + fprintf(stderr, "pEntryData=%p\n", pEntryData); +#endif + if (U_FAILURE(*pErrorCode)) { + retVal = NULL; + goto commonReturn; + } + if (pEntryData != NULL) { + pEntryData->length = length; + retVal = pEntryData; + goto commonReturn; + } + } + } + /* Data wasn't found. If we were looking for an ICUData item and there is + * more data available, load it and try again, + * otherwise break out of this loop. */ + if (!(isICUData && pCommonData && extendICUData(pCommonData, subErrorCode))) { + break; + } + } + +commonReturn: + return retVal; +} + +/* + * A note on the ownership of Mapped Memory + * + * For common format files, ownership resides with the UDataMemory object + * that lives in the cache of opened common data. These UDataMemorys are private + * to the udata implementation, and are never seen directly by users. + * + * The UDataMemory objects returned to users will have the address of some desired + * data within the mapped region, but they wont have the mapping info itself, and thus + * won't cause anything to be removed from memory when they are closed. + * + * For individual data files, the UDataMemory returned to the user holds the + * information necessary to unmap the data on close. If the user independently + * opens the same data file twice, two completely independent mappings will be made. + * (There is no cache of opened data items from individual files, only a cache of + * opened Common Data files, that is, files containing a collection of data items.) + * + * For common data passed in from the user via udata_setAppData() or + * udata_setCommonData(), ownership remains with the user. + * + * UDataMemory objects themselves, as opposed to the memory they describe, + * can be anywhere - heap, stack/local or global. + * They have a flag to indicate when they're heap allocated and thus + * must be deleted when closed. + */ + + +/*----------------------------------------------------------------------------* + * * + * main data loading functions * + * * + *----------------------------------------------------------------------------*/ +static UDataMemory * +doOpenChoice(const char *path, const char *type, const char *name, + UDataMemoryIsAcceptable *isAcceptable, void *context, + UErrorCode *pErrorCode) +{ + UDataMemory *retVal = NULL; + + TinyString tocEntryName; /* entry name in tree format. ex: 'icudt28b/coll/ar.res' */ + TinyString tocEntryPath; /* entry name in path format. ex: 'icudt28b\\coll\\ar.res' */ + + TinyString pkgName; + TinyString treeName; +#if (U_FILE_SEP_CHAR != U_FILE_ALT_SEP_CHAR) /* '/' vs '\' */ + TinyString altSepPath; +#endif + + const char *dataPath; + + int32_t tocEntrySuffixIndex; + const char *tocEntryPathSuffix; + UErrorCode subErrorCode=U_ZERO_ERROR; + const char *treeChar; + + UBool isICUData = FALSE; + + + /* Is this path ICU data? */ + if(path == NULL || + !strcmp(path, U_ICUDATA_ALIAS) || /* "ICUDATA" */ + !uprv_strncmp(path, U_ICUDATA_NAME U_TREE_SEPARATOR_STRING, /* "icudt26e-" */ + uprv_strlen(U_ICUDATA_NAME U_TREE_SEPARATOR_STRING)) || + !uprv_strncmp(path, U_ICUDATA_ALIAS U_TREE_SEPARATOR_STRING, /* "ICUDATA-" */ + uprv_strlen(U_ICUDATA_ALIAS U_TREE_SEPARATOR_STRING))) { + isICUData = TRUE; + } + +#if (U_FILE_SEP_CHAR != U_FILE_ALT_SEP_CHAR) /* Windows: try "foo\bar" and "foo/bar" */ + /* remap from alternate path char to the main one */ + TinyString_init(&altSepPath); + if(path) { + char *p; + if((p=uprv_strchr(path,U_FILE_ALT_SEP_CHAR))) { + TinyString_append(&altSepPath, path); + while((p=uprv_strchr(altSepPath.s,U_FILE_ALT_SEP_CHAR))) { + *p = U_FILE_SEP_CHAR; + } +#if defined (UDATA_DEBUG) + fprintf(stderr, "Changed path from [%s] to [%s]\n", path, altSepPath.s); +#endif + path = altSepPath.s; + } + } +#endif + + TinyString_init(&tocEntryName); + TinyString_init(&tocEntryPath); + + TinyString_init(&pkgName); + TinyString_init(&treeName); + + /* ======= Set up strings */ + if(path==NULL) { + TinyString_append(&pkgName, U_ICUDATA_NAME); + } else { + const char *pkg; + const char *first; + pkg = uprv_strrchr(path, U_FILE_SEP_CHAR); + first = uprv_strchr(path, U_FILE_SEP_CHAR); + if(uprv_pathIsAbsolute(path) || (pkg != first)) { /* more than one slash in the path- not a tree name */ + /* see if this is an /absolute/path/to/package path */ + if(pkg) { + TinyString_append(&pkgName, pkg+1); + } else { + TinyString_append(&pkgName, path); + } + } else { + treeChar = uprv_strchr(path, U_TREE_SEPARATOR); + if(treeChar) { + TinyString_append(&treeName, treeChar+1); /* following '-' */ + if(isICUData) { + TinyString_append(&pkgName, U_ICUDATA_NAME); + } else { + TinyString_appendn(&pkgName, path, (int32_t)(treeChar-path)); + if (first == NULL) { + /* + This user data has no path, but there is a tree name. + Look up the correct path from the data cache later. + */ + path = pkgName.s; + } + } + } else { + if(isICUData) { + TinyString_append(&pkgName, U_ICUDATA_NAME); + } else { + TinyString_append(&pkgName, path); + } + } + } + } + +#ifdef UDATA_DEBUG + fprintf(stderr, " P=%s T=%s\n", pkgName.s, treeName.s); +#endif + + /* setting up the entry name and file name + * Make up a full name by appending the type to the supplied + * name, assuming that a type was supplied. + */ + + /* prepend the package */ + TinyString_append(&tocEntryName, pkgName.s); + TinyString_append(&tocEntryPath, pkgName.s); + tocEntrySuffixIndex = tocEntryName.length; + + if(treeName.s[0]) { + TinyString_append(&tocEntryName, U_TREE_ENTRY_SEP_STRING); + TinyString_append(&tocEntryName, treeName.s); + + TinyString_append(&tocEntryPath, U_FILE_SEP_STRING); + TinyString_append(&tocEntryPath, treeName.s); + } + + TinyString_append(&tocEntryName, U_TREE_ENTRY_SEP_STRING); + TinyString_append(&tocEntryPath, U_FILE_SEP_STRING); + TinyString_append(&tocEntryName, name); + TinyString_append(&tocEntryPath, name); + if(type!=NULL && *type!=0) { + TinyString_append(&tocEntryName, "."); + TinyString_append(&tocEntryName, type); + TinyString_append(&tocEntryPath, "."); + TinyString_append(&tocEntryPath, type); + } + tocEntryPathSuffix = tocEntryPath.s+tocEntrySuffixIndex; /* suffix starts here */ + +#ifdef UDATA_DEBUG + fprintf(stderr, " tocEntryName = %s\n", tocEntryName.s); + fprintf(stderr, " tocEntryPath = %s\n", tocEntryName.s); +#endif + + if(path == NULL) { + path = COMMON_DATA_NAME; /* "icudt26e" */ + } + + /************************ Begin loop looking for ind. files ***************/ +#ifdef UDATA_DEBUG + fprintf(stderr, "IND: inBasename = %s, pkg=%s\n", inBasename, packageNameFromPath(path)); +#endif + + /* End of dealing with a null basename */ + dataPath = u_getDataDirectory(); + + /**** COMMON PACKAGE - only if packages are first. */ + if(gDataFileAccess == UDATA_PACKAGES_FIRST) { +#ifdef UDATA_DEBUG + fprintf(stderr, "Trying packages (UDATA_PACKAGES_FIRST)\n"); +#endif + /* #2 */ + retVal = doLoadFromCommonData(isICUData, + pkgName.s, dataPath, tocEntryPathSuffix, tocEntryName.s, + path, type, name, isAcceptable, context, &subErrorCode, pErrorCode); + if((retVal != NULL) || U_FAILURE(*pErrorCode)) { + goto commonReturn; + } + } + + /**** INDIVIDUAL FILES */ + if((gDataFileAccess==UDATA_PACKAGES_FIRST) || + (gDataFileAccess==UDATA_FILES_FIRST)) { +#ifdef UDATA_DEBUG + fprintf(stderr, "Trying individual files\n"); +#endif + /* Check to make sure that there is a dataPath to iterate over */ + if ((dataPath && *dataPath) || !isICUData) { + retVal = doLoadFromIndividualFiles(pkgName.s, dataPath, tocEntryPathSuffix, + path, type, name, isAcceptable, context, &subErrorCode, pErrorCode); + if((retVal != NULL) || U_FAILURE(*pErrorCode)) { + goto commonReturn; + } + } + } + + /**** COMMON PACKAGE */ + if((gDataFileAccess==UDATA_ONLY_PACKAGES) || + (gDataFileAccess==UDATA_FILES_FIRST)) { +#ifdef UDATA_DEBUG + fprintf(stderr, "Trying packages (UDATA_ONLY_PACKAGES || UDATA_FILES_FIRST)\n"); +#endif + retVal = doLoadFromCommonData(isICUData, + pkgName.s, dataPath, tocEntryPathSuffix, tocEntryName.s, + path, type, name, isAcceptable, context, &subErrorCode, pErrorCode); + if((retVal != NULL) || U_FAILURE(*pErrorCode)) { + goto commonReturn; + } + } + + /* Load from DLL. If we haven't attempted package load, we also haven't had any chance to + try a DLL (static or setCommonData/etc) load. + If we ever have a "UDATA_ONLY_FILES", add it to the or list here. */ + if(gDataFileAccess==UDATA_NO_FILES) { +#ifdef UDATA_DEBUG + fprintf(stderr, "Trying common data (UDATA_NO_FILES)\n"); +#endif + retVal = doLoadFromCommonData(isICUData, + pkgName.s, "", tocEntryPathSuffix, tocEntryName.s, + path, type, name, isAcceptable, context, &subErrorCode, pErrorCode); + if((retVal != NULL) || U_FAILURE(*pErrorCode)) { + goto commonReturn; + } + } + + /* data not found */ + if(U_SUCCESS(*pErrorCode)) { + if(U_SUCCESS(subErrorCode)) { + /* file not found */ + *pErrorCode=U_FILE_ACCESS_ERROR; + } else { + /* entry point not found or rejected */ + *pErrorCode=subErrorCode; + } + } + +commonReturn: + TinyString_dt(&tocEntryName); + TinyString_dt(&tocEntryPath); + TinyString_dt(&pkgName); + TinyString_dt(&treeName); +#if (U_FILE_SEP_CHAR != U_FILE_ALT_SEP_CHAR) + TinyString_dt(&altSepPath); +#endif + return retVal; +} + + + +/* API ---------------------------------------------------------------------- */ + +U_CAPI UDataMemory * U_EXPORT2 +udata_open(const char *path, const char *type, const char *name, + UErrorCode *pErrorCode) { +#ifdef UDATA_DEBUG + fprintf(stderr, "udata_open(): Opening: %s : %s . %s\n", (path?path:"NULL"), name, type); + fflush(stderr); +#endif + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return NULL; + } else if(name==NULL || *name==0) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } else { + return doOpenChoice(path, type, name, NULL, NULL, pErrorCode); + } +} + + + +U_CAPI UDataMemory * U_EXPORT2 +udata_openChoice(const char *path, const char *type, const char *name, + UDataMemoryIsAcceptable *isAcceptable, void *context, + UErrorCode *pErrorCode) { +#ifdef UDATA_DEBUG + fprintf(stderr, "udata_openChoice(): Opening: %s : %s . %s\n", (path?path:"NULL"), name, type); +#endif + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return NULL; + } else if(name==NULL || *name==0 || isAcceptable==NULL) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } else { + return doOpenChoice(path, type, name, isAcceptable, context, pErrorCode); + } +} + + + +U_CAPI void U_EXPORT2 +udata_getInfo(UDataMemory *pData, UDataInfo *pInfo) { + if(pInfo!=NULL) { + if(pData!=NULL && pData->pHeader!=NULL) { + const UDataInfo *info=&pData->pHeader->info; + uint16_t dataInfoSize=udata_getInfoSize(info); + if(pInfo->size>dataInfoSize) { + pInfo->size=dataInfoSize; + } + uprv_memcpy((uint16_t *)pInfo+1, (const uint16_t *)info+1, pInfo->size-2); + if(info->isBigEndian!=U_IS_BIG_ENDIAN) { + /* opposite endianness */ + uint16_t x=info->reservedWord; + pInfo->reservedWord=(uint16_t)((x<<8)|(x>>8)); + } + } else { + pInfo->size=0; + } + } +} + + +U_CAPI void U_EXPORT2 udata_setFileAccess(UDataFileAccess access, UErrorCode *status) +{ + gDataFileAccess = access; +} + diff --git a/Build/source/libs/icu/icu-xetex/common/udatamem.c b/Build/source/libs/icu/icu-xetex/common/udatamem.c new file mode 100644 index 00000000000..b0778a63807 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/udatamem.c @@ -0,0 +1,160 @@ +/* +****************************************************************************** +* +* Copyright (C) 1999-2004, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************/ + + +/*---------------------------------------------------------------------------------- + * + * UDataMemory A class-like struct that serves as a handle to a piece of memory + * that contains some ICU data (resource, converters, whatever.) + * + * When an application opens ICU data (with udata_open, for example, + * a UDataMemory * is returned. + * + *----------------------------------------------------------------------------------*/ + +#include "unicode/utypes.h" +#include "cmemory.h" +#include "unicode/udata.h" + +#include "udatamem.h" + +void UDataMemory_init(UDataMemory *This) { + uprv_memset(This, 0, sizeof(UDataMemory)); + This->length=-1; +} + + +void UDatamemory_assign(UDataMemory *dest, UDataMemory *source) { + /* UDataMemory Assignment. Destination UDataMemory must be initialized first. */ + UBool mallocedFlag = dest->heapAllocated; + uprv_memcpy(dest, source, sizeof(UDataMemory)); + dest->heapAllocated = mallocedFlag; +} + +UDataMemory *UDataMemory_createNewInstance(UErrorCode *pErr) { + UDataMemory *This; + + if (U_FAILURE(*pErr)) { + return NULL; + } + This = uprv_malloc(sizeof(UDataMemory)); + if (This == NULL) { + *pErr = U_MEMORY_ALLOCATION_ERROR; } + else { + UDataMemory_init(This); + This->heapAllocated = TRUE; + } + return This; +} + + +const DataHeader * +UDataMemory_normalizeDataPointer(const void *p) { + /* allow the data to be optionally prepended with an alignment-forcing double value */ + const DataHeader *pdh = (const DataHeader *)p; + if(pdh==NULL || (pdh->dataHeader.magic1==0xda && pdh->dataHeader.magic2==0x27)) { + return pdh; + } else { +#ifdef OS400 + /* + TODO: Fix this once the compiler implements this feature. Keep in sync with genccode.c + + This is here because this platform can't currently put + const data into the read-only pages of an object or + shared library (service program). Only strings are allowed in read-only + pages, so we use char * strings to store the data. + + In order to prevent the beginning of the data from ever matching the + magic numbers we must skip the initial double. + [grhoten 4/24/2003] + */ + return (const DataHeader *)*((const void **)p+1); +#else + return (const DataHeader *)((const double *)p+1); +#endif + } +} + + +void UDataMemory_setData (UDataMemory *This, const void *dataAddr) { + This->pHeader = UDataMemory_normalizeDataPointer(dataAddr); +} + + +U_CAPI void U_EXPORT2 +udata_close(UDataMemory *pData) { + if(pData!=NULL) { + uprv_unmapFile(pData); + if(pData->heapAllocated ) { + uprv_free(pData); + } else { + UDataMemory_init(pData); + } + } +} + +U_CAPI const void * U_EXPORT2 +udata_getMemory(UDataMemory *pData) { + if(pData!=NULL && pData->pHeader!=NULL) { + return (char *)(pData->pHeader)+udata_getHeaderSize(pData->pHeader); + } else { + return NULL; + } +} + +/** + * Get the length of the data item if possible. + * The length may be up to 15 bytes larger than the actual data. + * + * TODO Consider making this function public. + * It would have to return the actual length in more cases. + * For example, the length of the last item in a .dat package could be + * computed from the size of the whole .dat package minus the offset of the + * last item. + * The size of a file that was directly memory-mapped could be determined + * using some system API. + * + * In order to get perfect values for all data items, we may have to add a + * length field to UDataInfo, but that complicates data generation + * and may be overkill. + * + * @param pData The data item. + * @return the length of the data item, or -1 if not known + * @internal Currently used only in cintltst/udatatst.c + */ +U_CAPI int32_t U_EXPORT2 +udata_getLength(const UDataMemory *pData) { + if(pData!=NULL && pData->pHeader!=NULL && pData->length>=0) { + /* + * subtract the header size, + * return only the size of the actual data starting at udata_getMemory() + */ + return pData->length-udata_getHeaderSize(pData->pHeader); + } else { + return -1; + } +} + +/** + * Get the memory including the data header. + * Used in cintltst/udatatst.c + * @internal + */ +U_CAPI const void * U_EXPORT2 +udata_getRawMemory(const UDataMemory *pData) { + if(pData!=NULL && pData->pHeader!=NULL) { + return pData->pHeader; + } else { + return NULL; + } +} + +UBool UDataMemory_isLoaded(UDataMemory *This) { + return This->pHeader != NULL; +} + diff --git a/Build/source/libs/icu/icu-xetex/common/udatamem.h b/Build/source/libs/icu/icu-xetex/common/udatamem.h new file mode 100644 index 00000000000..504883d6aa1 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/udatamem.h @@ -0,0 +1,61 @@ +/* +****************************************************************************** +* +* Copyright (C) 1999-2004, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************/ + + +/*---------------------------------------------------------------------------------- + * + * UDataMemory A class-like struct that serves as a handle to a piece of memory + * that contains some ICU data (resource, converters, whatever.) + * + * When an application opens ICU data (with udata_open, for example, + * a UDataMemory * is returned. + * + *----------------------------------------------------------------------------------*/ +#ifndef __UDATAMEM_H__ +#define __UDATAMEM_H__ + +#include "unicode/udata.h" +#include "ucmndata.h" + +struct UDataMemory { + const commonDataFuncs *vFuncs; /* Function Pointers for accessing TOC */ + + const DataHeader *pHeader; /* Header of the memory being described by this */ + /* UDataMemory object. */ + const void *toc; /* For common memory, table of contents for */ + /* the pieces within. */ + UBool heapAllocated; /* True if this UDataMemory Object is on the */ + /* heap and thus needs to be deleted when closed. */ + + void *mapAddr; /* For mapped or allocated memory, the start addr. */ + /* Only non-null if a close operation should unmap */ + /* the associated data. */ + void *map; /* Handle, or other data, OS dependent. */ + /* Only non-null if a close operation should unmap */ + /* the associated data, and additional info */ + /* beyond the mapAddr is needed to do that. */ + int32_t length; /* Length of the data in bytes; -1 if unknown. */ +}; + +UDataMemory *UDataMemory_createNewInstance(UErrorCode *pErr); +void UDatamemory_assign (UDataMemory *dest, UDataMemory *source); +void UDataMemory_init (UDataMemory *This); +UBool UDataMemory_isLoaded(UDataMemory *This); +void UDataMemory_setData (UDataMemory *This, const void *dataAddr); + + +const DataHeader *UDataMemory_normalizeDataPointer(const void *p); + +U_CAPI int32_t U_EXPORT2 +udata_getLength(const UDataMemory *pData); + +U_CAPI const void * U_EXPORT2 +udata_getRawMemory(const UDataMemory *pData); + +#endif + diff --git a/Build/source/libs/icu/icu-xetex/common/udataswp.c b/Build/source/libs/icu/icu-xetex/common/udataswp.c new file mode 100644 index 00000000000..cacf716610e --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/udataswp.c @@ -0,0 +1,415 @@ +/* +******************************************************************************* +* +* Copyright (C) 2003, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: udataswp.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2003jun05 +* created by: Markus W. Scherer +* +* Definitions for ICU data transformations for different platforms, +* changing between big- and little-endian data and/or between +* charset families (ASCII<->EBCDIC). +*/ + +#include +#include "unicode/utypes.h" +#include "unicode/udata.h" /* UDataInfo */ +#include "ucmndata.h" /* DataHeader */ +#include "cmemory.h" +#include "udataswp.h" + +/* swapping primitives ------------------------------------------------------ */ + +static int32_t U_CALLCONV +uprv_swapArray16(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode) { + const uint16_t *p; + uint16_t *q; + int32_t count; + uint16_t x; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + if(ds==NULL || inData==NULL || length<0 || (length&1)!=0 || outData==NULL) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* setup and swapping */ + p=(const uint16_t *)inData; + q=(uint16_t *)outData; + count=length/2; + while(count>0) { + x=*p++; + *q++=(uint16_t)((x<<8)|(x>>8)); + --count; + } + + return length; +} + +static int32_t U_CALLCONV +uprv_copyArray16(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode) { + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + if(ds==NULL || inData==NULL || length<0 || (length&1)!=0 || outData==NULL) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + if(length>0 && inData!=outData) { + uprv_memcpy(outData, inData, length); + } + return length; +} + +static int32_t U_CALLCONV +uprv_swapArray32(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode) { + const uint32_t *p; + uint32_t *q; + int32_t count; + uint32_t x; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + if(ds==NULL || inData==NULL || length<0 || (length&3)!=0 || outData==NULL) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* setup and swapping */ + p=(const uint32_t *)inData; + q=(uint32_t *)outData; + count=length/4; + while(count>0) { + x=*p++; + *q++=(uint32_t)((x<<24)|((x<<8)&0xff0000)|((x>>8)&0xff00)|(x>>24)); + --count; + } + + return length; +} + +static int32_t U_CALLCONV +uprv_copyArray32(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode) { + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + if(ds==NULL || inData==NULL || length<0 || (length&3)!=0 || outData==NULL) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + if(length>0 && inData!=outData) { + uprv_memcpy(outData, inData, length); + } + return length; +} + +static uint16_t U_CALLCONV +uprv_readSwapUInt16(uint16_t x) { + return (uint16_t)((x<<8)|(x>>8)); +} + +static uint16_t U_CALLCONV +uprv_readDirectUInt16(uint16_t x) { + return x; +} + +static uint32_t U_CALLCONV +uprv_readSwapUInt32(uint32_t x) { + return (uint32_t)((x<<24)|((x<<8)&0xff0000)|((x>>8)&0xff00)|(x>>24)); +} + +static uint32_t U_CALLCONV +uprv_readDirectUInt32(uint32_t x) { + return x; +} + +static void U_CALLCONV +uprv_writeSwapUInt16(uint16_t *p, uint16_t x) { + *p=(uint16_t)((x<<8)|(x>>8)); +} + +static void U_CALLCONV +uprv_writeDirectUInt16(uint16_t *p, uint16_t x) { + *p=x; +} + +static void U_CALLCONV +uprv_writeSwapUInt32(uint32_t *p, uint32_t x) { + *p=(uint32_t)((x<<24)|((x<<8)&0xff0000)|((x>>8)&0xff00)|(x>>24)); +} + +static void U_CALLCONV +uprv_writeDirectUInt32(uint32_t *p, uint32_t x) { + *p=x; +} + +U_CAPI int16_t U_EXPORT2 +udata_readInt16(const UDataSwapper *ds, int16_t x) { + return (int16_t)ds->readUInt16((uint16_t)x); +} + +U_CAPI int32_t U_EXPORT2 +udata_readInt32(const UDataSwapper *ds, int32_t x) { + return (int32_t)ds->readUInt32((uint32_t)x); +} + +/** + * Swap a block of invariant, NUL-terminated strings, but not padding + * bytes after the last string. + * @internal + */ +U_CAPI int32_t U_EXPORT2 +udata_swapInvStringBlock(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode) { + const char *inChars; + int32_t stringsLength; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + if(ds==NULL || inData==NULL || length<0 || (length>0 && outData==NULL)) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* reduce the strings length to not include bytes after the last NUL */ + inChars=(const char *)inData; + stringsLength=length; + while(stringsLength>0 && inChars[stringsLength-1]!=0) { + --stringsLength; + } + + /* swap up to the last NUL */ + ds->swapInvChars(ds, inData, stringsLength, outData, pErrorCode); + + /* copy the bytes after the last NUL */ + if(inData!=outData && length>stringsLength) { + uprv_memcpy((char *)outData+stringsLength, inChars+stringsLength, length-stringsLength); + } + + /* return the length including padding bytes */ + if(U_SUCCESS(*pErrorCode)) { + return length; + } else { + return 0; + } +} + +U_CAPI void U_EXPORT2 +udata_printError(const UDataSwapper *ds, + const char *fmt, + ...) { + va_list args; + + if(ds->printError!=NULL) { + va_start(args, fmt); + ds->printError(ds->printErrorContext, fmt, args); + va_end(args); + } +} + +/* swap a data header ------------------------------------------------------- */ + +U_CAPI int32_t U_EXPORT2 +udata_swapDataHeader(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode) { + const DataHeader *pHeader; + uint16_t headerSize, infoSize; + + /* argument checking */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + if(ds==NULL || inData==NULL || length<-1 || (length>0 && outData==NULL)) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* check minimum length and magic bytes */ + pHeader=(const DataHeader *)inData; + if( (length>=0 && lengthdataHeader.magic1!=0xda || + pHeader->dataHeader.magic2!=0x27 || + pHeader->info.sizeofUChar!=2 + ) { + udata_printError(ds, "udata_swapDataHeader(): initial bytes do not look like ICU data\n"); + *pErrorCode=U_UNSUPPORTED_ERROR; + return 0; + } + + headerSize=ds->readUInt16(pHeader->dataHeader.headerSize); + infoSize=ds->readUInt16(pHeader->info.size); + + if( headerSizedataHeader)+infoSize) || + (length>=0 && length0) { + DataHeader *outHeader; + const char *s; + int32_t maxLength; + + /* Most of the fields are just bytes and need no swapping. */ + if(inData!=outData) { + uprv_memcpy(outData, inData, headerSize); + } + outHeader=(DataHeader *)outData; + + outHeader->info.isBigEndian = ds->outIsBigEndian; + outHeader->info.charsetFamily = ds->outCharset; + + /* swap headerSize */ + ds->swapArray16(ds, &pHeader->dataHeader.headerSize, 2, &outHeader->dataHeader.headerSize, pErrorCode); + + /* swap UDataInfo size and reservedWord */ + ds->swapArray16(ds, &pHeader->info.size, 4, &outHeader->info.size, pErrorCode); + + /* swap copyright statement after the UDataInfo */ + infoSize+=sizeof(pHeader->dataHeader); + s=(const char *)inData+infoSize; + maxLength=headerSize-infoSize; + /* get the length of the string */ + for(length=0; lengthswapInvChars(ds, s, length, (char *)outData+infoSize, pErrorCode); + } + + return headerSize; +} + +/* API functions ------------------------------------------------------------ */ + +U_CAPI UDataSwapper * U_EXPORT2 +udata_openSwapper(UBool inIsBigEndian, uint8_t inCharset, + UBool outIsBigEndian, uint8_t outCharset, + UErrorCode *pErrorCode) { + UDataSwapper *swapper; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return NULL; + } + if(inCharset>U_EBCDIC_FAMILY || outCharset>U_EBCDIC_FAMILY) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + + /* allocate the swapper */ + swapper=uprv_malloc(sizeof(UDataSwapper)); + if(swapper==NULL) { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + uprv_memset(swapper, 0, sizeof(UDataSwapper)); + + /* set values and functions pointers according to in/out parameters */ + swapper->inIsBigEndian=inIsBigEndian; + swapper->inCharset=inCharset; + swapper->outIsBigEndian=outIsBigEndian; + swapper->outCharset=outCharset; + + swapper->readUInt16= inIsBigEndian==U_IS_BIG_ENDIAN ? uprv_readDirectUInt16 : uprv_readSwapUInt16; + swapper->readUInt32= inIsBigEndian==U_IS_BIG_ENDIAN ? uprv_readDirectUInt32 : uprv_readSwapUInt32; + + swapper->writeUInt16= outIsBigEndian==U_IS_BIG_ENDIAN ? uprv_writeDirectUInt16 : uprv_writeSwapUInt16; + swapper->writeUInt32= outIsBigEndian==U_IS_BIG_ENDIAN ? uprv_writeDirectUInt32 : uprv_writeSwapUInt32; + + swapper->compareInvChars= outCharset==U_ASCII_FAMILY ? uprv_compareInvAscii : uprv_compareInvEbcdic; + + swapper->swapArray16= inIsBigEndian==outIsBigEndian ? uprv_copyArray16 : uprv_swapArray16; + swapper->swapArray32= inIsBigEndian==outIsBigEndian ? uprv_copyArray32 : uprv_swapArray32; + + if(inCharset==U_ASCII_FAMILY) { + swapper->swapInvChars= outCharset==U_ASCII_FAMILY ? uprv_copyAscii : uprv_ebcdicFromAscii; + } else /* U_EBCDIC_FAMILY */ { + swapper->swapInvChars= outCharset==U_EBCDIC_FAMILY ? uprv_copyEbcdic : uprv_asciiFromEbcdic; + } + + return swapper; +} + +U_CAPI UDataSwapper * U_EXPORT2 +udata_openSwapperForInputData(const void *data, int32_t length, + UBool outIsBigEndian, uint8_t outCharset, + UErrorCode *pErrorCode) { + const DataHeader *pHeader; + uint16_t headerSize, infoSize; + UBool inIsBigEndian; + int8_t inCharset; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return NULL; + } + if( data==NULL || + (length>=0 && lengthU_EBCDIC_FAMILY + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + + pHeader=(const DataHeader *)data; + if( (length>=0 && lengthdataHeader.magic1!=0xda || + pHeader->dataHeader.magic2!=0x27 || + pHeader->info.sizeofUChar!=2 + ) { + *pErrorCode=U_UNSUPPORTED_ERROR; + return 0; + } + + inIsBigEndian=(UBool)pHeader->info.isBigEndian; + inCharset=pHeader->info.charsetFamily; + + if(inIsBigEndian==U_IS_BIG_ENDIAN) { + headerSize=pHeader->dataHeader.headerSize; + infoSize=pHeader->info.size; + } else { + headerSize=uprv_readSwapUInt16(pHeader->dataHeader.headerSize); + infoSize=uprv_readSwapUInt16(pHeader->info.size); + } + + if( headerSizedataHeader)+infoSize) || + (length>=0 && lengthEBCDIC). +*/ + +#ifndef __UDATASWP_H__ +#define __UDATASWP_H__ + +#include +#include "unicode/utypes.h" + +/* forward declaration */ + +U_CDECL_BEGIN + +struct UDataSwapper; +typedef struct UDataSwapper UDataSwapper; + +/** + * Function type for data transformation. + * Transforms data, or just returns the length of the data if + * the input length is -1. + * Swap functions assume that their data pointers are aligned properly. + * + * Quick implementation outline: + * (best to copy and adapt and existing swapper implementation) + * check that the data looks like the expected format + * if(length<0) { + * preflight: + * never dereference outData + * read inData and determine the data size + * assume that inData is long enough for this + * } else { + * outData can be NULL if length==0 + * inData==outData (in-place swapping) possible but not required! + * verify that length>=(actual size) + * if there is a chance that not every byte up to size is reached + * due to padding etc.: + * if(inData!=outData) { + * memcpy(outData, inData, actual size); + * } + * swap contents + * } + * return actual size + * + * Further implementation notes: + * - read integers from inData before swapping them + * because in-place swapping can make them unreadable + * - compareInvChars compares a local Unicode string with already-swapped + * output charset strings + * + * @param ds Pointer to UDataSwapper containing global data about the + * transformation and function pointers for handling primitive + * types. + * @param inData Pointer to the input data to be transformed or examined. + * @param length Length of the data, counting bytes. May be -1 for preflighting. + * If length>=0, then transform the data. + * If length==-1, then only determine the length of the data. + * The length cannot be determined from the data itself for all + * types of data (e.g., not for simple arrays of integers). + * @param outData Pointer to the output data buffer. + * If length>=0 (transformation), then the output buffer must + * have a capacity of at least length. + * If length==-1, then outData will not be used and can be NULL. + * @param pErrorCode ICU UErrorCode parameter, must not be NULL and must + * fulfill U_SUCCESS on input. + * @return The actual length of the data. + * + * @see UDataSwapper + * @internal ICU 2.8 + */ +typedef int32_t U_CALLCONV +UDataSwapFn(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode); + +/** + * Convert one uint16_t from input to platform endianness. + * @internal ICU 2.8 + */ +typedef uint16_t U_CALLCONV +UDataReadUInt16(uint16_t x); + +/** + * Convert one uint32_t from input to platform endianness. + * @internal ICU 2.8 + */ +typedef uint32_t U_CALLCONV +UDataReadUInt32(uint32_t x); + +/** + * Convert one uint16_t from platform to input endianness. + * @internal ICU 2.8 + */ +typedef void U_CALLCONV +UDataWriteUInt16(uint16_t *p, uint16_t x); + +/** + * Convert one uint32_t from platform to input endianness. + * @internal ICU 2.8 + */ +typedef void U_CALLCONV +UDataWriteUInt32(uint32_t *p, uint32_t x); + +/** + * Compare invariant-character strings, one in the output data and the + * other one caller-provided in Unicode. + * An output data string is compared because strings are usually swapped + * before the rest of the data, to allow for sorting of string tables + * according to the output charset. + * You can use -1 for the length parameters of NUL-terminated strings as usual. + * Returns Unicode code point order for invariant characters. + * @internal ICU 2.8 + */ +typedef int32_t U_CALLCONV +UDataCompareInvChars(const UDataSwapper *ds, + const char *outString, int32_t outLength, + const UChar *localString, int32_t localLength); + +/** + * Function for message output when an error occurs during data swapping. + * A format string and variable number of arguments are passed + * like for vprintf(). + * + * @param context A function-specific context pointer. + * @param fmt The format string. + * @param args The arguments for format string inserts. + * + * @internal ICU 2.8 + */ +typedef void U_CALLCONV +UDataPrintError(void *context, const char *fmt, va_list args); + +struct UDataSwapper { + /** Input endianness. @internal ICU 2.8 */ + UBool inIsBigEndian; + /** Input charset family. @see U_CHARSET_FAMILY @internal ICU 2.8 */ + uint8_t inCharset; + /** Output endianness. @internal ICU 2.8 */ + UBool outIsBigEndian; + /** Output charset family. @see U_CHARSET_FAMILY @internal ICU 2.8 */ + uint8_t outCharset; + + /* basic functions for reading data values */ + + /** Convert one uint16_t from input to platform endianness. @internal ICU 2.8 */ + UDataReadUInt16 *readUInt16; + /** Convert one uint32_t from input to platform endianness. @internal ICU 2.8 */ + UDataReadUInt32 *readUInt32; + /** Compare an invariant-character output string with a local one. @internal ICU 2.8 */ + UDataCompareInvChars *compareInvChars; + + /* basic functions for writing data values */ + + /** Convert one uint16_t from platform to input endianness. @internal ICU 2.8 */ + UDataWriteUInt16 *writeUInt16; + /** Convert one uint32_t from platform to input endianness. @internal ICU 2.8 */ + UDataWriteUInt32 *writeUInt32; + + /* basic functions for data transformations */ + + /** Transform an array of 16-bit integers. @internal ICU 2.8 */ + UDataSwapFn *swapArray16; + /** Transform an array of 32-bit integers. @internal ICU 2.8 */ + UDataSwapFn *swapArray32; + /** Transform an invariant-character string. @internal ICU 2.8 */ + UDataSwapFn *swapInvChars; + + /** + * Function for message output when an error occurs during data swapping. + * Can be NULL. + * @internal ICU 2.8 + */ + UDataPrintError *printError; + /** Context pointer for printError. @internal ICU 2.8 */ + void *printErrorContext; +}; + +U_CDECL_END + +U_CAPI UDataSwapper * U_EXPORT2 +udata_openSwapper(UBool inIsBigEndian, uint8_t inCharset, + UBool outIsBigEndian, uint8_t outCharset, + UErrorCode *pErrorCode); + +/** + * Open a UDataSwapper for the given input data and the specified output + * characteristics. + * Values of -1 for any of the characteristics mean the local platform's + * characteristics. + * + * @see udata_swap + * @internal ICU 2.8 + */ +U_CAPI UDataSwapper * U_EXPORT2 +udata_openSwapperForInputData(const void *data, int32_t length, + UBool outIsBigEndian, uint8_t outCharset, + UErrorCode *pErrorCode); + +U_CAPI void U_EXPORT2 +udata_closeSwapper(UDataSwapper *ds); + +/** + * Read the beginning of an ICU data piece, recognize magic bytes, + * swap the structure. + * Set a U_UNSUPPORTED_ERROR if it does not look like an ICU data piece. + * + * @return The size of the data header, in bytes. + * + * @internal ICU 2.8 + */ +U_CAPI int32_t U_EXPORT2 +udata_swapDataHeader(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode); + +/** + * Convert one int16_t from input to platform endianness. + * @internal ICU 2.8 + */ +U_CAPI int16_t U_EXPORT2 +udata_readInt16(const UDataSwapper *ds, int16_t x); + +/** + * Convert one int32_t from input to platform endianness. + * @internal ICU 2.8 + */ +U_CAPI int32_t U_EXPORT2 +udata_readInt32(const UDataSwapper *ds, int32_t x); + +/** + * Swap a block of invariant, NUL-terminated strings, but not padding + * bytes after the last string. + * @internal + */ +U_CAPI int32_t U_EXPORT2 +udata_swapInvStringBlock(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode); + +U_CAPI void U_EXPORT2 +udata_printError(const UDataSwapper *ds, + const char *fmt, + ...); + +/* internal exports from putil.c -------------------------------------------- */ + +/* declared here to keep them out of the public putil.h */ + +/** + * Swap invariant char * strings ASCII->EBCDIC. + * @internal + */ +U_CAPI int32_t U_EXPORT2 +uprv_ebcdicFromAscii(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode); + +/** + * Copy invariant ASCII char * strings and verify they are invariant. + * @internal + */ +U_CFUNC int32_t +uprv_copyAscii(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode); + +/** + * Swap invariant char * strings EBCDIC->ASCII. + * @internal + */ +U_CFUNC int32_t +uprv_asciiFromEbcdic(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode); + +/** + * Copy invariant EBCDIC char * strings and verify they are invariant. + * @internal + */ +U_CFUNC int32_t +uprv_copyEbcdic(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode); + +/** + * Compare ASCII invariant char * with Unicode invariant UChar * + * @internal + */ +U_CFUNC int32_t +uprv_compareInvAscii(const UDataSwapper *ds, + const char *outString, int32_t outLength, + const UChar *localString, int32_t localLength); + +/** + * Compare EBCDIC invariant char * with Unicode invariant UChar * + * @internal + */ +U_CFUNC int32_t +uprv_compareInvEbcdic(const UDataSwapper *ds, + const char *outString, int32_t outLength, + const UChar *localString, int32_t localLength); + +/* material... -------------------------------------------------------------- */ + +#if 0 + +/* udata.h */ + +/** + * Public API function in udata.c + * + * Same as udata_openChoice() but automatically swaps the data. + * isAcceptable, if not NULL, may accept data with endianness and charset family + * different from the current platform's properties. + * If the data is acceptable and the platform properties do not match, then + * the swap function is called to swap an allocated version of the data. + * Preflighting may or may not be performed depending on whether the size of + * the loaded data item is known. + * + * @param isAcceptable Same as for udata_openChoice(). May be NULL. + * + * @internal ICU 2.8 + */ +U_CAPI UDataMemory * U_EXPORT2 +udata_openSwap(const char *path, const char *type, const char *name, + UDataMemoryIsAcceptable *isAcceptable, void *isAcceptableContext, + UDataSwapFn *swap, + UDataPrintError *printError, void *printErrorContext, + UErrorCode *pErrorCode); + +#endif + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/uenum.c b/Build/source/libs/icu/icu-xetex/common/uenum.c new file mode 100644 index 00000000000..549769c449d --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uenum.c @@ -0,0 +1,181 @@ +/* +******************************************************************************* +* +* Copyright (C) 2002-2004, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: uenum.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:2 +* +* created on: 2002jul08 +* created by: Vladimir Weinstein +*/ + +#include "unicode/putil.h" +#include "uenumimp.h" +#include "cmemory.h" + +/* Layout of the baseContext buffer. */ +typedef struct { + int32_t len; /* number of bytes available starting at 'data' */ + char data; /* actual data starts here */ +} _UEnumBuffer; + +/* Extra bytes to allocate in the baseContext buffer. */ +static const int32_t PAD = 8; + +/* Return a pointer to the baseContext buffer, possibly allocating + or reallocating it if at least 'capacity' bytes are not available. */ +static void* _getBuffer(UEnumeration* en, int32_t capacity) { + + if (en->baseContext != NULL) { + if (((_UEnumBuffer*) en->baseContext)->len < capacity) { + capacity += PAD; + en->baseContext = uprv_realloc(en->baseContext, + sizeof(int32_t) + capacity); + if (en->baseContext == NULL) { + return NULL; + } + ((_UEnumBuffer*) en->baseContext)->len = capacity; + } + } else { + capacity += PAD; + en->baseContext = uprv_malloc(sizeof(int32_t) + capacity); + if (en->baseContext == NULL) { + return NULL; + } + ((_UEnumBuffer*) en->baseContext)->len = capacity; + } + + return (void*) & ((_UEnumBuffer*) en->baseContext)->data; +} + +U_CAPI void U_EXPORT2 +uenum_close(UEnumeration* en) +{ + if (en) { + if (en->close != NULL) { + if (en->baseContext) { + uprv_free(en->baseContext); + } + en->close(en); + } else { /* this seems dangerous, but we better kill the object */ + uprv_free(en); + } + } +} + +U_CAPI int32_t U_EXPORT2 +uenum_count(UEnumeration* en, UErrorCode* status) +{ + if (!en || U_FAILURE(*status)) { + return -1; + } + if (en->count != NULL) { + return en->count(en, status); + } else { + *status = U_UNSUPPORTED_ERROR; + return -1; + } +} + +/* Don't call this directly. Only uenum_unext should be calling this. */ +U_CAPI const UChar* U_EXPORT2 +uenum_unextDefault(UEnumeration* en, + int32_t* resultLength, + UErrorCode* status) +{ + UChar *ustr = NULL; + int32_t len = 0; + if (en->next != NULL) { + const char *cstr = en->next(en, &len, status); + if (cstr != NULL) { + ustr = (UChar*) _getBuffer(en, (len+1) * sizeof(UChar)); + if (ustr == NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + } else { + u_charsToUChars(cstr, ustr, len+1); + } + } + } else { + *status = U_UNSUPPORTED_ERROR; + } + if (resultLength) { + *resultLength = len; + } + return ustr; +} + +/* Don't call this directly. Only uenum_next should be calling this. */ +U_CAPI const char* U_EXPORT2 +uenum_nextDefault(UEnumeration* en, + int32_t* resultLength, + UErrorCode* status) +{ + if (en->uNext != NULL) { + char *tempCharVal; + const UChar *tempUCharVal = en->uNext(en, resultLength, status); + if (tempUCharVal == NULL) { + return NULL; + } + tempCharVal = (char*) + _getBuffer(en, (*resultLength+1) * sizeof(char)); + if (!tempCharVal) { + *status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + u_UCharsToChars(tempUCharVal, tempCharVal, *resultLength + 1); + return tempCharVal; + } else { + *status = U_UNSUPPORTED_ERROR; + return NULL; + } +} + +U_CAPI const UChar* U_EXPORT2 +uenum_unext(UEnumeration* en, + int32_t* resultLength, + UErrorCode* status) +{ + if (!en || U_FAILURE(*status)) { + return NULL; + } + if (en->uNext != NULL) { + return en->uNext(en, resultLength, status); + } else { + *status = U_UNSUPPORTED_ERROR; + return NULL; + } +} + +U_CAPI const char* U_EXPORT2 +uenum_next(UEnumeration* en, + int32_t* resultLength, + UErrorCode* status) +{ + if (!en || U_FAILURE(*status)) { + return NULL; + } + if (en->next != NULL) { + return en->next(en, resultLength, status); + } else { + *status = U_UNSUPPORTED_ERROR; + return NULL; + } +} + +U_CAPI void U_EXPORT2 +uenum_reset(UEnumeration* en, UErrorCode* status) +{ + if (!en || U_FAILURE(*status)) { + return; + } + if (en->reset != NULL) { + en->reset(en, status); + } else { + *status = U_UNSUPPORTED_ERROR; + } +} diff --git a/Build/source/libs/icu/icu-xetex/common/uenumimp.h b/Build/source/libs/icu/icu-xetex/common/uenumimp.h new file mode 100644 index 00000000000..664bc686644 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uenumimp.h @@ -0,0 +1,153 @@ +/* +******************************************************************************* +* +* Copyright (C) 2002-2006, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: uenumimp.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:2 +* +* created on: 2002jul08 +* created by: Vladimir Weinstein +*/ + +#ifndef __UENUMIMP_H +#define __UENUMIMP_H + +#include "unicode/uenum.h" + +U_CDECL_BEGIN + +/** + * following are the type declarations for + * implementations of APIs. If any of these + * functions are NULL, U_UNSUPPORTED_ERROR + * is returned. If close is NULL, the enumeration + * object is going to be released. + * Initial error checking is done in the body + * of API function, so the implementations + * need not to check the initial error condition. + */ + +/** + * Function type declaration for uenum_close(). + * + * This function should cleanup the enumerator object + * + * @param en enumeration to be closed + */ +typedef void U_CALLCONV +UEnumClose(UEnumeration *en); + +/** + * Function type declaration for uenum_count(). + * + * This function should count the number of elements + * in this enumeration + * + * @param en enumeration to be counted + * @param status pointer to UErrorCode variable + * @return number of elements in enumeration + */ +typedef int32_t U_CALLCONV +UEnumCount(UEnumeration *en, UErrorCode *status); + +/** + * Function type declaration for uenum_unext(). + * + * This function returns the next element as a UChar *, + * or NULL after all elements haven been enumerated. + * + * @param en enumeration + * @param resultLength pointer to result length + * @param status pointer to UErrorCode variable + * @return next element as UChar *, + * or NULL after all elements haven been enumerated + */ +typedef const UChar* U_CALLCONV +UEnumUNext(UEnumeration* en, + int32_t* resultLength, + UErrorCode* status); + +/** + * Function type declaration for uenum_next(). + * + * This function returns the next element as a char *, + * or NULL after all elements haven been enumerated. + * + * @param en enumeration + * @param resultLength pointer to result length + * @param status pointer to UErrorCode variable + * @return next element as char *, + * or NULL after all elements haven been enumerated + */ +typedef const char* U_CALLCONV +UEnumNext(UEnumeration* en, + int32_t* resultLength, + UErrorCode* status); + +/** + * Function type declaration for uenum_reset(). + * + * This function should reset the enumeration + * object + * + * @param en enumeration + * @param status pointer to UErrorCode variable + */ +typedef void U_CALLCONV +UEnumReset(UEnumeration* en, + UErrorCode* status); + + +struct UEnumeration { + /* baseContext. For the base class only. Don't touch! */ + void *baseContext; + + /* context. Use it for what you need */ + void *context; + + /** + * these are functions that will + * be used for APIs + */ + /* called from uenum_close */ + UEnumClose *close; + /* called from uenum_count */ + UEnumCount *count; + /* called from uenum_unext */ + UEnumUNext *uNext; + /* called from uenum_next */ + UEnumNext *next; + /* called from uenum_reset */ + UEnumReset *reset; +}; + +U_CDECL_END + +/* This is the default implementation for uenum_unext(). + * It automatically converts the char * string to UChar *. + * Don't call this directly. This is called internally by uenum_unext + * when a UEnumeration is defined with 'uNext' pointing to this + * function. + */ +U_CAPI const UChar* U_EXPORT2 +uenum_unextDefault(UEnumeration* en, + int32_t* resultLength, + UErrorCode* status); + +/* This is the default implementation for uenum_next(). + * It automatically converts the UChar * string to char *. + * Don't call this directly. This is called internally by uenum_next + * when a UEnumeration is defined with 'next' pointing to this + * function. + */ +U_CAPI const char* U_EXPORT2 +uenum_nextDefault(UEnumeration* en, + int32_t* resultLength, + UErrorCode* status); + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/uhash.c b/Build/source/libs/icu/icu-xetex/common/uhash.c new file mode 100644 index 00000000000..fd1a6bfa968 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uhash.c @@ -0,0 +1,1001 @@ +/* +****************************************************************************** +* Copyright (C) 1997-2006, International Business Machines +* Corporation and others. All Rights Reserved. +****************************************************************************** +* Date Name Description +* 03/22/00 aliu Adapted from original C++ ICU Hashtable. +* 07/06/01 aliu Modified to support int32_t keys on +* platforms with sizeof(void*) < 32. +****************************************************************************** +*/ + +#include "uhash.h" +#include "unicode/ustring.h" +#include "cstring.h" +#include "cmemory.h" +#include "uassert.h" + +/* This hashtable is implemented as a double hash. All elements are + * stored in a single array with no secondary storage for collision + * resolution (no linked list, etc.). When there is a hash collision + * (when two unequal keys have the same hashcode) we resolve this by + * using a secondary hash. The secondary hash is an increment + * computed as a hash function (a different one) of the primary + * hashcode. This increment is added to the initial hash value to + * obtain further slots assigned to the same hash code. For this to + * work, the length of the array and the increment must be relatively + * prime. The easiest way to achieve this is to have the length of + * the array be prime, and the increment be any value from + * 1..length-1. + * + * Hashcodes are 32-bit integers. We make sure all hashcodes are + * non-negative by masking off the top bit. This has two effects: (1) + * modulo arithmetic is simplified. If we allowed negative hashcodes, + * then when we computed hashcode % length, we could get a negative + * result, which we would then have to adjust back into range. It's + * simpler to just make hashcodes non-negative. (2) It makes it easy + * to check for empty vs. occupied slots in the table. We just mark + * empty or deleted slots with a negative hashcode. + * + * The central function is _uhash_find(). This function looks for a + * slot matching the given key and hashcode. If one is found, it + * returns a pointer to that slot. If the table is full, and no match + * is found, it returns NULL -- in theory. This would make the code + * more complicated, since all callers of _uhash_find() would then + * have to check for a NULL result. To keep this from happening, we + * don't allow the table to fill. When there is only one + * empty/deleted slot left, uhash_put() will refuse to increase the + * count, and fail. This simplifies the code. In practice, one will + * seldom encounter this using default UHashtables. However, if a + * hashtable is set to a U_FIXED resize policy, or if memory is + * exhausted, then the table may fill. + * + * High and low water ratios control rehashing. They establish levels + * of fullness (from 0 to 1) outside of which the data array is + * reallocated and repopulated. Setting the low water ratio to zero + * means the table will never shrink. Setting the high water ratio to + * one means the table will never grow. The ratios should be + * coordinated with the ratio between successive elements of the + * PRIMES table, so that when the primeIndex is incremented or + * decremented during rehashing, it brings the ratio of count / length + * back into the desired range (between low and high water ratios). + */ + +/******************************************************************** + * PRIVATE Constants, Macros + ********************************************************************/ + +/* This is a list of non-consecutive primes chosen such that + * PRIMES[i+1] ~ 2*PRIMES[i]. (Currently, the ratio ranges from 1.81 + * to 2.18; the inverse ratio ranges from 0.459 to 0.552.) If this + * ratio is changed, the low and high water ratios should also be + * adjusted to suit. + * + * These prime numbers were also chosen so that they are the largest + * prime number while being less than a power of two. + */ +static const int32_t PRIMES[] = { + 13, 31, 61, 127, 251, 509, 1021, 2039, 4093, 8191, 16381, 32749, + 65521, 131071, 262139, 524287, 1048573, 2097143, 4194301, 8388593, + 16777213, 33554393, 67108859, 134217689, 268435399, 536870909, + 1073741789, 2147483647 /*, 4294967291 */ +}; + +#define PRIMES_LENGTH (sizeof(PRIMES) / sizeof(PRIMES[0])) +#define DEFAULT_PRIME_INDEX 3 + +/* These ratios are tuned to the PRIMES array such that a resize + * places the table back into the zone of non-resizing. That is, + * after a call to _uhash_rehash(), a subsequent call to + * _uhash_rehash() should do nothing (should not churn). This is only + * a potential problem with U_GROW_AND_SHRINK. + */ +static const float RESIZE_POLICY_RATIO_TABLE[6] = { + /* low, high water ratio */ + 0.0F, 0.5F, /* U_GROW: Grow on demand, do not shrink */ + 0.1F, 0.5F, /* U_GROW_AND_SHRINK: Grow and shrink on demand */ + 0.0F, 1.0F /* U_FIXED: Never change size */ +}; + +/* + Invariants for hashcode values: + + * DELETED < 0 + * EMPTY < 0 + * Real hashes >= 0 + + Hashcodes may not start out this way, but internally they are + adjusted so that they are always positive. We assume 32-bit + hashcodes; adjust these constants for other hashcode sizes. +*/ +#define HASH_DELETED ((int32_t) 0x80000000) +#define HASH_EMPTY ((int32_t) HASH_DELETED + 1) + +#define IS_EMPTY_OR_DELETED(x) ((x) < 0) + +/* This macro expects a UHashTok.pointer as its keypointer and + valuepointer parameters */ +#define HASH_DELETE_KEY_VALUE(hash, keypointer, valuepointer) \ + if (hash->keyDeleter != NULL && keypointer != NULL) { \ + (*hash->keyDeleter)(keypointer); \ + } \ + if (hash->valueDeleter != NULL && valuepointer != NULL) { \ + (*hash->valueDeleter)(valuepointer); \ + } + +/* + * Constants for hinting whether a key or value is an integer + * or a pointer. If a hint bit is zero, then the associated + * token is assumed to be an integer. + */ +#define HINT_KEY_POINTER (1) +#define HINT_VALUE_POINTER (2) + +/******************************************************************** + * PRIVATE Implementation + ********************************************************************/ + +static UHashTok +_uhash_setElement(UHashtable *hash, UHashElement* e, + int32_t hashcode, + UHashTok key, UHashTok value, int8_t hint) { + + UHashTok oldValue = e->value; + if (hash->keyDeleter != NULL && e->key.pointer != NULL && + e->key.pointer != key.pointer) { /* Avoid double deletion */ + (*hash->keyDeleter)(e->key.pointer); + } + if (hash->valueDeleter != NULL) { + if (oldValue.pointer != NULL && + oldValue.pointer != value.pointer) { /* Avoid double deletion */ + (*hash->valueDeleter)(oldValue.pointer); + } + oldValue.pointer = NULL; + } + /* Compilers should copy the UHashTok union correctly, but even if + * they do, memory heap tools (e.g. BoundsChecker) can get + * confused when a pointer is cloaked in a union and then copied. + * TO ALLEVIATE THIS, we use hints (based on what API the user is + * calling) to copy pointers when we know the user thinks + * something is a pointer. */ + if (hint & HINT_KEY_POINTER) { + e->key.pointer = key.pointer; + } else { + e->key = key; + } + if (hint & HINT_VALUE_POINTER) { + e->value.pointer = value.pointer; + } else { + e->value = value; + } + e->hashcode = hashcode; + return oldValue; +} + +/** + * Assumes that the given element is not empty or deleted. + */ +static UHashTok +_uhash_internalRemoveElement(UHashtable *hash, UHashElement* e) { + UHashTok empty; + U_ASSERT(!IS_EMPTY_OR_DELETED(e->hashcode)); + --hash->count; + empty.pointer = NULL; empty.integer = 0; + return _uhash_setElement(hash, e, HASH_DELETED, empty, empty, 0); +} + +static void +_uhash_internalSetResizePolicy(UHashtable *hash, enum UHashResizePolicy policy) { + U_ASSERT(hash != NULL); + U_ASSERT(((int32_t)policy) >= 0); + U_ASSERT(((int32_t)policy) < 3); + hash->lowWaterRatio = RESIZE_POLICY_RATIO_TABLE[policy * 2]; + hash->highWaterRatio = RESIZE_POLICY_RATIO_TABLE[policy * 2 + 1]; +} + +/** + * Allocate internal data array of a size determined by the given + * prime index. If the index is out of range it is pinned into range. + * If the allocation fails the status is set to + * U_MEMORY_ALLOCATION_ERROR and all array storage is freed. In + * either case the previous array pointer is overwritten. + * + * Caller must ensure primeIndex is in range 0..PRIME_LENGTH-1. + */ +static void +_uhash_allocate(UHashtable *hash, + int32_t primeIndex, + UErrorCode *status) { + + UHashElement *p, *limit; + UHashTok emptytok; + + if (U_FAILURE(*status)) return; + + U_ASSERT(primeIndex >= 0 && primeIndex < PRIMES_LENGTH); + + hash->primeIndex = primeIndex; + hash->length = PRIMES[primeIndex]; + + p = hash->elements = (UHashElement*) + uprv_malloc(sizeof(UHashElement) * hash->length); + + if (hash->elements == NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + return; + } + + emptytok.pointer = NULL; /* Only one of these two is needed */ + emptytok.integer = 0; /* but we don't know which one. */ + + limit = p + hash->length; + while (p < limit) { + p->key = emptytok; + p->value = emptytok; + p->hashcode = HASH_EMPTY; + ++p; + } + + hash->count = 0; + hash->lowWaterMark = (int32_t)(hash->length * hash->lowWaterRatio); + hash->highWaterMark = (int32_t)(hash->length * hash->highWaterRatio); +} + +static UHashtable* +_uhash_init(UHashtable *result, + UHashFunction *keyHash, + UKeyComparator *keyComp, + UValueComparator *valueComp, + int32_t primeIndex, + UErrorCode *status) +{ + if (U_FAILURE(*status)) return NULL; + U_ASSERT(keyHash != NULL); + U_ASSERT(keyComp != NULL); + + result->keyHasher = keyHash; + result->keyComparator = keyComp; + result->valueComparator = valueComp; + result->keyDeleter = NULL; + result->valueDeleter = NULL; + result->allocated = FALSE; + _uhash_internalSetResizePolicy(result, U_GROW); + + _uhash_allocate(result, primeIndex, status); + + if (U_FAILURE(*status)) { + return NULL; + } + + return result; +} + +static UHashtable* +_uhash_create(UHashFunction *keyHash, + UKeyComparator *keyComp, + UValueComparator *valueComp, + int32_t primeIndex, + UErrorCode *status) { + UHashtable *result; + + if (U_FAILURE(*status)) return NULL; + + result = (UHashtable*) uprv_malloc(sizeof(UHashtable)); + if (result == NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + + _uhash_init(result, keyHash, keyComp, valueComp, primeIndex, status); + result->allocated = TRUE; + + if (U_FAILURE(*status)) { + uprv_free(result); + return NULL; + } + + return result; +} + +/** + * Look for a key in the table, or if no such key exists, the first + * empty slot matching the given hashcode. Keys are compared using + * the keyComparator function. + * + * First find the start position, which is the hashcode modulo + * the length. Test it to see if it is: + * + * a. identical: First check the hash values for a quick check, + * then compare keys for equality using keyComparator. + * b. deleted + * c. empty + * + * Stop if it is identical or empty, otherwise continue by adding a + * "jump" value (moduloing by the length again to keep it within + * range) and retesting. For efficiency, there need enough empty + * values so that the searchs stop within a reasonable amount of time. + * This can be changed by changing the high/low water marks. + * + * In theory, this function can return NULL, if it is full (no empty + * or deleted slots) and if no matching key is found. In practice, we + * prevent this elsewhere (in uhash_put) by making sure the last slot + * in the table is never filled. + * + * The size of the table should be prime for this algorithm to work; + * otherwise we are not guaranteed that the jump value (the secondary + * hash) is relatively prime to the table length. + */ +static UHashElement* +_uhash_find(const UHashtable *hash, UHashTok key, + int32_t hashcode) { + + int32_t firstDeleted = -1; /* assume invalid index */ + int32_t theIndex, startIndex; + int32_t jump = 0; /* lazy evaluate */ + int32_t tableHash; + UHashElement *elements = hash->elements; + + hashcode &= 0x7FFFFFFF; /* must be positive */ + startIndex = theIndex = (hashcode ^ 0x4000000) % hash->length; + + do { + tableHash = elements[theIndex].hashcode; + if (tableHash == hashcode) { /* quick check */ + if ((*hash->keyComparator)(key, elements[theIndex].key)) { + return &(elements[theIndex]); + } + } else if (!IS_EMPTY_OR_DELETED(tableHash)) { + /* We have hit a slot which contains a key-value pair, + * but for which the hash code does not match. Keep + * looking. + */ + } else if (tableHash == HASH_EMPTY) { /* empty, end o' the line */ + break; + } else if (firstDeleted < 0) { /* remember first deleted */ + firstDeleted = theIndex; + } + if (jump == 0) { /* lazy compute jump */ + /* The jump value must be relatively prime to the table + * length. As long as the length is prime, then any value + * 1..length-1 will be relatively prime to it. + */ + jump = (hashcode % (hash->length - 1)) + 1; + } + theIndex = (theIndex + jump) % hash->length; + } while (theIndex != startIndex); + + if (firstDeleted >= 0) { + theIndex = firstDeleted; /* reset if had deleted slot */ + } else if (tableHash != HASH_EMPTY) { + /* We get to this point if the hashtable is full (no empty or + * deleted slots), and we've failed to find a match. THIS + * WILL NEVER HAPPEN as long as uhash_put() makes sure that + * count is always < length. + */ + U_ASSERT(FALSE); + return NULL; /* Never happens if uhash_put() behaves */ + } + return &(elements[theIndex]); +} + +/** + * Attempt to grow or shrink the data arrays in order to make the + * count fit between the high and low water marks. hash_put() and + * hash_remove() call this method when the count exceeds the high or + * low water marks. This method may do nothing, if memory allocation + * fails, or if the count is already in range, or if the length is + * already at the low or high limit. In any case, upon return the + * arrays will be valid. + */ +static void +_uhash_rehash(UHashtable *hash) { + + UHashElement *old = hash->elements; + int32_t oldLength = hash->length; + int32_t newPrimeIndex = hash->primeIndex; + int32_t i; + UErrorCode status = U_ZERO_ERROR; + + if (hash->count > hash->highWaterMark) { + if (++newPrimeIndex >= PRIMES_LENGTH) { + return; + } + } else if (hash->count < hash->lowWaterMark) { + if (--newPrimeIndex < 0) { + return; + } + } else { + return; + } + + _uhash_allocate(hash, newPrimeIndex, &status); + + if (U_FAILURE(status)) { + hash->elements = old; + hash->length = oldLength; + return; + } + + for (i = oldLength - 1; i >= 0; --i) { + if (!IS_EMPTY_OR_DELETED(old[i].hashcode)) { + UHashElement *e = _uhash_find(hash, old[i].key, old[i].hashcode); + U_ASSERT(e != NULL); + U_ASSERT(e->hashcode == HASH_EMPTY); + e->key = old[i].key; + e->value = old[i].value; + e->hashcode = old[i].hashcode; + ++hash->count; + } + } + + uprv_free(old); +} + +static UHashTok +_uhash_remove(UHashtable *hash, + UHashTok key) { + /* First find the position of the key in the table. If the object + * has not been removed already, remove it. If the user wanted + * keys deleted, then delete it also. We have to put a special + * hashcode in that position that means that something has been + * deleted, since when we do a find, we have to continue PAST any + * deleted values. + */ + UHashTok result; + UHashElement* e = _uhash_find(hash, key, hash->keyHasher(key)); + U_ASSERT(e != NULL); + result.pointer = NULL; result.integer = 0; + if (!IS_EMPTY_OR_DELETED(e->hashcode)) { + result = _uhash_internalRemoveElement(hash, e); + if (hash->count < hash->lowWaterMark) { + _uhash_rehash(hash); + } + } + return result; +} + +static UHashTok +_uhash_put(UHashtable *hash, + UHashTok key, + UHashTok value, + int8_t hint, + UErrorCode *status) { + + /* Put finds the position in the table for the new value. If the + * key is already in the table, it is deleted, if there is a + * non-NULL keyDeleter. Then the key, the hash and the value are + * all put at the position in their respective arrays. + */ + int32_t hashcode; + UHashElement* e; + UHashTok emptytok; + + if (U_FAILURE(*status)) { + goto err; + } + U_ASSERT(hash != NULL); + /* Cannot always check pointer here or iSeries sees NULL every time. */ + if ((hint & HINT_VALUE_POINTER) && value.pointer == NULL) { + /* Disallow storage of NULL values, since NULL is returned by + * get() to indicate an absent key. Storing NULL == removing. + */ + return _uhash_remove(hash, key); + } + if (hash->count > hash->highWaterMark) { + _uhash_rehash(hash); + } + + hashcode = (*hash->keyHasher)(key); + e = _uhash_find(hash, key, hashcode); + U_ASSERT(e != NULL); + + if (IS_EMPTY_OR_DELETED(e->hashcode)) { + /* Important: We must never actually fill the table up. If we + * do so, then _uhash_find() will return NULL, and we'll have + * to check for NULL after every call to _uhash_find(). To + * avoid this we make sure there is always at least one empty + * or deleted slot in the table. This only is a problem if we + * are out of memory and rehash isn't working. + */ + ++hash->count; + if (hash->count == hash->length) { + /* Don't allow count to reach length */ + --hash->count; + *status = U_MEMORY_ALLOCATION_ERROR; + goto err; + } + } + + /* We must in all cases handle storage properly. If there was an + * old key, then it must be deleted (if the deleter != NULL). + * Make hashcodes stored in table positive. + */ + return _uhash_setElement(hash, e, hashcode & 0x7FFFFFFF, key, value, hint); + + err: + /* If the deleters are non-NULL, this method adopts its key and/or + * value arguments, and we must be sure to delete the key and/or + * value in all cases, even upon failure. + */ + HASH_DELETE_KEY_VALUE(hash, key.pointer, value.pointer); + emptytok.pointer = NULL; emptytok.integer = 0; + return emptytok; +} + + +/******************************************************************** + * PUBLIC API + ********************************************************************/ + +U_CAPI UHashtable* U_EXPORT2 +uhash_open(UHashFunction *keyHash, + UKeyComparator *keyComp, + UValueComparator *valueComp, + UErrorCode *status) { + + return _uhash_create(keyHash, keyComp, valueComp, DEFAULT_PRIME_INDEX, status); +} + +U_CAPI UHashtable* U_EXPORT2 +uhash_openSize(UHashFunction *keyHash, + UKeyComparator *keyComp, + UValueComparator *valueComp, + int32_t size, + UErrorCode *status) { + + /* Find the smallest index i for which PRIMES[i] >= size. */ + int32_t i = 0; + while (i<(PRIMES_LENGTH-1) && PRIMES[i]elements != NULL) { + if (hash->keyDeleter != NULL || hash->valueDeleter != NULL) { + int32_t pos=-1; + UHashElement *e; + while ((e = (UHashElement*) uhash_nextElement(hash, &pos)) != NULL) { + HASH_DELETE_KEY_VALUE(hash, e->key.pointer, e->value.pointer); + } + } + uprv_free(hash->elements); + hash->elements = NULL; + } + if (hash->allocated) { + uprv_free(hash); + } +} + +U_CAPI UHashFunction *U_EXPORT2 +uhash_setKeyHasher(UHashtable *hash, UHashFunction *fn) { + UHashFunction *result = hash->keyHasher; + hash->keyHasher = fn; + return result; +} + +U_CAPI UKeyComparator *U_EXPORT2 +uhash_setKeyComparator(UHashtable *hash, UKeyComparator *fn) { + UKeyComparator *result = hash->keyComparator; + hash->keyComparator = fn; + return result; +} +U_CAPI UValueComparator *U_EXPORT2 +uhash_setValueComparator(UHashtable *hash, UValueComparator *fn){ + UValueComparator *result = hash->valueComparator; + hash->valueComparator = fn; + return result; +} + +U_CAPI UObjectDeleter *U_EXPORT2 +uhash_setKeyDeleter(UHashtable *hash, UObjectDeleter *fn) { + UObjectDeleter *result = hash->keyDeleter; + hash->keyDeleter = fn; + return result; +} + +U_CAPI UObjectDeleter *U_EXPORT2 +uhash_setValueDeleter(UHashtable *hash, UObjectDeleter *fn) { + UObjectDeleter *result = hash->valueDeleter; + hash->valueDeleter = fn; + return result; +} + +U_CAPI void U_EXPORT2 +uhash_setResizePolicy(UHashtable *hash, enum UHashResizePolicy policy) { + _uhash_internalSetResizePolicy(hash, policy); + hash->lowWaterMark = (int32_t)(hash->length * hash->lowWaterRatio); + hash->highWaterMark = (int32_t)(hash->length * hash->highWaterRatio); + _uhash_rehash(hash); +} + +U_CAPI int32_t U_EXPORT2 +uhash_count(const UHashtable *hash) { + return hash->count; +} + +U_CAPI void* U_EXPORT2 +uhash_get(const UHashtable *hash, + const void* key) { + UHashTok keyholder; + keyholder.pointer = (void*) key; + return _uhash_find(hash, keyholder, hash->keyHasher(keyholder))->value.pointer; +} + +U_CAPI void* U_EXPORT2 +uhash_iget(const UHashtable *hash, + int32_t key) { + UHashTok keyholder; + keyholder.integer = key; + return _uhash_find(hash, keyholder, hash->keyHasher(keyholder))->value.pointer; +} + +U_CAPI int32_t U_EXPORT2 +uhash_geti(const UHashtable *hash, + const void* key) { + UHashTok keyholder; + keyholder.pointer = (void*) key; + return _uhash_find(hash, keyholder, hash->keyHasher(keyholder))->value.integer; +} + +U_CAPI int32_t U_EXPORT2 +uhash_igeti(const UHashtable *hash, + int32_t key) { + UHashTok keyholder; + keyholder.integer = key; + return _uhash_find(hash, keyholder, hash->keyHasher(keyholder))->value.integer; +} + +U_CAPI void* U_EXPORT2 +uhash_put(UHashtable *hash, + void* key, + void* value, + UErrorCode *status) { + UHashTok keyholder, valueholder; + keyholder.pointer = key; + valueholder.pointer = value; + return _uhash_put(hash, keyholder, valueholder, + HINT_KEY_POINTER | HINT_VALUE_POINTER, + status).pointer; +} + +U_CAPI void* U_EXPORT2 +uhash_iput(UHashtable *hash, + int32_t key, + void* value, + UErrorCode *status) { + UHashTok keyholder, valueholder; + keyholder.integer = key; + valueholder.pointer = value; + return _uhash_put(hash, keyholder, valueholder, + HINT_VALUE_POINTER, + status).pointer; +} + +U_CAPI int32_t U_EXPORT2 +uhash_puti(UHashtable *hash, + void* key, + int32_t value, + UErrorCode *status) { + UHashTok keyholder, valueholder; + keyholder.pointer = key; + valueholder.integer = value; + return _uhash_put(hash, keyholder, valueholder, + HINT_KEY_POINTER, + status).integer; +} + + +U_CAPI int32_t U_EXPORT2 +uhash_iputi(UHashtable *hash, + int32_t key, + int32_t value, + UErrorCode *status) { + UHashTok keyholder, valueholder; + keyholder.integer = key; + valueholder.integer = value; + return _uhash_put(hash, keyholder, valueholder, + 0, /* neither is a ptr */ + status).integer; +} + +U_CAPI void* U_EXPORT2 +uhash_remove(UHashtable *hash, + const void* key) { + UHashTok keyholder; + keyholder.pointer = (void*) key; + return _uhash_remove(hash, keyholder).pointer; +} + +U_CAPI void* U_EXPORT2 +uhash_iremove(UHashtable *hash, + int32_t key) { + UHashTok keyholder; + keyholder.integer = key; + return _uhash_remove(hash, keyholder).pointer; +} + +U_CAPI int32_t U_EXPORT2 +uhash_removei(UHashtable *hash, + const void* key) { + UHashTok keyholder; + keyholder.pointer = (void*) key; + return _uhash_remove(hash, keyholder).integer; +} + +U_CAPI int32_t U_EXPORT2 +uhash_iremovei(UHashtable *hash, + int32_t key) { + UHashTok keyholder; + keyholder.integer = key; + return _uhash_remove(hash, keyholder).integer; +} + +U_CAPI void U_EXPORT2 +uhash_removeAll(UHashtable *hash) { + int32_t pos = -1; + const UHashElement *e; + U_ASSERT(hash != NULL); + if (hash->count != 0) { + while ((e = uhash_nextElement(hash, &pos)) != NULL) { + uhash_removeElement(hash, e); + } + } + U_ASSERT(hash->count == 0); +} + +U_CAPI const UHashElement* U_EXPORT2 +uhash_find(const UHashtable *hash, const void* key) { + UHashTok keyholder; + const UHashElement *e; + keyholder.pointer = (void*) key; + e = _uhash_find(hash, keyholder, hash->keyHasher(keyholder)); + return IS_EMPTY_OR_DELETED(e->hashcode) ? NULL : e; +} + +U_CAPI const UHashElement* U_EXPORT2 +uhash_nextElement(const UHashtable *hash, int32_t *pos) { + /* Walk through the array until we find an element that is not + * EMPTY and not DELETED. + */ + int32_t i; + U_ASSERT(hash != NULL); + for (i = *pos + 1; i < hash->length; ++i) { + if (!IS_EMPTY_OR_DELETED(hash->elements[i].hashcode)) { + *pos = i; + return &(hash->elements[i]); + } + } + + /* No more elements */ + return NULL; +} + +U_CAPI void* U_EXPORT2 +uhash_removeElement(UHashtable *hash, const UHashElement* e) { + U_ASSERT(hash != NULL); + U_ASSERT(e != NULL); + if (!IS_EMPTY_OR_DELETED(e->hashcode)) { + return _uhash_internalRemoveElement(hash, (UHashElement*) e).pointer; + } + return NULL; +} + +/******************************************************************** + * UHashTok convenience + ********************************************************************/ + +/** + * Return a UHashTok for an integer. + */ +/*U_CAPI UHashTok U_EXPORT2 +uhash_toki(int32_t i) { + UHashTok tok; + tok.integer = i; + return tok; +}*/ + +/** + * Return a UHashTok for a pointer. + */ +/*U_CAPI UHashTok U_EXPORT2 +uhash_tokp(void* p) { + UHashTok tok; + tok.pointer = p; + return tok; +}*/ + +/******************************************************************** + * PUBLIC Key Hash Functions + ********************************************************************/ + +/* + Compute the hash by iterating sparsely over about 32 (up to 63) + characters spaced evenly through the string. For each character, + multiply the previous hash value by a prime number and add the new + character in, like a linear congruential random number generator, + producing a pseudorandom deterministic value well distributed over + the output range. [LIU] +*/ + +#define STRING_HASH(TYPE, STR, STRLEN, DEREF) \ + int32_t hash = 0; \ + const TYPE *p = (const TYPE*) STR; \ + if (p != NULL) { \ + int32_t len = (int32_t)(STRLEN); \ + int32_t inc = ((len - 32) / 32) + 1; \ + const TYPE *limit = p + len; \ + while (pkeyComparator != hash2->keyComparator || + hash1->valueComparator != hash2->valueComparator || + hash1->valueComparator == NULL) + { + /* + Normally we would return an error here about incompatible hash tables, + but we return FALSE instead. + */ + return FALSE; + } + + count1 = uhash_count(hash1); + count2 = uhash_count(hash2); + if(count1!=count2){ + return FALSE; + } + + pos=-1; + for(i=0; ikey; + const UHashTok val1 = elem1->value; + /* here the keys are not compared, instead the key form hash1 is used to fetch + * value from hash2. If the hashes are equal then then both hashes should + * contain equal values for the same key! + */ + const UHashElement* elem2 = _uhash_find(hash2, key1, hash2->keyHasher(key1)); + const UHashTok val2 = elem2->value; + if(hash1->valueComparator(val1, val2)==FALSE){ + return FALSE; + } + } + return TRUE; +} + +/******************************************************************** + * PUBLIC Comparator Functions + ********************************************************************/ + +U_CAPI UBool U_EXPORT2 +uhash_compareUChars(const UHashTok key1, const UHashTok key2) { + const UChar *p1 = (const UChar*) key1.pointer; + const UChar *p2 = (const UChar*) key2.pointer; + if (p1 == p2) { + return TRUE; + } + if (p1 == NULL || p2 == NULL) { + return FALSE; + } + while (*p1 != 0 && *p1 == *p2) { + ++p1; + ++p2; + } + return (UBool)(*p1 == *p2); +} + +U_CAPI UBool U_EXPORT2 +uhash_compareChars(const UHashTok key1, const UHashTok key2) { + const char *p1 = (const char*) key1.pointer; + const char *p2 = (const char*) key2.pointer; + if (p1 == p2) { + return TRUE; + } + if (p1 == NULL || p2 == NULL) { + return FALSE; + } + while (*p1 != 0 && *p1 == *p2) { + ++p1; + ++p2; + } + return (UBool)(*p1 == *p2); +} + +U_CAPI UBool U_EXPORT2 +uhash_compareIChars(const UHashTok key1, const UHashTok key2) { + const char *p1 = (const char*) key1.pointer; + const char *p2 = (const char*) key2.pointer; + if (p1 == p2) { + return TRUE; + } + if (p1 == NULL || p2 == NULL) { + return FALSE; + } + while (*p1 != 0 && uprv_tolower(*p1) == uprv_tolower(*p2)) { + ++p1; + ++p2; + } + return (UBool)(*p1 == *p2); +} + +/******************************************************************** + * PUBLIC int32_t Support Functions + ********************************************************************/ + +U_CAPI int32_t U_EXPORT2 +uhash_hashLong(const UHashTok key) { + return key.integer; +} + +U_CAPI UBool U_EXPORT2 +uhash_compareLong(const UHashTok key1, const UHashTok key2) { + return (UBool)(key1.integer == key2.integer); +} + +/******************************************************************** + * PUBLIC Deleter Functions + ********************************************************************/ + +U_CAPI void U_EXPORT2 +uhash_freeBlock(void *obj) { + uprv_free(obj); +} + diff --git a/Build/source/libs/icu/icu-xetex/common/uhash.h b/Build/source/libs/icu/icu-xetex/common/uhash.h new file mode 100644 index 00000000000..b5f3a67db3f --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uhash.h @@ -0,0 +1,728 @@ +/* +****************************************************************************** +* Copyright (C) 1997-2006, International Business Machines +* Corporation and others. All Rights Reserved. +****************************************************************************** +* Date Name Description +* 03/22/00 aliu Adapted from original C++ ICU Hashtable. +* 07/06/01 aliu Modified to support int32_t keys on +* platforms with sizeof(void*) < 32. +****************************************************************************** +*/ + +#ifndef UHASH_H +#define UHASH_H + +#include "unicode/utypes.h" + +/** + * UHashtable stores key-value pairs and does moderately fast lookup + * based on keys. It provides a good tradeoff between access time and + * storage space. As elements are added to it, it grows to accomodate + * them. By default, the table never shrinks, even if all elements + * are removed from it. + * + * Keys and values are stored as void* pointers. These void* pointers + * may be actual pointers to strings, objects, or any other structure + * in memory, or they may simply be integral values cast to void*. + * UHashtable doesn't care and manipulates them via user-supplied + * functions. These functions hash keys, compare keys, delete keys, + * and delete values. Some function pointers are optional (may be + * NULL); others must be supplied. Several prebuilt functions exist + * to handle common key types. + * + * UHashtable ownership of keys and values is flexible, and controlled + * by whether or not the key deleter and value deleter functions are + * set. If a void* key is actually a pointer to a deletable object, + * then UHashtable can be made to delete that object by setting the + * key deleter function pointer to a non-NULL value. If this is done, + * then keys passed to uhash_put() are owned by the hashtable and will + * be deleted by it at some point, either as keys are replaced, or + * when uhash_close() is finally called. The same is true of values + * and the value deleter function pointer. Keys passed to methods + * other than uhash_put() are never owned by the hashtable. + * + * NULL values are not allowed. uhash_get() returns NULL to indicate + * a key that is not in the table, and having a NULL value in the + * table would generate an ambiguous result. If a key and a NULL + * value is passed to uhash_put(), this has the effect of doing a + * uhash_remove() on that key. This keeps uhash_get(), uhash_count(), + * and uhash_nextElement() consistent with one another. + * + * To see everything in a hashtable, use uhash_nextElement() to + * iterate through its contents. Each call to this function returns a + * UHashElement pointer. A hash element contains a key, value, and + * hashcode. During iteration an element may be deleted by calling + * uhash_removeElement(); iteration may safely continue thereafter. + * The uhash_remove() function may also be safely called in + * mid-iteration. However, if uhash_put() is called during iteration + * then the iteration will be out of sync. Under no circumstances + * should the UHashElement returned by uhash_nextElement be modified + * directly. + * + * By default, the hashtable grows when necessary, but never shrinks, + * even if all items are removed. For most applications this is + * optimal. However, in a highly dynamic usage where memory is at a + * premium, the table can be set to both grow and shrink by calling + * uhash_setResizePolicy() with the policy U_GROW_AND_SHRINK. In a + * situation where memory is critical and the client wants a table + * that does not grow at all, the constant U_FIXED can be used. + */ + +/******************************************************************** + * Data Structures + ********************************************************************/ + +U_CDECL_BEGIN + +/** + * A key or value within the hashtable. It may be either a 32-bit + * integral value or an opaque void* pointer. The void* pointer may + * be smaller than 32 bits (e.g. 24 bits) or may be larger (e.g. 64 + * bits). The hashing and comparison functions take a pointer to a + * UHashTok, but the deleter receives the void* pointer within it. + * + * Because a UHashTok is the size of a native pointer or a 32-bit + * integer, we pass it around by value. + */ +union UHashTok { + void* pointer; + int32_t integer; +}; +typedef union UHashTok UHashTok; + +/** + * This is a single hash element. + */ +struct UHashElement { + /* Reorder these elements to pack nicely if necessary */ + int32_t hashcode; + UHashTok value; + UHashTok key; +}; +typedef struct UHashElement UHashElement; + +/** + * A hashing function. + * @param key A key stored in a hashtable + * @return A NON-NEGATIVE hash code for parm. + */ +typedef int32_t U_CALLCONV UHashFunction(const UHashTok key); + +/** + * A key comparison function. + * @param key1 A key stored in a hashtable + * @param key2 A key stored in a hashtable + * @return TRUE if the two keys are equal. + */ +typedef UBool U_CALLCONV UKeyComparator(const UHashTok key1, + const UHashTok key2); +/** + * A key comparison function. + * @param val1 A key stored in a hashtable + * @param val2 A key stored in a hashtable + * @return TRUE if the two keys are equal. + */ +typedef UBool U_CALLCONV UValueComparator(const UHashTok val1, + const UHashTok val2); +/** + * A function called by uhash_remove, + * uhash_close, or uhash_put to delete + * an existing key or value. + * @param obj A key or value stored in a hashtable + */ +typedef void U_CALLCONV UObjectDeleter(void* obj); + +/** + * This specifies whether or not, and how, the hastable resizes itself. + * See uhash_setResizePolicy(). + */ +enum UHashResizePolicy { + U_GROW, /* Grow on demand, do not shrink */ + U_GROW_AND_SHRINK, /* Grow and shrink on demand */ + U_FIXED /* Never change size */ +}; + +/** + * The UHashtable struct. Clients should treat this as an opaque data + * type and manipulate it only through the uhash_... API. + */ +struct UHashtable { + + /* Main key-value pair storage array */ + + UHashElement *elements; + + /* Function pointers */ + + UHashFunction *keyHasher; /* Computes hash from key. + * Never null. */ + UKeyComparator *keyComparator; /* Compares keys for equality. + * Never null. */ + UValueComparator *valueComparator; /* Compares the values for equality */ + + UObjectDeleter *keyDeleter; /* Deletes keys when required. + * If NULL won't do anything */ + UObjectDeleter *valueDeleter; /* Deletes values when required. + * If NULL won't do anything */ + + /* Size parameters */ + + int32_t count; /* The number of key-value pairs in this table. + * 0 <= count <= length. In practice we + * never let count == length (see code). */ + int32_t length; /* The physical size of the arrays hashes, keys + * and values. Must be prime. */ + int32_t primeIndex; /* Index into our prime table for length. + * length == PRIMES[primeIndex] */ + + /* Rehashing thresholds */ + + int32_t highWaterMark; /* If count > highWaterMark, rehash */ + int32_t lowWaterMark; /* If count < lowWaterMark, rehash */ + float highWaterRatio; /* 0..1; high water as a fraction of length */ + float lowWaterRatio; /* 0..1; low water as a fraction of length */ + + UBool allocated; /* Was this UHashtable allocated? */ +}; +typedef struct UHashtable UHashtable; + +U_CDECL_END + +/******************************************************************** + * API + ********************************************************************/ + +/** + * Initialize a new UHashtable. + * @param keyHash A pointer to the key hashing function. Must not be + * NULL. + * @param keyComp A pointer to the function that compares keys. Must + * not be NULL. + * @param status A pointer to an UErrorCode to receive any errors. + * @return A pointer to a UHashtable, or 0 if an error occurred. + * @see uhash_openSize + */ +U_CAPI UHashtable* U_EXPORT2 +uhash_open(UHashFunction *keyHash, + UKeyComparator *keyComp, + UValueComparator *valueComp, + UErrorCode *status); + +/** + * Initialize a new UHashtable with a given initial size. + * @param keyHash A pointer to the key hashing function. Must not be + * NULL. + * @param keyComp A pointer to the function that compares keys. Must + * not be NULL. + * @param size The initial capacity of this hash table. + * @param status A pointer to an UErrorCode to receive any errors. + * @return A pointer to a UHashtable, or 0 if an error occurred. + * @see uhash_open + */ +U_CAPI UHashtable* U_EXPORT2 +uhash_openSize(UHashFunction *keyHash, + UKeyComparator *keyComp, + UValueComparator *valueComp, + int32_t size, + UErrorCode *status); + +/** + * Initialize an existing UHashtable. + * @param keyHash A pointer to the key hashing function. Must not be + * NULL. + * @param keyComp A pointer to the function that compares keys. Must + * not be NULL. + * @param status A pointer to an UErrorCode to receive any errors. + * @return A pointer to a UHashtable, or 0 if an error occurred. + * @see uhash_openSize + */ +U_CAPI UHashtable* U_EXPORT2 +uhash_init(UHashtable *hash, + UHashFunction *keyHash, + UKeyComparator *keyComp, + UValueComparator *valueComp, + UErrorCode *status); + +/** + * Close a UHashtable, releasing the memory used. + * @param hash The UHashtable to close. + */ +U_CAPI void U_EXPORT2 +uhash_close(UHashtable *hash); + + + +/** + * Set the function used to hash keys. + * @param hash The UHashtable to set + * @param fn the function to be used hash keys; must not be NULL + * @return the previous key hasher; non-NULL + */ +U_CAPI UHashFunction *U_EXPORT2 +uhash_setKeyHasher(UHashtable *hash, UHashFunction *fn); + +/** + * Set the function used to compare keys. The default comparison is a + * void* pointer comparison. + * @param hash The UHashtable to set + * @param fn the function to be used compare keys; must not be NULL + * @return the previous key comparator; non-NULL + */ +U_CAPI UKeyComparator *U_EXPORT2 +uhash_setKeyComparator(UHashtable *hash, UKeyComparator *fn); + +/** + * Set the function used to compare values. The default comparison is a + * void* pointer comparison. + * @param hash The UHashtable to set + * @param fn the function to be used compare keys; must not be NULL + * @return the previous key comparator; non-NULL + */ +U_CAPI UValueComparator *U_EXPORT2 +uhash_setValueComparator(UHashtable *hash, UValueComparator *fn); + +/** + * Set the function used to delete keys. If this function pointer is + * NULL, this hashtable does not delete keys. If it is non-NULL, this + * hashtable does delete keys. This function should be set once + * before any elements are added to the hashtable and should not be + * changed thereafter. + * @param hash The UHashtable to set + * @param fn the function to be used delete keys, or NULL + * @return the previous key deleter; may be NULL + */ +U_CAPI UObjectDeleter *U_EXPORT2 +uhash_setKeyDeleter(UHashtable *hash, UObjectDeleter *fn); + +/** + * Set the function used to delete values. If this function pointer + * is NULL, this hashtable does not delete values. If it is non-NULL, + * this hashtable does delete values. This function should be set + * once before any elements are added to the hashtable and should not + * be changed thereafter. + * @param hash The UHashtable to set + * @param fn the function to be used delete values, or NULL + * @return the previous value deleter; may be NULL + */ +U_CAPI UObjectDeleter *U_EXPORT2 +uhash_setValueDeleter(UHashtable *hash, UObjectDeleter *fn); + +/** + * Specify whether or not, and how, the hastable resizes itself. + * By default, tables grow but do not shrink (policy U_GROW). + * See enum UHashResizePolicy. + * @param hash The UHashtable to set + * @param policy The way the hashtable resizes itself, {U_GROW, U_GROW_AND_SHRINK, U_FIXED} + */ +U_CAPI void U_EXPORT2 +uhash_setResizePolicy(UHashtable *hash, enum UHashResizePolicy policy); + +/** + * Get the number of key-value pairs stored in a UHashtable. + * @param hash The UHashtable to query. + * @return The number of key-value pairs stored in hash. + */ +U_CAPI int32_t U_EXPORT2 +uhash_count(const UHashtable *hash); + +/** + * Put a (key=pointer, value=pointer) item in a UHashtable. If the + * keyDeleter is non-NULL, then the hashtable owns 'key' after this + * call. If the valueDeleter is non-NULL, then the hashtable owns + * 'value' after this call. Storing a NULL value is the same as + * calling uhash_remove(). + * @param hash The target UHashtable. + * @param key The key to store. + * @param value The value to store, may be NULL (see above). + * @param status A pointer to an UErrorCode to receive any errors. + * @return The previous value, or NULL if none. + * @see uhash_get + */ +U_CAPI void* U_EXPORT2 +uhash_put(UHashtable *hash, + void *key, + void *value, + UErrorCode *status); + +/** + * Put a (key=integer, value=pointer) item in a UHashtable. + * keyDeleter must be NULL. If the valueDeleter is non-NULL, then the + * hashtable owns 'value' after this call. Storing a NULL value is + * the same as calling uhash_remove(). + * @param hash The target UHashtable. + * @param key The integer key to store. + * @param value The value to store, may be NULL (see above). + * @param status A pointer to an UErrorCode to receive any errors. + * @return The previous value, or NULL if none. + * @see uhash_get + */ +U_CAPI void* U_EXPORT2 +uhash_iput(UHashtable *hash, + int32_t key, + void* value, + UErrorCode *status); + +/** + * Put a (key=pointer, value=integer) item in a UHashtable. If the + * keyDeleter is non-NULL, then the hashtable owns 'key' after this + * call. valueDeleter must be NULL. Storing a 0 value is the same as + * calling uhash_remove(). + * @param hash The target UHashtable. + * @param key The key to store. + * @param value The integer value to store. + * @param status A pointer to an UErrorCode to receive any errors. + * @return The previous value, or 0 if none. + * @see uhash_get + */ +U_CAPI int32_t U_EXPORT2 +uhash_puti(UHashtable *hash, + void* key, + int32_t value, + UErrorCode *status); + +/** + * Put a (key=integer, value=integer) item in a UHashtable. If the + * keyDeleter is non-NULL, then the hashtable owns 'key' after this + * call. valueDeleter must be NULL. Storing a 0 value is the same as + * calling uhash_remove(). + * @param hash The target UHashtable. + * @param key The key to store. + * @param value The integer value to store. + * @param status A pointer to an UErrorCode to receive any errors. + * @return The previous value, or 0 if none. + * @see uhash_get + */ +U_CAPI int32_t U_EXPORT2 +uhash_iputi(UHashtable *hash, + int32_t key, + int32_t value, + UErrorCode *status); + +/** + * Retrieve a pointer value from a UHashtable using a pointer key, + * as previously stored by uhash_put(). + * @param hash The target UHashtable. + * @param key A pointer key stored in a hashtable + * @return The requested item, or NULL if not found. + */ +U_CAPI void* U_EXPORT2 +uhash_get(const UHashtable *hash, + const void *key); + +/** + * Retrieve a pointer value from a UHashtable using a integer key, + * as previously stored by uhash_iput(). + * @param hash The target UHashtable. + * @param key An integer key stored in a hashtable + * @return The requested item, or NULL if not found. + */ +U_CAPI void* U_EXPORT2 +uhash_iget(const UHashtable *hash, + int32_t key); + +/** + * Retrieve an integer value from a UHashtable using a pointer key, + * as previously stored by uhash_puti(). + * @param hash The target UHashtable. + * @param key A pointer key stored in a hashtable + * @return The requested item, or 0 if not found. + */ +U_CAPI int32_t U_EXPORT2 +uhash_geti(const UHashtable *hash, + const void* key); +/** + * Retrieve an integer value from a UHashtable using an integer key, + * as previously stored by uhash_iputi(). + * @param hash The target UHashtable. + * @param key An integer key stored in a hashtable + * @return The requested item, or 0 if not found. + */ +U_CAPI int32_t U_EXPORT2 +uhash_igeti(const UHashtable *hash, + int32_t key); + +/** + * Remove an item from a UHashtable stored by uhash_put(). + * @param hash The target UHashtable. + * @param key A key stored in a hashtable + * @return The item removed, or NULL if not found. + */ +U_CAPI void* U_EXPORT2 +uhash_remove(UHashtable *hash, + const void *key); + +/** + * Remove an item from a UHashtable stored by uhash_iput(). + * @param hash The target UHashtable. + * @param key An integer key stored in a hashtable + * @return The item removed, or NULL if not found. + */ +U_CAPI void* U_EXPORT2 +uhash_iremove(UHashtable *hash, + int32_t key); + +/** + * Remove an item from a UHashtable stored by uhash_puti(). + * @param hash The target UHashtable. + * @param key An key stored in a hashtable + * @return The item removed, or 0 if not found. + */ +U_CAPI int32_t U_EXPORT2 +uhash_removei(UHashtable *hash, + const void* key); + +/** + * Remove an item from a UHashtable stored by uhash_iputi(). + * @param hash The target UHashtable. + * @param key An integer key stored in a hashtable + * @return The item removed, or 0 if not found. + */ +U_CAPI int32_t U_EXPORT2 +uhash_iremovei(UHashtable *hash, + int32_t key); + +/** + * Remove all items from a UHashtable. + * @param hash The target UHashtable. + */ +U_CAPI void U_EXPORT2 +uhash_removeAll(UHashtable *hash); + +/** + * Locate an element of a UHashtable. The caller must not modify the + * returned object. The primary use of this function is to obtain the + * stored key when it may not be identical to the search key. For + * example, if the compare function is a case-insensitive string + * compare, then the hash key may be desired in order to obtain the + * canonical case corresponding to a search key. + * @param hash The target UHashtable. + * @param key A key stored in a hashtable + * @return a hash element, or NULL if the key is not found. + */ +U_CAPI const UHashElement* U_EXPORT2 +uhash_find(const UHashtable *hash, const void* key); + +/** + * Iterate through the elements of a UHashtable. The caller must not + * modify the returned object. However, uhash_removeElement() may be + * called during iteration to remove an element from the table. + * Iteration may safely be resumed afterwards. If uhash_put() is + * called during iteration the iteration will then be out of sync and + * should be restarted. + * @param hash The target UHashtable. + * @param pos This should be set to -1 initially, and left untouched + * thereafter. + * @return a hash element, or NULL if no further key-value pairs + * exist in the table. + */ +U_CAPI const UHashElement* U_EXPORT2 +uhash_nextElement(const UHashtable *hash, + int32_t *pos); + +/** + * Remove an element, returned by uhash_nextElement(), from the table. + * Iteration may be safely continued afterwards. + * @param hash The hashtable + * @param e The element, returned by uhash_nextElement(), to remove. + * Must not be NULL. Must not be an empty or deleted element (as long + * as this was returned by uhash_nextElement() it will not be empty or + * deleted). Note: Although this parameter is const, it will be + * modified. + * @return the value that was removed. + */ +U_CAPI void* U_EXPORT2 +uhash_removeElement(UHashtable *hash, const UHashElement* e); + +/******************************************************************** + * UHashTok convenience + ********************************************************************/ + +/** + * Return a UHashTok for an integer. + * @param i The given integer + * @return a UHashTok for an integer. + */ +/*U_CAPI UHashTok U_EXPORT2 +uhash_toki(int32_t i);*/ + +/** + * Return a UHashTok for a pointer. + * @param p The given pointer + * @return a UHashTok for a pointer. + */ +/*U_CAPI UHashTok U_EXPORT2 +uhash_tokp(void* p);*/ + +/******************************************************************** + * UChar* and char* Support Functions + ********************************************************************/ + +/** + * Generate a hash code for a null-terminated UChar* string. If the + * string is not null-terminated do not use this function. Use + * together with uhash_compareUChars. + * @param key The string (const UChar*) to hash. + * @return A hash code for the key. + */ +U_CAPI int32_t U_EXPORT2 +uhash_hashUChars(const UHashTok key); + +/** + * Generate a hash code for a null-terminated char* string. If the + * string is not null-terminated do not use this function. Use + * together with uhash_compareChars. + * @param key The string (const char*) to hash. + * @return A hash code for the key. + */ +U_CAPI int32_t U_EXPORT2 +uhash_hashChars(const UHashTok key); + +/* Used by UnicodeString to compute its hashcode - Not public API. */ +U_CAPI int32_t U_EXPORT2 +uhash_hashUCharsN(const UChar *key, int32_t length); + +/** + * Generate a case-insensitive hash code for a null-terminated char* + * string. If the string is not null-terminated do not use this + * function. Use together with uhash_compareIChars. + * @param key The string (const char*) to hash. + * @return A hash code for the key. + */ +U_CAPI int32_t U_EXPORT2 +uhash_hashIChars(const UHashTok key); + +/** + * Comparator for null-terminated UChar* strings. Use together with + * uhash_hashUChars. + * @param key1 The string for comparison + * @param key2 The string for comparison + * @return true if key1 and key2 are equal, return false otherwise. + */ +U_CAPI UBool U_EXPORT2 +uhash_compareUChars(const UHashTok key1, const UHashTok key2); + +/** + * Comparator for null-terminated char* strings. Use together with + * uhash_hashChars. + * @param key1 The string for comparison + * @param key2 The string for comparison + * @return true if key1 and key2 are equal, return false otherwise. + */ +U_CAPI UBool U_EXPORT2 +uhash_compareChars(const UHashTok key1, const UHashTok key2); + +/** + * Case-insensitive comparator for null-terminated char* strings. Use + * together with uhash_hashIChars. + * @param key1 The string for comparison + * @param key2 The string for comparison + * @return true if key1 and key2 are equal, return false otherwise. + */ +U_CAPI UBool U_EXPORT2 +uhash_compareIChars(const UHashTok key1, const UHashTok key2); + +/******************************************************************** + * UnicodeString Support Functions + ********************************************************************/ + +/** + * Hash function for UnicodeString* keys. + * @param key The string (const char*) to hash. + * @return A hash code for the key. + */ +U_CAPI int32_t U_EXPORT2 +uhash_hashUnicodeString(const UHashTok key); + +/** + * Hash function for UnicodeString* keys (case insensitive). + * Make sure to use together with uhash_compareCaselessUnicodeString. + * @param key The string (const char*) to hash. + * @return A hash code for the key. + */ +U_CAPI int32_t U_EXPORT2 +uhash_hashCaselessUnicodeString(const UHashTok key); + +/** + * Comparator function for UnicodeString* keys. + * @param key1 The string for comparison + * @param key2 The string for comparison + * @return true if key1 and key2 are equal, return false otherwise. + */ +U_CAPI UBool U_EXPORT2 +uhash_compareUnicodeString(const UHashTok key1, const UHashTok key2); + +/** + * Comparator function for UnicodeString* keys (case insensitive). + * Make sure to use together with uhash_hashCaselessUnicodeString. + * @param key1 The string for comparison + * @param key2 The string for comparison + * @return true if key1 and key2 are equal, return false otherwise. + */ +U_CAPI UBool U_EXPORT2 +uhash_compareCaselessUnicodeString(const UHashTok key1, const UHashTok key2); + +/** + * Deleter function for UnicodeString* keys or values. + * @param obj The object to be deleted + */ +U_CAPI void U_EXPORT2 +uhash_deleteUnicodeString(void *obj); + +/******************************************************************** + * int32_t Support Functions + ********************************************************************/ + +/** + * Hash function for 32-bit integer keys. + * @param key The string (const char*) to hash. + * @return A hash code for the key. + */ +U_CAPI int32_t U_EXPORT2 +uhash_hashLong(const UHashTok key); + +/** + * Comparator function for 32-bit integer keys. + * @param key1 The integer for comparison + * @param Key2 The integer for comparison + * @return true if key1 and key2 are equal, return false otherwise + */ +U_CAPI UBool U_EXPORT2 +uhash_compareLong(const UHashTok key1, const UHashTok key2); + +/******************************************************************** + * Other Support Functions + ********************************************************************/ + +/** + * Deleter for Hashtable objects. + * @param obj The object to be deleted + */ +U_CAPI void U_EXPORT2 +uhash_deleteHashtable(void *obj); + +/** + * Deleter for UVector objects. + * @param obj The object to be deleted + */ +U_CAPI void U_EXPORT2 +uhash_deleteUVector(void *obj); + +/** + * Deleter for any key or value allocated using uprv_malloc. Calls + * uprv_free. + * @param obj The object to be deleted + */ +U_CAPI void U_EXPORT2 +uhash_freeBlock(void *obj); + +/** + * Checks if the given hash tables are equal or not. + * @param hash1 + * @param hash2 + * @return true if the hashtables are equal and false if not. + */ +U_CAPI UBool U_EXPORT2 +uhash_equals(const UHashtable* hash1, const UHashtable* hash2); + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/uhash_us.cpp b/Build/source/libs/icu/icu-xetex/common/uhash_us.cpp new file mode 100644 index 00000000000..0c6a3178904 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uhash_us.cpp @@ -0,0 +1,68 @@ +/* +****************************************************************************** +* Copyright (C) 1997-2004, International Business Machines +* Corporation and others. All Rights Reserved. +****************************************************************************** +* Date Name Description +* 03/22/00 aliu Creation. +* 07/06/01 aliu Modified to support int32_t keys on +* platforms with sizeof(void*) < 32. +****************************************************************************** +*/ + +#include "uhash.h" +#include "hash.h" +#include "uvector.h" +#include "unicode/unistr.h" +#include "unicode/uchar.h" + +/******************************************************************** + * PUBLIC UnicodeString support functions for UHashtable + ********************************************************************/ + +U_CAPI int32_t U_EXPORT2 +uhash_hashUnicodeString(const UHashTok key) { + U_NAMESPACE_USE + const UnicodeString *str = (const UnicodeString*) key.pointer; + return (str == NULL) ? 0 : str->hashCode(); +} + +U_CAPI void U_EXPORT2 +uhash_deleteUnicodeString(void *obj) { + U_NAMESPACE_USE + delete (UnicodeString*) obj; +} + +U_CAPI UBool U_EXPORT2 +uhash_compareUnicodeString(const UHashTok key1, const UHashTok key2) { + U_NAMESPACE_USE + const UnicodeString *str1 = (const UnicodeString*) key1.pointer; + const UnicodeString *str2 = (const UnicodeString*) key2.pointer; + if (str1 == str2) { + return TRUE; + } + if (str1 == NULL || str2 == NULL) { + return FALSE; + } + return *str1 == *str2; +} + +/** + * Deleter for Hashtable objects. + */ +U_CAPI void U_EXPORT2 +uhash_deleteHashtable(void *obj) { + U_NAMESPACE_USE + delete (Hashtable*) obj; +} + +/** + * Deleter for UVector objects. + */ +U_CAPI void U_EXPORT2 +uhash_deleteUVector(void *obj) { + U_NAMESPACE_USE + delete (UVector*) obj; +} + +//eof diff --git a/Build/source/libs/icu/icu-xetex/common/uidna.cpp b/Build/source/libs/icu/icu-xetex/common/uidna.cpp new file mode 100644 index 00000000000..b86ed26b673 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uidna.cpp @@ -0,0 +1,923 @@ +/* + ******************************************************************************* + * + * Copyright (C) 2003-2007, International Business Machines + * Corporation and others. All Rights Reserved. + * + ******************************************************************************* + * file name: uidna.cpp + * encoding: US-ASCII + * tab size: 8 (not used) + * indentation:4 + * + * created on: 2003feb1 + * created by: Ram Viswanadha + */ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_IDNA + +#include "unicode/uidna.h" +#include "unicode/ustring.h" +#include "unicode/usprep.h" +#include "punycode.h" +#include "ustr_imp.h" +#include "cmemory.h" +#include "uassert.h" +#include "sprpimpl.h" + +/* it is official IDNA ACE Prefix is "xn--" */ +static const UChar ACE_PREFIX[] ={ 0x0078,0x006E,0x002d,0x002d } ; +#define ACE_PREFIX_LENGTH 4 + +#define MAX_LABEL_LENGTH 63 +/* The Max length of the labels should not be more than MAX_LABEL_LENGTH */ +#define MAX_LABEL_BUFFER_SIZE 100 + +#define MAX_DOMAIN_NAME_LENGTH 255 +/* The Max length of the domain names should not be more than MAX_DOMAIN_NAME_LENGTH */ +#define MAX_IDN_BUFFER_SIZE MAX_DOMAIN_NAME_LENGTH+1 + +#define LOWER_CASE_DELTA 0x0020 +#define HYPHEN 0x002D +#define FULL_STOP 0x002E +#define CAPITAL_A 0x0041 +#define CAPITAL_Z 0x005A + +#define DATA_FILE_NAME "uidna" + +inline static UChar +toASCIILower(UChar ch){ + if(CAPITAL_A <= ch && ch <= CAPITAL_Z){ + return ch + LOWER_CASE_DELTA; + } + return ch; +} + +inline static UBool +startsWithPrefix(const UChar* src , int32_t srcLength){ + UBool startsWithPrefix = TRUE; + + if(srcLength < ACE_PREFIX_LENGTH){ + return FALSE; + } + + for(int8_t i=0; i< ACE_PREFIX_LENGTH; i++){ + if(toASCIILower(src[i]) != ACE_PREFIX[i]){ + startsWithPrefix = FALSE; + } + } + return startsWithPrefix; +} + + +inline static int32_t +compareCaseInsensitiveASCII(const UChar* s1, int32_t s1Len, + const UChar* s2, int32_t s2Len){ + + int32_t minLength; + int32_t lengthResult; + + // are we comparing different lengths? + if(s1Len != s2Len) { + if(s1Len < s2Len) { + minLength = s1Len; + lengthResult = -1; + } else { + minLength = s2Len; + lengthResult = 1; + } + } else { + // ok the lengths are equal + minLength = s1Len; + lengthResult = 0; + } + + UChar c1,c2; + int32_t rc; + + for(int32_t i =0;/* no condition */;i++) { + + /* If we reach the ends of both strings then they match */ + if(i == minLength) { + return lengthResult; + } + + c1 = s1[i]; + c2 = s2[i]; + + /* Case-insensitive comparison */ + if(c1!=c2) { + rc=(int32_t)toASCIILower(c1)-(int32_t)toASCIILower(c2); + if(rc!=0) { + lengthResult=rc; + break; + } + } + } + return lengthResult; +} + + +/** + * Ascertain if the given code point is a label separator as + * defined by the IDNA RFC + * + * @param ch The code point to be ascertained + * @return true if the char is a label separator + * @stable ICU 2.8 + */ +static inline UBool isLabelSeparator(UChar ch){ + switch(ch){ + case 0x002e: + case 0x3002: + case 0xFF0E: + case 0xFF61: + return TRUE; + default: + return FALSE; + } +} + +// returns the length of the label excluding the separator +// if *limit == separator then the length returned does not include +// the separtor. +static inline int32_t +getNextSeparator(UChar *src, int32_t srcLength, + UChar **limit, UBool *done){ + if(srcLength == -1){ + int32_t i; + for(i=0 ; ;i++){ + if(src[i] == 0){ + *limit = src + i; // point to null + *done = TRUE; + return i; + } + if(isLabelSeparator(src[i])){ + *limit = src + (i+1); // go past the delimiter + return i; + + } + } + }else{ + int32_t i; + for(i=0;i0x007A){ + return FALSE; + } + //[\\u002D \\u0030-\\u0039 \\u0041-\\u005A \\u0061-\\u007A] + if( (ch==0x002D) || + (0x0030 <= ch && ch <= 0x0039) || + (0x0041 <= ch && ch <= 0x005A) || + (0x0061 <= ch && ch <= 0x007A) + ){ + return TRUE; + } + return FALSE; +} + +static int32_t +_internal_toASCII(const UChar* src, int32_t srcLength, + UChar* dest, int32_t destCapacity, + int32_t options, + UStringPrepProfile* nameprep, + UParseError* parseError, + UErrorCode* status) +{ + + // TODO Revisit buffer handling. The label should not be over 63 ASCII characters. ICU4J may need to be updated too. + UChar b1Stack[MAX_LABEL_BUFFER_SIZE], b2Stack[MAX_LABEL_BUFFER_SIZE]; + //initialize pointers to stack buffers + UChar *b1 = b1Stack, *b2 = b2Stack; + int32_t b1Len=0, b2Len, + b1Capacity = MAX_LABEL_BUFFER_SIZE, + b2Capacity = MAX_LABEL_BUFFER_SIZE , + reqLength=0; + + int32_t namePrepOptions = ((options & UIDNA_ALLOW_UNASSIGNED) != 0) ? USPREP_ALLOW_UNASSIGNED: 0; + UBool* caseFlags = NULL; + + // the source contains all ascii codepoints + UBool srcIsASCII = TRUE; + // assume the source contains all LDH codepoints + UBool srcIsLDH = TRUE; + + int32_t j=0; + + //get the options + UBool useSTD3ASCIIRules = (UBool)((options & UIDNA_USE_STD3_RULES) != 0); + + int32_t failPos = -1; + + if(srcLength == -1){ + srcLength = u_strlen(src); + } + + if(srcLength > b1Capacity){ + b1 = (UChar*) uprv_malloc(srcLength * U_SIZEOF_UCHAR); + if(b1==NULL){ + *status = U_MEMORY_ALLOCATION_ERROR; + goto CLEANUP; + } + b1Capacity = srcLength; + } + + // step 1 + for( j=0;j 0x7F){ + srcIsASCII = FALSE; + } + b1[b1Len++] = src[j]; + } + + // step 2 is performed only if the source contains non ASCII + if(srcIsASCII == FALSE){ + + // step 2 + b1Len = usprep_prepare(nameprep, src, srcLength, b1, b1Capacity, namePrepOptions, parseError, status); + + if(*status == U_BUFFER_OVERFLOW_ERROR){ + // redo processing of string + // we do not have enough room so grow the buffer + if(b1 != b1Stack){ + uprv_free(b1); + } + b1 = (UChar*) uprv_malloc(b1Len * U_SIZEOF_UCHAR); + if(b1==NULL){ + *status = U_MEMORY_ALLOCATION_ERROR; + goto CLEANUP; + } + + *status = U_ZERO_ERROR; // reset error + + b1Len = usprep_prepare(nameprep, src, srcLength, b1, b1Len, namePrepOptions, parseError, status); + } + } + // error bail out + if(U_FAILURE(*status)){ + goto CLEANUP; + } + if(b1Len == 0){ + *status = U_IDNA_ZERO_LENGTH_LABEL_ERROR; + goto CLEANUP; + } + + // for step 3 & 4 + srcIsASCII = TRUE; + for( j=0;j 0x7F){ + srcIsASCII = FALSE; + }else if(isLDHChar(b1[j])==FALSE){ // if the char is in ASCII range verify that it is an LDH character + srcIsLDH = FALSE; + failPos = j; + } + } + if(useSTD3ASCIIRules == TRUE){ + // verify 3a and 3b + // 3(a) Verify the absence of non-LDH ASCII code points; that is, the + // absence of 0..2C, 2E..2F, 3A..40, 5B..60, and 7B..7F. + // 3(b) Verify the absence of leading and trailing hyphen-minus; that + // is, the absence of U+002D at the beginning and end of the + // sequence. + if( srcIsLDH == FALSE /* source at this point should not contain anyLDH characters */ + || b1[0] == HYPHEN || b1[b1Len-1] == HYPHEN){ + *status = U_IDNA_STD3_ASCII_RULES_ERROR; + + /* populate the parseError struct */ + if(srcIsLDH==FALSE){ + // failPos is always set the index of failure + uprv_syntaxError(b1,failPos, b1Len,parseError); + }else if(b1[0] == HYPHEN){ + // fail position is 0 + uprv_syntaxError(b1,0,b1Len,parseError); + }else{ + // the last index in the source is always length-1 + uprv_syntaxError(b1, (b1Len>0) ? b1Len-1 : b1Len, b1Len,parseError); + } + + goto CLEANUP; + } + } + // Step 4: if the source is ASCII then proceed to step 8 + if(srcIsASCII){ + if(b1Len <= destCapacity){ + uprv_memmove(dest, b1, b1Len * U_SIZEOF_UCHAR); + reqLength = b1Len; + }else{ + reqLength = b1Len; + goto CLEANUP; + } + }else{ + // step 5 : verify the sequence does not begin with ACE prefix + if(!startsWithPrefix(b1,b1Len)){ + + //step 6: encode the sequence with punycode + + // do not preserve the case flags for now! + // TODO: Preserve the case while implementing the RFE + // caseFlags = (UBool*) uprv_malloc(b1Len * sizeof(UBool)); + // uprv_memset(caseFlags,TRUE,b1Len); + + b2Len = u_strToPunycode(b1,b1Len,b2,b2Capacity,caseFlags, status); + + if(*status == U_BUFFER_OVERFLOW_ERROR){ + // redo processing of string + /* we do not have enough room so grow the buffer*/ + b2 = (UChar*) uprv_malloc(b2Len * U_SIZEOF_UCHAR); + if(b2 == NULL){ + *status = U_MEMORY_ALLOCATION_ERROR; + goto CLEANUP; + } + + *status = U_ZERO_ERROR; // reset error + + b2Len = u_strToPunycode(b1,b1Len,b2,b2Len,caseFlags, status); + } + //error bail out + if(U_FAILURE(*status)){ + goto CLEANUP; + } + // TODO : Reconsider while implementing the case preserve RFE + // convert all codepoints to lower case ASCII + // toASCIILower(b2,b2Len); + reqLength = b2Len+ACE_PREFIX_LENGTH; + + if(reqLength > destCapacity){ + *status = U_BUFFER_OVERFLOW_ERROR; + goto CLEANUP; + } + //Step 7: prepend the ACE prefix + uprv_memcpy(dest,ACE_PREFIX,ACE_PREFIX_LENGTH * U_SIZEOF_UCHAR); + //Step 6: copy the contents in b2 into dest + uprv_memcpy(dest+ACE_PREFIX_LENGTH, b2, b2Len * U_SIZEOF_UCHAR); + + }else{ + *status = U_IDNA_ACE_PREFIX_ERROR; + //position of failure is 0 + uprv_syntaxError(b1,0,b1Len,parseError); + goto CLEANUP; + } + } + // step 8: verify the length of label + if(reqLength > MAX_LABEL_LENGTH){ + *status = U_IDNA_LABEL_TOO_LONG_ERROR; + } + +CLEANUP: + if(b1 != b1Stack){ + uprv_free(b1); + } + if(b2 != b2Stack){ + uprv_free(b2); + } + uprv_free(caseFlags); + + return u_terminateUChars(dest, destCapacity, reqLength, status); +} + +static int32_t +_internal_toUnicode(const UChar* src, int32_t srcLength, + UChar* dest, int32_t destCapacity, + int32_t options, + UStringPrepProfile* nameprep, + UParseError* parseError, + UErrorCode* status) +{ + + //get the options + //UBool useSTD3ASCIIRules = (UBool)((options & UIDNA_USE_STD3_RULES) != 0); + int32_t namePrepOptions = ((options & UIDNA_ALLOW_UNASSIGNED) != 0) ? USPREP_ALLOW_UNASSIGNED: 0; + + // TODO Revisit buffer handling. The label should not be over 63 ASCII characters. ICU4J may need to be updated too. + UChar b1Stack[MAX_LABEL_BUFFER_SIZE], b2Stack[MAX_LABEL_BUFFER_SIZE], b3Stack[MAX_LABEL_BUFFER_SIZE]; + + //initialize pointers to stack buffers + UChar *b1 = b1Stack, *b2 = b2Stack, *b1Prime=NULL, *b3=b3Stack; + int32_t b1Len, b2Len, b1PrimeLen, b3Len, + b1Capacity = MAX_LABEL_BUFFER_SIZE, + b2Capacity = MAX_LABEL_BUFFER_SIZE, + b3Capacity = MAX_LABEL_BUFFER_SIZE, + reqLength=0; + + b1Len = 0; + UBool* caseFlags = NULL; + + UBool srcIsASCII = TRUE; + /*UBool srcIsLDH = TRUE; + int32_t failPos =0;*/ + + // step 1: find out if all the codepoints in src are ASCII + if(srcLength==-1){ + srcLength = 0; + for(;src[srcLength]!=0;){ + if(src[srcLength]> 0x7f){ + srcIsASCII = FALSE; + }/*else if(isLDHChar(src[srcLength])==FALSE){ + // here we do not assemble surrogates + // since we know that LDH code points + // are in the ASCII range only + srcIsLDH = FALSE; + failPos = srcLength; + }*/ + srcLength++; + } + }else if(srcLength > 0){ + for(int32_t j=0; j 0x7f){ + srcIsASCII = FALSE; + }/*else if(isLDHChar(src[j])==FALSE){ + // here we do not assemble surrogates + // since we know that LDH code points + // are in the ASCII range only + srcIsLDH = FALSE; + failPos = j; + }*/ + } + }else{ + return 0; + } + + if(srcIsASCII == FALSE){ + // step 2: process the string + b1Len = usprep_prepare(nameprep, src, srcLength, b1, b1Capacity, namePrepOptions, parseError, status); + if(*status == U_BUFFER_OVERFLOW_ERROR){ + // redo processing of string + /* we do not have enough room so grow the buffer*/ + b1 = (UChar*) uprv_malloc(b1Len * U_SIZEOF_UCHAR); + if(b1==NULL){ + *status = U_MEMORY_ALLOCATION_ERROR; + goto CLEANUP; + } + + *status = U_ZERO_ERROR; // reset error + + b1Len = usprep_prepare(nameprep, src, srcLength, b1, b1Len, namePrepOptions, parseError, status); + } + //bail out on error + if(U_FAILURE(*status)){ + goto CLEANUP; + } + }else{ + + //just point src to b1 + b1 = (UChar*) src; + b1Len = srcLength; + } + + // The RFC states that + // + // ToUnicode never fails. If any step fails, then the original input + // is returned immediately in that step. + // + + //step 3: verify ACE Prefix + if(startsWithPrefix(b1,b1Len)){ + + //step 4: Remove the ACE Prefix + b1Prime = b1 + ACE_PREFIX_LENGTH; + b1PrimeLen = b1Len - ACE_PREFIX_LENGTH; + + //step 5: Decode using punycode + b2Len = u_strFromPunycode(b1Prime, b1PrimeLen, b2, b2Capacity, caseFlags,status); + + if(*status == U_BUFFER_OVERFLOW_ERROR){ + // redo processing of string + /* we do not have enough room so grow the buffer*/ + b2 = (UChar*) uprv_malloc(b2Len * U_SIZEOF_UCHAR); + if(b2==NULL){ + *status = U_MEMORY_ALLOCATION_ERROR; + goto CLEANUP; + } + + *status = U_ZERO_ERROR; // reset error + + b2Len = u_strFromPunycode(b1Prime, b1PrimeLen, b2, b2Len, caseFlags, status); + } + + + //step 6:Apply toASCII + b3Len = uidna_toASCII(b2, b2Len, b3, b3Capacity, options, parseError, status); + + if(*status == U_BUFFER_OVERFLOW_ERROR){ + // redo processing of string + /* we do not have enough room so grow the buffer*/ + b3 = (UChar*) uprv_malloc(b3Len * U_SIZEOF_UCHAR); + if(b3==NULL){ + *status = U_MEMORY_ALLOCATION_ERROR; + goto CLEANUP; + } + + *status = U_ZERO_ERROR; // reset error + + b3Len = uidna_toASCII(b2,b2Len,b3,b3Len,options,parseError, status); + + } + //bail out on error + if(U_FAILURE(*status)){ + goto CLEANUP; + } + + //step 7: verify + if(compareCaseInsensitiveASCII(b1, b1Len, b3, b3Len) !=0){ + // Cause the original to be returned. + *status = U_IDNA_VERIFICATION_ERROR; + goto CLEANUP; + } + + //step 8: return output of step 5 + reqLength = b2Len; + if(b2Len <= destCapacity) { + uprv_memmove(dest, b2, b2Len * U_SIZEOF_UCHAR); + } + } + else{ + // See the start of this if statement for why this is commented out. + // verify that STD3 ASCII rules are satisfied + /*if(useSTD3ASCIIRules == TRUE){ + if( srcIsLDH == FALSE // source contains some non-LDH characters + || src[0] == HYPHEN || src[srcLength-1] == HYPHEN){ + *status = U_IDNA_STD3_ASCII_RULES_ERROR; + + // populate the parseError struct + if(srcIsLDH==FALSE){ + // failPos is always set the index of failure + uprv_syntaxError(src,failPos, srcLength,parseError); + }else if(src[0] == HYPHEN){ + // fail position is 0 + uprv_syntaxError(src,0,srcLength,parseError); + }else{ + // the last index in the source is always length-1 + uprv_syntaxError(src, (srcLength>0) ? srcLength-1 : srcLength, srcLength,parseError); + } + + goto CLEANUP; + } + }*/ + // just return the source + //copy the source to destination + if(srcLength <= destCapacity){ + uprv_memmove(dest,src,srcLength * U_SIZEOF_UCHAR); + } + reqLength = srcLength; + } + + +CLEANUP: + + if(b1 != b1Stack && b1!=src){ + uprv_free(b1); + } + if(b2 != b2Stack){ + uprv_free(b2); + } + uprv_free(caseFlags); + + // The RFC states that + // + // ToUnicode never fails. If any step fails, then the original input + // is returned immediately in that step. + // + // So if any step fails lets copy source to destination + if(U_FAILURE(*status)){ + //copy the source to destination + if(dest && srcLength <= destCapacity){ + // srcLength should have already been set earlier. + U_ASSERT(srcLength >= 0); + uprv_memmove(dest,src,srcLength * U_SIZEOF_UCHAR); + } + reqLength = srcLength; + *status = U_ZERO_ERROR; + } + + return u_terminateUChars(dest, destCapacity, reqLength, status); +} + +U_CAPI int32_t U_EXPORT2 +uidna_toASCII(const UChar* src, int32_t srcLength, + UChar* dest, int32_t destCapacity, + int32_t options, + UParseError* parseError, + UErrorCode* status){ + + if(status == NULL || U_FAILURE(*status)){ + return 0; + } + if((src==NULL) || (srcLength < -1) || (destCapacity<0) || (!dest && destCapacity > 0)){ + *status = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + UStringPrepProfile* nameprep = usprep_open(NULL,DATA_FILE_NAME, status); + + if(U_FAILURE(*status)){ + return -1; + } + + int32_t retLen = _internal_toASCII(src, srcLength, dest, destCapacity, options, nameprep, parseError, status); + + /* close the profile*/ + usprep_close(nameprep); + + return retLen; +} + +U_CAPI int32_t U_EXPORT2 +uidna_toUnicode(const UChar* src, int32_t srcLength, + UChar* dest, int32_t destCapacity, + int32_t options, + UParseError* parseError, + UErrorCode* status){ + + if(status == NULL || U_FAILURE(*status)){ + return 0; + } + if( (src==NULL) || (srcLength < -1) || (destCapacity<0) || (!dest && destCapacity > 0)){ + *status = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + UStringPrepProfile* nameprep = usprep_open(NULL, DATA_FILE_NAME, status); + + if(U_FAILURE(*status)){ + return -1; + } + + int32_t retLen = _internal_toUnicode(src, srcLength, dest, destCapacity, options, nameprep, parseError, status); + + usprep_close(nameprep); + + return retLen; +} + + +U_CAPI int32_t U_EXPORT2 +uidna_IDNToASCII( const UChar *src, int32_t srcLength, + UChar* dest, int32_t destCapacity, + int32_t options, + UParseError *parseError, + UErrorCode *status){ + + if(status == NULL || U_FAILURE(*status)){ + return 0; + } + if((src==NULL) || (srcLength < -1) || (destCapacity<0) || (!dest && destCapacity > 0)){ + *status = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + int32_t reqLength = 0; + + UStringPrepProfile* nameprep = usprep_open(NULL, DATA_FILE_NAME, status); + + if(U_FAILURE(*status)){ + return 0; + } + + //initialize pointers + UChar *delimiter = (UChar*)src; + UChar *labelStart = (UChar*)src; + UChar *currentDest = (UChar*) dest; + int32_t remainingLen = srcLength; + int32_t remainingDestCapacity = destCapacity; + int32_t labelLen = 0, labelReqLength = 0; + UBool done = FALSE; + + + for(;;){ + + labelLen = getNextSeparator(labelStart,remainingLen, &delimiter,&done); + labelReqLength = 0; + if(!(labelLen==0 && done)){// make sure this is not a root label separator. + + labelReqLength = _internal_toASCII( labelStart, labelLen, + currentDest, remainingDestCapacity, + options, nameprep, + parseError, status); + + if(*status == U_BUFFER_OVERFLOW_ERROR){ + + *status = U_ZERO_ERROR; // reset error + remainingDestCapacity = 0; + } + } + + + if(U_FAILURE(*status)){ + break; + } + + reqLength +=labelReqLength; + // adjust the destination pointer + if(labelReqLength < remainingDestCapacity){ + currentDest = currentDest + labelReqLength; + remainingDestCapacity -= labelReqLength; + }else{ + // should never occur + remainingDestCapacity = 0; + } + + if(done == TRUE){ + break; + } + + // add the label separator + if(remainingDestCapacity > 0){ + *currentDest++ = FULL_STOP; + remainingDestCapacity--; + } + reqLength++; + + labelStart = delimiter; + if(remainingLen >0 ){ + remainingLen = (int32_t)(srcLength - (delimiter - src)); + } + + } + + if(reqLength > MAX_DOMAIN_NAME_LENGTH){ + *status = U_IDNA_DOMAIN_NAME_TOO_LONG_ERROR; + } + + usprep_close(nameprep); + + return u_terminateUChars(dest, destCapacity, reqLength, status); +} + +U_CAPI int32_t U_EXPORT2 +uidna_IDNToUnicode( const UChar* src, int32_t srcLength, + UChar* dest, int32_t destCapacity, + int32_t options, + UParseError* parseError, + UErrorCode* status){ + + if(status == NULL || U_FAILURE(*status)){ + return 0; + } + if((src==NULL) || (srcLength < -1) || (destCapacity<0) || (!dest && destCapacity > 0)){ + *status = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + int32_t reqLength = 0; + + UStringPrepProfile* nameprep = usprep_open(NULL, DATA_FILE_NAME, status); + + if(U_FAILURE(*status)){ + return 0; + } + + //initialize pointers + UChar *delimiter = (UChar*)src; + UChar *labelStart = (UChar*)src; + UChar *currentDest = (UChar*) dest; + int32_t remainingLen = srcLength; + int32_t remainingDestCapacity = destCapacity; + int32_t labelLen = 0, labelReqLength = 0; + UBool done = FALSE; + + for(;;){ + + labelLen = getNextSeparator(labelStart,remainingLen, &delimiter,&done); + + // The RFC states that + // + // ToUnicode never fails. If any step fails, then the original input + // is returned immediately in that step. + // + // _internal_toUnicode will copy the label. + /*if(labelLen==0 && done==FALSE){ + *status = U_IDNA_ZERO_LENGTH_LABEL_ERROR; + break; + }*/ + + labelReqLength = _internal_toUnicode(labelStart, labelLen, + currentDest, remainingDestCapacity, + options, nameprep, + parseError, status); + + if(*status == U_BUFFER_OVERFLOW_ERROR){ + *status = U_ZERO_ERROR; // reset error + remainingDestCapacity = 0; + } + + if(U_FAILURE(*status)){ + break; + } + + reqLength +=labelReqLength; + // adjust the destination pointer + if(labelReqLength < remainingDestCapacity){ + currentDest = currentDest + labelReqLength; + remainingDestCapacity -= labelReqLength; + }else{ + // should never occur + remainingDestCapacity = 0; + } + + if(done == TRUE){ + break; + } + + // add the label separator + // Unlike the ToASCII operation we don't normalize the label separators + if(remainingDestCapacity > 0){ + *currentDest++ = *(labelStart + labelLen); + remainingDestCapacity--; + } + reqLength++; + + labelStart = delimiter; + if(remainingLen >0 ){ + remainingLen = (int32_t)(srcLength - (delimiter - src)); + } + + } + + if(reqLength > MAX_DOMAIN_NAME_LENGTH){ + *status = U_IDNA_DOMAIN_NAME_TOO_LONG_ERROR; + } + + usprep_close(nameprep); + + return u_terminateUChars(dest, destCapacity, reqLength, status); +} + +U_CAPI int32_t U_EXPORT2 +uidna_compare( const UChar *s1, int32_t length1, + const UChar *s2, int32_t length2, + int32_t options, + UErrorCode* status){ + + if(status == NULL || U_FAILURE(*status)){ + return -1; + } + + UChar b1Stack[MAX_IDN_BUFFER_SIZE], b2Stack[MAX_IDN_BUFFER_SIZE]; + UChar *b1 = b1Stack, *b2 = b2Stack; + int32_t b1Len, b2Len, b1Capacity = MAX_IDN_BUFFER_SIZE, b2Capacity = MAX_IDN_BUFFER_SIZE; + int32_t result=-1; + + UParseError parseError; + + b1Len = uidna_IDNToASCII(s1, length1, b1, b1Capacity, options, &parseError, status); + if(*status == U_BUFFER_OVERFLOW_ERROR){ + // redo processing of string + b1 = (UChar*) uprv_malloc(b1Len * U_SIZEOF_UCHAR); + if(b1==NULL){ + *status = U_MEMORY_ALLOCATION_ERROR; + goto CLEANUP; + } + + *status = U_ZERO_ERROR; // reset error + + b1Len = uidna_IDNToASCII(s1,length1,b1,b1Len, options, &parseError, status); + + } + + b2Len = uidna_IDNToASCII(s2,length2, b2,b2Capacity, options, &parseError, status); + if(*status == U_BUFFER_OVERFLOW_ERROR){ + // redo processing of string + b2 = (UChar*) uprv_malloc(b2Len * U_SIZEOF_UCHAR); + if(b2==NULL){ + *status = U_MEMORY_ALLOCATION_ERROR; + goto CLEANUP; + } + + *status = U_ZERO_ERROR; // reset error + + b2Len = uidna_IDNToASCII(s2, length2, b2, b2Len, options, &parseError, status); + + } + // when toASCII is applied all label separators are replaced with FULL_STOP + result = compareCaseInsensitiveASCII(b1,b1Len,b2,b2Len); + +CLEANUP: + if(b1 != b1Stack){ + uprv_free(b1); + } + + if(b2 != b2Stack){ + uprv_free(b2); + } + + return result; +} + +#endif /* #if !UCONFIG_NO_IDNA */ diff --git a/Build/source/libs/icu/icu-xetex/common/uinit.c b/Build/source/libs/icu/icu-xetex/common/uinit.c new file mode 100644 index 00000000000..1f2c6d6b8f8 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uinit.c @@ -0,0 +1,113 @@ +/* +****************************************************************************** +* * +* Copyright (C) 2001-2007, International Business Machines * +* Corporation and others. All Rights Reserved. * +* * +****************************************************************************** +* file name: uinit.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2001July05 +* created by: George Rhoten +*/ + +#include "unicode/utypes.h" +#include "unicode/uclean.h" +#include "utracimp.h" +#include "ustr_imp.h" +#include "unormimp.h" +#include "ucln_cmn.h" +#include "ucnv_io.h" +#include "umutex.h" +#include "ucln.h" +#include "cmemory.h" +#include "uassert.h" + +static UBool gICUInitialized = FALSE; +static UMTX gICUInitMutex = NULL; + + +/************************************************ + The cleanup order is important in this function. + Please be sure that you have read ucln.h + ************************************************/ +U_CAPI void U_EXPORT2 +u_cleanup(void) +{ + UTRACE_ENTRY_OC(UTRACE_U_CLEANUP); + umtx_lock(NULL); /* Force a memory barrier, so that we are sure to see */ + umtx_unlock(NULL); /* all state left around by any other threads. */ + + ucln_lib_cleanup(); + + umtx_destroy(&gICUInitMutex); + umtx_cleanup(); + cmemory_cleanup(); /* undo any heap functions set by u_setMemoryFunctions(). */ + gICUInitialized = FALSE; + UTRACE_EXIT(); /* Must be before utrace_cleanup(), which turns off tracing. */ +/*#if U_ENABLE_TRACING*/ + utrace_cleanup(); +/*#endif*/ +} + +/* + * + * ICU Initialization Function. Force loading and/or initialization of + * any shared data that could potentially be used concurrently + * by multiple threads. + */ +U_CAPI void U_EXPORT2 +u_init(UErrorCode *status) { + UTRACE_ENTRY_OC(UTRACE_U_INIT); + /* Make sure the global mutexes are initialized. */ + umtx_init(NULL); + umtx_lock(&gICUInitMutex); + if (gICUInitialized || U_FAILURE(*status)) { + umtx_unlock(&gICUInitMutex); + UTRACE_EXIT_STATUS(*status); + return; + } + +#if 1 + /* + * 2005-may-02 + * + * ICU4C 3.4 (jitterbug 4497) hardcodes the data for Unicode character + * properties for APIs that want to be fast. + * Therefore, we need not load them here nor check for errors. + * Instead, we load the converter alias table to see if any ICU data + * is available. + * Users should really open the service objects they need and check + * for errors there, to make sure that the actual items they need are + * available. + */ +#if !UCONFIG_NO_CONVERSION + ucnv_io_countKnownConverters(status); +#endif +#else + /* Do any required init for services that don't have open operations + * and use "only" the double-check initialization method for performance + * reasons (avoiding a mutex lock even for _checking_ whether the + * initialization had occurred). + */ + + /* Char Properties */ + uprv_haveProperties(status); + + /* load the case and bidi properties but don't fail if they are not available */ + u_isULowercase(0x61); + u_getIntPropertyValue(0x200D, UCHAR_JOINING_TYPE); /* ZERO WIDTH JOINER: Join_Causing */ + +#if !UCONFIG_NO_NORMALIZATION + /* Normalization */ + unorm_haveData(status); +#endif +#endif + gICUInitialized = TRUE; /* TODO: don't set if U_FAILURE? */ + umtx_unlock(&gICUInitMutex); + UTRACE_EXIT_STATUS(*status); +} + diff --git a/Build/source/libs/icu/icu-xetex/common/uinvchar.c b/Build/source/libs/icu/icu-xetex/common/uinvchar.c new file mode 100644 index 00000000000..325a09d1d1f --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uinvchar.c @@ -0,0 +1,515 @@ +/* +******************************************************************************* +* +* Copyright (C) 1999-2006, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: uinvchar.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:2 +* +* created on: 2004sep14 +* created by: Markus W. Scherer +* +* Functions for handling invariant characters, moved here from putil.c +* for better modularization. +*/ + +#include "unicode/utypes.h" +#include "unicode/ustring.h" +#include "udataswp.h" +#include "cstring.h" +#include "cmemory.h" +#include "uassert.h" +#include "uinvchar.h" + +/* invariant-character handling --------------------------------------------- */ + +/* + * These maps for ASCII to/from EBCDIC map invariant characters (see utypes.h) + * appropriately for most EBCDIC codepages. + * + * They currently also map most other ASCII graphic characters, + * appropriately for codepages 37 and 1047. + * Exceptions: The characters for []^ have different codes in 37 & 1047. + * Both versions are mapped to ASCII. + * + * ASCII 37 1047 + * [ 5B BA AD + * ] 5D BB BD + * ^ 5E B0 5F + * + * There are no mappings for variant characters from Unicode to EBCDIC. + * + * Currently, C0 control codes are also included in these maps. + * Exceptions: S/390 Open Edition swaps LF and NEL codes compared with other + * EBCDIC platforms; both codes (15 and 25) are mapped to ASCII LF (0A), + * but there is no mapping for ASCII LF back to EBCDIC. + * + * ASCII EBCDIC S/390-OE + * LF 0A 25 15 + * NEL 85 15 25 + * + * The maps below explicitly exclude the variant + * control and graphical characters that are in ASCII-based + * codepages at 0x80 and above. + * "No mapping" is expressed by mapping to a 00 byte. + * + * These tables do not establish a converter or a codepage. + */ + +static const uint8_t asciiFromEbcdic[256]={ + 0x00, 0x01, 0x02, 0x03, 0x00, 0x09, 0x00, 0x7f, 0x00, 0x00, 0x00, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x00, 0x0a, 0x08, 0x00, 0x18, 0x19, 0x00, 0x00, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x17, 0x1b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x06, 0x07, + 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x14, 0x15, 0x00, 0x1a, + + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2e, 0x3c, 0x28, 0x2b, 0x7c, + 0x26, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x24, 0x2a, 0x29, 0x3b, 0x5e, + 0x2d, 0x2f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2c, 0x25, 0x5f, 0x3e, 0x3f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x3a, 0x23, 0x40, 0x27, 0x3d, 0x22, + + 0x00, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x7e, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x00, 0x00, 0x00, 0x5b, 0x00, 0x00, + 0x5e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5b, 0x5d, 0x00, 0x5d, 0x00, 0x00, + + 0x7b, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x7d, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x5c, 0x00, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +static const uint8_t ebcdicFromAscii[256]={ + 0x00, 0x01, 0x02, 0x03, 0x37, 0x2d, 0x2e, 0x2f, 0x16, 0x05, 0x00, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x3c, 0x3d, 0x32, 0x26, 0x18, 0x19, 0x3f, 0x27, 0x1c, 0x1d, 0x1e, 0x1f, + 0x40, 0x00, 0x7f, 0x00, 0x00, 0x6c, 0x50, 0x7d, 0x4d, 0x5d, 0x5c, 0x4e, 0x6b, 0x60, 0x4b, 0x61, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0x7a, 0x5e, 0x4c, 0x7e, 0x6e, 0x6f, + + 0x00, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, + 0xd7, 0xd8, 0xd9, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0x00, 0x00, 0x00, 0x00, 0x6d, + 0x00, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, + 0x97, 0x98, 0x99, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0x00, 0x00, 0x00, 0x00, 0x07, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +/* + * Bit sets indicating which characters of the ASCII repertoire + * (by ASCII/Unicode code) are "invariant". + * See utypes.h for more details. + * + * As invariant are considered the characters of the ASCII repertoire except + * for the following: + * 21 '!' + * 23 '#' + * 24 '$' + * + * 40 '@' + * + * 5b '[' + * 5c '\' + * 5d ']' + * 5e '^' + * + * 60 '`' + * + * 7b '{' + * 7c '|' + * 7d '}' + * 7e '~' + */ +static const uint32_t invariantChars[4]={ + 0xfffffbff, /* 00..1f but not 0a */ + 0xffffffe5, /* 20..3f but not 21 23 24 */ + 0x87fffffe, /* 40..5f but not 40 5b..5e */ + 0x87fffffe /* 60..7f but not 60 7b..7e */ +}; + +/* + * test unsigned types (or values known to be non-negative) for invariant characters, + * tests ASCII-family character values + */ +#define UCHAR_IS_INVARIANT(c) (((c)<=0x7f) && (invariantChars[(c)>>5]&((uint32_t)1<<((c)&0x1f)))!=0) + +/* test signed types for invariant characters, adds test for positive values */ +#define SCHAR_IS_INVARIANT(c) ((0<=(c)) && UCHAR_IS_INVARIANT(c)) + +#if U_CHARSET_FAMILY==U_ASCII_FAMILY +#define CHAR_TO_UCHAR(c) c +#define UCHAR_TO_CHAR(c) c +#elif U_CHARSET_FAMILY==U_EBCDIC_FAMILY +#define CHAR_TO_UCHAR(u) asciiFromEbcdic[u] +#define UCHAR_TO_CHAR(u) ebcdicFromAscii[u] +#else +# error U_CHARSET_FAMILY is not valid +#endif + + +U_CAPI void U_EXPORT2 +u_charsToUChars(const char *cs, UChar *us, int32_t length) { + UChar u; + uint8_t c; + + /* + * Allow the entire ASCII repertoire to be mapped _to_ Unicode. + * For EBCDIC systems, this works for characters with codes from + * codepages 37 and 1047 or compatible. + */ + while(length>0) { + c=(uint8_t)(*cs++); + u=(UChar)CHAR_TO_UCHAR(c); + U_ASSERT((u!=0 || c==0)); /* only invariant chars converted? */ + *us++=u; + --length; + } +} + +U_CAPI void U_EXPORT2 +u_UCharsToChars(const UChar *us, char *cs, int32_t length) { + UChar u; + + while(length>0) { + u=*us++; + if(!UCHAR_IS_INVARIANT(u)) { + U_ASSERT(FALSE); /* Variant characters were used. These are not portable in ICU. */ + u=0; + } + *cs++=(char)UCHAR_TO_CHAR(u); + --length; + } +} + +U_CAPI UBool U_EXPORT2 +uprv_isInvariantString(const char *s, int32_t length) { + uint8_t c; + + for(;;) { + if(length<0) { + /* NUL-terminated */ + c=(uint8_t)*s++; + if(c==0) { + break; + } + } else { + /* count length */ + if(length==0) { + break; + } + --length; + c=(uint8_t)*s++; + if(c==0) { + continue; /* NUL is invariant */ + } + } + /* c!=0 now, one branch below checks c==0 for variant characters */ + + /* + * no assertions here because these functions are legitimately called + * for strings with variant characters + */ +#if U_CHARSET_FAMILY==U_ASCII_FAMILY + if(!UCHAR_IS_INVARIANT(c)) { + return FALSE; /* found a variant char */ + } +#elif U_CHARSET_FAMILY==U_EBCDIC_FAMILY + c=CHAR_TO_UCHAR(c); + if(c==0 || !UCHAR_IS_INVARIANT(c)) { + return FALSE; /* found a variant char */ + } +#else +# error U_CHARSET_FAMILY is not valid +#endif + } + return TRUE; +} + +U_CAPI UBool U_EXPORT2 +uprv_isInvariantUString(const UChar *s, int32_t length) { + UChar c; + + for(;;) { + if(length<0) { + /* NUL-terminated */ + c=*s++; + if(c==0) { + break; + } + } else { + /* count length */ + if(length==0) { + break; + } + --length; + c=*s++; + } + + /* + * no assertions here because these functions are legitimately called + * for strings with variant characters + */ + if(!UCHAR_IS_INVARIANT(c)) { + return FALSE; /* found a variant char */ + } + } + return TRUE; +} + +/* UDataSwapFn implementations used in udataswp.c ------- */ + +/* convert ASCII to EBCDIC and verify that all characters are invariant */ +U_CAPI int32_t U_EXPORT2 +uprv_ebcdicFromAscii(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode) { + const uint8_t *s; + uint8_t *t; + uint8_t c; + + int32_t count; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + if(ds==NULL || inData==NULL || length<0 || (length>0 && outData==NULL)) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* setup and swapping */ + s=(const uint8_t *)inData; + t=(uint8_t *)outData; + count=length; + while(count>0) { + c=*s++; + if(!UCHAR_IS_INVARIANT(c)) { + udata_printError(ds, "uprv_ebcdicFromAscii() string[%d] contains a variant character in position %d\n", + length, length-count); + *pErrorCode=U_INVALID_CHAR_FOUND; + return 0; + } + *t++=ebcdicFromAscii[c]; + --count; + } + + return length; +} + +/* this function only checks and copies ASCII strings without conversion */ +U_CFUNC int32_t +uprv_copyAscii(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode) { + const uint8_t *s; + uint8_t c; + + int32_t count; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + if(ds==NULL || inData==NULL || length<0 || (length>0 && outData==NULL)) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* setup and checking */ + s=(const uint8_t *)inData; + count=length; + while(count>0) { + c=*s++; + if(!UCHAR_IS_INVARIANT(c)) { + udata_printError(ds, "uprv_copyFromAscii() string[%d] contains a variant character in position %d\n", + length, length-count); + *pErrorCode=U_INVALID_CHAR_FOUND; + return 0; + } + --count; + } + + if(length>0 && inData!=outData) { + uprv_memcpy(outData, inData, length); + } + + return length; +} + +/* convert EBCDIC to ASCII and verify that all characters are invariant */ +U_CFUNC int32_t +uprv_asciiFromEbcdic(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode) { + const uint8_t *s; + uint8_t *t; + uint8_t c; + + int32_t count; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + if(ds==NULL || inData==NULL || length<0 || (length>0 && outData==NULL)) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* setup and swapping */ + s=(const uint8_t *)inData; + t=(uint8_t *)outData; + count=length; + while(count>0) { + c=*s++; + if(c!=0 && ((c=asciiFromEbcdic[c])==0 || !UCHAR_IS_INVARIANT(c))) { + udata_printError(ds, "uprv_asciiFromEbcdic() string[%d] contains a variant character in position %d\n", + length, length-count); + *pErrorCode=U_INVALID_CHAR_FOUND; + return 0; + } + *t++=c; + --count; + } + + return length; +} + +/* this function only checks and copies EBCDIC strings without conversion */ +U_CFUNC int32_t +uprv_copyEbcdic(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode) { + const uint8_t *s; + uint8_t c; + + int32_t count; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + if(ds==NULL || inData==NULL || length<0 || (length>0 && outData==NULL)) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* setup and checking */ + s=(const uint8_t *)inData; + count=length; + while(count>0) { + c=*s++; + if(c!=0 && ((c=asciiFromEbcdic[c])==0 || !UCHAR_IS_INVARIANT(c))) { + udata_printError(ds, "uprv_copyEbcdic() string[%] contains a variant character in position %d\n", + length, length-count); + *pErrorCode=U_INVALID_CHAR_FOUND; + return 0; + } + --count; + } + + if(length>0 && inData!=outData) { + uprv_memcpy(outData, inData, length); + } + + return length; +} + +/* compare invariant strings; variant characters compare less than others and unlike each other */ +U_CFUNC int32_t +uprv_compareInvAscii(const UDataSwapper *ds, + const char *outString, int32_t outLength, + const UChar *localString, int32_t localLength) { + int32_t minLength; + UChar32 c1, c2; + uint8_t c; + + if(outString==NULL || outLength<-1 || localString==NULL || localLength<-1) { + return 0; + } + + if(outLength<0) { + outLength=(int32_t)uprv_strlen(outString); + } + if(localLength<0) { + localLength=u_strlen(localString); + } + + minLength= outLength0) { + c=(uint8_t)*outString++; + if(UCHAR_IS_INVARIANT(c)) { + c1=c; + } else { + c1=-1; + } + + c2=*localString++; + if(!UCHAR_IS_INVARIANT(c2)) { + c1=-2; + } + + if((c1-=c2)!=0) { + return c1; + } + + --minLength; + } + + /* strings start with same prefix, compare lengths */ + return outLength-localLength; +} + +U_CFUNC int32_t +uprv_compareInvEbcdic(const UDataSwapper *ds, + const char *outString, int32_t outLength, + const UChar *localString, int32_t localLength) { + int32_t minLength; + UChar32 c1, c2; + uint8_t c; + + if(outString==NULL || outLength<-1 || localString==NULL || localLength<-1) { + return 0; + } + + if(outLength<0) { + outLength=(int32_t)uprv_strlen(outString); + } + if(localLength<0) { + localLength=u_strlen(localString); + } + + minLength= outLength0) { + c=(uint8_t)*outString++; + if(c==0) { + c1=0; + } else if((c1=asciiFromEbcdic[c])!=0 && UCHAR_IS_INVARIANT(c1)) { + /* c1 is set */ + } else { + c1=-1; + } + + c2=*localString++; + if(!UCHAR_IS_INVARIANT(c2)) { + c1=-2; + } + + if((c1-=c2)!=0) { + return c1; + } + + --minLength; + } + + /* strings start with same prefix, compare lengths */ + return outLength-localLength; +} diff --git a/Build/source/libs/icu/icu-xetex/common/uinvchar.h b/Build/source/libs/icu/icu-xetex/common/uinvchar.h new file mode 100644 index 00000000000..2a333b81ee4 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uinvchar.h @@ -0,0 +1,66 @@ +/* +******************************************************************************* +* +* Copyright (C) 1999-2004, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: uinvchar.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:2 +* +* created on: 2004sep14 +* created by: Markus W. Scherer +* +* Definitions for handling invariant characters, moved here from putil.c +* for better modularization. +*/ + +#ifndef __UINVCHAR_H__ +#define __UINVCHAR_H__ + +#include "unicode/utypes.h" + +/** + * Check if a char string only contains invariant characters. + * See utypes.h for details. + * + * @param s Input string pointer. + * @param length Length of the string, can be -1 if NUL-terminated. + * @return TRUE if s contains only invariant characters. + * + * @internal (ICU 2.8) + */ +U_INTERNAL UBool U_EXPORT2 +uprv_isInvariantString(const char *s, int32_t length); + +/** + * Check if a Unicode string only contains invariant characters. + * See utypes.h for details. + * + * @param s Input string pointer. + * @param length Length of the string, can be -1 if NUL-terminated. + * @return TRUE if s contains only invariant characters. + * + * @internal (ICU 2.8) + */ +U_INTERNAL UBool U_EXPORT2 +uprv_isInvariantUString(const UChar *s, int32_t length); + +/** + * \def U_UPPER_ORDINAL + * Get the ordinal number of an uppercase invariant character + * @internal + */ +#if U_CHARSET_FAMILY==U_ASCII_FAMILY +# define U_UPPER_ORDINAL(x) ((x)-'A') +#elif U_CHARSET_FAMILY==U_EBCDIC_FAMILY +# define U_UPPER_ORDINAL(x) (((x) < 'J') ? ((x)-'A') : \ + (((x) < 'S') ? ((x)-'J'+9) : \ + ((x)-'S'+18))) +#else +# error Unknown charset family! +#endif + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/uiter.cpp b/Build/source/libs/icu/icu-xetex/common/uiter.cpp new file mode 100644 index 00000000000..bec7190c635 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uiter.cpp @@ -0,0 +1,1121 @@ +/* +******************************************************************************* +* +* Copyright (C) 2002-2006, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: uiter.cpp +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2002jan18 +* created by: Markus W. Scherer +*/ + +#include "unicode/utypes.h" +#include "unicode/ustring.h" +#include "unicode/chariter.h" +#include "unicode/rep.h" +#include "unicode/uiter.h" +#include "cstring.h" + +U_NAMESPACE_USE + +#define IS_EVEN(n) (((n)&1)==0) +#define IS_POINTER_EVEN(p) IS_EVEN((size_t)p) + +U_CDECL_BEGIN + +/* No-Op UCharIterator implementation for illegal input --------------------- */ + +static int32_t U_CALLCONV +noopGetIndex(UCharIterator * /*iter*/, UCharIteratorOrigin /*origin*/) { + return 0; +} + +static int32_t U_CALLCONV +noopMove(UCharIterator * /*iter*/, int32_t /*delta*/, UCharIteratorOrigin /*origin*/) { + return 0; +} + +static UBool U_CALLCONV +noopHasNext(UCharIterator * /*iter*/) { + return FALSE; +} + +static UChar32 U_CALLCONV +noopCurrent(UCharIterator * /*iter*/) { + return U_SENTINEL; +} + +static uint32_t U_CALLCONV +noopGetState(const UCharIterator * /*iter*/) { + return UITER_NO_STATE; +} + +static void U_CALLCONV +noopSetState(UCharIterator * /*iter*/, uint32_t /*state*/, UErrorCode *pErrorCode) { + *pErrorCode=U_UNSUPPORTED_ERROR; +} + +static const UCharIterator noopIterator={ + 0, 0, 0, 0, 0, 0, + noopGetIndex, + noopMove, + noopHasNext, + noopHasNext, + noopCurrent, + noopCurrent, + noopCurrent, + NULL, + noopGetState, + noopSetState +}; + +/* UCharIterator implementation for simple strings -------------------------- */ + +/* + * This is an implementation of a code unit (UChar) iterator + * for UChar * strings. + * + * The UCharIterator.context field holds a pointer to the string. + */ + +static int32_t U_CALLCONV +stringIteratorGetIndex(UCharIterator *iter, UCharIteratorOrigin origin) { + switch(origin) { + case UITER_ZERO: + return 0; + case UITER_START: + return iter->start; + case UITER_CURRENT: + return iter->index; + case UITER_LIMIT: + return iter->limit; + case UITER_LENGTH: + return iter->length; + default: + /* not a valid origin */ + /* Should never get here! */ + return -1; + } +} + +static int32_t U_CALLCONV +stringIteratorMove(UCharIterator *iter, int32_t delta, UCharIteratorOrigin origin) { + int32_t pos; + + switch(origin) { + case UITER_ZERO: + pos=delta; + break; + case UITER_START: + pos=iter->start+delta; + break; + case UITER_CURRENT: + pos=iter->index+delta; + break; + case UITER_LIMIT: + pos=iter->limit+delta; + break; + case UITER_LENGTH: + pos=iter->length+delta; + break; + default: + return -1; /* Error */ + } + + if(posstart) { + pos=iter->start; + } else if(pos>iter->limit) { + pos=iter->limit; + } + + return iter->index=pos; +} + +static UBool U_CALLCONV +stringIteratorHasNext(UCharIterator *iter) { + return iter->indexlimit; +} + +static UBool U_CALLCONV +stringIteratorHasPrevious(UCharIterator *iter) { + return iter->index>iter->start; +} + +static UChar32 U_CALLCONV +stringIteratorCurrent(UCharIterator *iter) { + if(iter->indexlimit) { + return ((const UChar *)(iter->context))[iter->index]; + } else { + return U_SENTINEL; + } +} + +static UChar32 U_CALLCONV +stringIteratorNext(UCharIterator *iter) { + if(iter->indexlimit) { + return ((const UChar *)(iter->context))[iter->index++]; + } else { + return U_SENTINEL; + } +} + +static UChar32 U_CALLCONV +stringIteratorPrevious(UCharIterator *iter) { + if(iter->index>iter->start) { + return ((const UChar *)(iter->context))[--iter->index]; + } else { + return U_SENTINEL; + } +} + +static uint32_t U_CALLCONV +stringIteratorGetState(const UCharIterator *iter) { + return (uint32_t)iter->index; +} + +static void U_CALLCONV +stringIteratorSetState(UCharIterator *iter, uint32_t state, UErrorCode *pErrorCode) { + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + /* do nothing */ + } else if(iter==NULL) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + } else if((int32_t)statestart || iter->limit<(int32_t)state) { + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + } else { + iter->index=(int32_t)state; + } +} + +static const UCharIterator stringIterator={ + 0, 0, 0, 0, 0, 0, + stringIteratorGetIndex, + stringIteratorMove, + stringIteratorHasNext, + stringIteratorHasPrevious, + stringIteratorCurrent, + stringIteratorNext, + stringIteratorPrevious, + NULL, + stringIteratorGetState, + stringIteratorSetState +}; + +U_CAPI void U_EXPORT2 +uiter_setString(UCharIterator *iter, const UChar *s, int32_t length) { + if(iter!=0) { + if(s!=0 && length>=-1) { + *iter=stringIterator; + iter->context=s; + if(length>=0) { + iter->length=length; + } else { + iter->length=u_strlen(s); + } + iter->limit=iter->length; + } else { + *iter=noopIterator; + } + } +} + +/* UCharIterator implementation for UTF-16BE strings ------------------------ */ + +/* + * This is an implementation of a code unit (UChar) iterator + * for UTF-16BE strings, i.e., strings in byte-vectors where + * each UChar is stored as a big-endian pair of bytes. + * + * The UCharIterator.context field holds a pointer to the string. + * Everything works just like with a normal UChar iterator (uiter_setString), + * except that UChars are assembled from byte pairs. + */ + +/* internal helper function */ +static inline UChar32 +utf16BEIteratorGet(UCharIterator *iter, int32_t index) { + const uint8_t *p=(const uint8_t *)iter->context; + return ((UChar)p[2*index]<<8)|(UChar)p[2*index+1]; +} + +static UChar32 U_CALLCONV +utf16BEIteratorCurrent(UCharIterator *iter) { + int32_t index; + + if((index=iter->index)limit) { + return utf16BEIteratorGet(iter, index); + } else { + return U_SENTINEL; + } +} + +static UChar32 U_CALLCONV +utf16BEIteratorNext(UCharIterator *iter) { + int32_t index; + + if((index=iter->index)limit) { + iter->index=index+1; + return utf16BEIteratorGet(iter, index); + } else { + return U_SENTINEL; + } +} + +static UChar32 U_CALLCONV +utf16BEIteratorPrevious(UCharIterator *iter) { + int32_t index; + + if((index=iter->index)>iter->start) { + iter->index=--index; + return utf16BEIteratorGet(iter, index); + } else { + return U_SENTINEL; + } +} + +static const UCharIterator utf16BEIterator={ + 0, 0, 0, 0, 0, 0, + stringIteratorGetIndex, + stringIteratorMove, + stringIteratorHasNext, + stringIteratorHasPrevious, + utf16BEIteratorCurrent, + utf16BEIteratorNext, + utf16BEIteratorPrevious, + NULL, + stringIteratorGetState, + stringIteratorSetState +}; + +/* + * Count the number of UChars in a UTF-16BE string before a terminating UChar NUL, + * i.e., before a pair of 0 bytes where the first 0 byte is at an even + * offset from s. + */ +static int32_t +utf16BE_strlen(const char *s) { + if(IS_POINTER_EVEN(s)) { + /* + * even-aligned, call u_strlen(s) + * we are probably on a little-endian machine, but searching for UChar NUL + * does not care about endianness + */ + return u_strlen((const UChar *)s); + } else { + /* odd-aligned, search for pair of 0 bytes */ + const char *p=s; + + while(!(*p==0 && p[1]==0)) { + p+=2; + } + return (int32_t)((p-s)/2); + } +} + +U_CAPI void U_EXPORT2 +uiter_setUTF16BE(UCharIterator *iter, const char *s, int32_t length) { + if(iter!=NULL) { + /* allow only even-length strings (the input length counts bytes) */ + if(s!=NULL && (length==-1 || (length>=0 && IS_EVEN(length)))) { + /* length/=2, except that >>=1 also works for -1 (-1/2==0, -1>>1==-1) */ + length>>=1; + + if(U_IS_BIG_ENDIAN && IS_POINTER_EVEN(s)) { + /* big-endian machine and 2-aligned UTF-16BE string: use normal UChar iterator */ + uiter_setString(iter, (const UChar *)s, length); + return; + } + + *iter=utf16BEIterator; + iter->context=s; + if(length>=0) { + iter->length=length; + } else { + iter->length=utf16BE_strlen(s); + } + iter->limit=iter->length; + } else { + *iter=noopIterator; + } + } +} + +/* UCharIterator wrapper around CharacterIterator --------------------------- */ + +/* + * This is wrapper code around a C++ CharacterIterator to + * look like a C UCharIterator. + * + * The UCharIterator.context field holds a pointer to the CharacterIterator. + */ + +static int32_t U_CALLCONV +characterIteratorGetIndex(UCharIterator *iter, UCharIteratorOrigin origin) { + switch(origin) { + case UITER_ZERO: + return 0; + case UITER_START: + return ((CharacterIterator *)(iter->context))->startIndex(); + case UITER_CURRENT: + return ((CharacterIterator *)(iter->context))->getIndex(); + case UITER_LIMIT: + return ((CharacterIterator *)(iter->context))->endIndex(); + case UITER_LENGTH: + return ((CharacterIterator *)(iter->context))->getLength(); + default: + /* not a valid origin */ + /* Should never get here! */ + return -1; + } +} + +static int32_t U_CALLCONV +characterIteratorMove(UCharIterator *iter, int32_t delta, UCharIteratorOrigin origin) { + switch(origin) { + case UITER_ZERO: + ((CharacterIterator *)(iter->context))->setIndex(delta); + return ((CharacterIterator *)(iter->context))->getIndex(); + case UITER_START: + case UITER_CURRENT: + case UITER_LIMIT: + return ((CharacterIterator *)(iter->context))->move(delta, (CharacterIterator::EOrigin)origin); + case UITER_LENGTH: + ((CharacterIterator *)(iter->context))->setIndex(((CharacterIterator *)(iter->context))->getLength()+delta); + return ((CharacterIterator *)(iter->context))->getIndex(); + default: + /* not a valid origin */ + /* Should never get here! */ + return -1; + } +} + +static UBool U_CALLCONV +characterIteratorHasNext(UCharIterator *iter) { + return ((CharacterIterator *)(iter->context))->hasNext(); +} + +static UBool U_CALLCONV +characterIteratorHasPrevious(UCharIterator *iter) { + return ((CharacterIterator *)(iter->context))->hasPrevious(); +} + +static UChar32 U_CALLCONV +characterIteratorCurrent(UCharIterator *iter) { + UChar32 c; + + c=((CharacterIterator *)(iter->context))->current(); + if(c!=0xffff || ((CharacterIterator *)(iter->context))->hasNext()) { + return c; + } else { + return U_SENTINEL; + } +} + +static UChar32 U_CALLCONV +characterIteratorNext(UCharIterator *iter) { + if(((CharacterIterator *)(iter->context))->hasNext()) { + return ((CharacterIterator *)(iter->context))->nextPostInc(); + } else { + return U_SENTINEL; + } +} + +static UChar32 U_CALLCONV +characterIteratorPrevious(UCharIterator *iter) { + if(((CharacterIterator *)(iter->context))->hasPrevious()) { + return ((CharacterIterator *)(iter->context))->previous(); + } else { + return U_SENTINEL; + } +} + +static uint32_t U_CALLCONV +characterIteratorGetState(const UCharIterator *iter) { + return ((CharacterIterator *)(iter->context))->getIndex(); +} + +static void U_CALLCONV +characterIteratorSetState(UCharIterator *iter, uint32_t state, UErrorCode *pErrorCode) { + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + /* do nothing */ + } else if(iter==NULL || iter->context==NULL) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + } else if((int32_t)state<((CharacterIterator *)(iter->context))->startIndex() || ((CharacterIterator *)(iter->context))->endIndex()<(int32_t)state) { + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + } else { + ((CharacterIterator *)(iter->context))->setIndex((int32_t)state); + } +} + +static const UCharIterator characterIteratorWrapper={ + 0, 0, 0, 0, 0, 0, + characterIteratorGetIndex, + characterIteratorMove, + characterIteratorHasNext, + characterIteratorHasPrevious, + characterIteratorCurrent, + characterIteratorNext, + characterIteratorPrevious, + NULL, + characterIteratorGetState, + characterIteratorSetState +}; + +U_CAPI void U_EXPORT2 +uiter_setCharacterIterator(UCharIterator *iter, CharacterIterator *charIter) { + if(iter!=0) { + if(charIter!=0) { + *iter=characterIteratorWrapper; + iter->context=charIter; + } else { + *iter=noopIterator; + } + } +} + +/* UCharIterator wrapper around Replaceable --------------------------------- */ + +/* + * This is an implementation of a code unit (UChar) iterator + * based on a Replaceable object. + * + * The UCharIterator.context field holds a pointer to the Replaceable. + * UCharIterator.length and UCharIterator.index hold Replaceable.length() + * and the iteration index. + */ + +static UChar32 U_CALLCONV +replaceableIteratorCurrent(UCharIterator *iter) { + if(iter->indexlimit) { + return ((Replaceable *)(iter->context))->charAt(iter->index); + } else { + return U_SENTINEL; + } +} + +static UChar32 U_CALLCONV +replaceableIteratorNext(UCharIterator *iter) { + if(iter->indexlimit) { + return ((Replaceable *)(iter->context))->charAt(iter->index++); + } else { + return U_SENTINEL; + } +} + +static UChar32 U_CALLCONV +replaceableIteratorPrevious(UCharIterator *iter) { + if(iter->index>iter->start) { + return ((Replaceable *)(iter->context))->charAt(--iter->index); + } else { + return U_SENTINEL; + } +} + +static const UCharIterator replaceableIterator={ + 0, 0, 0, 0, 0, 0, + stringIteratorGetIndex, + stringIteratorMove, + stringIteratorHasNext, + stringIteratorHasPrevious, + replaceableIteratorCurrent, + replaceableIteratorNext, + replaceableIteratorPrevious, + NULL, + stringIteratorGetState, + stringIteratorSetState +}; + +U_CAPI void U_EXPORT2 +uiter_setReplaceable(UCharIterator *iter, const Replaceable *rep) { + if(iter!=0) { + if(rep!=0) { + *iter=replaceableIterator; + iter->context=rep; + iter->limit=iter->length=rep->length(); + } else { + *iter=noopIterator; + } + } +} + +/* UCharIterator implementation for UTF-8 strings --------------------------- */ + +/* + * Possible, probably necessary only for an implementation for arbitrary + * converters: + * Maintain a buffer (ring buffer?) for a piece of converted 16-bit text. + * This would require to turn reservedFn into a close function and + * to introduce a uiter_close(iter). + */ + +#define UITER_CNV_CAPACITY 16 + +/* + * Minimal implementation: + * Maintain a single-UChar buffer for an additional surrogate. + * The caller must not modify start and limit because they are used internally. + * + * Use UCharIterator fields as follows: + * context pointer to UTF-8 string + * length UTF-16 length of the string; -1 until lazy evaluation + * start current UTF-8 index + * index current UTF-16 index; may be -1="unknown" after setState() + * limit UTF-8 length of the string + * reservedField supplementary code point + * + * Since UCharIterator delivers 16-bit code units, the iteration can be + * currently in the middle of the byte sequence for a supplementary code point. + * In this case, reservedField will contain that code point and start will + * point to after the corresponding byte sequence. The UTF-16 index will be + * one less than what it would otherwise be corresponding to the UTF-8 index. + * Otherwise, reservedField will be 0. + */ + +/* + * Possible optimization for NUL-terminated UTF-8 and UTF-16 strings: + * Add implementations that do not call strlen() for iteration but check for NUL. + */ + +static int32_t U_CALLCONV +utf8IteratorGetIndex(UCharIterator *iter, UCharIteratorOrigin origin) { + switch(origin) { + case UITER_ZERO: + case UITER_START: + return 0; + case UITER_CURRENT: + if(iter->index<0) { + /* the current UTF-16 index is unknown after setState(), count from the beginning */ + const uint8_t *s; + UChar32 c; + int32_t i, limit, index; + + s=(const uint8_t *)iter->context; + i=index=0; + limit=iter->start; /* count up to the UTF-8 index */ + while(istart=i; /* just in case setState() did not get us to a code point boundary */ + if(i==iter->limit) { + iter->length=index; /* in case it was <0 or wrong */ + } + if(iter->reservedField!=0) { + --index; /* we are in the middle of a supplementary code point */ + } + iter->index=index; + } + return iter->index; + case UITER_LIMIT: + case UITER_LENGTH: + if(iter->length<0) { + const uint8_t *s; + UChar32 c; + int32_t i, limit, length; + + s=(const uint8_t *)iter->context; + if(iter->index<0) { + /* + * the current UTF-16 index is unknown after setState(), + * we must first count from the beginning to here + */ + i=length=0; + limit=iter->start; + + /* count from the beginning to the current index */ + while(istart, set the UTF-16 index */ + iter->start=i; /* just in case setState() did not get us to a code point boundary */ + iter->index= iter->reservedField!=0 ? length-1 : length; + } else { + i=iter->start; + length=iter->index; + if(iter->reservedField!=0) { + ++length; + } + } + + /* count from the current index to the end */ + limit=iter->limit; + while(ilength=length; + } + return iter->length; + default: + /* not a valid origin */ + /* Should never get here! */ + return -1; + } +} + +static int32_t U_CALLCONV +utf8IteratorMove(UCharIterator *iter, int32_t delta, UCharIteratorOrigin origin) { + const uint8_t *s; + UChar32 c; + int32_t pos; /* requested UTF-16 index */ + int32_t i; /* UTF-8 index */ + UBool havePos; + + /* calculate the requested UTF-16 index */ + switch(origin) { + case UITER_ZERO: + case UITER_START: + pos=delta; + havePos=TRUE; + /* iter->index<0 (unknown) is possible */ + break; + case UITER_CURRENT: + if(iter->index>=0) { + pos=iter->index+delta; + havePos=TRUE; + } else { + /* the current UTF-16 index is unknown after setState(), use only delta */ + pos=0; + havePos=FALSE; + } + break; + case UITER_LIMIT: + case UITER_LENGTH: + if(iter->length>=0) { + pos=iter->length+delta; + havePos=TRUE; + } else { + /* pin to the end, avoid counting the length */ + iter->index=-1; + iter->start=iter->limit; + iter->reservedField=0; + if(delta>=0) { + return UITER_UNKNOWN_INDEX; + } else { + /* the current UTF-16 index is unknown, use only delta */ + pos=0; + havePos=FALSE; + } + } + break; + default: + return -1; /* Error */ + } + + if(havePos) { + /* shortcuts: pinning to the edges of the string */ + if(pos<=0) { + iter->index=iter->start=iter->reservedField=0; + return 0; + } else if(iter->length>=0 && pos>=iter->length) { + iter->index=iter->length; + iter->start=iter->limit; + iter->reservedField=0; + return iter->index; + } + + /* minimize the number of U8_NEXT/PREV operations */ + if(iter->index<0 || posindex/2) { + /* go forward from the start instead of backward from the current index */ + iter->index=iter->start=iter->reservedField=0; + } else if(iter->length>=0 && (iter->length-pos)<(pos-iter->index)) { + /* + * if we have the UTF-16 index and length and the new position is + * closer to the end than the current index, + * then go backward from the end instead of forward from the current index + */ + iter->index=iter->length; + iter->start=iter->limit; + iter->reservedField=0; + } + + delta=pos-iter->index; + if(delta==0) { + return iter->index; /* nothing to do */ + } + } else { + /* move relative to unknown UTF-16 index */ + if(delta==0) { + return UITER_UNKNOWN_INDEX; /* nothing to do */ + } else if(-delta>=iter->start) { + /* moving backwards by more UChars than there are UTF-8 bytes, pin to 0 */ + iter->index=iter->start=iter->reservedField=0; + return 0; + } else if(delta>=(iter->limit-iter->start)) { + /* moving forward by more UChars than the remaining UTF-8 bytes, pin to the end */ + iter->index=iter->length; /* may or may not be <0 (unknown) */ + iter->start=iter->limit; + iter->reservedField=0; + return iter->index>=0 ? iter->index : (int32_t)UITER_UNKNOWN_INDEX; + } + } + + /* delta!=0 */ + + /* move towards the requested position, pin to the edges of the string */ + s=(const uint8_t *)iter->context; + pos=iter->index; /* could be <0 (unknown) */ + i=iter->start; + if(delta>0) { + /* go forward */ + int32_t limit=iter->limit; + if(iter->reservedField!=0) { + iter->reservedField=0; + ++pos; + --delta; + } + while(delta>0 && i=2) { + pos+=2; + delta-=2; + } else /* delta==1 */ { + /* stop in the middle of a supplementary code point */ + iter->reservedField=c; + ++pos; + break; /* delta=0; */ + } + } + if(i==limit) { + if(iter->length<0 && iter->index>=0) { + iter->length= iter->reservedField==0 ? pos : pos+1; + } else if(iter->index<0 && iter->length>=0) { + iter->index= iter->reservedField==0 ? iter->length : iter->length-1; + } + } + } else /* delta<0 */ { + /* go backward */ + if(iter->reservedField!=0) { + iter->reservedField=0; + i-=4; /* we stayed behind the supplementary code point; go before it now */ + --pos; + ++delta; + } + while(delta<0 && i>0) { + U8_PREV(s, 0, i, c); + if(c<0xffff) { + --pos; + ++delta; + } else if(delta<=-2) { + pos-=2; + delta+=2; + } else /* delta==-1 */ { + /* stop in the middle of a supplementary code point */ + i+=4; /* back to behind this supplementary code point for consistent state */ + iter->reservedField=c; + --pos; + break; /* delta=0; */ + } + } + } + + iter->start=i; + if(iter->index>=0) { + return iter->index=pos; + } else { + /* we started with index<0 (unknown) so pos is bogus */ + if(i<=1) { + return iter->index=i; /* reached the beginning */ + } else { + /* we still don't know the UTF-16 index */ + return UITER_UNKNOWN_INDEX; + } + } +} + +static UBool U_CALLCONV +utf8IteratorHasNext(UCharIterator *iter) { + return iter->startlimit || iter->reservedField!=0; +} + +static UBool U_CALLCONV +utf8IteratorHasPrevious(UCharIterator *iter) { + return iter->start>0; +} + +static UChar32 U_CALLCONV +utf8IteratorCurrent(UCharIterator *iter) { + if(iter->reservedField!=0) { + return U16_TRAIL(iter->reservedField); + } else if(iter->startlimit) { + const uint8_t *s=(const uint8_t *)iter->context; + UChar32 c; + int32_t i=iter->start; + + U8_NEXT(s, i, iter->limit, c); + if(c<0) { + return 0xfffd; + } else if(c<=0xffff) { + return c; + } else { + return U16_LEAD(c); + } + } else { + return U_SENTINEL; + } +} + +static UChar32 U_CALLCONV +utf8IteratorNext(UCharIterator *iter) { + int32_t index; + + if(iter->reservedField!=0) { + UChar trail=U16_TRAIL(iter->reservedField); + iter->reservedField=0; + if((index=iter->index)>=0) { + iter->index=index+1; + } + return trail; + } else if(iter->startlimit) { + const uint8_t *s=(const uint8_t *)iter->context; + UChar32 c; + + U8_NEXT(s, iter->start, iter->limit, c); + if((index=iter->index)>=0) { + iter->index=++index; + if(iter->length<0 && iter->start==iter->limit) { + iter->length= c<=0xffff ? index : index+1; + } + } else if(iter->start==iter->limit && iter->length>=0) { + iter->index= c<=0xffff ? iter->length : iter->length-1; + } + if(c<0) { + return 0xfffd; + } else if(c<=0xffff) { + return c; + } else { + iter->reservedField=c; + return U16_LEAD(c); + } + } else { + return U_SENTINEL; + } +} + +static UChar32 U_CALLCONV +utf8IteratorPrevious(UCharIterator *iter) { + int32_t index; + + if(iter->reservedField!=0) { + UChar lead=U16_LEAD(iter->reservedField); + iter->reservedField=0; + iter->start-=4; /* we stayed behind the supplementary code point; go before it now */ + if((index=iter->index)>0) { + iter->index=index-1; + } + return lead; + } else if(iter->start>0) { + const uint8_t *s=(const uint8_t *)iter->context; + UChar32 c; + + U8_PREV(s, 0, iter->start, c); + if((index=iter->index)>0) { + iter->index=index-1; + } else if(iter->start<=1) { + iter->index= c<=0xffff ? iter->start : iter->start+1; + } + if(c<0) { + return 0xfffd; + } else if(c<=0xffff) { + return c; + } else { + iter->start+=4; /* back to behind this supplementary code point for consistent state */ + iter->reservedField=c; + return U16_TRAIL(c); + } + } else { + return U_SENTINEL; + } +} + +static uint32_t U_CALLCONV +utf8IteratorGetState(const UCharIterator *iter) { + uint32_t state=(uint32_t)(iter->start<<1); + if(iter->reservedField!=0) { + state|=1; + } + return state; +} + +static void U_CALLCONV +utf8IteratorSetState(UCharIterator *iter, + uint32_t state, + UErrorCode *pErrorCode) +{ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + /* do nothing */ + } else if(iter==NULL) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + } else if(state==utf8IteratorGetState(iter)) { + /* setting to the current state: no-op */ + } else { + int32_t index=(int32_t)(state>>1); /* UTF-8 index */ + state&=1; /* 1 if in surrogate pair, must be index>=4 */ + + if((state==0 ? index<0 : index<4) || iter->limitstart=index; /* restore UTF-8 byte index */ + if(index<=1) { + iter->index=index; + } else { + iter->index=-1; /* unknown UTF-16 index */ + } + if(state==0) { + iter->reservedField=0; + } else { + /* verified index>=4 above */ + UChar32 c; + U8_PREV((const uint8_t *)iter->context, 0, index, c); + if(c<=0xffff) { + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + } else { + iter->reservedField=c; + } + } + } + } +} + +static const UCharIterator utf8Iterator={ + 0, 0, 0, 0, 0, 0, + utf8IteratorGetIndex, + utf8IteratorMove, + utf8IteratorHasNext, + utf8IteratorHasPrevious, + utf8IteratorCurrent, + utf8IteratorNext, + utf8IteratorPrevious, + NULL, + utf8IteratorGetState, + utf8IteratorSetState +}; + +U_CAPI void U_EXPORT2 +uiter_setUTF8(UCharIterator *iter, const char *s, int32_t length) { + if(iter!=0) { + if(s!=0 && length>=-1) { + *iter=utf8Iterator; + iter->context=s; + if(length>=0) { + iter->limit=length; + } else { + iter->limit=(int32_t)uprv_strlen(s); + } + iter->length= iter->limit<=1 ? iter->limit : -1; + } else { + *iter=noopIterator; + } + } +} + +/* Helper functions --------------------------------------------------------- */ + +U_CAPI UChar32 U_EXPORT2 +uiter_current32(UCharIterator *iter) { + UChar32 c, c2; + + c=iter->current(iter); + if(UTF_IS_SURROGATE(c)) { + if(UTF_IS_SURROGATE_FIRST(c)) { + /* + * go to the next code unit + * we know that we are not at the limit because c!=U_SENTINEL + */ + iter->move(iter, 1, UITER_CURRENT); + if(UTF_IS_SECOND_SURROGATE(c2=iter->current(iter))) { + c=UTF16_GET_PAIR_VALUE(c, c2); + } + + /* undo index movement */ + iter->move(iter, -1, UITER_CURRENT); + } else { + if(UTF_IS_FIRST_SURROGATE(c2=iter->previous(iter))) { + c=UTF16_GET_PAIR_VALUE(c2, c); + } + if(c2>=0) { + /* undo index movement */ + iter->move(iter, 1, UITER_CURRENT); + } + } + } + return c; +} + +U_CAPI UChar32 U_EXPORT2 +uiter_next32(UCharIterator *iter) { + UChar32 c, c2; + + c=iter->next(iter); + if(UTF_IS_FIRST_SURROGATE(c)) { + if(UTF_IS_SECOND_SURROGATE(c2=iter->next(iter))) { + c=UTF16_GET_PAIR_VALUE(c, c2); + } else if(c2>=0) { + /* unmatched first surrogate, undo index movement */ + iter->move(iter, -1, UITER_CURRENT); + } + } + return c; +} + +U_CAPI UChar32 U_EXPORT2 +uiter_previous32(UCharIterator *iter) { + UChar32 c, c2; + + c=iter->previous(iter); + if(UTF_IS_SECOND_SURROGATE(c)) { + if(UTF_IS_FIRST_SURROGATE(c2=iter->previous(iter))) { + c=UTF16_GET_PAIR_VALUE(c2, c); + } else if(c2>=0) { + /* unmatched second surrogate, undo index movement */ + iter->move(iter, 1, UITER_CURRENT); + } + } + return c; +} + +U_CAPI uint32_t U_EXPORT2 +uiter_getState(const UCharIterator *iter) { + if(iter==NULL || iter->getState==NULL) { + return UITER_NO_STATE; + } else { + return iter->getState(iter); + } +} + +U_CAPI void U_EXPORT2 +uiter_setState(UCharIterator *iter, uint32_t state, UErrorCode *pErrorCode) { + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + /* do nothing */ + } else if(iter==NULL) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + } else if(iter->setState==NULL) { + *pErrorCode=U_UNSUPPORTED_ERROR; + } else { + iter->setState(iter, state, pErrorCode); + } +} + +U_CDECL_END diff --git a/Build/source/libs/icu/icu-xetex/common/uloc.c b/Build/source/libs/icu/icu-xetex/common/uloc.c new file mode 100644 index 00000000000..309e2d0ea7c --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uloc.c @@ -0,0 +1,3046 @@ +/* +********************************************************************** +* Copyright (C) 1997-2007, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* +* File ULOC.CPP +* +* Modification History: +* +* Date Name Description +* 04/01/97 aliu Creation. +* 08/21/98 stephen JDK 1.2 sync +* 12/08/98 rtg New Locale implementation and C API +* 03/15/99 damiba overhaul. +* 04/06/99 stephen changed setDefault() to realloc and copy +* 06/14/99 stephen Changed calls to ures_open for new params +* 07/21/99 stephen Modified setDefault() to propagate to C++ +* 05/14/04 alan 7 years later: refactored, cleaned up, fixed bugs, +* brought canonicalization code into line with spec +*****************************************************************************/ + +/* + POSIX's locale format, from putil.c: [no spaces] + + ll [ _CC ] [ . MM ] [ @ VV] + + l = lang, C = ctry, M = charmap, V = variant +*/ + +#include "unicode/utypes.h" +#include "unicode/ustring.h" +#include "unicode/uloc.h" +#include "unicode/ures.h" + +#include "putilimp.h" +#include "ustr_imp.h" +#include "ulocimp.h" +#include "uresimp.h" +#include "umutex.h" +#include "cstring.h" +#include "cmemory.h" +#include "ucln_cmn.h" +#include "locmap.h" +#include "uarrsort.h" +#include "uenumimp.h" +#include "uassert.h" + +#include /* for sprintf */ + +/* ### Declarations **************************************************/ + +/* Locale stuff from locid.cpp */ +U_CFUNC void locale_set_default(const char *id); +U_CFUNC const char *locale_get_default(void); +U_CFUNC int32_t +locale_getKeywords(const char *localeID, + char prev, + char *keywords, int32_t keywordCapacity, + char *values, int32_t valuesCapacity, int32_t *valLen, + UBool valuesToo, + UErrorCode *status); + +/* ### Constants **************************************************/ + +/* These strings describe the resources we attempt to load from + the locale ResourceBundle data file.*/ +static const char _kLanguages[] = "Languages"; +static const char _kScripts[] = "Scripts"; +static const char _kCountries[] = "Countries"; +static const char _kVariants[] = "Variants"; +static const char _kKeys[] = "Keys"; +static const char _kTypes[] = "Types"; +static const char _kIndexLocaleName[] = "res_index"; +static const char _kRootName[] = "root"; +static const char _kIndexTag[] = "InstalledLocales"; +static const char _kCurrency[] = "currency"; +static const char _kCurrencies[] = "Currencies"; +static char** _installedLocales = NULL; +static int32_t _installedLocalesCount = 0; + +/* ### Data tables **************************************************/ + +/** + * Table of language codes, both 2- and 3-letter, with preference + * given to 2-letter codes where possible. Includes 3-letter codes + * that lack a 2-letter equivalent. + * + * This list must be in sorted order. This list is returned directly + * to the user by some API. + * + * This list must be kept in sync with LANGUAGES_3, with corresponding + * entries matched. + * + * This table should be terminated with a NULL entry, followed by a + * second list, and another NULL entry. The first list is visible to + * user code when this array is returned by API. The second list + * contains codes we support, but do not expose through user API. + * + * Notes + * + * Tables updated per http://lcweb.loc.gov/standards/iso639-2/ to + * include the revisions up to 2001/7/27 *CWB* + * + * The 3 character codes are the terminology codes like RFC 3066. This + * is compatible with prior ICU codes + * + * "in" "iw" "ji" "jw" & "sh" have been withdrawn but are still in the + * table but now at the end of the table because 3 character codes are + * duplicates. This avoids bad searches going from 3 to 2 character + * codes. + * + * The range qaa-qtz is reserved for local use + */ +static const char * const LANGUAGES[] = { + "aa", "ab", "ace", "ach", "ada", "ady", "ae", "af", "afa", + "afh", "ain", "ak", "akk", "ale", "alg", "alt", "am", "an", + "ang", "anp", "apa", + "ar", "arc", "arn", "arp", "art", "arw", "as", "ast", + "ath", "aus", "av", "awa", "ay", "az", "ba", "bad", + "bai", "bal", "ban", "bas", "bat", "be", "bej", + "bem", "ber", "bg", "bh", "bho", "bi", "bik", "bin", + "bla", "bm", "bn", "bnt", "bo", "br", "bra", "bs", + "btk", "bua", "bug", "byn", "ca", "cad", "cai", "car", "cau", + "cch", "ce", "ceb", "cel", "ch", "chb", "chg", "chk", "chm", + "chn", "cho", "chp", "chr", "chy", "cmc", "co", "cop", + "cpe", "cpf", "cpp", "cr", "crh", "crp", "cs", "csb", "cu", "cus", + "cv", "cy", "da", "dak", "dar", "day", "de", "del", "den", + "dgr", "din", "doi", "dra", "dsb", "dua", "dum", "dv", "dyu", + "dz", "ee", "efi", "egy", "eka", "el", "elx", "en", + "enm", "eo", "es", "et", "eu", "ewo", "fa", + "fan", "fat", "ff", "fi", "fil", "fiu", "fj", "fo", "fon", + "fr", "frm", "fro", "frr", "frs", "fur", "fy", + "ga", "gaa", "gay", "gba", "gd", "gem", "gez", "gil", + "gl", "gmh", "gn", "goh", "gon", "gor", "got", "grb", + "grc", "gsw", "gu", "gv", "gwi", + "ha", "hai", "haw", "he", "hi", "hil", "him", + "hit", "hmn", "ho", "hr", "hsb", "ht", "hu", "hup", "hy", "hz", + "ia", "iba", "id", "ie", "ig", "ii", "ijo", "ik", + "ilo", "inc", "ine", "inh", "io", "ira", "iro", "is", "it", + "iu", "ja", "jbo", "jpr", "jrb", "jv", "ka", "kaa", "kab", + "kac", "kaj", "kam", "kar", "kaw", "kbd", "kcg", "kfo", "kg", "kha", "khi", + "kho", "ki", "kj", "kk", "kl", "km", "kmb", "kn", + "ko", "kok", "kos", "kpe", "kr", "krc", "krl", "kro", "kru", "ks", + "ku", "kum", "kut", "kv", "kw", "ky", "la", "lad", + "lah", "lam", "lb", "lez", "lg", "li", "ln", "lo", "lol", + "loz", "lt", "lu", "lua", "lui", "lun", "luo", "lus", + "lv", "mad", "mag", "mai", "mak", "man", "map", "mas", + "mdf", "mdr", "men", "mg", "mga", "mh", "mi", "mic", "min", + "mis", "mk", "mkh", "ml", "mn", "mnc", "mni", "mno", + "mo", "moh", "mos", "mr", "ms", "mt", "mul", "mun", + "mus", "mwl", "mwr", "my", "myn", "myv", "na", "nah", "nai", "nap", + "nb", "nd", "nds", "ne", "new", "ng", "nia", "nic", + "niu", "nl", "nn", "no", "nog", "non", "nqo", "nr", "nso", "nub", + "nv", "nwc", "ny", "nym", "nyn", "nyo", "nzi", "oc", "oj", + "om", "or", "os", "osa", "ota", "oto", "pa", "paa", + "pag", "pal", "pam", "pap", "pau", "peo", "phi", "phn", + "pi", "pl", "pon", "pra", "pro", "ps", "pt", "qu", + "raj", "rap", "rar", "rm", "rn", "ro", "roa", "rom", + "ru", "rup", "rw", "sa", "sad", "sah", "sai", "sal", "sam", + "sas", "sat", "sc", "scn", "sco", "sd", "se", "sel", "sem", + "sg", "sga", "sgn", "shn", "si", "sid", "sio", "sit", + "sk", "sl", "sla", "sm", "sma", "smi", "smj", "smn", + "sms", "sn", "snk", "so", "sog", "son", "sq", "sr", + "srn", "srr", "ss", "ssa", "st", "su", "suk", "sus", "sux", + "sv", "sw", "syc", "syr", "ta", "tai", "te", "tem", "ter", + "tet", "tg", "th", "ti", "tig", "tiv", "tk", "tkl", + "tl", "tlh", "tli", "tmh", "tn", "to", "tog", "tpi", "tr", + "ts", "tsi", "tt", "tum", "tup", "tut", "tvl", "tw", + "ty", "tyv", "udm", "ug", "uga", "uk", "umb", "und", "ur", + "uz", "vai", "ve", "vi", "vo", "vot", "wa", "wak", + "wal", "war", "was", "wen", "wo", "xal", "xh", "yao", "yap", + "yi", "yo", "ypk", "za", "zap", "zbl", "zen", "zh", "znd", + "zu", "zun", "zxx", "zza", +NULL, + "in", "iw", "ji", "jw", "sh", /* obsolete language codes */ +NULL +}; +static const char* const DEPRECATED_LANGUAGES[]={ + "in", "iw", "ji", "jw", NULL, NULL +}; +static const char* const REPLACEMENT_LANGUAGES[]={ + "id", "he", "yi", "jv", NULL, NULL +}; + +/** + * Table of 3-letter language codes. + * + * This is a lookup table used to convert 3-letter language codes to + * their 2-letter equivalent, where possible. It must be kept in sync + * with LANGUAGES. For all valid i, LANGUAGES[i] must refer to the + * same language as LANGUAGES_3[i]. The commented-out lines are + * copied from LANGUAGES to make eyeballing this baby easier. + * + * Where a 3-letter language code has no 2-letter equivalent, the + * 3-letter code occupies both LANGUAGES[i] and LANGUAGES_3[i]. + * + * This table should be terminated with a NULL entry, followed by a + * second list, and another NULL entry. The two lists correspond to + * the two lists in LANGUAGES. + */ +static const char * const LANGUAGES_3[] = { +/* "aa", "ab", "ace", "ach", "ada", "ady", "ae", "af", "afa", */ + "aar", "abk", "ace", "ach", "ada", "ady", "ave", "afr", "afa", +/* "afh", "ain", "ak", "akk", "ale", "alg", "alt", "am", "an", "ang", "anp", "apa", */ + "afh", "ain", "aka", "akk", "ale", "alg", "alt", "amh", "arg", "ang", "anp", "apa", +/* "ar", "arc", "arn", "arp", "art", "arw", "as", "ast", */ + "ara", "arc", "arn", "arp", "art", "arw", "asm", "ast", +/* "ath", "aus", "av", "awa", "ay", "az", "ba", "bad", */ + "ath", "aus", "ava", "awa", "aym", "aze", "bak", "bad", +/* "bai", "bal", "ban", "bas", "bat", "be", "bej", */ + "bai", "bal", "ban", "bas", "bat", "bel", "bej", +/* "bem", "ber", "bg", "bh", "bho", "bi", "bik", "bin", */ + "bem", "ber", "bul", "bih", "bho", "bis", "bik", "bin", +/* "bla", "bm", "bn", "bnt", "bo", "br", "bra", "bs", */ + "bla", "bam", "ben", "bnt", "bod", "bre", "bra", "bos", +/* "btk", "bua", "bug", "byn", "ca", "cad", "cai", "car", "cau", */ + "btk", "bua", "bug", "byn", "cat", "cad", "cai", "car", "cau", +/* "cch", "ce", "ceb", "cel", "ch", "chb", "chg", "chk", "chm", */ + "cch", "che", "ceb", "cel", "cha", "chb", "chg", "chk", "chm", +/* "chn", "cho", "chp", "chr", "chy", "cmc", "co", "cop", */ + "chn", "cho", "chp", "chr", "chy", "cmc", "cos", "cop", +/* "cpe", "cpf", "cpp", "cr", "crh", "crp", "cs", "csb", "cu", "cus", */ + "cpe", "cpf", "cpp", "cre", "crh", "crp", "ces", "csb", "chu", "cus", +/* "cv", "cy", "da", "dak", "dar", "day", "de", "del", "den", */ + "chv", "cym", "dan", "dak", "dar", "day", "deu", "del", "den", +/* "dgr", "din", "doi", "dra", "dsb", "dua", "dum", "dv", "dyu", */ + "dgr", "din", "doi", "dra", "dsb", "dua", "dum", "div", "dyu", +/* "dz", "ee", "efi", "egy", "eka", "el", "elx", "en", */ + "dzo", "ewe", "efi", "egy", "eka", "ell", "elx", "eng", +/* "enm", "eo", "es", "et", "eu", "ewo", "fa", */ + "enm", "epo", "spa", "est", "eus", "ewo", "fas", +/* "fan", "fat", "ff", "fi", "fil", "fiu", "fj", "fo", "fon", */ + "fan", "fat", "ful", "fin", "fil", "fiu", "fij", "fao", "fon", +/* "fr", "frm", "fro", "fur", "frr", "frs", "fy", "ga", "gaa", "gay", */ + "fra", "frm", "fro", "fur", "frr", "frs", "fry", "gle", "gaa", "gay", +/* "gba", "gd", "gem", "gez", "gil", "gl", "gmh", "gn", */ + "gba", "gla", "gem", "gez", "gil", "glg", "gmh", "grn", +/* "goh", "gon", "gor", "got", "grb", "grc", "gsw", "gu", "gv", */ + "goh", "gon", "gor", "got", "grb", "grc", "gsw", "guj", "glv", +/* "gwi", "ha", "hai", "haw", "he", "hi", "hil", "him", */ + "gwi", "hau", "hai", "haw", "heb", "hin", "hil", "him", +/* "hit", "hmn", "ho", "hr", "hsb", "ht", "hu", "hup", "hy", "hz", */ + "hit", "hmn", "hmo", "hrv", "hsb", "hat", "hun", "hup", "hye", "her", +/* "ia", "iba", "id", "ie", "ig", "ii", "ijo", "ik", */ + "ina", "iba", "ind", "ile", "ibo", "iii", "ijo", "ipk", +/* "ilo", "inc", "ine", "inh", "io", "ira", "iro", "is", "it", */ + "ilo", "inc", "ine", "inh", "ido", "ira", "iro", "isl", "ita", +/* "iu", "ja", "jbo", "jpr", "jrb", "jv", "ka", "kaa", "kab", */ + "iku", "jpn", "jbo", "jpr", "jrb", "jav", "kat", "kaa", "kab", +/* "kac", "kaj", "kam", "kar", "kaw", "kbd", "kcg", "kfo", "kg", "kha", "khi",*/ + "kac", "kaj", "kam", "kar", "kaw", "kbd", "kcg", "kfo", "kg", "kha", "khi", +/* "kho", "ki", "kj", "kk", "kl", "km", "kmb", "kn", */ + "kho", "kik", "kua", "kaz", "kal", "khm", "kmb", "kan", +/* "ko", "kok", "kos", "kpe", "kr", "krc", "krl", "kro", "kru", "ks", */ + "kor", "kok", "kos", "kpe", "kau", "krc", "krl", "kro", "kru", "kas", +/* "ku", "kum", "kut", "kv", "kw", "ky", "la", "lad", */ + "kur", "kum", "kut", "kom", "cor", "kir", "lat", "lad", +/* "lah", "lam", "lb", "lez", "lg", "li", "ln", "lo", "lol", */ + "lah", "lam", "ltz", "lez", "lug", "lim", "lin", "lao", "lol", +/* "loz", "lt", "lu", "lua", "lui", "lun", "luo", "lus", */ + "loz", "lit", "lub", "lua", "lui", "lun", "luo", "lus", +/* "lv", "mad", "mag", "mai", "mak", "man", "map", "mas", */ + "lav", "mad", "mag", "mai", "mak", "man", "map", "mas", +/* "mdf", "mdr", "men", "mg", "mga", "mh", "mi", "mic", "min", */ + "mdf", "mdr", "men", "mlg", "mga", "mah", "mri", "mic", "min", +/* "mis", "mk", "mkh", "ml", "mn", "mnc", "mni", "mno", */ + "mis", "mkd", "mkh", "mal", "mon", "mnc", "mni", "mno", +/* "mo", "moh", "mos", "mr", "ms", "mt", "mul", "mun", */ + "mol", "moh", "mos", "mar", "msa", "mlt", "mul", "mun", +/* "mus", "mwl", "mwr", "my", "myn", "myv", "na", "nah", "nai", "nap", */ + "mus", "mwl", "mwr", "mya", "myn", "myv", "nau", "nah", "nai", "nap", +/* "nb", "nd", "nds", "ne", "new", "ng", "nia", "nic", */ + "nob", "nde", "nds", "nep", "new", "ndo", "nia", "nic", +/* "niu", "nl", "nn", "no", "nog", "non", "nqo", "nr", "nso", "nub", */ + "niu", "nld", "nno", "nor", "nog", "non", "nqo", "nbl", "nso", "nub", +/* "nv", "nwc", "ny", "nym", "nyn", "nyo", "nzi", "oc", "oj", */ + "nav", "nwc", "nya", "nym", "nyn", "nyo", "nzi", "oci", "oji", +/* "om", "or", "os", "osa", "ota", "oto", "pa", "paa", */ + "orm", "ori", "oss", "osa", "ota", "oto", "pan", "paa", +/* "pag", "pal", "pam", "pap", "pau", "peo", "phi", "phn", */ + "pag", "pal", "pam", "pap", "pau", "peo", "phi", "phn", +/* "pi", "pl", "pon", "pra", "pro", "ps", "pt", "qu", */ + "pli", "pol", "pon", "pra", "pro", "pus", "por", "que", +/* "raj", "rap", "rar", "rm", "rn", "ro", "roa", "rom", */ + "raj", "rap", "rar", "roh", "run", "ron", "roa", "rom", +/* "ru", "rup", "rw", "sa", "sad", "sah", "sai", "sal", "sam", */ + "rus", "rup", "kin", "san", "sad", "sah", "sai", "sal", "sam", +/* "sas", "sat", "sc", "scn", "sco", "sd", "se", "sel", "sem", */ + "sas", "sat", "srd", "scn", "sco", "snd", "sme", "sel", "sem", +/* "sg", "sga", "sgn", "shn", "si", "sid", "sio", "sit", */ + "sag", "sga", "sgn", "shn", "sin", "sid", "sio", "sit", +/* "sk", "sl", "sla", "sm", "sma", "smi", "smj", "smn", */ + "slk", "slv", "sla", "smo", "sma", "smi", "smj", "smn", +/* "sms", "sn", "snk", "so", "sog", "son", "sq", "sr", */ + "sms", "sna", "snk", "som", "sog", "son", "sqi", "srp", +/* "srn", "srr", "ss", "ssa", "st", "su", "suk", "sus", "sux", */ + "srn", "srr", "ssw", "ssa", "sot", "sun", "suk", "sus", "sux", +/* "sv", "sw", "syc", "syr", "ta", "tai", "te", "tem", "ter", */ + "swe", "swa", "syc", "syr", "tam", "tai", "tel", "tem", "ter", +/* "tet", "tg", "th", "ti", "tig", "tiv", "tk", "tkl", */ + "tet", "tgk", "tha", "tir", "tig", "tiv", "tuk", "tkl", +/* "tl", "tlh", "tli", "tmh", "tn", "to", "tog", "tpi", "tr", */ + "tgl", "tlh", "tli", "tmh", "tsn", "ton", "tog", "tpi", "tur", +/* "ts", "tsi", "tt", "tum", "tup", "tut", "tvl", "tw", */ + "tso", "tsi", "tat", "tum", "tup", "tut", "tvl", "twi", +/* "ty", "tyv", "udm", "ug", "uga", "uk", "umb", "und", "ur", */ + "tah", "tyv", "udm", "uig", "uga", "ukr", "umb", "und", "urd", +/* "uz", "vai", "ve", "vi", "vo", "vot", "wa", "wak", */ + "uzb", "vai", "ven", "vie", "vol", "vot", "wln", "wak", +/* "wal", "war", "was", "wen", "wo", "xal", "xh", "yao", "yap", */ + "wal", "war", "was", "wen", "wol", "xal", "xho", "yao", "yap", +/* "yi", "yo", "ypk", "za", "zap", "zbl", "zen", "zh", "znd", */ + "yid", "yor", "ypk", "zha", "zap", "zbl", "zen", "zho", "znd", +/* "zu", "zun", "zxx", "zza", */ + "zul", "zun", "zxx", "zza", +NULL, +/* "in", "iw", "ji", "jw", "sh", */ + "ind", "heb", "yid", "jaw", "srp", +NULL +}; + +/** + * Table of 2-letter country codes. + * + * This list must be in sorted order. This list is returned directly + * to the user by some API. + * + * This list must be kept in sync with COUNTRIES_3, with corresponding + * entries matched. + * + * This table should be terminated with a NULL entry, followed by a + * second list, and another NULL entry. The first list is visible to + * user code when this array is returned by API. The second list + * contains codes we support, but do not expose through user API. + * + * Notes: + * + * ZR(ZAR) is now CD(COD) and FX(FXX) is PS(PSE) as per + * http://www.evertype.com/standards/iso3166/iso3166-1-en.html added + * new codes keeping the old ones for compatibility updated to include + * 1999/12/03 revisions *CWB* + * + * RO(ROM) is now RO(ROU) according to + * http://www.iso.org/iso/en/prods-services/iso3166ma/03updates-on-iso-3166/nlv3e-rou.html + */ +static const char * const COUNTRIES[] = { + "AD", "AE", "AF", "AG", "AI", "AL", "AM", "AN", + "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ", + "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", + "BJ", "BL", "BM", "BN", "BO", "BR", "BS", "BT", "BV", + "BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", + "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CR", + "CU", "CV", "CX", "CY", "CZ", "DE", "DJ", "DK", + "DM", "DO", "DZ", "EC", "EE", "EG", "EH", "ER", + "ES", "ET", "FI", "FJ", "FK", "FM", "FO", "FR", + "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", + "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", + "GW", "GY", "HK", "HM", "HN", "HR", "HT", "HU", + "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", "IS", + "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", + "KM", "KN", "KP", "KR", "KW", "KY", "KZ", "LA", + "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", + "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", + "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", + "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", + "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", + "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", + "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", + "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW", "SA", + "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", + "SK", "SL", "SM", "SN", "SO", "SR", "ST", "SV", + "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", + "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", + "TW", "TZ", "UA", "UG", "UM", "US", "UY", "UZ", + "VA", "VC", "VE", "VG", "VI", "VN", "VU", "WF", + "WS", "YE", "YT", "ZA", "ZM", "ZW", +NULL, + "FX", "CS", "RO", "TP", "YU", "ZR", /* obsolete country codes */ +NULL +}; + +static const char* const DEPRECATED_COUNTRIES[] ={ + "BU", "CS", "DY", "FX", "HV", "NH", "RH", "TP", "YU", "ZR", NULL, NULL /* deprecated country list */ +}; +static const char* const REPLACEMENT_COUNTRIES[] = { +/* "BU", "CS", "DY", "FX", "HV", "NH", "RH", "TP", "YU", "ZR" */ + "MM", "RS", "BJ", "FR", "BF", "VU", "ZW", "TL", "RS", "CD", NULL, NULL /* replacement country codes */ +}; + +/** + * Table of 3-letter country codes. + * + * This is a lookup table used to convert 3-letter country codes to + * their 2-letter equivalent. It must be kept in sync with COUNTRIES. + * For all valid i, COUNTRIES[i] must refer to the same country as + * COUNTRIES_3[i]. The commented-out lines are copied from COUNTRIES + * to make eyeballing this baby easier. + * + * This table should be terminated with a NULL entry, followed by a + * second list, and another NULL entry. The two lists correspond to + * the two lists in COUNTRIES. + */ +static const char * const COUNTRIES_3[] = { +/* "AD", "AE", "AF", "AG", "AI", "AL", "AM", "AN", */ + "AND", "ARE", "AFG", "ATG", "AIA", "ALB", "ARM", "ANT", +/* "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ", */ + "AGO", "ATA", "ARG", "ASM", "AUT", "AUS", "ABW", "ALA", "AZE", +/* "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", */ + "BIH", "BRB", "BGD", "BEL", "BFA", "BGR", "BHR", "BDI", +/* "BJ", "BL", "BM", "BN", "BO", "BR", "BS", "BT", "BV", */ + "BEN", "BLM", "BMU", "BRN", "BOL", "BRA", "BHS", "BTN", "BVT", +/* "BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", */ + "BWA", "BLR", "BLZ", "CAN", "CCK", "COD", "CAF", "COG", +/* "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CR", */ + "CHE", "CIV", "COK", "CHL", "CMR", "CHN", "COL", "CRI", +/* "CU", "CV", "CX", "CY", "CZ", "DE", "DJ", "DK", */ + "CUB", "CPV", "CXR", "CYP", "CZE", "DEU", "DJI", "DNK", +/* "DM", "DO", "DZ", "EC", "EE", "EG", "EH", "ER", */ + "DMA", "DOM", "DZA", "ECU", "EST", "EGY", "ESH", "ERI", +/* "ES", "ET", "FI", "FJ", "FK", "FM", "FO", "FR", */ + "ESP", "ETH", "FIN", "FJI", "FLK", "FSM", "FRO", "FRA", +/* "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", */ + "GAB", "GBR", "GRD", "GEO", "GUF", "GGY", "GHA", "GIB", "GRL", +/* "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", */ + "GMB", "GIN", "GLP", "GNQ", "GRC", "SGS", "GTM", "GUM", +/* "GW", "GY", "HK", "HM", "HN", "HR", "HT", "HU", */ + "GNB", "GUY", "HKG", "HMD", "HND", "HRV", "HTI", "HUN", +/* "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", "IS" */ + "IDN", "IRL", "ISR", "IMN", "IND", "IOT", "IRQ", "IRN", "ISL", +/* "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", */ + "ITA", "JEY", "JAM", "JOR", "JPN", "KEN", "KGZ", "KHM", "KIR", +/* "KM", "KN", "KP", "KR", "KW", "KY", "KZ", "LA", */ + "COM", "KNA", "PRK", "KOR", "KWT", "CYM", "KAZ", "LAO", +/* "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", */ + "LBN", "LCA", "LIE", "LKA", "LBR", "LSO", "LTU", "LUX", +/* "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", */ + "LVA", "LBY", "MAR", "MCO", "MDA", "MNE", "MAF", "MDG", "MHL", "MKD", +/* "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", */ + "MLI", "MMR", "MNG", "MAC", "MNP", "MTQ", "MRT", "MSR", +/* "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", */ + "MLT", "MUS", "MDV", "MWI", "MEX", "MYS", "MOZ", "NAM", +/* "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", */ + "NCL", "NER", "NFK", "NGA", "NIC", "NLD", "NOR", "NPL", +/* "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", */ + "NRU", "NIU", "NZL", "OMN", "PAN", "PER", "PYF", "PNG", +/* "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", */ + "PHL", "PAK", "POL", "SPM", "PCN", "PRI", "PSE", "PRT", +/* "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW", "SA", */ + "PLW", "PRY", "QAT", "REU", "ROU", "SRB", "RUS", "RWA", "SAU", +/* "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", */ + "SLB", "SYC", "SDN", "SWE", "SGP", "SHN", "SVN", "SJM", +/* "SK", "SL", "SM", "SN", "SO", "SR", "ST", "SV", */ + "SVK", "SLE", "SMR", "SEN", "SOM", "SUR", "STP", "SLV", +/* "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", */ + "SYR", "SWZ", "TCA", "TCD", "ATF", "TGO", "THA", "TJK", +/* "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", */ + "TKL", "TLS", "TKM", "TUN", "TON", "TUR", "TTO", "TUV", +/* "TW", "TZ", "UA", "UG", "UM", "US", "UY", "UZ", */ + "TWN", "TZA", "UKR", "UGA", "UMI", "USA", "URY", "UZB", +/* "VA", "VC", "VE", "VG", "VI", "VN", "VU", "WF", */ + "VAT", "VCT", "VEN", "VGB", "VIR", "VNM", "VUT", "WLF", +/* "WS", "YE", "YT", "ZA", "ZM", "ZW", */ + "WSM", "YEM", "MYT", "ZAF", "ZMB", "ZWE", +NULL, +/* "FX", "CS", "RO", "TP", "YU", "ZR", */ + "FXX", "SCG", "ROM", "TMP", "YUG", "ZAR", +NULL +}; + +typedef struct CanonicalizationMap { + const char *id; /* input ID */ + const char *canonicalID; /* canonicalized output ID */ + const char *keyword; /* keyword, or NULL if none */ + const char *value; /* keyword value, or NULL if kw==NULL */ +} CanonicalizationMap; + +/** + * A map to canonicalize locale IDs. This handles a variety of + * different semantic kinds of transformations. + */ +static const CanonicalizationMap CANONICALIZE_MAP[] = { + { "", "en_US_POSIX", NULL, NULL }, /* .NET name */ + { "C", "en_US_POSIX", NULL, NULL }, /* POSIX name */ + { "posix", "en_US_POSIX", NULL, NULL }, /* POSIX name (alias of C) */ + { "art_LOJBAN", "jbo", NULL, NULL }, /* registered name */ + { "az_AZ_CYRL", "az_Cyrl_AZ", NULL, NULL }, /* .NET name */ + { "az_AZ_LATN", "az_Latn_AZ", NULL, NULL }, /* .NET name */ + { "ca_ES_PREEURO", "ca_ES", "currency", "ESP" }, + { "cel_GAULISH", "cel__GAULISH", NULL, NULL }, /* registered name */ + { "de_1901", "de__1901", NULL, NULL }, /* registered name */ + { "de_1906", "de__1906", NULL, NULL }, /* registered name */ + { "de__PHONEBOOK", "de", "collation", "phonebook" }, + { "de_AT_PREEURO", "de_AT", "currency", "ATS" }, + { "de_DE_PREEURO", "de_DE", "currency", "DEM" }, + { "de_LU_PREEURO", "de_LU", "currency", "LUF" }, + { "el_GR_PREEURO", "el_GR", "currency", "GRD" }, + { "en_BOONT", "en__BOONT", NULL, NULL }, /* registered name */ + { "en_SCOUSE", "en__SCOUSE", NULL, NULL }, /* registered name */ + { "en_BE_PREEURO", "en_BE", "currency", "BEF" }, + { "en_IE_PREEURO", "en_IE", "currency", "IEP" }, + { "es__TRADITIONAL", "es", "collation", "traditional" }, + { "es_ES_PREEURO", "es_ES", "currency", "ESP" }, + { "eu_ES_PREEURO", "eu_ES", "currency", "ESP" }, + { "fi_FI_PREEURO", "fi_FI", "currency", "FIM" }, + { "fr_BE_PREEURO", "fr_BE", "currency", "BEF" }, + { "fr_FR_PREEURO", "fr_FR", "currency", "FRF" }, + { "fr_LU_PREEURO", "fr_LU", "currency", "LUF" }, + { "ga_IE_PREEURO", "ga_IE", "currency", "IEP" }, + { "gl_ES_PREEURO", "gl_ES", "currency", "ESP" }, + { "hi__DIRECT", "hi", "collation", "direct" }, + { "it_IT_PREEURO", "it_IT", "currency", "ITL" }, + { "ja_JP_TRADITIONAL", "ja_JP", "calendar", "japanese" }, + { "nb_NO_NY", "nn_NO", NULL, NULL }, /* "markus said this was ok" :-) */ + { "nl_BE_PREEURO", "nl_BE", "currency", "BEF" }, + { "nl_NL_PREEURO", "nl_NL", "currency", "NLG" }, + { "pt_PT_PREEURO", "pt_PT", "currency", "PTE" }, + { "sl_ROZAJ", "sl__ROZAJ", NULL, NULL }, /* registered name */ + { "sr_SP_CYRL", "sr_Cyrl_CS", NULL, NULL }, /* .NET name */ + { "sr_SP_LATN", "sr_Latn_CS", NULL, NULL }, /* .NET name */ + { "sr_YU_CYRILLIC", "sr_Cyrl_CS", NULL, NULL }, /* Linux name */ + { "uz_UZ_CYRILLIC", "uz_Cyrl_UZ", NULL, NULL }, /* Linux name */ + { "uz_UZ_CYRL", "uz_Cyrl_UZ", NULL, NULL }, /* .NET name */ + { "uz_UZ_LATN", "uz_Latn_UZ", NULL, NULL }, /* .NET name */ + { "zh_CHS", "zh_Hans", NULL, NULL }, /* .NET name */ + { "zh_CHT", "zh_Hant", NULL, NULL }, /* .NET name TODO: This should be zh_Hant once the locale structure is fixed. */ + { "zh_GAN", "zh__GAN", NULL, NULL }, /* registered name */ + { "zh_GUOYU", "zh", NULL, NULL }, /* registered name */ + { "zh_HAKKA", "zh__HAKKA", NULL, NULL }, /* registered name */ + { "zh_MIN", "zh__MIN", NULL, NULL }, /* registered name */ + { "zh_MIN_NAN", "zh__MINNAN", NULL, NULL }, /* registered name */ + { "zh_WUU", "zh__WUU", NULL, NULL }, /* registered name */ + { "zh_XIANG", "zh__XIANG", NULL, NULL }, /* registered name */ + { "zh_YUE", "zh__YUE", NULL, NULL }, /* registered name */ + { "th_TH_TRADITIONAL", "th_TH", "calendar", "buddhist" }, + { "hi_IN_TRADITIONAL", "hi_IN", "calendar", "indian" }, + { "zh_TW_STROKE", "zh_Hant_TW", "collation", "stroke" }, + { "zh__PINYIN", "zh", "collation", "pinyin" } +}; + +/* ### Keywords **************************************************/ + +#define ULOC_KEYWORD_BUFFER_LEN 25 +#define ULOC_MAX_NO_KEYWORDS 25 + +static const char * +locale_getKeywordsStart(const char *localeID) { + const char *result = NULL; + if((result = uprv_strchr(localeID, '@')) != NULL) { + return result; + } +#if (U_CHARSET_FAMILY == U_EBCDIC_FAMILY) + else { + /* We do this because the @ sign is variant, and the @ sign used on one + EBCDIC machine won't be compiled the same way on other EBCDIC based + machines. */ + static const uint8_t ebcdicSigns[] = { 0x7C, 0x44, 0x66, 0x80, 0xAC, 0xAE, 0xAF, 0xB5, 0xEC, 0xEF, 0x00 }; + const uint8_t *charToFind = ebcdicSigns; + while(*charToFind) { + if((result = uprv_strchr(localeID, *charToFind)) != NULL) { + return result; + } + charToFind++; + } + } +#endif + return NULL; +} + +/** + * @param buf buffer of size [ULOC_KEYWORD_BUFFER_LEN] + * @param keywordName incoming name to be canonicalized + * @param status return status (keyword too long) + * @return length of the keyword name + */ +static int32_t locale_canonKeywordName(char *buf, const char *keywordName, UErrorCode *status) +{ + int32_t i; + int32_t keywordNameLen = (int32_t)uprv_strlen(keywordName); + + if(keywordNameLen >= ULOC_KEYWORD_BUFFER_LEN) { + /* keyword name too long for internal buffer */ + *status = U_INTERNAL_PROGRAM_ERROR; + return 0; + } + + /* normalize the keyword name */ + for(i = 0; i < keywordNameLen; i++) { + buf[i] = uprv_tolower(keywordName[i]); + } + buf[i] = 0; + + return keywordNameLen; +} + +typedef struct { + char keyword[ULOC_KEYWORD_BUFFER_LEN]; + int32_t keywordLen; + const char *valueStart; + int32_t valueLen; +} KeywordStruct; + +static int32_t U_CALLCONV +compareKeywordStructs(const void *context, const void *left, const void *right) { + const char* leftString = ((const KeywordStruct *)left)->keyword; + const char* rightString = ((const KeywordStruct *)right)->keyword; + return uprv_strcmp(leftString, rightString); +} + +/** + * Both addKeyword and addValue must already be in canonical form. + * Either both addKeyword and addValue are NULL, or neither is NULL. + * If they are not NULL they must be zero terminated. + * If addKeyword is not NULL is must have length small enough to fit in KeywordStruct.keyword. + */ +static int32_t +_getKeywords(const char *localeID, + char prev, + char *keywords, int32_t keywordCapacity, + char *values, int32_t valuesCapacity, int32_t *valLen, + UBool valuesToo, + const char* addKeyword, + const char* addValue, + UErrorCode *status) +{ + KeywordStruct keywordList[ULOC_MAX_NO_KEYWORDS]; + + int32_t maxKeywords = ULOC_MAX_NO_KEYWORDS; + int32_t numKeywords = 0; + const char* pos = localeID; + const char* equalSign = NULL; + const char* semicolon = NULL; + int32_t i = 0, j, n; + int32_t keywordsLen = 0; + int32_t valuesLen = 0; + + if(prev == '@') { /* start of keyword definition */ + /* we will grab pairs, trim spaces, lowercase keywords, sort and return */ + do { + UBool duplicate = FALSE; + /* skip leading spaces */ + while(*pos == ' ') { + pos++; + } + if (!*pos) { /* handle trailing "; " */ + break; + } + if(numKeywords == maxKeywords) { + *status = U_INTERNAL_PROGRAM_ERROR; + return 0; + } + equalSign = uprv_strchr(pos, '='); + semicolon = uprv_strchr(pos, ';'); + /* lack of '=' [foo@currency] is illegal */ + /* ';' before '=' [foo@currency;collation=pinyin] is illegal */ + if(!equalSign || (semicolon && semicolon= ULOC_KEYWORD_BUFFER_LEN) { + /* keyword name too long for internal buffer */ + *status = U_INTERNAL_PROGRAM_ERROR; + return 0; + } + for(i = 0, n = 0; i < equalSign - pos; ++i) { + if (pos[i] != ' ') { + keywordList[numKeywords].keyword[n++] = uprv_tolower(pos[i]); + } + } + keywordList[numKeywords].keyword[n] = 0; + keywordList[numKeywords].keywordLen = n; + /* now grab the value part. First we skip the '=' */ + equalSign++; + /* then we leading spaces */ + while(*equalSign == ' ') { + equalSign++; + } + keywordList[numKeywords].valueStart = equalSign; + + pos = semicolon; + i = 0; + if(pos) { + while(*(pos - i - 1) == ' ') { + i++; + } + keywordList[numKeywords].valueLen = (int32_t)(pos - equalSign - i); + pos++; + } else { + i = (int32_t)uprv_strlen(equalSign); + while(equalSign[i-1] == ' ') { + i--; + } + keywordList[numKeywords].valueLen = i; + } + /* If this is a duplicate keyword, then ignore it */ + for (j=0; j1) { + bufLen = (int32_t)uprv_strlen(buffer); + } else { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + if(bufferCapacity= bufferCapacity) { + *status = U_BUFFER_OVERFLOW_ERROR; + return needLen; /* no change */ + } + *startSearchHere = '@'; + startSearchHere++; + uprv_strcpy(startSearchHere, keywordNameBuffer); + startSearchHere += keywordNameLen; + *startSearchHere = '='; + startSearchHere++; + uprv_strcpy(startSearchHere, keywordValue); + startSearchHere+=keywordValueLen; + return needLen; + } /* end shortcut - no @ */ + + keywordStart = startSearchHere; + /* search for keyword */ + while(keywordStart) { + keywordStart++; + /* skip leading spaces (allowed?) */ + while(*keywordStart == ' ') { + keywordStart++; + } + nextEqualsign = uprv_strchr(keywordStart, '='); + /* need to normalize both keyword and keyword name */ + if(!nextEqualsign) { + break; + } + if(nextEqualsign - keywordStart >= ULOC_KEYWORD_BUFFER_LEN) { + /* keyword name too long for internal buffer */ + *status = U_INTERNAL_PROGRAM_ERROR; + return 0; + } + for(i = 0; i < nextEqualsign - keywordStart; i++) { + localeKeywordNameBuffer[i] = uprv_tolower(keywordStart[i]); + } + /* trim trailing spaces */ + while(keywordStart[i-1] == ' ') { + i--; + } + localeKeywordNameBuffer[i] = 0; + + nextSeparator = uprv_strchr(nextEqualsign, ';'); + rc = uprv_strcmp(keywordNameBuffer, localeKeywordNameBuffer); + if(rc == 0) { + nextEqualsign++; + while(*nextEqualsign == ' ') { + nextEqualsign++; + } + /* we actually found the keyword. Change the value */ + if (nextSeparator) { + keywordAtEnd = 0; + foundValueLen = (int32_t)(nextSeparator - nextEqualsign); + } else { + keywordAtEnd = 1; + foundValueLen = (int32_t)uprv_strlen(nextEqualsign); + } + if(keywordValue) { /* adding a value - not removing */ + if(foundValueLen == keywordValueLen) { + uprv_strncpy(nextEqualsign, keywordValue, keywordValueLen); + return bufLen; /* no change in size */ + } else if(foundValueLen > keywordValueLen) { + int32_t delta = foundValueLen - keywordValueLen; + if(nextSeparator) { /* RH side */ + uprv_memmove(nextSeparator - delta, nextSeparator, bufLen-(nextSeparator-buffer)); + } + uprv_strncpy(nextEqualsign, keywordValue, keywordValueLen); + bufLen -= delta; + buffer[bufLen]=0; + return bufLen; + } else { /* FVL < KVL */ + int32_t delta = keywordValueLen - foundValueLen; + if((bufLen+delta) >= bufferCapacity) { + *status = U_BUFFER_OVERFLOW_ERROR; + return bufLen+delta; + } + if(nextSeparator) { /* RH side */ + uprv_memmove(nextSeparator+delta,nextSeparator, bufLen-(nextSeparator-buffer)); + } + uprv_strncpy(nextEqualsign, keywordValue, keywordValueLen); + bufLen += delta; + buffer[bufLen]=0; + return bufLen; + } + } else { /* removing a keyword */ + if(keywordAtEnd) { + /* zero out the ';' or '@' just before startSearchhere */ + keywordStart[-1] = 0; + return (int32_t)((keywordStart-buffer)-1); /* (string length without keyword) minus separator */ + } else { + uprv_memmove(keywordStart, nextSeparator+1, bufLen-((nextSeparator+1)-buffer)); + keywordStart[bufLen-((nextSeparator+1)-buffer)]=0; + return (int32_t)(bufLen-((nextSeparator+1)-keywordStart)); + } + } + } else if(rc<0){ /* end match keyword */ + /* could insert at this location. */ + insertHere = keywordStart; + } + keywordStart = nextSeparator; + } /* end loop searching */ + + if(!keywordValue) { + return bufLen; /* removal of non-extant keyword - no change */ + } + + /* we know there is at least one keyword. */ + needLen = bufLen+1+keywordNameLen+1+keywordValueLen; + if(needLen >= bufferCapacity) { + *status = U_BUFFER_OVERFLOW_ERROR; + return needLen; /* no change */ + } + + if(insertHere) { + uprv_memmove(insertHere+(1+keywordNameLen+1+keywordValueLen), insertHere, bufLen-(insertHere-buffer)); + keywordStart = insertHere; + } else { + keywordStart = buffer+bufLen; + *keywordStart = ';'; + keywordStart++; + } + uprv_strncpy(keywordStart, keywordNameBuffer, keywordNameLen); + keywordStart += keywordNameLen; + *keywordStart = '='; + keywordStart++; + uprv_strncpy(keywordStart, keywordValue, keywordValueLen); /* terminates. */ + keywordStart+=keywordValueLen; + if(insertHere) { + *keywordStart = ';'; + keywordStart++; + } + buffer[needLen]=0; + return needLen; +} + +/* ### ID parsing implementation **************************************************/ + +/*returns TRUE if a is an ID separator FALSE otherwise*/ +#define _isIDSeparator(a) (a == '_' || a == '-') + +#define _isPrefixLetter(a) ((a=='x')||(a=='X')||(a=='i')||(a=='I')) + +/*returns TRUE if one of the special prefixes is here (s=string) + 'x-' or 'i-' */ +#define _isIDPrefix(s) (_isPrefixLetter(s[0])&&_isIDSeparator(s[1])) + +/* Dot terminates it because of POSIX form where dot precedes the codepage + * except for variant + */ +#define _isTerminator(a) ((a==0)||(a=='.')||(a=='@')) + +static char* _strnchr(const char* str, int32_t len, char c) { + U_ASSERT(str != 0 && len >= 0); + while (len-- != 0) { + char d = *str; + if (d == c) { + return (char*) str; + } else if (d == 0) { + break; + } + ++str; + } + return NULL; +} + +/** + * Lookup 'key' in the array 'list'. The array 'list' should contain + * a NULL entry, followed by more entries, and a second NULL entry. + * + * The 'list' param should be LANGUAGES, LANGUAGES_3, COUNTRIES, or + * COUNTRIES_3. + */ +static int16_t _findIndex(const char* const* list, const char* key) +{ + const char* const* anchor = list; + int32_t pass = 0; + + /* Make two passes through two NULL-terminated arrays at 'list' */ + while (pass++ < 2) { + while (*list) { + if (uprv_strcmp(key, *list) == 0) { + return (int16_t)(list - anchor); + } + list++; + } + ++list; /* skip final NULL *CWB*/ + } + return -1; +} + +/* count the length of src while copying it to dest; return strlen(src) */ +static U_INLINE int32_t +_copyCount(char *dest, int32_t destCapacity, const char *src) { + const char *anchor; + char c; + + anchor=src; + for(;;) { + if((c=*src)==0) { + return (int32_t)(src-anchor); + } + if(destCapacity<=0) { + return (int32_t)((src-anchor)+uprv_strlen(src)); + } + ++src; + *dest++=c; + --destCapacity; + } +} + +static const char* +uloc_getCurrentCountryID(const char* oldID){ + int32_t offset = _findIndex(DEPRECATED_COUNTRIES, oldID); + if (offset >= 0) { + return REPLACEMENT_COUNTRIES[offset]; + } + return oldID; +} +static const char* +uloc_getCurrentLanguageID(const char* oldID){ + int32_t offset = _findIndex(DEPRECATED_LANGUAGES, oldID); + if (offset >= 0) { + return REPLACEMENT_LANGUAGES[offset]; + } + return oldID; +} +/* + * the internal functions _getLanguage(), _getCountry(), _getVariant() + * avoid duplicating code to handle the earlier locale ID pieces + * in the functions for the later ones by + * setting the *pEnd pointer to where they stopped parsing + * + * TODO try to use this in Locale + */ +static int32_t +_getLanguage(const char *localeID, + char *language, int32_t languageCapacity, + const char **pEnd) { + int32_t i=0; + int32_t offset; + char lang[4]={ 0, 0, 0, 0 }; /* temporary buffer to hold language code for searching */ + + /* if it starts with i- or x- then copy that prefix */ + if(_isIDPrefix(localeID)) { + if(i=0) { + i=_copyCount(language, languageCapacity, LANGUAGES[offset]); + } + } + + if(pEnd!=NULL) { + *pEnd=localeID; + } + return i; +} + +static int32_t +_getScript(const char *localeID, + char *script, int32_t scriptCapacity, + const char **pEnd) +{ + int32_t idLen = 0; + + if (pEnd != NULL) { + *pEnd = localeID; + } + + /* copy the second item as far as possible and count its length */ + while(!_isTerminator(localeID[idLen]) && !_isIDSeparator(localeID[idLen])) { + idLen++; + } + + /* If it's exactly 4 characters long, then it's a script and not a country. */ + if (idLen == 4) { + int32_t i; + if (pEnd != NULL) { + *pEnd = localeID+idLen; + } + if(idLen > scriptCapacity) { + idLen = scriptCapacity; + } + if (idLen >= 1) { + script[0]=(char)uprv_toupper(*(localeID++)); + } + for (i = 1; i < idLen; i++) { + script[i]=(char)uprv_tolower(*(localeID++)); + } + } + else { + idLen = 0; + } + return idLen; +} + +static int32_t +_getCountry(const char *localeID, + char *country, int32_t countryCapacity, + const char **pEnd) +{ + int32_t i=0; + char cnty[ULOC_COUNTRY_CAPACITY]={ 0, 0, 0, 0 }; + int32_t offset; + + /* copy the country as far as possible and count its length */ + while(!_isTerminator(*localeID) && !_isIDSeparator(*localeID)) { + if(i=0) { + i=_copyCount(country, countryCapacity, COUNTRIES[offset]); + } + } + + if(pEnd!=NULL) { + *pEnd=localeID; + } + return i; +} + +/** + * @param needSeparator if true, then add leading '_' if any variants + * are added to 'variant' + */ +static int32_t +_getVariantEx(const char *localeID, + char prev, + char *variant, int32_t variantCapacity, + UBool needSeparator) { + int32_t i=0; + + /* get one or more variant tags and separate them with '_' */ + if(_isIDSeparator(prev)) { + /* get a variant string after a '-' or '_' */ + while(!_isTerminator(*localeID)) { + if (needSeparator) { + if (i "FOO_BAR". + * @param variants the source string of one or more variants, + * separated by '_'. This will be MODIFIED IN PLACE. Not zero + * terminated; if it is, trailing zero will NOT be maintained. + * @param variantsLen length of variants + * @param toDelete variant to delete, without separators, e.g. "EURO" + * or "PREEURO"; not zero terminated + * @param toDeleteLen length of toDelete + * @return number of characters deleted from variants + */ +static int32_t +_deleteVariant(char* variants, int32_t variantsLen, + const char* toDelete, int32_t toDeleteLen) { + int32_t delta = 0; /* number of chars deleted */ + for (;;) { + UBool flag = FALSE; + if (variantsLen < toDeleteLen) { + return delta; + } + if (uprv_strncmp(variants, toDelete, toDeleteLen) == 0 && + (variantsLen == toDeleteLen || + (flag=(variants[toDeleteLen] == '_')))) { + int32_t d = toDeleteLen + (flag?1:0); + variantsLen -= d; + delta += d; + if (variantsLen > 0) { + uprv_memmove(variants, variants+d, variantsLen); + } + } else { + char* p = _strnchr(variants, variantsLen, '_'); + if (p == NULL) { + return delta; + } + ++p; + variantsLen -= (int32_t)(p - variants); + variants = p; + } + } +} + +/* Keyword enumeration */ + +typedef struct UKeywordsContext { + char* keywords; + char* current; +} UKeywordsContext; + +static void U_CALLCONV +uloc_kw_closeKeywords(UEnumeration *enumerator) { + uprv_free(((UKeywordsContext *)enumerator->context)->keywords); + uprv_free(enumerator->context); + uprv_free(enumerator); +} + +static int32_t U_CALLCONV +uloc_kw_countKeywords(UEnumeration *en, UErrorCode *status) { + char *kw = ((UKeywordsContext *)en->context)->keywords; + int32_t result = 0; + while(*kw) { + result++; + kw += uprv_strlen(kw)+1; + } + return result; +} + +static const char* U_CALLCONV +uloc_kw_nextKeyword(UEnumeration* en, + int32_t* resultLength, + UErrorCode* status) { + const char* result = ((UKeywordsContext *)en->context)->current; + int32_t len = 0; + if(*result) { + len = (int32_t)uprv_strlen(((UKeywordsContext *)en->context)->current); + ((UKeywordsContext *)en->context)->current += len+1; + } else { + result = NULL; + } + if (resultLength) { + *resultLength = len; + } + return result; +} + +static void U_CALLCONV +uloc_kw_resetKeywords(UEnumeration* en, + UErrorCode* status) { + ((UKeywordsContext *)en->context)->current = ((UKeywordsContext *)en->context)->keywords; +} + +static const UEnumeration gKeywordsEnum = { + NULL, + NULL, + uloc_kw_closeKeywords, + uloc_kw_countKeywords, + uenum_unextDefault, + uloc_kw_nextKeyword, + uloc_kw_resetKeywords +}; + +U_CAPI UEnumeration* U_EXPORT2 +uloc_openKeywordList(const char *keywordList, int32_t keywordListSize, UErrorCode* status) +{ + UKeywordsContext *myContext = NULL; + UEnumeration *result = NULL; + + if(U_FAILURE(*status)) { + return NULL; + } + result = (UEnumeration *)uprv_malloc(sizeof(UEnumeration)); + uprv_memcpy(result, &gKeywordsEnum, sizeof(UEnumeration)); + myContext = uprv_malloc(sizeof(UKeywordsContext)); + if (myContext == NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + uprv_free(result); + return NULL; + } + myContext->keywords = (char *)uprv_malloc(keywordListSize+1); + uprv_memcpy(myContext->keywords, keywordList, keywordListSize); + myContext->keywords[keywordListSize] = 0; + myContext->current = myContext->keywords; + result->context = myContext; + return result; +} + +U_CAPI UEnumeration* U_EXPORT2 +uloc_openKeywords(const char* localeID, + UErrorCode* status) +{ + int32_t i=0; + char keywords[256]; + int32_t keywordsCapacity = 256; + if(status==NULL || U_FAILURE(*status)) { + return 0; + } + + if(localeID==NULL) { + localeID=uloc_getDefault(); + } + + /* Skip the language */ + _getLanguage(localeID, NULL, 0, &localeID); + if(_isIDSeparator(*localeID)) { + const char *scriptID; + /* Skip the script if available */ + _getScript(localeID+1, NULL, 0, &scriptID); + if(scriptID != localeID+1) { + /* Found optional script */ + localeID = scriptID; + } + /* Skip the Country */ + if (_isIDSeparator(*localeID)) { + _getCountry(localeID+1, NULL, 0, &localeID); + if(_isIDSeparator(*localeID)) { + _getVariant(localeID+1, *localeID, NULL, 0); + } + } + } + + /* keywords are located after '@' */ + if((localeID = locale_getKeywordsStart(localeID)) != NULL) { + i=locale_getKeywords(localeID+1, '@', keywords, keywordsCapacity, NULL, 0, NULL, FALSE, status); + } + + if(i) { + return uloc_openKeywordList(keywords, i, status); + } else { + return NULL; + } +} + + +/* bit-flags for 'options' parameter of _canonicalize */ +#define _ULOC_STRIP_KEYWORDS 0x2 +#define _ULOC_CANONICALIZE 0x1 + +#define OPTION_SET(options, mask) ((options & mask) != 0) + +static const char i_default[] = {'i', '-', 'd', 'e', 'f', 'a', 'u', 'l', 't'}; +#define I_DEFAULT_LENGTH (sizeof i_default / sizeof i_default[0]) + +/** + * Canonicalize the given localeID, to level 1 or to level 2, + * depending on the options. To specify level 1, pass in options=0. + * To specify level 2, pass in options=_ULOC_CANONICALIZE. + * + * This is the code underlying uloc_getName and uloc_canonicalize. + */ +static int32_t +_canonicalize(const char* localeID, + char* result, + int32_t resultCapacity, + uint32_t options, + UErrorCode* err) { + int32_t j, len, fieldCount=0, scriptSize=0, variantSize=0, nameCapacity; + char localeBuffer[ULOC_FULLNAME_CAPACITY]; + const char* origLocaleID; + const char* keywordAssign = NULL; + const char* separatorIndicator = NULL; + const char* addKeyword = NULL; + const char* addValue = NULL; + char* name; + char* variant = NULL; /* pointer into name, or NULL */ + int32_t sawEuro = 0; + + if (U_FAILURE(*err)) { + return 0; + } + + if (localeID==NULL) { + localeID=uloc_getDefault(); + } + origLocaleID=localeID; + + /* if we are doing a full canonicalization, then put results in + localeBuffer, if necessary; otherwise send them to result. */ + if (OPTION_SET(options, _ULOC_CANONICALIZE) && + (result == NULL || resultCapacity < sizeof(localeBuffer))) { + name = localeBuffer; + nameCapacity = sizeof(localeBuffer); + } else { + name = result; + nameCapacity = resultCapacity; + } + + /* get all pieces, one after another, and separate with '_' */ + len=_getLanguage(localeID, name, nameCapacity, &localeID); + + if(len == I_DEFAULT_LENGTH && uprv_strncmp(origLocaleID, i_default, len) == 0) { + const char *d = uloc_getDefault(); + + len = uprv_strlen(d); + + if (name != NULL) { + uprv_strncpy(name, d, len); + } + } else if(_isIDSeparator(*localeID)) { + const char *scriptID; + + ++fieldCount; + if(len 0) { + /* Found optional script */ + localeID = scriptID; + ++fieldCount; + len+=scriptSize; + if (_isIDSeparator(*localeID)) { + /* If there is something else, then we add the _ */ + if(len 0) { + variant = name+len; + len += variantSize; + localeID += variantSize + 1; /* skip '_' and variant */ + } + } + } + } + + /* Copy POSIX-style charset specifier, if any [mr.utf8] */ + if (!OPTION_SET(options, _ULOC_CANONICALIZE) && *localeID == '.') { + UBool done = FALSE; + do { + char c = *localeID; + switch (c) { + case 0: + case '@': + done = TRUE; + break; + default: + if (len 0)) { + do { + if(len 0)); + if (posixVariantSize > 0) { + if (variant == NULL) { + variant = name+len; + } + len += posixVariantSize; + variantSize += posixVariantSize; + } + } + + /* Check for EURO variants. */ + sawEuro = _deleteVariant(variant, uprv_min(variantSize, (nameCapacity-len)), "EURO", 4); + len -= sawEuro; + if (sawEuro > 0 && name[len-1] == '_') { /* delete trailing '_' */ + --len; + } + + /* Look up the ID in the canonicalization map */ + for (j=0; j<(int32_t)(sizeof(CANONICALIZE_MAP)/sizeof(CANONICALIZE_MAP[0])); j++) { + const char* id = CANONICALIZE_MAP[j].id; + int32_t n = (int32_t)uprv_strlen(id); + if (len == n && uprv_strncmp(name, id, n) == 0) { + if (n == 0 && localeID != NULL) { + break; /* Don't remap "" if keywords present */ + } + len = _copyCount(name, nameCapacity, CANONICALIZE_MAP[j].canonicalID); + addKeyword = CANONICALIZE_MAP[j].keyword; + addValue = CANONICALIZE_MAP[j].value; + break; + } + } + + /* Explicit EURO variant overrides keyword in CANONICALIZE_MAP */ + if (sawEuro > 0) { + addKeyword = "currency"; + addValue = "EUR"; + } + } + + if (!OPTION_SET(options, _ULOC_STRIP_KEYWORDS)) { + if (localeID!=NULL && keywordAssign!=NULL && + (!separatorIndicator || separatorIndicator > keywordAssign)) { + if(len localeCapacity) { + *status = U_BUFFER_OVERFLOW_ERROR; + } + else { + uprv_strcpy(locale, posix); + } + return length; +} + +/* ### Default locale **************************************************/ + +U_CAPI const char* U_EXPORT2 +uloc_getDefault() +{ + return locale_get_default(); +} + +U_CAPI void U_EXPORT2 +uloc_setDefault(const char* newDefaultLocale, + UErrorCode* err) +{ + if (U_FAILURE(*err)) + return; + /* the error code isn't currently used for anything by this function*/ + + /* propagate change to C++ */ + locale_set_default(newDefaultLocale); +} + +/* ### Display name **************************************************/ + +/* + * Lookup a resource bundle table item with fallback on the table level. + * Regular resource bundle lookups perform fallback to parent locale bundles + * and eventually the root bundle, but only for top-level items. + * This function takes the name of a top-level table and of an item in that table + * and performs a lookup of both, falling back until a bundle contains a table + * with this item. + * + * Note: Only the opening of entire bundles falls back through the default locale + * before root. Once a bundle is open, item lookups do not go through the + * default locale because that would result in a mix of languages that is + * unpredictable to the programmer and most likely useless. + */ +static const UChar * +_res_getTableStringWithFallback(const char *path, const char *locale, + const char *tableKey, const char *subTableKey, + const char *itemKey, + int32_t *pLength, + UErrorCode *pErrorCode) +{ +/* char localeBuffer[ULOC_FULLNAME_CAPACITY*4];*/ + UResourceBundle *rb=NULL, table, subTable; + const UChar *item=NULL; + UErrorCode errorCode; + char explicitFallbackName[ULOC_FULLNAME_CAPACITY] = {0}; + + /* + * open the bundle for the current locale + * this falls back through the locale's chain to root + */ + errorCode=U_ZERO_ERROR; + rb=ures_open(path, locale, &errorCode); + if(U_FAILURE(errorCode)) { + /* total failure, not even root could be opened */ + *pErrorCode=errorCode; + return NULL; + } else if(errorCode==U_USING_DEFAULT_WARNING || + (errorCode==U_USING_FALLBACK_WARNING && *pErrorCode!=U_USING_DEFAULT_WARNING) + ) { + /* set the "strongest" error code (success->fallback->default->failure) */ + *pErrorCode=errorCode; + } + + for(;;){ + ures_initStackObject(&table); + ures_initStackObject(&subTable); + ures_getByKeyWithFallback(rb, tableKey, &table, &errorCode); + if (subTableKey != NULL) { + /* + ures_getByKeyWithFallback(&table,subTableKey, &subTable, &errorCode); + item = ures_getStringByKeyWithFallback(&subTable, itemKey, pLength, &errorCode); + if(U_FAILURE(errorCode)){ + *pErrorCode = errorCode; + } + + break;*/ + + ures_getByKeyWithFallback(&table,subTableKey, &table, &errorCode); + } + if(U_SUCCESS(errorCode)){ + item = ures_getStringByKeyWithFallback(&table, itemKey, pLength, &errorCode); + if(U_FAILURE(errorCode)){ + const char* replacement = NULL; + *pErrorCode = errorCode; /*save the errorCode*/ + errorCode = U_ZERO_ERROR; + /* may be a deprecated code */ + if(uprv_strcmp(tableKey, "Countries")==0){ + replacement = uloc_getCurrentCountryID(itemKey); + }else if(uprv_strcmp(tableKey, "Languages")==0){ + replacement = uloc_getCurrentLanguageID(itemKey); + } + /*pointer comparison is ok since uloc_getCurrentCountryID & uloc_getCurrentLanguageID return the key itself is replacement is not found*/ + if(replacement!=NULL && itemKey != replacement){ + item = ures_getStringByKeyWithFallback(&table, replacement, pLength, &errorCode); + if(U_SUCCESS(errorCode)){ + *pErrorCode = errorCode; + break; + } + } + }else{ + break; + } + } + + if(U_FAILURE(errorCode)){ + + /* still can't figure out ?.. try the fallback mechanism */ + int32_t len = 0; + const UChar* fallbackLocale = NULL; + *pErrorCode = errorCode; + errorCode = U_ZERO_ERROR; + + fallbackLocale = ures_getStringByKeyWithFallback(&table, "Fallback", &len, &errorCode); + if(U_FAILURE(errorCode)){ + *pErrorCode = errorCode; + break; + } + + u_UCharsToChars(fallbackLocale, explicitFallbackName, len); + + /* guard against recursive fallback */ + if(uprv_strcmp(explicitFallbackName, locale)==0){ + *pErrorCode = U_INTERNAL_PROGRAM_ERROR; + break; + } + ures_close(rb); + rb = ures_open(NULL, explicitFallbackName, &errorCode); + if(U_FAILURE(errorCode)){ + *pErrorCode = errorCode; + break; + } + /* succeeded in opening the fallback bundle .. continue and try to fetch the item */ + }else{ + break; + } + } + /* done with the locale string - ready to close table and rb */ + ures_close(&subTable); + ures_close(&table); + ures_close(rb); + return item; +} + +static int32_t +_getStringOrCopyKey(const char *path, const char *locale, + const char *tableKey, + const char* subTableKey, + const char *itemKey, + const char *substitute, + UChar *dest, int32_t destCapacity, + UErrorCode *pErrorCode) { + const UChar *s = NULL; + int32_t length = 0; + + if(itemKey==NULL) { + /* top-level item: normal resource bundle access */ + UResourceBundle *rb; + + rb=ures_open(path, locale, pErrorCode); + if(U_SUCCESS(*pErrorCode)) { + s=ures_getStringByKey(rb, tableKey, &length, pErrorCode); + /* see comment about closing rb near "return item;" in _res_getTableStringWithFallback() */ + ures_close(rb); + } + } else { + /* second-level item, use special fallback */ + s=_res_getTableStringWithFallback(path, locale, + tableKey, + subTableKey, + itemKey, + &length, + pErrorCode); + } + if(U_SUCCESS(*pErrorCode)) { + int32_t copyLength=uprv_min(length, destCapacity); + if(copyLength>0 && s != NULL) { + u_memcpy(dest, s, copyLength); + } + } else { + /* no string from a resource bundle: convert the substitute */ + length=(int32_t)uprv_strlen(substitute); + u_charsToUChars(substitute, dest, uprv_min(length, destCapacity)); + *pErrorCode=U_USING_DEFAULT_WARNING; + } + + return u_terminateUChars(dest, destCapacity, length, pErrorCode); +} + +static int32_t +_getDisplayNameForComponent(const char *locale, + const char *displayLocale, + UChar *dest, int32_t destCapacity, + int32_t (*getter)(const char *, char *, int32_t, UErrorCode *), + const char *tag, + UErrorCode *pErrorCode) { + char localeBuffer[ULOC_FULLNAME_CAPACITY*4]; + int32_t length; + UErrorCode localStatus; + + /* argument checking */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + if(destCapacity<0 || (destCapacity>0 && dest==NULL)) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + localStatus = U_ZERO_ERROR; + length=(*getter)(locale, localeBuffer, sizeof(localeBuffer), &localStatus); + if(U_FAILURE(localStatus) || localStatus==U_STRING_NOT_TERMINATED_WARNING) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + if(length==0) { + return u_terminateUChars(dest, destCapacity, 0, pErrorCode); + } + + return _getStringOrCopyKey(NULL, displayLocale, + tag, NULL, localeBuffer, + localeBuffer, + dest, destCapacity, + pErrorCode); +} + +U_CAPI int32_t U_EXPORT2 +uloc_getDisplayLanguage(const char *locale, + const char *displayLocale, + UChar *dest, int32_t destCapacity, + UErrorCode *pErrorCode) { + return _getDisplayNameForComponent(locale, displayLocale, dest, destCapacity, + uloc_getLanguage, _kLanguages, pErrorCode); +} + +U_CAPI int32_t U_EXPORT2 +uloc_getDisplayScript(const char* locale, + const char* displayLocale, + UChar *dest, int32_t destCapacity, + UErrorCode *pErrorCode) +{ + return _getDisplayNameForComponent(locale, displayLocale, dest, destCapacity, + uloc_getScript, _kScripts, pErrorCode); +} + +U_CAPI int32_t U_EXPORT2 +uloc_getDisplayCountry(const char *locale, + const char *displayLocale, + UChar *dest, int32_t destCapacity, + UErrorCode *pErrorCode) { + return _getDisplayNameForComponent(locale, displayLocale, dest, destCapacity, + uloc_getCountry, _kCountries, pErrorCode); +} + +/* + * TODO separate variant1_variant2_variant3... + * by getting each tag's display string and concatenating them with ", " + * in between - similar to uloc_getDisplayName() + */ +U_CAPI int32_t U_EXPORT2 +uloc_getDisplayVariant(const char *locale, + const char *displayLocale, + UChar *dest, int32_t destCapacity, + UErrorCode *pErrorCode) { + return _getDisplayNameForComponent(locale, displayLocale, dest, destCapacity, + uloc_getVariant, _kVariants, pErrorCode); +} + +U_CAPI int32_t U_EXPORT2 +uloc_getDisplayName(const char *locale, + const char *displayLocale, + UChar *dest, int32_t destCapacity, + UErrorCode *pErrorCode) +{ + int32_t length, length2, length3 = 0; + UBool hasLanguage, hasScript, hasCountry, hasVariant, hasKeywords; + UEnumeration* keywordEnum = NULL; + int32_t keywordCount = 0; + const char *keyword = NULL; + int32_t keywordLen = 0; + char keywordValue[256]; + int32_t keywordValueLen = 0; + + /* argument checking */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + if(destCapacity<0 || (destCapacity>0 && dest==NULL)) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* + * if there is a language, then write "language (country, variant)" + * otherwise write "country, variant" + */ + + /* write the language */ + length=uloc_getDisplayLanguage(locale, displayLocale, + dest, destCapacity, + pErrorCode); + hasLanguage= length>0; + + if(hasLanguage) { + /* append " (" */ + if(length0; + length+=length2; + + if(hasScript) { + /* append ", " */ + if(length0; + length+=length2; + + if(hasCountry) { + /* append ", " */ + if(length0; + length+=length2; + + if(hasVariant) { + /* append ", " */ + if(length 0 ; keywordCount--){ + if(U_FAILURE(*pErrorCode)){ + break; + } + /* the uenum_next returns NUL terminated string */ + keyword = uenum_next(keywordEnum, &keywordLen, pErrorCode); + if(length + length3 < destCapacity) { + length3 += uloc_getDisplayKeyword(keyword, displayLocale, dest+length+length3, destCapacity-length-length3, pErrorCode); + } else { + length3 += uloc_getDisplayKeyword(keyword, displayLocale, NULL, 0, pErrorCode); + } + if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) { + /* keep preflighting */ + *pErrorCode=U_ZERO_ERROR; + } + keywordValueLen = uloc_getKeywordValue(locale, keyword, keywordValue, 256, pErrorCode); + if(keywordValueLen) { + if(length + length3 < destCapacity) { + dest[length + length3] = 0x3D; + } + length3++; + if(length + length3 < destCapacity) { + length3 += uloc_getDisplayKeywordValue(locale, keyword, displayLocale, dest+length+length3, destCapacity-length-length3, pErrorCode); + } else { + length3 += uloc_getDisplayKeywordValue(locale, keyword, displayLocale, NULL, 0, pErrorCode); + } + if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) { + /* keep preflighting */ + *pErrorCode=U_ZERO_ERROR; + } + } + if(keywordCount > 1) { + if(length + length3 + 1 < destCapacity && keywordCount) { + dest[length + length3]=0x2c; + dest[length + length3+1]=0x20; + } + length3++; /* ',' */ + length3++; /* ' ' */ + } + } + uenum_close(keywordEnum); + + hasKeywords = length3 > 0; + length += length3; + + + + if ((hasScript && !hasCountry) + || ((hasScript || hasCountry) && !hasVariant && !hasKeywords) + || ((hasScript || hasCountry || hasVariant) && !hasKeywords) + || (hasLanguage && !hasScript && !hasCountry && !hasVariant && !hasKeywords)) + { + /* remove ", " or " (" */ + length-=2; + } + + if (hasLanguage && (hasScript || hasCountry || hasVariant || hasKeywords)) { + /* append ")" */ + if(length0 && dest==NULL)) { + *status=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + + /* pass itemKey=NULL to look for a top-level item */ + return _getStringOrCopyKey(NULL, displayLocale, + _kKeys, NULL, + keyword, + keyword, + dest, destCapacity, + status); + +} + + +#define UCURRENCY_DISPLAY_NAME_INDEX 1 + +U_CAPI int32_t U_EXPORT2 +uloc_getDisplayKeywordValue( const char* locale, + const char* keyword, + const char* displayLocale, + UChar* dest, + int32_t destCapacity, + UErrorCode* status){ + + + char keywordValue[ULOC_FULLNAME_CAPACITY*4]; + int32_t capacity = ULOC_FULLNAME_CAPACITY*4; + int32_t keywordValueLen =0; + + /* argument checking */ + if(status==NULL || U_FAILURE(*status)) { + return 0; + } + + if(destCapacity<0 || (destCapacity>0 && dest==NULL)) { + *status=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* get the keyword value */ + keywordValue[0]=0; + keywordValueLen = uloc_getKeywordValue(locale, keyword, keywordValue, capacity, status); + + /* + * if the keyword is equal to currency .. then to get the display name + * we need to do the fallback ourselves + */ + if(uprv_stricmp(keyword, _kCurrency)==0){ + + int32_t dispNameLen = 0; + const UChar *dispName = NULL; + + UResourceBundle *bundle = ures_open(NULL, displayLocale, status); + UResourceBundle *currencies = ures_getByKey(bundle, _kCurrencies, NULL, status); + UResourceBundle *currency = ures_getByKeyWithFallback(currencies, keywordValue, NULL, status); + + dispName = ures_getStringByIndex(currency, UCURRENCY_DISPLAY_NAME_INDEX, &dispNameLen, status); + + /*close the bundles */ + ures_close(currency); + ures_close(currencies); + ures_close(bundle); + + if(U_FAILURE(*status)){ + if(*status == U_MISSING_RESOURCE_ERROR){ + /* we just want to write the value over if nothing is available */ + *status = U_USING_DEFAULT_WARNING; + }else{ + return 0; + } + } + + /* now copy the dispName over if not NULL */ + if(dispName != NULL){ + if(dispNameLen <= destCapacity){ + uprv_memcpy(dest, dispName, dispNameLen * U_SIZEOF_UCHAR); + return u_terminateUChars(dest, destCapacity, dispNameLen, status); + }else{ + *status = U_BUFFER_OVERFLOW_ERROR; + return dispNameLen; + } + }else{ + /* we have not found the display name for the value .. just copy over */ + if(keywordValueLen <= destCapacity){ + u_charsToUChars(keywordValue, dest, keywordValueLen); + return u_terminateUChars(dest, destCapacity, keywordValueLen, status); + }else{ + *status = U_BUFFER_OVERFLOW_ERROR; + return keywordValueLen; + } + } + + + }else{ + + return _getStringOrCopyKey(NULL, displayLocale, + _kTypes, keyword, + keywordValue, + keywordValue, + dest, destCapacity, + status); + } +} + +/* ### Get available **************************************************/ + +static UBool U_CALLCONV uloc_cleanup(void) { + char ** temp; + + if (_installedLocales) { + temp = _installedLocales; + _installedLocales = NULL; + + _installedLocalesCount = 0; + + uprv_free(temp); + } + return TRUE; +} + +static void _load_installedLocales() +{ + UBool localesLoaded; + + UMTX_CHECK(NULL, _installedLocales != NULL, localesLoaded); + + if (localesLoaded == FALSE) { + UResourceBundle *index = NULL; + UResourceBundle installed; + UErrorCode status = U_ZERO_ERROR; + char ** temp; + int32_t i = 0; + int32_t localeCount; + + ures_initStackObject(&installed); + index = ures_openDirect(NULL, _kIndexLocaleName, &status); + ures_getByKey(index, _kIndexTag, &installed, &status); + + if(U_SUCCESS(status)) { + localeCount = ures_getSize(&installed); + temp = (char **) uprv_malloc(sizeof(char*) * (localeCount+1)); + + ures_resetIterator(&installed); + while(ures_hasNext(&installed)) { + ures_getNextString(&installed, NULL, (const char **)&temp[i++], &status); + } + temp[i] = NULL; + + umtx_lock(NULL); + if (_installedLocales == NULL) + { + _installedLocales = temp; + _installedLocalesCount = localeCount; + temp = NULL; + ucln_common_registerCleanup(UCLN_COMMON_ULOC, uloc_cleanup); + } + umtx_unlock(NULL); + + uprv_free(temp); + ures_close(&installed); + } + ures_close(index); + } +} + +U_CAPI const char* U_EXPORT2 +uloc_getAvailable(int32_t offset) +{ + + _load_installedLocales(); + + if (offset > _installedLocalesCount) + return NULL; + return _installedLocales[offset]; +} + +U_CAPI int32_t U_EXPORT2 +uloc_countAvailable() +{ + _load_installedLocales(); + return _installedLocalesCount; +} + +/** + * Returns a list of all language codes defined in ISO 639. This is a pointer + * to an array of pointers to arrays of char. All of these pointers are owned + * by ICU-- do not delete them, and do not write through them. The array is + * terminated with a null pointer. + */ +U_CAPI const char* const* U_EXPORT2 +uloc_getISOLanguages() +{ + return LANGUAGES; +} + +/** + * Returns a list of all 2-letter country codes defined in ISO 639. This is a + * pointer to an array of pointers to arrays of char. All of these pointers are + * owned by ICU-- do not delete them, and do not write through them. The array is + * terminated with a null pointer. + */ +U_CAPI const char* const* U_EXPORT2 +uloc_getISOCountries() +{ + return COUNTRIES; +} + + +/* this function to be moved into cstring.c later */ +static char gDecimal = 0; + +static /* U_CAPI */ +double +/* U_EXPORT2 */ +_uloc_strtod(const char *start, char **end) { + char *decimal; + char *myEnd; + char buf[30]; + double rv; + if (!gDecimal) { + char rep[5]; + /* For machines that decide to change the decimal on you, + and try to be too smart with localization. + This normally should be just a '.'. */ + sprintf(rep, "%+1.1f", 1.0); + gDecimal = rep[2]; + } + + if(gDecimal == '.') { + return uprv_strtod(start, end); /* fall through to OS */ + } else { + uprv_strncpy(buf, start, 29); + buf[29]=0; + decimal = uprv_strchr(buf, '.'); + if(decimal) { + *decimal = gDecimal; + } else { + return uprv_strtod(start, end); /* no decimal point */ + } + rv = uprv_strtod(buf, &myEnd); + if(end) { + *end = (char*)(start+(myEnd-buf)); /* cast away const (to follow uprv_strtod API.) */ + } + return rv; + } +} + +typedef struct { + float q; + int32_t dummy; /* to avoid uninitialized memory copy from qsort */ + char *locale; +} _acceptLangItem; + +static int32_t U_CALLCONV +uloc_acceptLanguageCompare(const void *context, const void *a, const void *b) +{ + const _acceptLangItem *aa = (const _acceptLangItem*)a; + const _acceptLangItem *bb = (const _acceptLangItem*)b; + + int32_t rc = 0; + if(bb->q < aa->q) { + rc = -1; /* A > B */ + } else if(bb->q > aa->q) { + rc = 1; /* A < B */ + } else { + rc = 0; /* A = B */ + } + + if(rc==0) { + rc = uprv_stricmp(aa->locale, bb->locale); + } + +#if defined(ULOC_DEBUG) + /* fprintf(stderr, "a:[%s:%g], b:[%s:%g] -> %d\n", + aa->locale, aa->q, + bb->locale, bb->q, + rc);*/ +#endif + + return rc; +} + +/* +mt-mt, ja;q=0.76, en-us;q=0.95, en;q=0.92, en-gb;q=0.89, fr;q=0.87, iu-ca;q=0.84, iu;q=0.82, ja-jp;q=0.79, mt;q=0.97, de-de;q=0.74, de;q=0.71, es;q=0.68, it-it;q=0.66, it;q=0.63, vi-vn;q=0.61, vi;q=0.58, nl-nl;q=0.55, nl;q=0.53 +*/ + +U_CAPI int32_t U_EXPORT2 +uloc_acceptLanguageFromHTTP(char *result, int32_t resultAvailable, UAcceptResult *outResult, + const char *httpAcceptLanguage, + UEnumeration* availableLocales, + UErrorCode *status) +{ + _acceptLangItem *j; + _acceptLangItem smallBuffer[30]; + char **strs; + char tmp[ULOC_FULLNAME_CAPACITY +1]; + int32_t n = 0; + const char *itemEnd; + const char *paramEnd; + const char *s; + const char *t; + int32_t res; + int32_t i; + int32_t l = (int32_t)uprv_strlen(httpAcceptLanguage); + int32_t jSize; + + j = smallBuffer; + jSize = sizeof(smallBuffer)/sizeof(smallBuffer[0]); + if(U_FAILURE(*status)) { + return -1; + } + + for(s=httpAcceptLanguage;s&&*s;) { + while(isspace(*s)) /* eat space at the beginning */ + s++; + itemEnd=uprv_strchr(s,','); + paramEnd=uprv_strchr(s,';'); + if(!itemEnd) { + itemEnd = httpAcceptLanguage+l; /* end of string */ + } + if(paramEnd && paramEnds)&&isspace(*t);t--) + ; + j[n].locale = uprv_strndup(s,(int32_t)((t+1)-s)); + uloc_canonicalize(j[n].locale,tmp,sizeof(tmp)/sizeof(tmp[0]),status); + if(strcmp(j[n].locale,tmp)) { + uprv_free(j[n].locale); + j[n].locale=uprv_strdup(tmp); + } +#if defined(ULOC_DEBUG) + /*fprintf(stderr,"%d: s <%s> q <%g>\n", n, j[n].locale, j[n].q);*/ +#endif + n++; + s = itemEnd; + while(*s==',') { /* eat duplicate commas */ + s++; + } + if(n>=jSize) { + if(j==smallBuffer) { /* overflowed the small buffer. */ + j = uprv_malloc(sizeof(j[0])*(jSize*2)); + if(j!=NULL) { + uprv_memcpy(j,smallBuffer,sizeof(j[0])*jSize); + } +#if defined(ULOC_DEBUG) + fprintf(stderr,"malloced at size %d\n", jSize); +#endif + } else { + j = uprv_realloc(j, sizeof(j[0])*jSize*2); +#if defined(ULOC_DEBUG) + fprintf(stderr,"re-alloced at size %d\n", jSize); +#endif + } + jSize *= 2; + if(j==NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + return -1; + } + } + } + uprv_sortArray(j, n, sizeof(j[0]), uloc_acceptLanguageCompare, NULL, TRUE, status); + if(U_FAILURE(*status)) { + if(j != smallBuffer) { +#if defined(ULOC_DEBUG) + fprintf(stderr,"freeing j %p\n", j); +#endif + uprv_free(j); + } + return -1; + } + strs = uprv_malloc((size_t)(sizeof(strs[0])*n)); + for(i=0;i q <%g>\n", i, j[i].locale, j[i].q);*/ +#endif + strs[i]=j[i].locale; + } + res = uloc_acceptLanguage(result, resultAvailable, outResult, + (const char**)strs, n, availableLocales, status); + for(i=0;i0) { + uprv_strncpy(result, l, uprv_min(len, resultAvailable)); + } + for(j=0;jmaxLen) { + maxLen = len; + } + } + uenum_reset(availableLocales, status); + /* save off parent info */ + if(uloc_getParent(acceptList[i], tmp, sizeof(tmp)/sizeof(tmp[0]), status)!=0) { + fallbackList[i] = uprv_strdup(tmp); + } else { + fallbackList[i]=0; + } + } + + for(maxLen--;maxLen>0;maxLen--) { + for(i=0;i0) { + uprv_strncpy(result, l, uprv_min(len, resultAvailable)); + } + for(j=0;j +# include "cmemory.h" + + typedef HANDLE MemoryMap; + +# define IS_MAP(map) ((map)!=NULL) + +# define MAP_IMPLEMENTATION MAP_WIN32 + +#elif U_HAVE_MMAP || defined(OS390) + typedef size_t MemoryMap; + +# define IS_MAP(map) ((map)!=0) + +# include +# include +# include +# include + +# ifndef MAP_FAILED +# define MAP_FAILED ((void*)-1) +# endif + +# if defined(OS390) && defined (OS390_STUBDATA) + /* No memory mapping for 390 batch mode. Fake it using dll loading. */ +# include +# include "cstring.h" +# include "cmemory.h" +# include "unicode/udata.h" +# define LIB_PREFIX "lib" +# define LIB_SUFFIX ".dll" +# define MAP_IMPLEMENTATION MAP_390DLL + +/* This is inconvienient until we figure out what to do with U_ICUDATA_NAME in utypes.h */ +# define U_ICUDATA_ENTRY_NAME "icudt" U_ICU_VERSION_SHORT U_LIB_SUFFIX_C_NAME_STRING "_dat" +# else +# define MAP_IMPLEMENTATION MAP_POSIX +# endif + +#else /* unknown platform, no memory map implementation: use stdio.h and uprv_malloc() instead */ + +# include +# include "cmemory.h" + + typedef void *MemoryMap; + +# define IS_MAP(map) ((map)!=NULL) + +# define MAP_IMPLEMENTATION MAP_STDIO + +#endif + + + + +/*----------------------------------------------------------------------------* + * * + * Memory Mapped File support. Platform dependent implementation of * + * functions used by the rest of the implementation.* + * * + *----------------------------------------------------------------------------*/ +#if MAP_IMPLEMENTATION==MAP_NONE + UBool + uprv_mapFile(UDataMemory *pData, const char *path) { + UDataMemory_init(pData); /* Clear the output struct. */ + return FALSE; /* no file access */ + } + + void uprv_unmapFile(UDataMemory *pData) { + /* nothing to do */ + } +#elif MAP_IMPLEMENTATION==MAP_WIN32 + UBool + uprv_mapFile( + UDataMemory *pData, /* Fill in with info on the result doing the mapping. */ + /* Output only; any original contents are cleared. */ + const char *path /* File path to be opened/mapped */ + ) + { + HANDLE map; + HANDLE file; + SECURITY_ATTRIBUTES mappingAttributes; + SECURITY_ATTRIBUTES *mappingAttributesPtr = NULL; + SECURITY_DESCRIPTOR securityDesc; + + UDataMemory_init(pData); /* Clear the output struct. */ + + /* open the input file */ + file=CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, NULL, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL|FILE_FLAG_RANDOM_ACCESS, NULL); + if(file==INVALID_HANDLE_VALUE) { + return FALSE; + } + + /* Declare and initialize a security descriptor. + This is required for multiuser systems on Windows 2000 SP4 and beyond */ + if (InitializeSecurityDescriptor(&securityDesc, SECURITY_DESCRIPTOR_REVISION)) { + /* give the security descriptor a Null Dacl done using the "TRUE, (PACL)NULL" here */ + if (SetSecurityDescriptorDacl(&securityDesc, TRUE, (PACL)NULL, FALSE)) { + /* Make the security attributes point to the security descriptor */ + uprv_memset(&mappingAttributes, 0, sizeof(mappingAttributes)); + mappingAttributes.nLength = sizeof(mappingAttributes); + mappingAttributes.lpSecurityDescriptor = &securityDesc; + mappingAttributes.bInheritHandle = FALSE; /* object uninheritable */ + mappingAttributesPtr = &mappingAttributes; + } + } + /* else creating security descriptors can fail when we are on Windows 98, + and mappingAttributesPtr == NULL for that case. */ + + /* create an unnamed Windows file-mapping object for the specified file */ + map=CreateFileMapping(file, mappingAttributesPtr, PAGE_READONLY, 0, 0, NULL); + CloseHandle(file); + if(map==NULL) { + return FALSE; + } + + /* map a view of the file into our address space */ + pData->pHeader=(const DataHeader *)MapViewOfFile(map, FILE_MAP_READ, 0, 0, 0); + if(pData->pHeader==NULL) { + CloseHandle(map); + return FALSE; + } + pData->map=map; + return TRUE; + } + + + void + uprv_unmapFile(UDataMemory *pData) { + if(pData!=NULL && pData->map!=NULL) { + UnmapViewOfFile(pData->pHeader); + CloseHandle(pData->map); + pData->pHeader=NULL; + pData->map=NULL; + } + } + + + +#elif MAP_IMPLEMENTATION==MAP_POSIX + UBool + uprv_mapFile(UDataMemory *pData, const char *path) { + int fd; + int length; + struct stat mystat; + void *data; + + UDataMemory_init(pData); /* Clear the output struct. */ + + /* determine the length of the file */ + if(stat(path, &mystat)!=0 || mystat.st_size<=0) { + return FALSE; + } + length=mystat.st_size; + + /* open the file */ + fd=open(path, O_RDONLY); + if(fd==-1) { + return FALSE; + } + + /* get a view of the mapping */ +#ifndef U_HPUX + data=mmap(0, length, PROT_READ, MAP_SHARED, fd, 0); +#else + data=mmap(0, length, PROT_READ, MAP_PRIVATE, fd, 0); +#endif + close(fd); /* no longer needed */ + if(data==MAP_FAILED) { + return FALSE; + } + + pData->map = (char *)data + length; + pData->pHeader=(const DataHeader *)data; + pData->mapAddr = data; + return TRUE; + } + + + + void + uprv_unmapFile(UDataMemory *pData) { + if(pData!=NULL && pData->map!=NULL) { + size_t dataLen = (char *)pData->map - (char *)pData->mapAddr; + if(munmap(pData->mapAddr, dataLen)==-1) { + } + pData->pHeader=NULL; + pData->map=0; + pData->mapAddr=NULL; + } + } + + + +#elif MAP_IMPLEMENTATION==MAP_STDIO + /* copy of the filestrm.c/T_FileStream_size() implementation */ + static int32_t + umap_fsize(FILE *f) { + int32_t savedPos = ftell(f); + int32_t size = 0; + + /*Changes by Bertrand A. D. doesn't affect the current position + goes to the end of the file before ftell*/ + fseek(f, 0, SEEK_END); + size = (int32_t)ftell(f); + fseek(f, savedPos, SEEK_SET); + return size; + } + + UBool + uprv_mapFile(UDataMemory *pData, const char *path) { + FILE *file; + int32_t fileLength; + void *p; + + UDataMemory_init(pData); /* Clear the output struct. */ + /* open the input file */ + file=fopen(path, "rb"); + if(file==NULL) { + return FALSE; + } + + /* get the file length */ + fileLength=umap_fsize(file); + if(ferror(file) || fileLength<=20) { + fclose(file); + return FALSE; + } + + /* allocate the memory to hold the file data */ + p=uprv_malloc(fileLength); + if(p==NULL) { + fclose(file); + return FALSE; + } + + /* read the file */ + if(fileLength!=fread(p, 1, fileLength, file)) { + uprv_free(p); + fclose(file); + return FALSE; + } + + fclose(file); + pData->map=p; + pData->pHeader=(const DataHeader *)p; + pData->mapAddr=p; + return TRUE; + } + + void + uprv_unmapFile(UDataMemory *pData) { + if(pData!=NULL && pData->map!=NULL) { + uprv_free(pData->map); + pData->map = NULL; + pData->mapAddr = NULL; + pData->pHeader = NULL; + } + } + + +#elif MAP_IMPLEMENTATION==MAP_390DLL + /* 390 specific Library Loading. + * This is the only platform left that dynamically loads an ICU Data Library. + * All other platforms use .data files when dynamic loading is required, but + * this turn out to be awkward to support in 390 batch mode. + * + * The idea here is to hide the fact that 390 is using dll loading from the + * rest of ICU, and make it look like there is file loading happening. + * + */ + + static char *strcpy_returnEnd(char *dest, const char *src) + { + while((*dest=*src)!=0) { + ++dest; + ++src; + } + return dest; + } + + /*------------------------------------------------------------------------------ + * + * computeDirPath given a user-supplied path of an item to be opened, + * compute and return + * - the full directory path to be used + * when opening the file. + * - Pointer to null at end of above returned path + * + * Parameters: + * path: input path. Buffer is not altered. + * pathBuffer: Output buffer. Any contents are overwritten. + * + * Returns: + * Pointer to null termination in returned pathBuffer. + * + * TODO: This works the way ICU historically has, but the + * whole data fallback search path is so complicated that + * proabably almost no one will ever really understand it, + * the potential for confusion is large. (It's not just + * this one function, but the whole scheme.) + * + *------------------------------------------------------------------------------*/ + static char *uprv_computeDirPath(const char *path, char *pathBuffer) + { + char *finalSlash; /* Ptr to last dir separator in input path, or null if none. */ + int32_t pathLen; /* Length of the returned directory path */ + + finalSlash = 0; + if (path != 0) { + finalSlash = uprv_strrchr(path, U_FILE_SEP_CHAR); + } + + *pathBuffer = 0; + if (finalSlash == 0) { + /* No user-supplied path. + * Copy the ICU_DATA path to the path buffer and return that*/ + const char *icuDataDir; + icuDataDir=u_getDataDirectory(); + if(icuDataDir!=NULL && *icuDataDir!=0) { + return strcpy_returnEnd(pathBuffer, icuDataDir); + } else { + /* there is no icuDataDir either. Just return the empty pathBuffer. */ + return pathBuffer; + } + } + + /* User supplied path did contain a directory portion. + * Copy it to the output path buffer */ + pathLen = (int32_t)(finalSlash - path + 1); + uprv_memcpy(pathBuffer, path, pathLen); + *(pathBuffer+pathLen) = 0; + return pathBuffer+pathLen; + } + + +# define DATA_TYPE "dat" + + UBool uprv_mapFile(UDataMemory *pData, const char *path) { + const char *inBasename; + char *basename; + char pathBuffer[1024]; + const DataHeader *pHeader; + dllhandle *handle; + void *val=0; + + inBasename=uprv_strrchr(path, U_FILE_SEP_CHAR); + if(inBasename==NULL) { + inBasename = path; + } else { + inBasename++; + } + basename=uprv_computeDirPath(path, pathBuffer); + if(uprv_strcmp(inBasename, U_ICUDATA_NAME".dat") != 0) { + /* must mmap file... for build */ + int fd; + int length; + struct stat mystat; + void *data; + UDataMemory_init(pData); /* Clear the output struct. */ + + /* determine the length of the file */ + if(stat(path, &mystat)!=0 || mystat.st_size<=0) { + return FALSE; + } + length=mystat.st_size; + + /* open the file */ + fd=open(path, O_RDONLY); + if(fd==-1) { + return FALSE; + } + + /* get a view of the mapping */ + data=mmap(0, length, PROT_READ, MAP_PRIVATE, fd, 0); + close(fd); /* no longer needed */ + if(data==MAP_FAILED) { + return FALSE; + } + pData->map = (char *)data + length; + pData->pHeader=(const DataHeader *)data; + pData->mapAddr = data; + return TRUE; + } + +# ifdef OS390BATCH + /* ### hack: we still need to get u_getDataDirectory() fixed + for OS/390 (batch mode - always return "//"? ) + and this here straightened out with LIB_PREFIX and LIB_SUFFIX (both empty?!) + This is probably due to the strange file system on OS/390. It's more like + a database with short entry names than a typical file system. */ + /* U_ICUDATA_NAME should always have the correct name */ + /* BUT FOR BATCH MODE IT IS AN EXCEPTION BECAUSE */ + /* THE FIRST THREE LETTERS ARE PREASSIGNED TO THE */ + /* PROJECT!!!!! */ + uprv_strcpy(pathBuffer, "IXMI" U_ICU_VERSION_SHORT "DA"); +# else + /* set up the library name */ + uprv_strcpy(basename, LIB_PREFIX U_LIBICUDATA_NAME U_ICU_VERSION_SHORT LIB_SUFFIX); +# endif + +# ifdef UDATA_DEBUG + fprintf(stderr, "dllload: %s ", pathBuffer); +# endif + + handle=dllload(pathBuffer); + +# ifdef UDATA_DEBUG + fprintf(stderr, " -> %08X\n", handle ); +# endif + + if(handle != NULL) { + /* we have a data DLL - what kind of lookup do we need here? */ + /* try to find the Table of Contents */ + UDataMemory_init(pData); /* Clear the output struct. */ + val=dllqueryvar((dllhandle*)handle, U_ICUDATA_ENTRY_NAME); + if(val == 0) { + /* failed... so keep looking */ + return FALSE; + } +# ifdef UDATA_DEBUG + fprintf(stderr, "dllqueryvar(%08X, %s) -> %08X\n", handle, U_ICUDATA_ENTRY_NAME, val); +# endif + + pData->pHeader=(const DataHeader *)val; + return TRUE; + } else { + return FALSE; /* no handle */ + } + } + + + + void uprv_unmapFile(UDataMemory *pData) { + if(pData!=NULL && pData->map!=NULL) { + uprv_free(pData->map); + pData->map = NULL; + pData->mapAddr = NULL; + pData->pHeader = NULL; + } + } + +#else +# error MAP_IMPLEMENTATION is set incorrectly +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/umapfile.h b/Build/source/libs/icu/icu-xetex/common/umapfile.h new file mode 100644 index 00000000000..c8c3de9332b --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/umapfile.h @@ -0,0 +1,32 @@ +/* +****************************************************************************** +* +* Copyright (C) 1999-2004, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************/ + + +/*---------------------------------------------------------------------------------- + * + * Memory mapped file wrappers for use by the ICU Data Implementation + * + * Porting note: The implementation of these functions is very platform specific. + * Not all platforms can do real memory mapping. Those that can't + * still must implement these functions, getting the data into memory using + * whatever means are available. + * + * These functions are part of the ICU internal implementation, and + * are not inteded to be used directly by applications. + * + *----------------------------------------------------------------------------------*/ + +#ifndef __UMAPFILE_H__ +#define __UMAPFILE_H__ + +#include "unicode/udata.h" + +UBool uprv_mapFile(UDataMemory *pdm, const char *path); +void uprv_unmapFile(UDataMemory *pData); + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/umath.c b/Build/source/libs/icu/icu-xetex/common/umath.c new file mode 100644 index 00000000000..4a571143168 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/umath.c @@ -0,0 +1,24 @@ +/* +****************************************************************************** +* +* Copyright (C) 1997-2006, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* This file contains platform independent math. +*/ + +#include "putilimp.h" + +U_CAPI int32_t U_EXPORT2 +uprv_max(int32_t x, int32_t y) +{ + return (x > y ? x : y); +} + +U_CAPI int32_t U_EXPORT2 +uprv_min(int32_t x, int32_t y) +{ + return (x > y ? y : x); +} + diff --git a/Build/source/libs/icu/icu-xetex/common/umutex.c b/Build/source/libs/icu/icu-xetex/common/umutex.c new file mode 100644 index 00000000000..22396f5e1d9 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/umutex.c @@ -0,0 +1,636 @@ +/* +****************************************************************************** +* +* Copyright (C) 1997-2006, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* +* File CMUTEX.C +* +* Modification History: +* +* Date Name Description +* 04/02/97 aliu Creation. +* 04/07/99 srl updated +* 05/13/99 stephen Changed to umutex (from cmutex). +* 11/22/99 aliu Make non-global mutex autoinitialize [j151] +****************************************************************************** +*/ + +#include "unicode/utypes.h" +#include "uassert.h" +#include "ucln_cmn.h" + +#if defined(U_DARWIN) +#include +#if (ICU_USE_THREADS == 1) && defined(MAC_OS_X_VERSION_10_4) && defined(MAC_OS_X_VERSION_MIN_REQUIRED) && (MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_4) +#include +#define USE_MAC_OS_ATOMIC_INCREMENT 1 +#endif +#endif + +/* Assume POSIX, and modify as necessary below */ +#define POSIX + +#if defined(U_WINDOWS) +#undef POSIX +#endif +#if defined(macintosh) +#undef POSIX +#endif +#if defined(OS2) +#undef POSIX +#endif + +#if defined(POSIX) && (ICU_USE_THREADS==1) +# include /* must be first, so that we get the multithread versions of things. */ + +#endif /* POSIX && (ICU_USE_THREADS==1) */ + +#ifdef U_WINDOWS +# define WIN32_LEAN_AND_MEAN +# define VC_EXTRALEAN +# define NOUSER +# define NOSERVICE +# define NOIME +# define NOMCX +# include +#endif + +#include "umutex.h" +#include "cmemory.h" + +/* + * A note on ICU Mutex Initialization and ICU startup: + * + * ICU mutexes, as used through the rest of the ICU code, are self-initializing. + * To make this work, ICU uses the _ICU GLobal Mutex_ to synchronize the lazy init + * of other ICU mutexes. For the global mutex itself, we need some other mechanism + * to safely initialize it on first use. This becomes important if two or more + * threads were more or less simultaenously the first to use ICU in a process, and + * were racing into the mutex initialization code. + * + * The solution for the global mutex init is platform dependent. + * On POSIX systems, C-style init can be used on a mutex, with the + * macro PTHREAD_MUTEX_INITIALIZER. The mutex is then ready for use, without + * first calling pthread_mutex_init(). + * + * Windows has no equivalent statically initialized mutex or CRITICAL SECION. + * InitializeCriticalSection() must be called. If the global mutex does not + * appear to be initialized, a thread will create and initialize a new + * CRITICAL_SECTION, then use a Windows InterlockedCompareAndExchange to + * avoid problems with race conditions. + * + * If an application has overridden the ICU mutex implementation + * by calling u_setMutexFunctions(), the user supplied init function must + * be safe in the event that multiple threads concurrently attempt to init + * the same mutex. The first thread should do the init, and the others should + * have no effect. + * + */ + +#define MAX_MUTEXES 30 +static UMTX gGlobalMutex = NULL; +static UMTX gIncDecMutex = NULL; +#if (ICU_USE_THREADS == 1) +static UBool gMutexPoolInitialized = FALSE; +static char gMutexesInUse[MAX_MUTEXES]; + +#if defined(U_WINDOWS) +/*------------------------------------------------------------- + * + * WINDOWS platform variable declarations + * + *-------------------------------------------------------------*/ +static CRITICAL_SECTION gMutexes[MAX_MUTEXES]; +static CRITICAL_SECTION gGlobalWinMutex; + + +/* On WIN32 mutexes are reentrant. This makes it difficult to debug + * deadlocking problems that show up on POSIXy platforms, where + * mutexes deadlock upon reentry. ICU contains checking code for + * the global mutex as well as for other mutexes in the pool. + * + * This is for debugging purposes. + * + * This has no effect on non-WIN32 platforms, non-DEBUG builds, and + * non-ICU_USE_THREADS builds. + * + * Note: The CRITICAL_SECTION structure already has a RecursionCount + * member that can be used for this purpose, but portability to + * Win98/NT/2K needs to be tested before use. Works fine on XP. + * After portability is confirmed, the built-in RecursionCount can be + * used, and the gRecursionCountPool can be removed. + * + * Note: Non-global mutex checking only happens if there is no custom + * pMutexLockFn defined. Use one function, not two (don't use + * pMutexLockFn and pMutexUnlockFn) so the increment and decrement of + * the recursion count don't get out of sync. Users might set just + * one function, e.g., to perform a custom action, followed by a + * standard call to EnterCriticalSection. + */ +#if defined(U_DEBUG) && (ICU_USE_THREADS==1) +static int32_t gRecursionCount = 0; /* detect global mutex locking */ +static int32_t gRecursionCountPool[MAX_MUTEXES]; /* ditto for non-global */ +#endif + + +#elif defined(POSIX) +/*------------------------------------------------------------- + * + * POSIX platform variable declarations + * + *-------------------------------------------------------------*/ +static pthread_mutex_t gMutexes[MAX_MUTEXES] = { + PTHREAD_MUTEX_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, + PTHREAD_MUTEX_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, + PTHREAD_MUTEX_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, + PTHREAD_MUTEX_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, + PTHREAD_MUTEX_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, + PTHREAD_MUTEX_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, + PTHREAD_MUTEX_INITIALIZER, PTHREAD_MUTEX_INITIALIZER +}; + +#else +/*------------------------------------------------------------- + * + * UNKNOWN platform declarations + * + *-------------------------------------------------------------*/ +static void *gMutexes[MAX_MUTEXES] = { + NULL, NULL, NULL, + NULL, NULL, NULL, + NULL, NULL, NULL, + NULL, NULL, NULL, + NULL, NULL, NULL, + NULL, NULL, NULL, + NULL, NULL }; + +/* Unknown platform. OK so long as ICU_USE_THREAD is not set. + Note that user can still set mutex functions at run time, + and that the global mutex variable is still needed in that case. */ +#if (ICU_USE_THREADS == 1) +#error no ICU mutex implementation for this platform +#endif +#endif +#endif /* ICU_USE_THREADS==1 */ + + + + +/* + * User mutex implementation functions. If non-null, call back to these rather than + * directly using the system (Posix or Windows) APIs. + * (declarations are in uclean.h) + */ +static UMtxInitFn *pMutexInitFn = NULL; +static UMtxFn *pMutexDestroyFn = NULL; +static UMtxFn *pMutexLockFn = NULL; +static UMtxFn *pMutexUnlockFn = NULL; +static const void *gMutexContext = NULL; + + + +/* + * umtx_lock + */ +U_CAPI void U_EXPORT2 +umtx_lock(UMTX *mutex) +{ + if (mutex == NULL) { + mutex = &gGlobalMutex; + } + + if (*mutex == NULL) { + /* Lock of an uninitialized mutex. Initialize it before proceeding. */ + umtx_init(mutex); + } + + if (pMutexLockFn != NULL) { + (*pMutexLockFn)(gMutexContext, mutex); + } else { + +#if (ICU_USE_THREADS == 1) +#if defined(U_WINDOWS) + EnterCriticalSection((CRITICAL_SECTION*) *mutex); +#elif defined(POSIX) + pthread_mutex_lock((pthread_mutex_t*) *mutex); +#endif /* cascade of platforms */ +#endif /* ICU_USE_THREADS==1 */ + } + +#if defined(U_WINDOWS) && defined(U_DEBUG) && (ICU_USE_THREADS==1) + if (mutex == &gGlobalMutex) { /* Detect Reentrant locking of the global mutex. */ + gRecursionCount++; /* Recursion causes deadlocks on Unixes. */ + U_ASSERT(gRecursionCount == 1); /* Detection works on Windows. Debug problems there. */ + } + /* This handles gGlobalMutex too, but only if there is no pMutexLockFn */ + else if (pMutexLockFn == NULL) { /* see comments above */ + size_t i = ((CRITICAL_SECTION*)*mutex) - &gMutexes[0]; + U_ASSERT(i >= 0 && i < MAX_MUTEXES); + ++gRecursionCountPool[i]; + + U_ASSERT(gRecursionCountPool[i] == 1); /* !Detect Deadlock! */ + + /* This works and is fast, but needs testing on Win98/NT/2K. + See comments above. [alan] + U_ASSERT((CRITICAL_SECTION*)*mutex >= &gMutexes[0] && + (CRITICAL_SECTION*)*mutex <= &gMutexes[MAX_MUTEXES]); + U_ASSERT(((CRITICAL_SECTION*)*mutex)->RecursionCount == 1); + */ + } +#endif /*U_DEBUG*/ +} + + + +/* + * umtx_unlock + */ +U_CAPI void U_EXPORT2 +umtx_unlock(UMTX* mutex) +{ + if(mutex == NULL) { + mutex = &gGlobalMutex; + } + + if(*mutex == NULL) { +#if (ICU_USE_THREADS == 1) + U_ASSERT(FALSE); /* This mutex is not initialized. */ +#endif + return; + } + +#if defined (U_WINDOWS) && defined (U_DEBUG) && (ICU_USE_THREADS==1) + if (mutex == &gGlobalMutex) { + gRecursionCount--; + U_ASSERT(gRecursionCount == 0); /* Detect unlock of an already unlocked mutex */ + } + /* This handles gGlobalMutex too, but only if there is no pMutexLockFn */ + else if (pMutexLockFn == NULL) { /* see comments above */ + size_t i = ((CRITICAL_SECTION*)*mutex) - &gMutexes[0]; + U_ASSERT(i >= 0 && i < MAX_MUTEXES); + --gRecursionCountPool[i]; + + U_ASSERT(gRecursionCountPool[i] == 0); /* !Detect Deadlock! */ + + /* This works and is fast, but needs testing on Win98/NT/2K. + Note that RecursionCount will be 1, not 0, since we haven't + left the CRITICAL_SECTION yet. See comments above. [alan] + U_ASSERT((CRITICAL_SECTION*)*mutex >= &gMutexes[0] && + (CRITICAL_SECTION*)*mutex <= &gMutexes[MAX_MUTEXES]); + U_ASSERT(((CRITICAL_SECTION*)*mutex)->RecursionCount == 1); + */ + } +#endif + + if (pMutexUnlockFn) { + (*pMutexUnlockFn)(gMutexContext, mutex); + } else { +#if (ICU_USE_THREADS==1) +#if defined (U_WINDOWS) + LeaveCriticalSection((CRITICAL_SECTION*)*mutex); +#elif defined (POSIX) + pthread_mutex_unlock((pthread_mutex_t*)*mutex); +#endif /* cascade of platforms */ +#endif /* ICU_USE_THREADS == 1 */ + } +} + + + + +/* + * initGlobalMutex Do the platform specific initialization of the ICU global mutex. + * Separated out from the other mutexes because it is different: + * Mutex storage is static for POSIX, init must be thread safe + * without the use of another mutex. + */ +static void initGlobalMutex() { + /* + * If User Supplied mutex functions are in use + * init the icu global mutex using them. + */ + if (pMutexInitFn != NULL) { + if (gGlobalMutex==NULL) { + UErrorCode status = U_ZERO_ERROR; + (*pMutexInitFn)(gMutexContext, &gGlobalMutex, &status); + if (U_FAILURE(status)) { + /* TODO: how should errors here be handled? */ + return; + } + } + return; + } + + /* No user override of mutex functions. + * Use default ICU mutex implementations. + */ +#if (ICU_USE_THREADS == 1) + /* + * for Windows, init the pool of critical sections that we + * will use as needed for ICU mutexes. + */ +#if defined (U_WINDOWS) + if (gMutexPoolInitialized == FALSE) { + int i; + for (i=0; iMethod(); + * umtx_unlock(NULL); + * } + * + * an alternative C++ mutex API is defined in the file common/mutex.h + */ + +/* Lock a mutex. + * @param mutex The given mutex to be locked. Pass NULL to specify + * the global ICU mutex. Recursive locks are an error + * and may cause a deadlock on some platforms. + */ +U_CAPI void U_EXPORT2 umtx_lock ( UMTX* mutex ); + +/* Unlock a mutex. Pass in NULL if you want the single global + mutex. + * @param mutex The given mutex to be unlocked. Pass NULL to specify + * the global ICU mutex. + */ +U_CAPI void U_EXPORT2 umtx_unlock ( UMTX* mutex ); + +/* Initialize a mutex. Use it this way: + umtx_init( &aMutex ); + * ICU Mutexes do not need explicit initialization before use. Use of this + * function is not necessary. + * Initialization of an already initialized mutex has no effect, and is safe to do. + * Initialization of mutexes is thread safe. Two threads can concurrently + * initialize the same mutex without causing problems. + * @param mutex The given mutex to be initialized + */ +U_CAPI void U_EXPORT2 umtx_init ( UMTX* mutex ); + +/* Destroy a mutex. This will free the resources of a mutex. + * Use it this way: + * umtx_destroy( &aMutex ); + * Destroying an already destroyed mutex has no effect, and causes no problems. + * This function is not thread safe. Two threads must not attempt to concurrently + * destroy the same mutex. + * @param mutex The given mutex to be destroyed. + */ +U_CAPI void U_EXPORT2 umtx_destroy( UMTX *mutex ); + + + +/* + * Atomic Increment and Decrement of an int32_t value. + * + * Return Values: + * If the result of the operation is zero, the return zero. + * If the result of the operation is not zero, the sign of returned value + * is the same as the sign of the result, but the returned value itself may + * be different from the result of the operation. + */ +U_CAPI int32_t U_EXPORT2 umtx_atomic_inc(int32_t *); +U_CAPI int32_t U_EXPORT2 umtx_atomic_dec(int32_t *); + + +#endif /*_CMUTEX*/ +/*eof*/ + + + diff --git a/Build/source/libs/icu/icu-xetex/common/unames.c b/Build/source/libs/icu/icu-xetex/common/unames.c new file mode 100644 index 00000000000..c12033fa47d --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unames.c @@ -0,0 +1,2083 @@ +/* +****************************************************************************** +* +* Copyright (C) 1999-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: unames.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 1999oct04 +* created by: Markus W. Scherer +*/ + +#include "unicode/utypes.h" +#include "unicode/putil.h" +#include "unicode/uchar.h" +#include "unicode/udata.h" +#include "ustr_imp.h" +#include "umutex.h" +#include "cmemory.h" +#include "cstring.h" +#include "ucln_cmn.h" +#include "udataswp.h" +#include "uprops.h" + +/* prototypes ------------------------------------------------------------- */ + +#define LENGTHOF(array) (sizeof(array)/sizeof((array)[0])) + +static const char DATA_NAME[] = "unames"; +static const char DATA_TYPE[] = "icu"; + +#define GROUP_SHIFT 5 +#define LINES_PER_GROUP (1UL<size>=20 && + pInfo->isBigEndian==U_IS_BIG_ENDIAN && + pInfo->charsetFamily==U_CHARSET_FAMILY && + pInfo->dataFormat[0]==0x75 && /* dataFormat="unam" */ + pInfo->dataFormat[1]==0x6e && + pInfo->dataFormat[2]==0x61 && + pInfo->dataFormat[3]==0x6d && + pInfo->formatVersion[0]==1); +} + +static UBool +isDataLoaded(UErrorCode *pErrorCode) { + /* load UCharNames from file if necessary */ + UBool isCached; + + /* do this because double-checked locking is broken */ + UMTX_CHECK(NULL, (uCharNames!=NULL), isCached); + + if(!isCached) { + UCharNames *names; + UDataMemory *data; + + /* check error code from previous attempt */ + if(U_FAILURE(gLoadErrorCode)) { + *pErrorCode=gLoadErrorCode; + return FALSE; + } + + /* open the data outside the mutex block */ + data=udata_openChoice(NULL, DATA_TYPE, DATA_NAME, isAcceptable, NULL, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + gLoadErrorCode=*pErrorCode; + return FALSE; + } + + names=(UCharNames *)udata_getMemory(data); + + /* in the mutex block, set the data for this process */ + { + umtx_lock(NULL); + if(uCharNames==NULL) { + uCharNames=names; + uCharNamesData=data; + data=NULL; + names=NULL; + ucln_common_registerCleanup(UCLN_COMMON_UNAMES, unames_cleanup); + } + umtx_unlock(NULL); + } + + /* if a different thread set it first, then close the extra data */ + if(data!=NULL) { + udata_close(data); /* NULL if it was set correctly */ + } + } + return TRUE; +} + +#define WRITE_CHAR(buffer, bufferLength, bufferPos, c) { \ + if((bufferLength)>0) { \ + *(buffer)++=c; \ + --(bufferLength); \ + } \ + ++(bufferPos); \ +} + +#define U_ISO_COMMENT U_CHAR_NAME_CHOICE_COUNT + +/* + * Important: expandName() and compareName() are almost the same - + * apply fixes to both. + * + * UnicodeData.txt uses ';' as a field separator, so no + * field can contain ';' as part of its contents. + * In unames.dat, it is marked as token[';']==-1 only if the + * semicolon is used in the data file - which is iff we + * have Unicode 1.0 names or ISO comments. + * So, it will be token[';']==-1 if we store U1.0 names/ISO comments + * although we know that it will never be part of a name. + */ +static uint16_t +expandName(UCharNames *names, + const uint8_t *name, uint16_t nameLength, UCharNameChoice nameChoice, + char *buffer, uint16_t bufferLength) { + uint16_t *tokens=(uint16_t *)names+8; + uint16_t token, tokenCount=*tokens++, bufferPos=0; + uint8_t *tokenStrings=(uint8_t *)names+names->tokenStringOffset; + uint8_t c; + + if(nameChoice==U_UNICODE_10_CHAR_NAME || nameChoice==U_ISO_COMMENT) { + /* + * skip the modern name if it is not requested _and_ + * if the semicolon byte value is a character, not a token number + */ + if((uint8_t)';'>=tokenCount || tokens[(uint8_t)';']==(uint16_t)(-1)) { + while(nameLength>0) { + --nameLength; + if(*name++==';') { + break; + } + } + if(nameChoice==U_ISO_COMMENT) { + /* skip the Unicode 1.0 name as well to get the ISO comment */ + while(nameLength>0) { + --nameLength; + if(*name++==';') { + break; + } + } + } + } else { + /* + * the semicolon byte value is a token number, therefore + * only modern names are stored in unames.dat and there is no + * such requested Unicode 1.0 name here + */ + nameLength=0; + } + } + + /* write each letter directly, and write a token word per token */ + while(nameLength>0) { + --nameLength; + c=*name++; + + if(c>=tokenCount) { + if(c!=';') { + /* implicit letter */ + WRITE_CHAR(buffer, bufferLength, bufferPos, c); + } else { + /* finished */ + break; + } + } else { + token=tokens[c]; + if(token==(uint16_t)(-2)) { + /* this is a lead byte for a double-byte token */ + token=tokens[c<<8|*name++]; + --nameLength; + } + if(token==(uint16_t)(-1)) { + if(c!=';') { + /* explicit letter */ + WRITE_CHAR(buffer, bufferLength, bufferPos, c); + } else { + /* stop, but skip the semicolon if we are seeking + extended names and there was no 2.0 name but there + is a 1.0 name. */ + if(!bufferPos && nameChoice == U_EXTENDED_CHAR_NAME) { + if ((uint8_t)';'>=tokenCount || tokens[(uint8_t)';']==(uint16_t)(-1)) { + continue; + } + } + /* finished */ + break; + } + } else { + /* write token word */ + uint8_t *tokenString=tokenStrings+token; + while((c=*tokenString++)!=0) { + WRITE_CHAR(buffer, bufferLength, bufferPos, c); + } + } + } + } + + /* zero-terminate */ + if(bufferLength>0) { + *buffer=0; + } + + return bufferPos; +} + +/* + * compareName() is almost the same as expandName() except that it compares + * the currently expanded name to an input name. + * It returns the match/no match result as soon as possible. + */ +static UBool +compareName(UCharNames *names, + const uint8_t *name, uint16_t nameLength, UCharNameChoice nameChoice, + const char *otherName) { + uint16_t *tokens=(uint16_t *)names+8; + uint16_t token, tokenCount=*tokens++; + uint8_t *tokenStrings=(uint8_t *)names+names->tokenStringOffset; + uint8_t c; + const char *origOtherName = otherName; + + if(nameChoice==U_UNICODE_10_CHAR_NAME) { + /* + * skip the modern name if it is not requested _and_ + * if the semicolon byte value is a character, not a token number + */ + if((uint8_t)';'>=tokenCount || tokens[(uint8_t)';']==(uint16_t)(-1)) { + while(nameLength>0) { + --nameLength; + if(*name++==';') { + break; + } + } + } else { + /* + * the semicolon byte value is a token number, therefore + * only modern names are stored in unames.dat and there is no + * such requested Unicode 1.0 name here + */ + nameLength=0; + } + } + + /* compare each letter directly, and compare a token word per token */ + while(nameLength>0) { + --nameLength; + c=*name++; + + if(c>=tokenCount) { + if(c!=';') { + /* implicit letter */ + if((char)c!=*otherName++) { + return FALSE; + } + } else { + /* finished */ + break; + } + } else { + token=tokens[c]; + if(token==(uint16_t)(-2)) { + /* this is a lead byte for a double-byte token */ + token=tokens[c<<8|*name++]; + --nameLength; + } + if(token==(uint16_t)(-1)) { + if(c!=';') { + /* explicit letter */ + if((char)c!=*otherName++) { + return FALSE; + } + } else { + /* stop, but skip the semicolon if we are seeking + extended names and there was no 2.0 name but there + is a 1.0 name. */ + if(otherName == origOtherName && nameChoice == U_EXTENDED_CHAR_NAME) { + if ((uint8_t)';'>=tokenCount || tokens[(uint8_t)';']==(uint16_t)(-1)) { + continue; + } + } + /* finished */ + break; + } + } else { + /* write token word */ + uint8_t *tokenString=tokenStrings+token; + while((c=*tokenString++)!=0) { + if((char)c!=*otherName++) { + return FALSE; + } + } + } + } + } + + /* complete match? */ + return (UBool)(*otherName==0); +} + +static uint8_t getCharCat(UChar32 cp) { + uint8_t cat; + + if (UTF_IS_UNICODE_NONCHAR(cp)) { + return U_NONCHARACTER_CODE_POINT; + } + + if ((cat = u_charType(cp)) == U_SURROGATE) { + cat = UTF_IS_LEAD(cp) ? U_LEAD_SURROGATE : U_TRAIL_SURROGATE; + } + + return cat; +} + +static const char *getCharCatName(UChar32 cp) { + uint8_t cat = getCharCat(cp); + + /* Return unknown if the table of names above is not up to + date. */ + + if (cat >= LENGTHOF(charCatNames)) { + return "unknown"; + } else { + return charCatNames[cat]; + } +} + +static uint16_t getExtName(uint32_t code, char *buffer, uint16_t bufferLength) { + const char *catname = getCharCatName(code); + uint16_t length = 0; + + UChar32 cp; + int ndigits, i; + + WRITE_CHAR(buffer, bufferLength, length, '<'); + while (catname[length - 1]) { + WRITE_CHAR(buffer, bufferLength, length, catname[length - 1]); + } + WRITE_CHAR(buffer, bufferLength, length, '-'); + for (cp = code, ndigits = 0; cp; ++ndigits, cp >>= 4) + ; + if (ndigits < 4) + ndigits = 4; + for (cp = code, i = ndigits; (cp || i > 0) && bufferLength; cp >>= 4, bufferLength--) { + uint8_t v = (uint8_t)(cp & 0xf); + buffer[--i] = (v < 10 ? '0' + v : 'A' + v - 10); + } + buffer += ndigits; + length += ndigits; + WRITE_CHAR(buffer, bufferLength, length, '>'); + + return length; +} + +/* + * getGroup() does a binary search for the group that contains the + * Unicode code point "code". + * The return value is always a valid Group* that may contain "code" + * or else is the highest group before "code". + * If the lowest group is after "code", then that one is returned. + */ +static Group * +getGroup(UCharNames *names, uint32_t code) { + uint16_t groupMSB=(uint16_t)(code>>GROUP_SHIFT), + start=0, + limit=*(uint16_t *)((char *)names+names->groupsOffset), + number; + Group *groups=(Group *)((char *)names+names->groupsOffset+2); + + /* binary search for the group of names that contains the one for code */ + while(start=0xc, then it forms a length value with the following nibble. + * Calculation see below. + * The offsets and lengths arrays must be at least 33 (one more) long because + * there is no check here at the end if the last nibble is still used. + */ +static const uint8_t * +expandGroupLengths(const uint8_t *s, + uint16_t offsets[LINES_PER_GROUP+1], uint16_t lengths[LINES_PER_GROUP+1]) { + /* read the lengths of the 32 strings in this group and get each string's offset */ + uint16_t i=0, offset=0, length=0; + uint8_t lengthByte; + + /* all 32 lengths must be read to get the offset of the first group string */ + while(i=12) { + /* double-nibble length spread across two bytes */ + length=(uint16_t)(((length&0x3)<<4|lengthByte>>4)+12); + lengthByte&=0xf; + } else if((lengthByte /* &0xf0 */)>=0xc0) { + /* double-nibble length spread across this one byte */ + length=(uint16_t)((lengthByte&0x3f)+12); + } else { + /* single-nibble length in MSBs */ + length=(uint16_t)(lengthByte>>4); + lengthByte&=0xf; + } + + *offsets++=offset; + *lengths++=length; + + offset+=length; + ++i; + + /* read odd nibble - LSBs of lengthByte */ + if((lengthByte&0xf0)==0) { + /* this nibble was not consumed for a double-nibble length above */ + length=lengthByte; + if(length<12) { + /* single-nibble length in LSBs */ + *offsets++=offset; + *lengths++=length; + + offset+=length; + ++i; + } + } else { + length=0; /* prevent double-nibble detection in the next iteration */ + } + } + + /* now, s is at the first group string */ + return s; +} + +static uint16_t +expandGroupName(UCharNames *names, Group *group, + uint16_t lineNumber, UCharNameChoice nameChoice, + char *buffer, uint16_t bufferLength) { + uint16_t offsets[LINES_PER_GROUP+2], lengths[LINES_PER_GROUP+2]; + const uint8_t *s=(uint8_t *)names+names->groupStringOffset+ + (group->offsetHigh<<16|group->offsetLow); + s=expandGroupLengths(s, offsets, lengths); + return expandName(names, s+offsets[lineNumber], lengths[lineNumber], nameChoice, + buffer, bufferLength); +} + +static uint16_t +getName(UCharNames *names, uint32_t code, UCharNameChoice nameChoice, + char *buffer, uint16_t bufferLength) { + Group *group=getGroup(names, code); + if((uint16_t)(code>>GROUP_SHIFT)==group->groupMSB) { + return expandGroupName(names, group, (uint16_t)(code&GROUP_MASK), nameChoice, + buffer, bufferLength); + } else { + /* group not found */ + /* zero-terminate */ + if(bufferLength>0) { + *buffer=0; + } + return 0; + } +} + +/* + * enumGroupNames() enumerates all the names in a 32-group + * and either calls the enumerator function or finds a given input name. + */ +static UBool +enumGroupNames(UCharNames *names, Group *group, + UChar32 start, UChar32 end, + UEnumCharNamesFn *fn, void *context, + UCharNameChoice nameChoice) { + uint16_t offsets[LINES_PER_GROUP+2], lengths[LINES_PER_GROUP+2]; + const uint8_t *s=(uint8_t *)names+names->groupStringOffset+ + (group->offsetHigh<<16|group->offsetLow); + + s=expandGroupLengths(s, offsets, lengths); + if(fn!=DO_FIND_NAME) { + char buffer[200]; + uint16_t length; + + while(start<=end) { + length=expandName(names, s+offsets[start&GROUP_MASK], lengths[start&GROUP_MASK], nameChoice, buffer, sizeof(buffer)); + if (!length && nameChoice == U_EXTENDED_CHAR_NAME) { + buffer[length = getExtName(start, buffer, sizeof(buffer))] = 0; + } + /* here, we assume that the buffer is large enough */ + if(length>0) { + if(!fn(context, start, nameChoice, buffer, length)) { + return FALSE; + } + } + ++start; + } + } else { + const char *otherName=((FindName *)context)->otherName; + while(start<=end) { + if(compareName(names, s+offsets[start&GROUP_MASK], lengths[start&GROUP_MASK], nameChoice, otherName)) { + ((FindName *)context)->code=start; + return FALSE; + } + ++start; + } + } + return TRUE; +} + +/* + * enumExtNames enumerate extended names. + * It only needs to do it if it is called with a real function and not + * with the dummy DO_FIND_NAME, because u_charFromName() does a check + * for extended names by itself. + */ +static UBool +enumExtNames(UChar32 start, UChar32 end, + UEnumCharNamesFn *fn, void *context) +{ + if(fn!=DO_FIND_NAME) { + char buffer[200]; + uint16_t length; + + while(start<=end) { + buffer[length = getExtName(start, buffer, sizeof(buffer))] = 0; + /* here, we assume that the buffer is large enough */ + if(length>0) { + if(!fn(context, start, U_EXTENDED_CHAR_NAME, buffer, length)) { + return FALSE; + } + } + ++start; + } + } + + return TRUE; +} + +static UBool +enumNames(UCharNames *names, + UChar32 start, UChar32 limit, + UEnumCharNamesFn *fn, void *context, + UCharNameChoice nameChoice) { + uint16_t startGroupMSB, endGroupMSB, groupCount; + Group *group, *groupLimit; + + startGroupMSB=(uint16_t)(start>>GROUP_SHIFT); + endGroupMSB=(uint16_t)((limit-1)>>GROUP_SHIFT); + + /* find the group that contains start, or the highest before it */ + group=getGroup(names, start); + + if(startGroupMSB==endGroupMSB) { + if(startGroupMSB==group->groupMSB) { + /* if start and limit-1 are in the same group, then enumerate only in that one */ + return enumGroupNames(names, group, start, limit-1, fn, context, nameChoice); + } + } else { + groupCount=*(uint16_t *)((char *)names+names->groupsOffset); + groupLimit=(Group *)((char *)names+names->groupsOffset+2)+groupCount; + + if(startGroupMSB==group->groupMSB) { + /* enumerate characters in the partial start group */ + if((start&GROUP_MASK)!=0) { + if(!enumGroupNames(names, group, + start, ((UChar32)startGroupMSB<group->groupMSB) { + /* make sure that we start enumerating with the first group after start */ + if (group + 1 < groupLimit && (group + 1)->groupMSB > startGroupMSB && nameChoice == U_EXTENDED_CHAR_NAME) { + UChar32 end = (group + 1)->groupMSB << GROUP_SHIFT; + if (end > limit) { + end = limit; + } + if (!enumExtNames(start, end - 1, fn, context)) { + return FALSE; + } + } + ++group; + } + + /* enumerate entire groups between the start- and end-groups */ + while(groupgroupMSBgroupMSB<groupMSB > group->groupMSB + 1 && nameChoice == U_EXTENDED_CHAR_NAME) { + UChar32 end = (group + 1)->groupMSB << GROUP_SHIFT; + if (end > limit) { + end = limit; + } + if (!enumExtNames((group->groupMSB + 1) << GROUP_SHIFT, end - 1, fn, context)) { + return FALSE; + } + } + ++group; + } + + /* enumerate within the end group (group->groupMSB==endGroupMSB) */ + if(groupgroupMSB==endGroupMSB) { + return enumGroupNames(names, group, (limit-1)&~GROUP_MASK, limit-1, fn, context, nameChoice); + } else if (nameChoice == U_EXTENDED_CHAR_NAME && group == groupLimit) { + UChar32 next = ((group - 1)->groupMSB + 1) << GROUP_SHIFT; + if (next > start) { + start = next; + } + } else { + return TRUE; + } + } + + /* we have not found a group, which means everything is made of + extended names. */ + if (nameChoice == U_EXTENDED_CHAR_NAME) { + if (limit > UCHAR_MAX_VALUE + 1) { + limit = UCHAR_MAX_VALUE + 1; + } + return enumExtNames(start, limit - 1, fn, context); + } + + return TRUE; +} + +static uint16_t +writeFactorSuffix(const uint16_t *factors, uint16_t count, + const char *s, /* suffix elements */ + uint32_t code, + uint16_t indexes[8], /* output fields from here */ + const char *elementBases[8], const char *elements[8], + char *buffer, uint16_t bufferLength) { + uint16_t i, factor, bufferPos=0; + char c; + + /* write elements according to the factors */ + + /* + * the factorized elements are determined by modulo arithmetic + * with the factors of this algorithm + * + * note that for fewer operations, count is decremented here + */ + --count; + for(i=count; i>0; --i) { + factor=factors[i]; + indexes[i]=(uint16_t)(code%factor); + code/=factor; + } + /* + * we don't need to calculate the last modulus because start<=code<=end + * guarantees here that code<=factors[0] + */ + indexes[0]=(uint16_t)code; + + /* write each element */ + for(;;) { + if(elementBases!=NULL) { + *elementBases++=s; + } + + /* skip indexes[i] strings */ + factor=indexes[i]; + while(factor>0) { + while(*s++!=0) {} + --factor; + } + if(elements!=NULL) { + *elements++=s; + } + + /* write element */ + while((c=*s++)!=0) { + WRITE_CHAR(buffer, bufferLength, bufferPos, c); + } + + /* we do not need to perform the rest of this loop for i==count - break here */ + if(i>=count) { + break; + } + + /* skip the rest of the strings for this factors[i] */ + factor=(uint16_t)(factors[i]-indexes[i]-1); + while(factor>0) { + while(*s++!=0) {} + --factor; + } + + ++i; + } + + /* zero-terminate */ + if(bufferLength>0) { + *buffer=0; + } + + return bufferPos; +} + +/* + * Important: + * Parts of findAlgName() are almost the same as some of getAlgName(). + * Fixes must be applied to both. + */ +static uint16_t +getAlgName(AlgorithmicRange *range, uint32_t code, UCharNameChoice nameChoice, + char *buffer, uint16_t bufferLength) { + uint16_t bufferPos=0; + + /* + * Do not write algorithmic Unicode 1.0 names because + * Unihan names are the same as the modern ones, + * extension A was only introduced with Unicode 3.0, and + * the Hangul syllable block was moved and changed around Unicode 1.1.5. + */ + if(nameChoice==U_UNICODE_10_CHAR_NAME) { + /* zero-terminate */ + if(bufferLength>0) { + *buffer=0; + } + return 0; + } + + switch(range->type) { + case 0: { + /* name = prefix hex-digits */ + const char *s=(const char *)(range+1); + char c; + + uint16_t i, count; + + /* copy prefix */ + while((c=*s++)!=0) { + WRITE_CHAR(buffer, bufferLength, bufferPos, c); + } + + /* write hexadecimal code point value */ + count=range->variant; + + /* zero-terminate */ + if(count0;) { + if(--i>=4; + } + + bufferPos+=count; + break; + } + case 1: { + /* name = prefix factorized-elements */ + uint16_t indexes[8]; + const uint16_t *factors=(const uint16_t *)(range+1); + uint16_t count=range->variant; + const char *s=(const char *)(factors+count); + char c; + + /* copy prefix */ + while((c=*s++)!=0) { + WRITE_CHAR(buffer, bufferLength, bufferPos, c); + } + + bufferPos+=writeFactorSuffix(factors, count, + s, code-range->start, indexes, NULL, NULL, buffer, bufferLength); + break; + } + default: + /* undefined type */ + /* zero-terminate */ + if(bufferLength>0) { + *buffer=0; + } + break; + } + + return bufferPos; +} + +/* + * Important: enumAlgNames() and findAlgName() are almost the same. + * Any fix must be applied to both. + */ +static UBool +enumAlgNames(AlgorithmicRange *range, + UChar32 start, UChar32 limit, + UEnumCharNamesFn *fn, void *context, + UCharNameChoice nameChoice) { + char buffer[200]; + uint16_t length; + + if(nameChoice==U_UNICODE_10_CHAR_NAME) { + return TRUE; + } + + switch(range->type) { + case 0: { + char *s, *end; + char c; + + /* get the full name of the start character */ + length=getAlgName(range, (uint32_t)start, nameChoice, buffer, sizeof(buffer)); + if(length<=0) { + return TRUE; + } + + /* call the enumerator function with this first character */ + if(!fn(context, start, nameChoice, buffer, length)) { + return FALSE; + } + + /* go to the end of the name; all these names have the same length */ + end=buffer; + while(*end!=0) { + ++end; + } + + /* enumerate the rest of the names */ + while(++startvariant; + const char *s=(const char *)(factors+count); + char *suffix, *t; + uint16_t prefixLength, i, index; + + char c; + + /* name = prefix factorized-elements */ + + /* copy prefix */ + suffix=buffer; + prefixLength=0; + while((c=*s++)!=0) { + *suffix++=c; + ++prefixLength; + } + + /* append the suffix of the start character */ + length=(uint16_t)(prefixLength+writeFactorSuffix(factors, count, + s, (uint32_t)start-range->start, + indexes, elementBases, elements, + suffix, (uint16_t)(sizeof(buffer)-prefixLength))); + + /* call the enumerator function with this first character */ + if(!fn(context, start, nameChoice, buffer, length)) { + return FALSE; + } + + /* enumerate the rest of the names */ + while(++starttype) { + case 0: { + /* name = prefix hex-digits */ + const char *s=(const char *)(range+1); + char c; + + uint16_t i, count; + + /* compare prefix */ + while((c=*s++)!=0) { + if((char)c!=*otherName++) { + return 0xffff; + } + } + + /* read hexadecimal code point value */ + count=range->variant; + code=0; + for(i=0; istart<=(uint32_t)code && (uint32_t)code<=range->end) { + return code; + } + break; + } + case 1: { + char buffer[64]; + uint16_t indexes[8]; + const char *elementBases[8], *elements[8]; + const uint16_t *factors=(const uint16_t *)(range+1); + uint16_t count=range->variant; + const char *s=(const char *)(factors+count), *t; + UChar32 start, limit; + uint16_t i, index; + + char c; + + /* name = prefix factorized-elements */ + + /* compare prefix */ + while((c=*s++)!=0) { + if((char)c!=*otherName++) { + return 0xffff; + } + } + + start=(UChar32)range->start; + limit=(UChar32)(range->end+1); + + /* initialize the suffix elements for enumeration; indexes should all be set to 0 */ + writeFactorSuffix(factors, count, s, 0, + indexes, elementBases, elements, buffer, sizeof(buffer)); + + /* compare the first suffix */ + if(0==uprv_strcmp(otherName, buffer)) { + return start; + } + + /* enumerate and compare the rest of the suffixes */ + while(++start>5]|=((uint32_t)1<<((uint8_t)c&0x1f))) +#define SET_CONTAINS(set, c) (((set)[(uint8_t)c>>5]&((uint32_t)1<<((uint8_t)c&0x1f)))!=0) + +static int32_t +calcStringSetLength(uint32_t set[8], const char *s) { + int32_t length=0; + char c; + + while((c=*s++)!=0) { + SET_ADD(set, c); + ++length; + } + return length; +} + +static int32_t +calcAlgNameSetsLengths(int32_t maxNameLength) { + AlgorithmicRange *range; + uint32_t *p; + uint32_t rangeCount; + int32_t length; + + /* enumerate algorithmic ranges */ + p=(uint32_t *)((uint8_t *)uCharNames+uCharNames->algNamesOffset); + rangeCount=*p; + range=(AlgorithmicRange *)(p+1); + while(rangeCount>0) { + switch(range->type) { + case 0: + /* name = prefix + (range->variant times) hex-digits */ + /* prefix */ + length=calcStringSetLength(gNameSet, (const char *)(range+1))+range->variant; + if(length>maxNameLength) { + maxNameLength=length; + } + break; + case 1: { + /* name = prefix factorized-elements */ + const uint16_t *factors=(const uint16_t *)(range+1); + const char *s; + int32_t i, count=range->variant, factor, factorLength, maxFactorLength; + + /* prefix length */ + s=(const char *)(factors+count); + length=calcStringSetLength(gNameSet, s); + s+=length+1; /* start of factor suffixes */ + + /* get the set and maximum factor suffix length for each factor */ + for(i=0; i0; --factor) { + factorLength=calcStringSetLength(gNameSet, s); + s+=factorLength+1; + if(factorLength>maxFactorLength) { + maxFactorLength=factorLength; + } + } + length+=maxFactorLength; + } + + if(length>maxNameLength) { + maxNameLength=length; + } + break; + } + default: + /* unknown type */ + break; + } + + range=(AlgorithmicRange *)((uint8_t *)range+range->size); + --rangeCount; + } + return maxNameLength; +} + +static int32_t +calcExtNameSetsLengths(int32_t maxNameLength) { + int32_t i, length; + + for(i=0; i + * 1 for - + * 6 for most hex digits per code point + */ + length=9+calcStringSetLength(gNameSet, charCatNames[i]); + if(length>maxNameLength) { + maxNameLength=length; + } + } + return maxNameLength; +} + +static int32_t +calcNameSetLength(const uint16_t *tokens, uint16_t tokenCount, const uint8_t *tokenStrings, int8_t *tokenLengths, + uint32_t set[8], + const uint8_t **pLine, const uint8_t *lineLimit) { + const uint8_t *line=*pLine; + int32_t length=0, tokenLength; + uint16_t c, token; + + while(line!=lineLimit && (c=*line++)!=(uint8_t)';') { + if(c>=tokenCount) { + /* implicit letter */ + SET_ADD(set, c); + ++length; + } else { + token=tokens[c]; + if(token==(uint16_t)(-2)) { + /* this is a lead byte for a double-byte token */ + c=c<<8|*line++; + token=tokens[c]; + } + if(token==(uint16_t)(-1)) { + /* explicit letter */ + SET_ADD(set, c); + ++length; + } else { + /* count token word */ + if(tokenLengths!=NULL) { + /* use cached token length */ + tokenLength=tokenLengths[c]; + if(tokenLength==0) { + tokenLength=calcStringSetLength(set, (const char *)tokenStrings+token); + tokenLengths[c]=(int8_t)tokenLength; + } + } else { + tokenLength=calcStringSetLength(set, (const char *)tokenStrings+token); + } + length+=tokenLength; + } + } + } + + *pLine=line; + return length; +} + +static void +calcGroupNameSetsLengths(int32_t maxNameLength) { + uint16_t offsets[LINES_PER_GROUP+2], lengths[LINES_PER_GROUP+2]; + + uint16_t *tokens=(uint16_t *)uCharNames+8; + uint16_t tokenCount=*tokens++; + uint8_t *tokenStrings=(uint8_t *)uCharNames+uCharNames->tokenStringOffset; + + int8_t *tokenLengths; + + uint16_t *groups; + Group *group; + const uint8_t *s, *line, *lineLimit; + + int32_t groupCount, lineNumber, length; + + tokenLengths=(int8_t *)uprv_malloc(tokenCount); + if(tokenLengths!=NULL) { + uprv_memset(tokenLengths, 0, tokenCount); + } + + groups=(uint16_t *)((char *)uCharNames+uCharNames->groupsOffset); + groupCount=*groups++; + group=(Group *)groups; + + /* enumerate all groups */ + while(groupCount>0) { + s=(uint8_t *)uCharNames+uCharNames->groupStringOffset+ + ((int32_t)group->offsetHigh<<16|group->offsetLow); + s=expandGroupLengths(s, offsets, lengths); + + /* enumerate all lines in each group */ + for(lineNumber=0; lineNumbermaxNameLength) { + maxNameLength=length; + } + if(line==lineLimit) { + continue; + } + + /* read Unicode 1.0 name */ + length=calcNameSetLength(tokens, tokenCount, tokenStrings, tokenLengths, gNameSet, &line, lineLimit); + if(length>maxNameLength) { + maxNameLength=length; + } + if(line==lineLimit) { + continue; + } + + /* read ISO comment */ + /*length=calcNameSetLength(tokens, tokenCount, tokenStrings, tokenLengths, gISOCommentSet, &line, lineLimit);*/ + } + + ++group; + --groupCount; + } + + if(tokenLengths!=NULL) { + uprv_free(tokenLengths); + } + + /* set gMax... - name length last for threading */ + gMaxNameLength=maxNameLength; +} + +static UBool +calcNameSetsLengths(UErrorCode *pErrorCode) { + static const char extChars[]="0123456789ABCDEF<>-"; + int32_t i, maxNameLength; + + if(gMaxNameLength!=0) { + return TRUE; + } + + if(!isDataLoaded(pErrorCode)) { + return FALSE; + } + + /* set hex digits, used in various names, and <>-, used in extended names */ + for(i=0; i=U_CHAR_NAME_CHOICE_COUNT || + bufferLength<0 || (bufferLength>0 && buffer==NULL) + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + if((uint32_t)code>UCHAR_MAX_VALUE || !isDataLoaded(pErrorCode)) { + return u_terminateChars(buffer, bufferLength, 0, pErrorCode); + } + + length=0; + + /* try algorithmic names first */ + p=(uint32_t *)((uint8_t *)uCharNames+uCharNames->algNamesOffset); + i=*p; + algRange=(AlgorithmicRange *)(p+1); + while(i>0) { + if(algRange->start<=(uint32_t)code && (uint32_t)code<=algRange->end) { + length=getAlgName(algRange, (uint32_t)code, nameChoice, buffer, (uint16_t)bufferLength); + break; + } + algRange=(AlgorithmicRange *)((uint8_t *)algRange+algRange->size); + --i; + } + + if(i==0) { + if (nameChoice == U_EXTENDED_CHAR_NAME) { + length = getName(uCharNames, (uint32_t )code, U_EXTENDED_CHAR_NAME, buffer, (uint16_t) bufferLength); + if (!length) { + /* extended character name */ + length = getExtName((uint32_t) code, buffer, (uint16_t) bufferLength); + } + } else { + /* normal character name */ + length=getName(uCharNames, (uint32_t)code, nameChoice, buffer, (uint16_t)bufferLength); + } + } + + return u_terminateChars(buffer, bufferLength, length, pErrorCode); +} + +U_CAPI int32_t U_EXPORT2 +u_getISOComment(UChar32 c, + char *dest, int32_t destCapacity, + UErrorCode *pErrorCode) { + int32_t length; + + /* check the argument values */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } else if(destCapacity<0 || (destCapacity>0 && dest==NULL)) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + if((uint32_t)c>UCHAR_MAX_VALUE || !isDataLoaded(pErrorCode)) { + return u_terminateChars(dest, destCapacity, 0, pErrorCode); + } + + /* the ISO comment is stored like a normal character name */ + length=getName(uCharNames, (uint32_t)c, U_ISO_COMMENT, dest, (uint16_t)destCapacity); + return u_terminateChars(dest, destCapacity, length, pErrorCode); +} + +U_CAPI UChar32 U_EXPORT2 +u_charFromName(UCharNameChoice nameChoice, + const char *name, + UErrorCode *pErrorCode) { + char upper[120], lower[120]; + FindName findName; + AlgorithmicRange *algRange; + uint32_t *p; + uint32_t i; + UChar32 cp = 0; + char c0; + UChar32 error = 0xffff; /* Undefined, but use this for backwards compatibility. */ + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return error; + } + + if(nameChoice>=U_CHAR_NAME_CHOICE_COUNT || name==NULL || *name==0) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return error; + } + + if(!isDataLoaded(pErrorCode)) { + return error; + } + + /* construct the uppercase and lowercase of the name first */ + for(i=0; i') { + for (--i; lower[i] && lower[i] != '-'; --i) { + } + + if (lower[i] == '-') { /* We've got a category. */ + uint32_t cIdx; + + lower[i] = 0; + + for (++i; lower[i] != '>'; ++i) { + if (lower[i] >= '0' && lower[i] <= '9') { + cp = (cp << 4) + lower[i] - '0'; + } else if (lower[i] >= 'a' && lower[i] <= 'f') { + cp = (cp << 4) + lower[i] - 'a' + 10; + } else { + *pErrorCode = U_ILLEGAL_CHAR_FOUND; + return error; + } + } + + /* Now validate the category name. + We could use a binary search, or a trie, if + we really wanted to. */ + + for (lower[i] = 0, cIdx = 0; cIdx < LENGTHOF(charCatNames); ++cIdx) { + + if (!uprv_strcmp(lower + 1, charCatNames[cIdx])) { + if (getCharCat(cp) == cIdx) { + return cp; + } + break; + } + } + } + } + } + + *pErrorCode = U_ILLEGAL_CHAR_FOUND; + return error; + } + + /* try algorithmic names now */ + p=(uint32_t *)((uint8_t *)uCharNames+uCharNames->algNamesOffset); + i=*p; + algRange=(AlgorithmicRange *)(p+1); + while(i>0) { + if((cp=findAlgName(algRange, nameChoice, upper))!=0xffff) { + return cp; + } + algRange=(AlgorithmicRange *)((uint8_t *)algRange+algRange->size); + --i; + } + + /* normal character name */ + findName.otherName=upper; + findName.code=error; + enumNames(uCharNames, 0, UCHAR_MAX_VALUE + 1, DO_FIND_NAME, &findName, nameChoice); + if (findName.code == error) { + *pErrorCode = U_ILLEGAL_CHAR_FOUND; + } + return findName.code; +} + +U_CAPI void U_EXPORT2 +u_enumCharNames(UChar32 start, UChar32 limit, + UEnumCharNamesFn *fn, + void *context, + UCharNameChoice nameChoice, + UErrorCode *pErrorCode) { + AlgorithmicRange *algRange; + uint32_t *p; + uint32_t i; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return; + } + + if(nameChoice>=U_CHAR_NAME_CHOICE_COUNT || fn==NULL) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + if((uint32_t) limit > UCHAR_MAX_VALUE + 1) { + limit = UCHAR_MAX_VALUE + 1; + } + if((uint32_t)start>=(uint32_t)limit) { + return; + } + + if(!isDataLoaded(pErrorCode)) { + return; + } + + /* interleave the data-driven ones with the algorithmic ones */ + /* iterate over all algorithmic ranges; assume that they are in ascending order */ + p=(uint32_t *)((uint8_t *)uCharNames+uCharNames->algNamesOffset); + i=*p; + algRange=(AlgorithmicRange *)(p+1); + while(i>0) { + /* enumerate the character names before the current algorithmic range */ + /* here: startstart) { + if((uint32_t)limit<=algRange->start) { + enumNames(uCharNames, start, limit, fn, context, nameChoice); + return; + } + if(!enumNames(uCharNames, start, (UChar32)algRange->start, fn, context, nameChoice)) { + return; + } + start=(UChar32)algRange->start; + } + /* enumerate the character names in the current algorithmic range */ + /* here: algRange->start<=startend) { + if((uint32_t)limit<=(algRange->end+1)) { + enumAlgNames(algRange, start, limit, fn, context, nameChoice); + return; + } + if(!enumAlgNames(algRange, start, (UChar32)algRange->end+1, fn, context, nameChoice)) { + return; + } + start=(UChar32)algRange->end+1; + } + /* continue to the next algorithmic range (here: startsize); + --i; + } + /* enumerate the character names after the last algorithmic range */ + enumNames(uCharNames, start, limit, fn, context, nameChoice); +} + +U_CAPI int32_t U_EXPORT2 +uprv_getMaxCharNameLength() { + UErrorCode errorCode=U_ZERO_ERROR; + if(calcNameSetsLengths(&errorCode)) { + return gMaxNameLength; + } else { + return 0; + } +} + +/** + * Converts the char set cset into a Unicode set uset. + * @param cset Set of 256 bit flags corresponding to a set of chars. + * @param uset USet to receive characters. Existing contents are deleted. + */ +static void +charSetToUSet(uint32_t cset[8], const USetAdder *sa) { + UChar us[256]; + char cs[256]; + + int32_t i, length; + UErrorCode errorCode; + + errorCode=U_ZERO_ERROR; + + if(!calcNameSetsLengths(&errorCode)) { + return; + } + + /* build a char string with all chars that are used in character names */ + length=0; + for(i=0; i<256; ++i) { + if(SET_CONTAINS(cset, i)) { + cs[length++]=(char)i; + } + } + + /* convert the char string to a UChar string */ + u_charsToUChars(cs, us, length); + + /* add each UChar to the USet */ + for(i=0; iadd(sa->set, us[i]); + } + } +} + +/** + * Fills set with characters that are used in Unicode character names. + * @param set USet to receive characters. + */ +U_CAPI void U_EXPORT2 +uprv_getCharNameCharacters(const USetAdder *sa) { + charSetToUSet(gNameSet, sa); +} + +/* data swapping ------------------------------------------------------------ */ + +/* + * The token table contains non-negative entries for token bytes, + * and -1 for bytes that represent themselves in the data file's charset. + * -2 entries are used for lead bytes. + * + * Direct bytes (-1 entries) must be translated from the input charset family + * to the output charset family. + * makeTokenMap() writes a permutation mapping for this. + * Use it once for single-/lead-byte tokens and once more for all trail byte + * tokens. (';' is an unused trail byte marked with -1.) + */ +static void +makeTokenMap(const UDataSwapper *ds, + int16_t tokens[], uint16_t tokenCount, + uint8_t map[256], + UErrorCode *pErrorCode) { + UBool usedOutChar[256]; + uint16_t i, j; + uint8_t c1, c2; + + if(U_FAILURE(*pErrorCode)) { + return; + } + + if(ds->inCharset==ds->outCharset) { + /* Same charset family: identity permutation */ + for(i=0; i<256; ++i) { + map[i]=(uint8_t)i; + } + } else { + uprv_memset(map, 0, 256); + uprv_memset(usedOutChar, 0, 256); + + if(tokenCount>256) { + tokenCount=256; + } + + /* set the direct bytes (byte 0 always maps to itself) */ + for(i=1; iswapInvChars(ds, &c1, 1, &c2, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + udata_printError(ds, "unames/makeTokenMap() finds variant character 0x%02x used (input charset family %d)\n", + i, ds->inCharset); + return; + } + + /* enter the converted character into the map and mark it used */ + map[c1]=c2; + usedOutChar[c2]=TRUE; + } + } + + /* set the mappings for the rest of the permutation */ + for(i=j=1; idataFormat[0]==0x75 && /* dataFormat="unam" */ + pInfo->dataFormat[1]==0x6e && + pInfo->dataFormat[2]==0x61 && + pInfo->dataFormat[3]==0x6d && + pInfo->formatVersion[0]==1 + )) { + udata_printError(ds, "uchar_swapNames(): data format %02x.%02x.%02x.%02x (format version %02x) is not recognized as unames.icu\n", + pInfo->dataFormat[0], pInfo->dataFormat[1], + pInfo->dataFormat[2], pInfo->dataFormat[3], + pInfo->formatVersion[0]); + *pErrorCode=U_UNSUPPORTED_ERROR; + return 0; + } + + inBytes=(const uint8_t *)inData+headerSize; + outBytes=(uint8_t *)outData+headerSize; + if(length<0) { + algNamesOffset=ds->readUInt32(((const uint32_t *)inBytes)[3]); + } else { + length-=headerSize; + if( length<20 || + (uint32_t)length<(algNamesOffset=ds->readUInt32(((const uint32_t *)inBytes)[3])) + ) { + udata_printError(ds, "uchar_swapNames(): too few bytes (%d after header) for unames.icu\n", + length); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + } + + if(length<0) { + /* preflighting: iterate through algorithmic ranges */ + offset=algNamesOffset; + count=ds->readUInt32(*((const uint32_t *)(inBytes+offset))); + offset+=4; + + for(i=0; ireadUInt16(inRange->size); + } + } else { + /* swap data */ + const uint16_t *p; + uint16_t *q, *temp; + + int16_t tokens[512]; + uint16_t tokenCount; + + uint8_t map[256], trailMap[256]; + + /* copy the data for inaccessible bytes */ + if(inBytes!=outBytes) { + uprv_memcpy(outBytes, inBytes, length); + } + + /* the initial 4 offsets first */ + tokenStringOffset=ds->readUInt32(((const uint32_t *)inBytes)[0]); + groupsOffset=ds->readUInt32(((const uint32_t *)inBytes)[1]); + groupStringOffset=ds->readUInt32(((const uint32_t *)inBytes)[2]); + ds->swapArray32(ds, inBytes, 16, outBytes, pErrorCode); + + /* + * now the tokens table + * it needs to be permutated along with the compressed name strings + */ + p=(const uint16_t *)(inBytes+16); + q=(uint16_t *)(outBytes+16); + + /* read and swap the tokenCount */ + tokenCount=ds->readUInt16(*p); + ds->swapArray16(ds, p, 2, q, pErrorCode); + ++p; + ++q; + + /* read the first 512 tokens and make the token maps */ + if(tokenCount<=512) { + count=tokenCount; + } else { + count=512; + } + for(i=0; i256 ? tokenCount-256 : 0), trailMap, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return 0; + } + + /* + * swap and permutate the tokens + * go through a temporary array to support in-place swapping + */ + temp=(uint16_t *)uprv_malloc(tokenCount*2); + if(temp==NULL) { + udata_printError(ds, "out of memory swapping %u unames.icu tokens\n", + tokenCount); + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return 0; + } + + /* swap and permutate single-/lead-byte tokens */ + for(i=0; iswapArray16(ds, p+i, 2, temp+map[i], pErrorCode); + } + + /* swap and permutate trail-byte tokens */ + for(; iswapArray16(ds, p+i, 2, temp+(i&0xffffff00)+trailMap[i&0xff], pErrorCode); + } + + /* copy the result into the output and free the temporary array */ + uprv_memcpy(q, temp, tokenCount*2); + uprv_free(temp); + + /* + * swap the token strings but not a possible padding byte after + * the terminating NUL of the last string + */ + udata_swapInvStringBlock(ds, inBytes+tokenStringOffset, (int32_t)(groupsOffset-tokenStringOffset), + outBytes+tokenStringOffset, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + udata_printError(ds, "uchar_swapNames(token strings) failed\n"); + return 0; + } + + /* swap the group table */ + count=ds->readUInt16(*((const uint16_t *)(inBytes+groupsOffset))); + ds->swapArray16(ds, inBytes+groupsOffset, (int32_t)((1+count*3)*2), + outBytes+groupsOffset, pErrorCode); + + /* + * swap the group strings + * swap the string bytes but not the nibble-encoded string lengths + */ + if(ds->inCharset!=ds->outCharset) { + uint16_t offsets[LINES_PER_GROUP+1], lengths[LINES_PER_GROUP+1]; + + const uint8_t *inStrings, *nextInStrings; + uint8_t *outStrings; + + uint8_t c; + + inStrings=inBytes+groupStringOffset; + outStrings=outBytes+groupStringOffset; + + stringsCount=algNamesOffset-groupStringOffset; + + /* iterate through string groups until only a few padding bytes are left */ + while(stringsCount>32) { + nextInStrings=expandGroupLengths(inStrings, offsets, lengths); + + /* move past the length bytes */ + stringsCount-=(uint32_t)(nextInStrings-inStrings); + outStrings+=nextInStrings-inStrings; + inStrings=nextInStrings; + + count=offsets[31]+lengths[31]; /* total number of string bytes in this group */ + stringsCount-=count; + + /* swap the string bytes using map[] and trailMap[] */ + while(count>0) { + c=*inStrings++; + *outStrings++=map[c]; + if(tokens[c]!=-2) { + --count; + } else { + /* token lead byte: swap the trail byte, too */ + *outStrings++=trailMap[*inStrings++]; + count-=2; + } + } + } + } + + /* swap the algorithmic ranges */ + offset=algNamesOffset; + count=ds->readUInt32(*((const uint32_t *)(inBytes+offset))); + ds->swapArray32(ds, inBytes+offset, 4, outBytes+offset, pErrorCode); + offset+=4; + + for(i=0; i(uint32_t)length) { + udata_printError(ds, "uchar_swapNames(): too few bytes (%d after header) for unames.icu algorithmic range %u\n", + length, i); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + + inRange=(const AlgorithmicRange *)(inBytes+offset); + outRange=(AlgorithmicRange *)(outBytes+offset); + offset+=ds->readUInt16(inRange->size); + + ds->swapArray32(ds, inRange, 8, outRange, pErrorCode); + ds->swapArray16(ds, &inRange->size, 2, &outRange->size, pErrorCode); + switch(inRange->type) { + case 0: + /* swap prefix string */ + ds->swapInvChars(ds, inRange+1, (int32_t)uprv_strlen((const char *)(inRange+1)), + outRange+1, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + udata_printError(ds, "uchar_swapNames(prefix string of algorithmic range %u) failed\n", + i); + return 0; + } + break; + case 1: + { + /* swap factors and the prefix and factor strings */ + uint32_t factorsCount; + + factorsCount=inRange->variant; + p=(const uint16_t *)(inRange+1); + q=(uint16_t *)(outRange+1); + ds->swapArray16(ds, p, (int32_t)(factorsCount*2), q, pErrorCode); + + /* swap the strings, up to the last terminating NUL */ + p+=factorsCount; + q+=factorsCount; + stringsCount=(uint32_t)((inBytes+offset)-(const uint8_t *)p); + while(stringsCount>0 && ((const uint8_t *)p)[stringsCount-1]!=0) { + --stringsCount; + } + ds->swapInvChars(ds, p, (int32_t)stringsCount, q, pErrorCode); + } + break; + default: + udata_printError(ds, "uchar_swapNames(): unknown type %u of algorithmic range %u\n", + inRange->type, i); + *pErrorCode=U_UNSUPPORTED_ERROR; + return 0; + } + } + } + + return headerSize+(int32_t)offset; +} + +/* + * Hey, Emacs, please set the following: + * + * Local Variables: + * indent-tabs-mode: nil + * End: + * + */ diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/brkiter.h b/Build/source/libs/icu/icu-xetex/common/unicode/brkiter.h new file mode 100644 index 00000000000..7df5f140838 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/brkiter.h @@ -0,0 +1,557 @@ +/* +******************************************************************************** +* Copyright (C) 1997-2007, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************** +* +* File brkiter.h +* +* Modification History: +* +* Date Name Description +* 02/18/97 aliu Added typedef for TextCount. Made DONE const. +* 05/07/97 aliu Fixed DLL declaration. +* 07/09/97 jfitz Renamed BreakIterator and interface synced with JDK +* 08/11/98 helena Sync-up JDK1.2. +* 01/13/2000 helena Added UErrorCode parameter to createXXXInstance methods. +******************************************************************************** +*/ + +#ifndef BRKITER_H +#define BRKITER_H + +#include "unicode/utypes.h" + +/** + * \file + * \brief C++ API: Break Iterator. + */ + +#if UCONFIG_NO_BREAK_ITERATION + +U_NAMESPACE_BEGIN + +/* + * Allow the declaration of APIs with pointers to BreakIterator + * even when break iteration is removed from the build. + */ +class BreakIterator; + +U_NAMESPACE_END + +#else + +#include "unicode/uobject.h" +#include "unicode/unistr.h" +#include "unicode/chariter.h" +#include "unicode/locid.h" +#include "unicode/ubrk.h" +#include "unicode/strenum.h" +#include "unicode/utext.h" +#include "unicode/umisc.h" + +U_NAMESPACE_BEGIN + +/** + * The BreakIterator class implements methods for finding the location + * of boundaries in text. BreakIterator is an abstract base class. + * Instances of BreakIterator maintain a current position and scan over + * text returning the index of characters where boundaries occur. + *

+ * Line boundary analysis determines where a text string can be broken + * when line-wrapping. The mechanism correctly handles punctuation and + * hyphenated words. + *

+ * Sentence boundary analysis allows selection with correct + * interpretation of periods within numbers and abbreviations, and + * trailing punctuation marks such as quotation marks and parentheses. + *

+ * Word boundary analysis is used by search and replace functions, as + * well as within text editing applications that allow the user to + * select words with a double click. Word selection provides correct + * interpretation of punctuation marks within and following + * words. Characters that are not part of a word, such as symbols or + * punctuation marks, have word-breaks on both sides. + *

+ * Character boundary analysis allows users to interact with + * characters as they expect to, for example, when moving the cursor + * through a text string. Character boundary analysis provides correct + * navigation of through character strings, regardless of how the + * character is stored. For example, an accented character might be + * stored as a base character and a diacritical mark. What users + * consider to be a character can differ between languages. + *

+ * The text boundary positions are found according to the rules + * described in Unicode Standard Annex #29, Text Boundaries, and + * Unicode Standard Annex #14, Line Breaking Properties. These + * are available at http://www.unicode.org/reports/tr14/ and + * http://www.unicode.org/reports/tr29/. + *

+ * In addition to the C++ API defined in this header file, a + * plain C API with equivalent functionality is defined in the + * file ubrk.h + *

+ * Code snippits illustrating the use of the Break Iterator APIs + * are available in the ICU User Guide, + * http://icu-project.org/userguide/boundaryAnalysis.html + * and in the sample program icu/source/samples/break/break.cpp" + * + */ +class U_COMMON_API BreakIterator : public UObject { +public: + /** + * destructor + * @stable ICU 2.0 + */ + virtual ~BreakIterator(); + + /** + * Return true if another object is semantically equal to this + * one. The other object should be an instance of the same subclass of + * BreakIterator. Objects of different subclasses are considered + * unequal. + *

+ * Return true if this BreakIterator is at the same position in the + * same text, and is the same class and type (word, line, etc.) of + * BreakIterator, as the argument. Text is considered the same if + * it contains the same characters, it need not be the same + * object, and styles are not considered. + * @stable ICU 2.0 + */ + virtual UBool operator==(const BreakIterator&) const = 0; + + /** + * Returns the complement of the result of operator== + * @param rhs The BreakIterator to be compared for inequality + * @return the complement of the result of operator== + * @stable ICU 2.0 + */ + UBool operator!=(const BreakIterator& rhs) const { return !operator==(rhs); } + + /** + * Return a polymorphic copy of this object. This is an abstract + * method which subclasses implement. + * @stable ICU 2.0 + */ + virtual BreakIterator* clone(void) const = 0; + + /** + * Return a polymorphic class ID for this object. Different subclasses + * will return distinct unequal values. + * @stable ICU 2.0 + */ + virtual UClassID getDynamicClassID(void) const = 0; + + /** + * Return a CharacterIterator over the text being analyzed. + * @stable ICU 2.0 + */ + virtual CharacterIterator& getText(void) const = 0; + + + /** + * Get a UText for the text being analyzed. + * The returned UText is a shallow clone of the UText used internally + * by the break iterator implementation. It can safely be used to + * access the text without impacting any break iterator operations, + * but the underlying text itself must not be altered. + * + * @param fillIn A UText to be filled in. If NULL, a new UText will be + * allocated to hold the result. + * @param status receives any error codes. + * @return The current UText for this break iterator. If an input + * UText was provided, it will always be returned. + * @stable ICU 3.4 + */ + virtual UText *getUText(UText *fillIn, UErrorCode &status) const = 0; + + /** + * Change the text over which this operates. The text boundary is + * reset to the start. + * @param text The UnicodeString used to change the text. + * @stable ICU 2.0 + */ + virtual void setText(const UnicodeString &text) = 0; + + /** + * Reset the break iterator to operate over the text represented by + * the UText. The iterator position is reset to the start. + * + * This function makes a shallow clone of the supplied UText. This means + * that the caller is free to immediately close or otherwise reuse the + * Utext that was passed as a parameter, but that the underlying text itself + * must not be altered while being referenced by the break iterator. + * + * @param text The UText used to change the text. + * @param status receives any error codes. + * @stable ICU 3.4 + */ + virtual void setText(UText *text, UErrorCode &status) = 0; + + /** + * Change the text over which this operates. The text boundary is + * reset to the start. + * Note that setText(UText *) provides similar functionality to this function, + * and is more efficient. + * @param it The CharacterIterator used to change the text. + * @stable ICU 2.0 + */ + virtual void adoptText(CharacterIterator* it) = 0; + + enum { + /** + * DONE is returned by previous() and next() after all valid + * boundaries have been returned. + * @stable ICU 2.0 + */ + DONE = (int32_t)-1 + }; + + /** + * Return the index of the first character in the text being scanned. + * @stable ICU 2.0 + */ + virtual int32_t first(void) = 0; + + /** + * Return the index immediately BEYOND the last character in the text being scanned. + * @stable ICU 2.0 + */ + virtual int32_t last(void) = 0; + + /** + * Return the boundary preceding the current boundary. + * @return The character index of the previous text boundary or DONE if all + * boundaries have been returned. + * @stable ICU 2.0 + */ + virtual int32_t previous(void) = 0; + + /** + * Return the boundary following the current boundary. + * @return The character index of the next text boundary or DONE if all + * boundaries have been returned. + * @stable ICU 2.0 + */ + virtual int32_t next(void) = 0; + + /** + * Return character index of the current interator position within the text. + * @return The boundary most recently returned. + * @stable ICU 2.0 + */ + virtual int32_t current(void) const = 0; + + /** + * Return the first boundary following the specified offset. + * The value returned is always greater than the offset or + * the value BreakIterator.DONE + * @param offset the offset to begin scanning. + * @return The first boundary after the specified offset. + * @stable ICU 2.0 + */ + virtual int32_t following(int32_t offset) = 0; + + /** + * Return the first boundary preceding the specified offset. + * The value returned is always smaller than the offset or + * the value BreakIterator.DONE + * @param offset the offset to begin scanning. + * @return The first boundary before the specified offset. + * @stable ICU 2.0 + */ + virtual int32_t preceding(int32_t offset) = 0; + + /** + * Return true if the specfied position is a boundary position. + * As a side effect, the current position of the iterator is set + * to the first boundary position at or following the specified offset. + * @param offset the offset to check. + * @return True if "offset" is a boundary position. + * @stable ICU 2.0 + */ + virtual UBool isBoundary(int32_t offset) = 0; + + /** + * Return the nth boundary from the current boundary + * @param n which boundary to return. A value of 0 + * does nothing. Negative values move to previous boundaries + * and positive values move to later boundaries. + * @return The index of the nth boundary from the current position, or + * DONE if there are fewer than |n| boundaries in the specfied direction. + * @stable ICU 2.0 + */ + virtual int32_t next(int32_t n) = 0; + + /** + * Create BreakIterator for word-breaks using the given locale. + * Returns an instance of a BreakIterator implementing word breaks. + * WordBreak is useful for word selection (ex. double click) + * @param where the locale. + * @param status the error code + * @return A BreakIterator for word-breaks. The UErrorCode& status + * parameter is used to return status information to the user. + * To check whether the construction succeeded or not, you should check + * the value of U_SUCCESS(err). If you wish more detailed information, you + * can check for informational error results which still indicate success. + * U_USING_FALLBACK_WARNING indicates that a fall back locale was used. For + * example, 'de_CH' was requested, but nothing was found there, so 'de' was + * used. U_USING_DEFAULT_WARNING indicates that the default locale data was + * used; neither the requested locale nor any of its fall back locales + * could be found. + * The caller owns the returned object and is responsible for deleting it. + * @stable ICU 2.0 + */ + static BreakIterator* U_EXPORT2 + createWordInstance(const Locale& where, UErrorCode& status); + + /** + * Create BreakIterator for line-breaks using specified locale. + * Returns an instance of a BreakIterator implementing line breaks. Line + * breaks are logically possible line breaks, actual line breaks are + * usually determined based on display width. + * LineBreak is useful for word wrapping text. + * @param where the locale. + * @param status The error code. + * @return A BreakIterator for line-breaks. The UErrorCode& status + * parameter is used to return status information to the user. + * To check whether the construction succeeded or not, you should check + * the value of U_SUCCESS(err). If you wish more detailed information, you + * can check for informational error results which still indicate success. + * U_USING_FALLBACK_WARNING indicates that a fall back locale was used. For + * example, 'de_CH' was requested, but nothing was found there, so 'de' was + * used. U_USING_DEFAULT_WARNING indicates that the default locale data was + * used; neither the requested locale nor any of its fall back locales + * could be found. + * The caller owns the returned object and is responsible for deleting it. + * @stable ICU 2.0 + */ + static BreakIterator* U_EXPORT2 + createLineInstance(const Locale& where, UErrorCode& status); + + /** + * Create BreakIterator for character-breaks using specified locale + * Returns an instance of a BreakIterator implementing character breaks. + * Character breaks are boundaries of combining character sequences. + * @param where the locale. + * @param status The error code. + * @return A BreakIterator for character-breaks. The UErrorCode& status + * parameter is used to return status information to the user. + * To check whether the construction succeeded or not, you should check + * the value of U_SUCCESS(err). If you wish more detailed information, you + * can check for informational error results which still indicate success. + * U_USING_FALLBACK_WARNING indicates that a fall back locale was used. For + * example, 'de_CH' was requested, but nothing was found there, so 'de' was + * used. U_USING_DEFAULT_WARNING indicates that the default locale data was + * used; neither the requested locale nor any of its fall back locales + * could be found. + * The caller owns the returned object and is responsible for deleting it. + * @stable ICU 2.0 + */ + static BreakIterator* U_EXPORT2 + createCharacterInstance(const Locale& where, UErrorCode& status); + + /** + * Create BreakIterator for sentence-breaks using specified locale + * Returns an instance of a BreakIterator implementing sentence breaks. + * @param where the locale. + * @param status The error code. + * @return A BreakIterator for sentence-breaks. The UErrorCode& status + * parameter is used to return status information to the user. + * To check whether the construction succeeded or not, you should check + * the value of U_SUCCESS(err). If you wish more detailed information, you + * can check for informational error results which still indicate success. + * U_USING_FALLBACK_WARNING indicates that a fall back locale was used. For + * example, 'de_CH' was requested, but nothing was found there, so 'de' was + * used. U_USING_DEFAULT_WARNING indicates that the default locale data was + * used; neither the requested locale nor any of its fall back locales + * could be found. + * The caller owns the returned object and is responsible for deleting it. + * @stable ICU 2.0 + */ + static BreakIterator* U_EXPORT2 + createSentenceInstance(const Locale& where, UErrorCode& status); + + /** + * Create BreakIterator for title-casing breaks using the specified locale + * Returns an instance of a BreakIterator implementing title breaks. + * The iterator returned locates title boundaries as described for + * Unicode 3.2 only. For Unicode 4.0 and above title boundary iteration, + * please use Word Boundary iterator.{@link #createWordInstance } + * + * @param where the locale. + * @param status The error code. + * @return A BreakIterator for title-breaks. The UErrorCode& status + * parameter is used to return status information to the user. + * To check whether the construction succeeded or not, you should check + * the value of U_SUCCESS(err). If you wish more detailed information, you + * can check for informational error results which still indicate success. + * U_USING_FALLBACK_WARNING indicates that a fall back locale was used. For + * example, 'de_CH' was requested, but nothing was found there, so 'de' was + * used. U_USING_DEFAULT_WARNING indicates that the default locale data was + * used; neither the requested locale nor any of its fall back locales + * could be found. + * The caller owns the returned object and is responsible for deleting it. + * @stable ICU 2.1 + */ + static BreakIterator* U_EXPORT2 + createTitleInstance(const Locale& where, UErrorCode& status); + + /** + * Get the set of Locales for which TextBoundaries are installed. + *

Note: this will not return locales added through the register + * call. To see the registered locales too, use the getAvailableLocales + * function that returns a StringEnumeration object

+ * @param count the output parameter of number of elements in the locale list + * @return available locales + * @stable ICU 2.0 + */ + static const Locale* U_EXPORT2 getAvailableLocales(int32_t& count); + + /** + * Get name of the object for the desired Locale, in the desired langauge. + * @param objectLocale must be from getAvailableLocales. + * @param displayLocale specifies the desired locale for output. + * @param name the fill-in parameter of the return value + * Uses best match. + * @return user-displayable name + * @stable ICU 2.0 + */ + static UnicodeString& U_EXPORT2 getDisplayName(const Locale& objectLocale, + const Locale& displayLocale, + UnicodeString& name); + + /** + * Get name of the object for the desired Locale, in the langauge of the + * default locale. + * @param objectLocale must be from getMatchingLocales + * @param name the fill-in parameter of the return value + * @return user-displayable name + * @stable ICU 2.0 + */ + static UnicodeString& U_EXPORT2 getDisplayName(const Locale& objectLocale, + UnicodeString& name); + + /** + * Thread safe client-buffer-based cloning operation + * Do NOT call delete on a safeclone, since 'new' is not used to create it. + * @param stackBuffer user allocated space for the new clone. If NULL new memory will be allocated. + * If buffer is not large enough, new memory will be allocated. + * @param BufferSize reference to size of allocated space. + * If BufferSize == 0, a sufficient size for use in cloning will + * be returned ('pre-flighting') + * If BufferSize is not enough for a stack-based safe clone, + * new memory will be allocated. + * @param status to indicate whether the operation went on smoothly or there were errors + * An informational status value, U_SAFECLONE_ALLOCATED_ERROR, is used if any allocations were + * necessary. + * @return pointer to the new clone + * + * @stable ICU 2.0 + */ + virtual BreakIterator * createBufferClone(void *stackBuffer, + int32_t &BufferSize, + UErrorCode &status) = 0; + + /** + * Determine whether the BreakIterator was created in user memory by + * createBufferClone(), and thus should not be deleted. Such objects + * must be closed by an explicit call to the destructor (not delete). + * @stable ICU 2.0 + */ + inline UBool isBufferClone(void); + +#if !UCONFIG_NO_SERVICE + /** + * Register a new break iterator of the indicated kind, to use in the given locale. + * The break iterator will be adopted. Clones of the iterator will be returned + * if a request for a break iterator of the given kind matches or falls back to + * this locale. + * @param toAdopt the BreakIterator instance to be adopted + * @param locale the Locale for which this instance is to be registered + * @param kind the type of iterator for which this instance is to be registered + * @param status the in/out status code, no special meanings are assigned + * @return a registry key that can be used to unregister this instance + * @stable ICU 2.4 + */ + static URegistryKey U_EXPORT2 registerInstance(BreakIterator* toAdopt, + const Locale& locale, + UBreakIteratorType kind, + UErrorCode& status); + + /** + * Unregister a previously-registered BreakIterator using the key returned from the + * register call. Key becomes invalid after a successful call and should not be used again. + * The BreakIterator corresponding to the key will be deleted. + * @param key the registry key returned by a previous call to registerInstance + * @param status the in/out status code, no special meanings are assigned + * @return TRUE if the iterator for the key was successfully unregistered + * @stable ICU 2.4 + */ + static UBool U_EXPORT2 unregister(URegistryKey key, UErrorCode& status); + + /** + * Return a StringEnumeration over the locales available at the time of the call, + * including registered locales. + * @return a StringEnumeration over the locales available at the time of the call + * @stable ICU 2.4 + */ + static StringEnumeration* U_EXPORT2 getAvailableLocales(void); +#endif + + /** + * Returns the locale for this break iterator. Two flavors are available: valid and + * actual locale. + * @stable ICU 2.8 + */ + Locale getLocale(ULocDataLocaleType type, UErrorCode& status) const; + + /** Get the locale for this break iterator object. You can choose between valid and actual locale. + * @param type type of the locale we're looking for (valid or actual) + * @param status error code for the operation + * @return the locale + * @internal + */ + const char *getLocaleID(ULocDataLocaleType type, UErrorCode& status) const; + + private: + static BreakIterator* buildInstance(const Locale& loc, const char *type, int32_t kind, UErrorCode& status); + static BreakIterator* createInstance(const Locale& loc, int32_t kind, UErrorCode& status); + static BreakIterator* makeInstance(const Locale& loc, int32_t kind, UErrorCode& status); + + friend class ICUBreakIteratorFactory; + friend class ICUBreakIteratorService; + +protected: + /** @internal */ + BreakIterator(); + /** @internal */ + UBool fBufferClone; + /** @internal */ + BreakIterator (const BreakIterator &other) : UObject(other), fBufferClone(FALSE) {} + +private: + + /** @internal */ + char actualLocale[ULOC_FULLNAME_CAPACITY]; + char validLocale[ULOC_FULLNAME_CAPACITY]; + + /** + * The assignment operator has no real implementation. + * It's provided to make the compiler happy. Do not call. + */ + BreakIterator& operator=(const BreakIterator&); +}; + +inline UBool BreakIterator::isBufferClone() +{ + return fBufferClone; +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_BREAK_ITERATION */ + +#endif // _BRKITER +//eof + diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/caniter.h b/Build/source/libs/icu/icu-xetex/common/unicode/caniter.h new file mode 100644 index 00000000000..84a65958d16 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/caniter.h @@ -0,0 +1,201 @@ +/* + ******************************************************************************* + * Copyright (C) 1996-2006, International Business Machines Corporation and * + * others. All Rights Reserved. * + ******************************************************************************* + */ + +#ifndef CANITER_H +#define CANITER_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_NORMALIZATION + +#include "unicode/uobject.h" +#include "unicode/unistr.h" + +/** + * \file + * \brief C++ API: Canonical Iterator + */ + +/** Should permutation skip characters with combining class zero + * Should be either TRUE or FALSE. This is a compile time option + * @stable ICU 2.4 + */ +#ifndef CANITER_SKIP_ZEROES +#define CANITER_SKIP_ZEROES TRUE +#endif + +U_NAMESPACE_BEGIN + +class Hashtable; + +/** + * This class allows one to iterate through all the strings that are canonically equivalent to a given + * string. For example, here are some sample results: +Results for: {LATIN CAPITAL LETTER A WITH RING ABOVE}{LATIN SMALL LETTER D}{COMBINING DOT ABOVE}{COMBINING CEDILLA} +1: \\u0041\\u030A\\u0064\\u0307\\u0327 + = {LATIN CAPITAL LETTER A}{COMBINING RING ABOVE}{LATIN SMALL LETTER D}{COMBINING DOT ABOVE}{COMBINING CEDILLA} +2: \\u0041\\u030A\\u0064\\u0327\\u0307 + = {LATIN CAPITAL LETTER A}{COMBINING RING ABOVE}{LATIN SMALL LETTER D}{COMBINING CEDILLA}{COMBINING DOT ABOVE} +3: \\u0041\\u030A\\u1E0B\\u0327 + = {LATIN CAPITAL LETTER A}{COMBINING RING ABOVE}{LATIN SMALL LETTER D WITH DOT ABOVE}{COMBINING CEDILLA} +4: \\u0041\\u030A\\u1E11\\u0307 + = {LATIN CAPITAL LETTER A}{COMBINING RING ABOVE}{LATIN SMALL LETTER D WITH CEDILLA}{COMBINING DOT ABOVE} +5: \\u00C5\\u0064\\u0307\\u0327 + = {LATIN CAPITAL LETTER A WITH RING ABOVE}{LATIN SMALL LETTER D}{COMBINING DOT ABOVE}{COMBINING CEDILLA} +6: \\u00C5\\u0064\\u0327\\u0307 + = {LATIN CAPITAL LETTER A WITH RING ABOVE}{LATIN SMALL LETTER D}{COMBINING CEDILLA}{COMBINING DOT ABOVE} +7: \\u00C5\\u1E0B\\u0327 + = {LATIN CAPITAL LETTER A WITH RING ABOVE}{LATIN SMALL LETTER D WITH DOT ABOVE}{COMBINING CEDILLA} +8: \\u00C5\\u1E11\\u0307 + = {LATIN CAPITAL LETTER A WITH RING ABOVE}{LATIN SMALL LETTER D WITH CEDILLA}{COMBINING DOT ABOVE} +9: \\u212B\\u0064\\u0307\\u0327 + = {ANGSTROM SIGN}{LATIN SMALL LETTER D}{COMBINING DOT ABOVE}{COMBINING CEDILLA} +10: \\u212B\\u0064\\u0327\\u0307 + = {ANGSTROM SIGN}{LATIN SMALL LETTER D}{COMBINING CEDILLA}{COMBINING DOT ABOVE} +11: \\u212B\\u1E0B\\u0327 + = {ANGSTROM SIGN}{LATIN SMALL LETTER D WITH DOT ABOVE}{COMBINING CEDILLA} +12: \\u212B\\u1E11\\u0307 + = {ANGSTROM SIGN}{LATIN SMALL LETTER D WITH CEDILLA}{COMBINING DOT ABOVE} + *
Note: the code is intended for use with small strings, and is not suitable for larger ones, + * since it has not been optimized for that situation. + * Note, CanonicalIterator is not intended to be subclassed. + * @author M. Davis + * @author C++ port by V. Weinstein + * @stable ICU 2.4 + */ +class U_COMMON_API CanonicalIterator : public UObject { +public: + /** + * Construct a CanonicalIterator object + * @param source string to get results for + * @param status Fill-in parameter which receives the status of this operation. + * @stable ICU 2.4 + */ + CanonicalIterator(const UnicodeString &source, UErrorCode &status); + + /** Destructor + * Cleans pieces + * @stable ICU 2.4 + */ + virtual ~CanonicalIterator(); + + /** + * Gets the NFD form of the current source we are iterating over. + * @return gets the source: NOTE: it is the NFD form of source + * @stable ICU 2.4 + */ + UnicodeString getSource(); + + /** + * Resets the iterator so that one can start again from the beginning. + * @stable ICU 2.4 + */ + void reset(); + + /** + * Get the next canonically equivalent string. + *
Warning: The strings are not guaranteed to be in any particular order. + * @return the next string that is canonically equivalent. A bogus string is returned when + * the iteration is done. + * @stable ICU 2.4 + */ + UnicodeString next(); + + /** + * Set a new source for this iterator. Allows object reuse. + * @param newSource the source string to iterate against. This allows the same iterator to be used + * while changing the source string, saving object creation. + * @param status Fill-in parameter which receives the status of this operation. + * @stable ICU 2.4 + */ + void setSource(const UnicodeString &newSource, UErrorCode &status); + + /** + * Dumb recursive implementation of permutation. + * TODO: optimize + * @param source the string to find permutations for + * @param skipZeros determine if skip zeros + * @param result the results in a set. + * @param status Fill-in parameter which receives the status of this operation. + * @internal + */ + static void U_EXPORT2 permute(UnicodeString &source, UBool skipZeros, Hashtable *result, UErrorCode &status); + + /** + * ICU "poor man's RTTI", returns a UClassID for this class. + * + * @stable ICU 2.2 + */ + static UClassID U_EXPORT2 getStaticClassID(); + + /** + * ICU "poor man's RTTI", returns a UClassID for the actual class. + * + * @stable ICU 2.2 + */ + virtual UClassID getDynamicClassID() const; + +private: + // ===================== PRIVATES ============================== + // private default constructor + CanonicalIterator(); + + + /** + * Copy constructor. Private for now. + * @internal + */ + CanonicalIterator(const CanonicalIterator& other); + + /** + * Assignment operator. Private for now. + * @internal + */ + CanonicalIterator& operator=(const CanonicalIterator& other); + + // fields + UnicodeString source; + UBool done; + + // 2 dimensional array holds the pieces of the string with + // their different canonically equivalent representations + UnicodeString **pieces; + int32_t pieces_length; + int32_t *pieces_lengths; + + // current is used in iterating to combine pieces + int32_t *current; + int32_t current_length; + + // transient fields + UnicodeString buffer; + + // we have a segment, in NFD. Find all the strings that are canonically equivalent to it. + UnicodeString *getEquivalents(const UnicodeString &segment, int32_t &result_len, UErrorCode &status); //private String[] getEquivalents(String segment) + + //Set getEquivalents2(String segment); + Hashtable *getEquivalents2(Hashtable *fillinResult, const UChar *segment, int32_t segLen, UErrorCode &status); + //Hashtable *getEquivalents2(const UnicodeString &segment, int32_t segLen, UErrorCode &status); + + /** + * See if the decomposition of cp2 is at segment starting at segmentPos + * (with canonical rearrangment!) + * If so, take the remainder, and return the equivalents + */ + //Set extract(int comp, String segment, int segmentPos, StringBuffer buffer); + Hashtable *extract(Hashtable *fillinResult, UChar32 comp, const UChar *segment, int32_t segLen, int32_t segmentPos, UErrorCode &status); + //Hashtable *extract(UChar32 comp, const UnicodeString &segment, int32_t segLen, int32_t segmentPos, UErrorCode &status); + + void cleanPieces(); + +}; + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_NORMALIZATION */ + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/chariter.h b/Build/source/libs/icu/icu-xetex/common/unicode/chariter.h new file mode 100644 index 00000000000..12fc9248a23 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/chariter.h @@ -0,0 +1,716 @@ +/* +******************************************************************** +* +* Copyright (C) 1997-2005, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************** +*/ + +#ifndef CHARITER_H +#define CHARITER_H + +#include "unicode/utypes.h" +#include "unicode/uobject.h" +#include "unicode/unistr.h" +/** + * \file + * \brief C++ API: Character Iterator + */ + +U_NAMESPACE_BEGIN +/** + * Abstract class that defines an API for forward-only iteration + * on text objects. + * This is a minimal interface for iteration without random access + * or backwards iteration. It is especially useful for wrapping + * streams with converters into an object for collation or + * normalization. + * + *

Characters can be accessed in two ways: as code units or as + * code points. + * Unicode code points are 21-bit integers and are the scalar values + * of Unicode characters. ICU uses the type UChar32 for them. + * Unicode code units are the storage units of a given + * Unicode/UCS Transformation Format (a character encoding scheme). + * With UTF-16, all code points can be represented with either one + * or two code units ("surrogates"). + * String storage is typically based on code units, while properties + * of characters are typically determined using code point values. + * Some processes may be designed to work with sequences of code units, + * or it may be known that all characters that are important to an + * algorithm can be represented with single code units. + * Other processes will need to use the code point access functions.

+ * + *

ForwardCharacterIterator provides nextPostInc() to access + * a code unit and advance an internal position into the text object, + * similar to a return text[position++].
+ * It provides next32PostInc() to access a code point and advance an internal + * position.

+ * + *

next32PostInc() assumes that the current position is that of + * the beginning of a code point, i.e., of its first code unit. + * After next32PostInc(), this will be true again. + * In general, access to code units and code points in the same + * iteration loop should not be mixed. In UTF-16, if the current position + * is on a second code unit (Low Surrogate), then only that code unit + * is returned even by next32PostInc().

+ * + *

For iteration with either function, there are two ways to + * check for the end of the iteration. When there are no more + * characters in the text object: + *

    + *
  • The hasNext() function returns FALSE.
  • + *
  • nextPostInc() and next32PostInc() return DONE + * when one attempts to read beyond the end of the text object.
  • + *
+ * + * Example: + * \code + * void function1(ForwardCharacterIterator &it) { + * UChar32 c; + * while(it.hasNext()) { + * c=it.next32PostInc(); + * // use c + * } + * } + * + * void function1(ForwardCharacterIterator &it) { + * UChar c; + * while((c=it.nextPostInc())!=ForwardCharacterIterator::DONE) { + * // use c + * } + * } + * \endcode + *

+ * + * @stable ICU 2.0 + */ +class U_COMMON_API ForwardCharacterIterator : public UObject { +public: + /** + * Value returned by most of ForwardCharacterIterator's functions + * when the iterator has reached the limits of its iteration. + * @stable ICU 2.0 + */ + enum { DONE = 0xffff }; + + /** + * Destructor. + * @stable ICU 2.0 + */ + virtual ~ForwardCharacterIterator(); + + /** + * Returns true when both iterators refer to the same + * character in the same character-storage object. + * @param that The ForwardCharacterIterator to be compared for equality + * @return true when both iterators refer to the same + * character in the same character-storage object + * @stable ICU 2.0 + */ + virtual UBool operator==(const ForwardCharacterIterator& that) const = 0; + + /** + * Returns true when the iterators refer to different + * text-storage objects, or to different characters in the + * same text-storage object. + * @param that The ForwardCharacterIterator to be compared for inequality + * @return true when the iterators refer to different + * text-storage objects, or to different characters in the + * same text-storage object + * @stable ICU 2.0 + */ + inline UBool operator!=(const ForwardCharacterIterator& that) const; + + /** + * Generates a hash code for this iterator. + * @return the hash code. + * @stable ICU 2.0 + */ + virtual int32_t hashCode(void) const = 0; + + /** + * Returns a UClassID for this ForwardCharacterIterator ("poor man's + * RTTI").

Despite the fact that this function is public, + * DO NOT CONSIDER IT PART OF CHARACTERITERATOR'S API! + * @return a UClassID for this ForwardCharacterIterator + * @stable ICU 2.0 + */ + virtual UClassID getDynamicClassID(void) const = 0; + + /** + * Gets the current code unit for returning and advances to the next code unit + * in the iteration range + * (toward endIndex()). If there are + * no more code units to return, returns DONE. + * @return the current code unit. + * @stable ICU 2.0 + */ + virtual UChar nextPostInc(void) = 0; + + /** + * Gets the current code point for returning and advances to the next code point + * in the iteration range + * (toward endIndex()). If there are + * no more code points to return, returns DONE. + * @return the current code point. + * @stable ICU 2.0 + */ + virtual UChar32 next32PostInc(void) = 0; + + /** + * Returns FALSE if there are no more code units or code points + * at or after the current position in the iteration range. + * This is used with nextPostInc() or next32PostInc() in forward + * iteration. + * @returns FALSE if there are no more code units or code points + * at or after the current position in the iteration range. + * @stable ICU 2.0 + */ + virtual UBool hasNext() = 0; + +protected: + /** Default constructor to be overridden in the implementing class. @stable ICU 2.0*/ + ForwardCharacterIterator(); + + /** Copy constructor to be overridden in the implementing class. @stable ICU 2.0*/ + ForwardCharacterIterator(const ForwardCharacterIterator &other); + + /** + * Assignment operator to be overridden in the implementing class. + * @stable ICU 2.0 + */ + ForwardCharacterIterator &operator=(const ForwardCharacterIterator&) { return *this; } +}; + +/** + * Abstract class that defines an API for iteration + * on text objects. + * This is an interface for forward and backward iteration + * and random access into a text object. + * + *

The API provides backward compatibility to the Java and older ICU + * CharacterIterator classes but extends them significantly: + *

    + *
  1. CharacterIterator is now a subclass of ForwardCharacterIterator.
  2. + *
  3. While the old API functions provided forward iteration with + * "pre-increment" semantics, the new one also provides functions + * with "post-increment" semantics. They are more efficient and should + * be the preferred iterator functions for new implementations. + * The backward iteration always had "pre-decrement" semantics, which + * are efficient.
  4. + *
  5. Just like ForwardCharacterIterator, it provides access to + * both code units and code points. Code point access versions are available + * for the old and the new iteration semantics.
  6. + *
  7. There are new functions for setting and moving the current position + * without returning a character, for efficiency.
  8. + *
+ * + * See ForwardCharacterIterator for examples for using the new forward iteration + * functions. For backward iteration, there is also a hasPrevious() function + * that can be used analogously to hasNext(). + * The old functions work as before and are shown below.

+ * + *

Examples for some of the new functions:

+ * + * Forward iteration with hasNext(): + * \code + * void forward1(CharacterIterator &it) { + * UChar32 c; + * for(it.setToStart(); it.hasNext();) { + * c=it.next32PostInc(); + * // use c + * } + * } + * \endcode + * Forward iteration more similar to loops with the old forward iteration, + * showing a way to convert simple for() loops: + * \code + * void forward2(CharacterIterator &it) { + * UChar c; + * for(c=it.firstPostInc(); c!=CharacterIterator::DONE; c=it.nextPostInc()) { + * // use c + * } + * } + * \endcode + * Backward iteration with setToEnd() and hasPrevious(): + * \code + * void backward1(CharacterIterator &it) { + * UChar32 c; + * for(it.setToEnd(); it.hasPrevious();) { + * c=it.previous32(); + * // use c + * } + * } + * \endcode + * Backward iteration with a more traditional for() loop: + * \code + * void backward2(CharacterIterator &it) { + * UChar c; + * for(c=it.last(); c!=CharacterIterator::DONE; c=it.previous()) { + * // use c + * } + * } + * \endcode + * + * Example for random access: + * \code + * void random(CharacterIterator &it) { + * // set to the third code point from the beginning + * it.move32(3, CharacterIterator::kStart); + * // get a code point from here without moving the position + * UChar32 c=it.current32(); + * // get the position + * int32_t pos=it.getIndex(); + * // get the previous code unit + * UChar u=it.previous(); + * // move back one more code unit + * it.move(-1, CharacterIterator::kCurrent); + * // set the position back to where it was + * // and read the same code point c and move beyond it + * it.setIndex(pos); + * if(c!=it.next32PostInc()) { + * exit(1); // CharacterIterator inconsistent + * } + * } + * \endcode + * + *

Examples, especially for the old API:

+ * + * Function processing characters, in this example simple output + *
+ * \code
+ *  void processChar( UChar c )
+ *  {
+ *      cout << " " << c;
+ *  }
+ * \endcode
+ * 
+ * Traverse the text from start to finish + *
 
+ * \code
+ *  void traverseForward(CharacterIterator& iter)
+ *  {
+ *      for(UChar c = iter.first(); c != CharacterIterator.DONE; c = iter.next()) {
+ *          processChar(c);
+ *      }
+ *  }
+ * \endcode
+ * 
+ * Traverse the text backwards, from end to start + *
+ * \code
+ *  void traverseBackward(CharacterIterator& iter)
+ *  {
+ *      for(UChar c = iter.last(); c != CharacterIterator.DONE; c = iter.previous()) {
+ *          processChar(c);
+ *      }
+ *  }
+ * \endcode
+ * 
+ * Traverse both forward and backward from a given position in the text. + * Calls to notBoundary() in this example represents some additional stopping criteria. + *
+ * \code
+ * void traverseOut(CharacterIterator& iter, int32_t pos)
+ * {
+ *      UChar c;
+ *      for (c = iter.setIndex(pos);
+ *      c != CharacterIterator.DONE && (Unicode::isLetter(c) || Unicode::isDigit(c));
+ *          c = iter.next()) {}
+ *      int32_t end = iter.getIndex();
+ *      for (c = iter.setIndex(pos);
+ *          c != CharacterIterator.DONE && (Unicode::isLetter(c) || Unicode::isDigit(c));
+ *          c = iter.previous()) {}
+ *      int32_t start = iter.getIndex() + 1;
+ *  
+ *      cout << "start: " << start << " end: " << end << endl;
+ *      for (c = iter.setIndex(start); iter.getIndex() < end; c = iter.next() ) {
+ *          processChar(c);
+ *     }
+ *  }
+ * \endcode
+ * 
+ * Creating a StringCharacterIterator and calling the test functions + *
+ * \code
+ *  void CharacterIterator_Example( void )
+ *   {
+ *       cout << endl << "===== CharacterIterator_Example: =====" << endl;
+ *       UnicodeString text("Ein kleiner Satz.");
+ *       StringCharacterIterator iterator(text);
+ *       cout << "----- traverseForward: -----------" << endl;
+ *       traverseForward( iterator );
+ *       cout << endl << endl << "----- traverseBackward: ----------" << endl;
+ *       traverseBackward( iterator );
+ *       cout << endl << endl << "----- traverseOut: ---------------" << endl;
+ *       traverseOut( iterator, 7 );
+ *       cout << endl << endl << "-----" << endl;
+ *   }
+ * \endcode
+ * 
+ * + * @stable ICU 2.0 + */ +class U_COMMON_API CharacterIterator : public ForwardCharacterIterator { +public: + /** + * Origin enumeration for the move() and move32() functions. + * @stable ICU 2.0 + */ + enum EOrigin { kStart, kCurrent, kEnd }; + + /** + * Returns a pointer to a new CharacterIterator of the same + * concrete class as this one, and referring to the same + * character in the same text-storage object as this one. The + * caller is responsible for deleting the new clone. + * @return a pointer to a new CharacterIterator + * @stable ICU 2.0 + */ + virtual CharacterIterator* clone(void) const = 0; + + /** + * Sets the iterator to refer to the first code unit in its + * iteration range, and returns that code unit. + * This can be used to begin an iteration with next(). + * @return the first code unit in its iteration range. + * @stable ICU 2.0 + */ + virtual UChar first(void) = 0; + + /** + * Sets the iterator to refer to the first code unit in its + * iteration range, returns that code unit, and moves the position + * to the second code unit. This is an alternative to setToStart() + * for forward iteration with nextPostInc(). + * @return the first code unit in its iteration range. + * @stable ICU 2.0 + */ + virtual UChar firstPostInc(void); + + /** + * Sets the iterator to refer to the first code point in its + * iteration range, and returns that code unit, + * This can be used to begin an iteration with next32(). + * Note that an iteration with next32PostInc(), beginning with, + * e.g., setToStart() or firstPostInc(), is more efficient. + * @return the first code point in its iteration range. + * @stable ICU 2.0 + */ + virtual UChar32 first32(void) = 0; + + /** + * Sets the iterator to refer to the first code point in its + * iteration range, returns that code point, and moves the position + * to the second code point. This is an alternative to setToStart() + * for forward iteration with next32PostInc(). + * @return the first code point in its iteration range. + * @stable ICU 2.0 + */ + virtual UChar32 first32PostInc(void); + + /** + * Sets the iterator to refer to the first code unit or code point in its + * iteration range. This can be used to begin a forward + * iteration with nextPostInc() or next32PostInc(). + * @return the start position of the iteration range + * @stable ICU 2.0 + */ + inline int32_t setToStart(); + + /** + * Sets the iterator to refer to the last code unit in its + * iteration range, and returns that code unit. + * This can be used to begin an iteration with previous(). + * @return the last code unit. + * @stable ICU 2.0 + */ + virtual UChar last(void) = 0; + + /** + * Sets the iterator to refer to the last code point in its + * iteration range, and returns that code unit. + * This can be used to begin an iteration with previous32(). + * @return the last code point. + * @stable ICU 2.0 + */ + virtual UChar32 last32(void) = 0; + + /** + * Sets the iterator to the end of its iteration range, just behind + * the last code unit or code point. This can be used to begin a backward + * iteration with previous() or previous32(). + * @return the end position of the iteration range + * @stable ICU 2.0 + */ + inline int32_t setToEnd(); + + /** + * Sets the iterator to refer to the "position"-th code unit + * in the text-storage object the iterator refers to, and + * returns that code unit. + * @param position the "position"-th code unit in the text-storage object + * @return the "position"-th code unit. + * @stable ICU 2.0 + */ + virtual UChar setIndex(int32_t position) = 0; + + /** + * Sets the iterator to refer to the beginning of the code point + * that contains the "position"-th code unit + * in the text-storage object the iterator refers to, and + * returns that code point. + * The current position is adjusted to the beginning of the code point + * (its first code unit). + * @param position the "position"-th code unit in the text-storage object + * @return the "position"-th code point. + * @stable ICU 2.0 + */ + virtual UChar32 setIndex32(int32_t position) = 0; + + /** + * Returns the code unit the iterator currently refers to. + * @return the current code unit. + * @stable ICU 2.0 + */ + virtual UChar current(void) const = 0; + + /** + * Returns the code point the iterator currently refers to. + * @return the current code point. + * @stable ICU 2.0 + */ + virtual UChar32 current32(void) const = 0; + + /** + * Advances to the next code unit in the iteration range + * (toward endIndex()), and returns that code unit. If there are + * no more code units to return, returns DONE. + * @return the next code unit. + * @stable ICU 2.0 + */ + virtual UChar next(void) = 0; + + /** + * Advances to the next code point in the iteration range + * (toward endIndex()), and returns that code point. If there are + * no more code points to return, returns DONE. + * Note that iteration with "pre-increment" semantics is less + * efficient than iteration with "post-increment" semantics + * that is provided by next32PostInc(). + * @return the next code point. + * @stable ICU 2.0 + */ + virtual UChar32 next32(void) = 0; + + /** + * Advances to the previous code unit in the iteration range + * (toward startIndex()), and returns that code unit. If there are + * no more code units to return, returns DONE. + * @return the previous code unit. + * @stable ICU 2.0 + */ + virtual UChar previous(void) = 0; + + /** + * Advances to the previous code point in the iteration range + * (toward startIndex()), and returns that code point. If there are + * no more code points to return, returns DONE. + * @return the previous code point. + * @stable ICU 2.0 + */ + virtual UChar32 previous32(void) = 0; + + /** + * Returns FALSE if there are no more code units or code points + * before the current position in the iteration range. + * This is used with previous() or previous32() in backward + * iteration. + * @return FALSE if there are no more code units or code points + * before the current position in the iteration range, return TRUE otherwise. + * @stable ICU 2.0 + */ + virtual UBool hasPrevious() = 0; + + /** + * Returns the numeric index in the underlying text-storage + * object of the character returned by first(). Since it's + * possible to create an iterator that iterates across only + * part of a text-storage object, this number isn't + * necessarily 0. + * @returns the numeric index in the underlying text-storage + * object of the character returned by first(). + * @stable ICU 2.0 + */ + inline int32_t startIndex(void) const; + + /** + * Returns the numeric index in the underlying text-storage + * object of the position immediately BEYOND the character + * returned by last(). + * @return the numeric index in the underlying text-storage + * object of the position immediately BEYOND the character + * returned by last(). + * @stable ICU 2.0 + */ + inline int32_t endIndex(void) const; + + /** + * Returns the numeric index in the underlying text-storage + * object of the character the iterator currently refers to + * (i.e., the character returned by current()). + * @return the numberic index in the text-storage object of + * the character the iterator currently refers to + * @stable ICU 2.0 + */ + inline int32_t getIndex(void) const; + + /** + * Returns the length of the entire text in the underlying + * text-storage object. + * @return the length of the entire text in the text-storage object + * @stable ICU 2.0 + */ + inline int32_t getLength() const; + + /** + * Moves the current position relative to the start or end of the + * iteration range, or relative to the current position itself. + * The movement is expressed in numbers of code units forward + * or backward by specifying a positive or negative delta. + * @param delta the position relative to origin. A positive delta means forward; + * a negative delta means backward. + * @param origin Origin enumeration {kStart, kCurrent, kEnd} + * @return the new position + * @stable ICU 2.0 + */ + virtual int32_t move(int32_t delta, EOrigin origin) = 0; + + /** + * Moves the current position relative to the start or end of the + * iteration range, or relative to the current position itself. + * The movement is expressed in numbers of code points forward + * or backward by specifying a positive or negative delta. + * @param delta the position relative to origin. A positive delta means forward; + * a negative delta means backward. + * @param origin Origin enumeration {kStart, kCurrent, kEnd} + * @return the new position + * @stable ICU 2.0 + */ + virtual int32_t move32(int32_t delta, EOrigin origin) = 0; + + /** + * Copies the text under iteration into the UnicodeString + * referred to by "result". + * @param result Receives a copy of the text under iteration. + * @stable ICU 2.0 + */ + virtual void getText(UnicodeString& result) = 0; + +protected: + /** + * Empty constructor. + * @stable ICU 2.0 + */ + CharacterIterator(); + + /** + * Constructor, just setting the length field in this base class. + * @stable ICU 2.0 + */ + CharacterIterator(int32_t length); + + /** + * Constructor, just setting the length and position fields in this base class. + * @stable ICU 2.0 + */ + CharacterIterator(int32_t length, int32_t position); + + /** + * Constructor, just setting the length, start, end, and position fields in this base class. + * @stable ICU 2.0 + */ + CharacterIterator(int32_t length, int32_t textBegin, int32_t textEnd, int32_t position); + + /** + * Copy constructor. + * + * @param that The CharacterIterator to be copied + * @stable ICU 2.0 + */ + CharacterIterator(const CharacterIterator &that); + + /** + * Assignment operator. Sets this CharacterIterator to have the same behavior, + * as the one passed in. + * @param that The CharacterIterator passed in. + * @return the newly set CharacterIterator. + * @stable ICU 2.0 + */ + CharacterIterator &operator=(const CharacterIterator &that); + + /** + * Base class text length field. + * Necessary this for correct getText() and hashCode(). + * @stable ICU 2.0 + */ + int32_t textLength; + + /** + * Base class field for the current position. + * @stable ICU 2.0 + */ + int32_t pos; + + /** + * Base class field for the start of the iteration range. + * @stable ICU 2.0 + */ + int32_t begin; + + /** + * Base class field for the end of the iteration range. + * @stable ICU 2.0 + */ + int32_t end; +}; + +inline UBool +ForwardCharacterIterator::operator!=(const ForwardCharacterIterator& that) const { + return !operator==(that); +} + +inline int32_t +CharacterIterator::setToStart() { + return move(0, kStart); +} + +inline int32_t +CharacterIterator::setToEnd() { + return move(0, kEnd); +} + +inline int32_t +CharacterIterator::startIndex(void) const { + return begin; +} + +inline int32_t +CharacterIterator::endIndex(void) const { + return end; +} + +inline int32_t +CharacterIterator::getIndex(void) const { + return pos; +} + +inline int32_t +CharacterIterator::getLength(void) const { + return textLength; +} + +U_NAMESPACE_END +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/dbbi.h b/Build/source/libs/icu/icu-xetex/common/unicode/dbbi.h new file mode 100644 index 00000000000..c7984ef862f --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/dbbi.h @@ -0,0 +1,41 @@ +/* +********************************************************************** +* Copyright (C) 1999-2006 IBM Corp. All rights reserved. +********************************************************************** +* Date Name Description +* 12/1/99 rgillam Complete port from Java. +* 01/13/2000 helena Added UErrorCode to ctors. +********************************************************************** +*/ + +#ifndef DBBI_H +#define DBBI_H + +#include "unicode/rbbi.h" + +#if !UCONFIG_NO_BREAK_ITERATION + +/** + * \file + * \brief C++ API: Dictionary Based Break Iterator + */ + +U_NAMESPACE_BEGIN + +/** + * An obsolete subclass of RuleBasedBreakIterator. Handling of dictionary- + * based break iteration has been folded into the base class. This class + * is deprecated as of ICU 3.6. + */ + +#ifndef U_HIDE_DEPRECATED_API + +typedef RuleBasedBreakIterator DictionaryBasedBreakIterator; + +#endif + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_BREAK_ITERATION */ + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/docmain.h b/Build/source/libs/icu/icu-xetex/common/unicode/docmain.h new file mode 100644 index 00000000000..973ebea7a4a --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/docmain.h @@ -0,0 +1,202 @@ +/******************************************************************** + * COPYRIGHT: + * Copyright (c) 1997-2007, International Business Machines Corporation and + * others. All Rights Reserved. + * + * FILE NAME: DOCMAIN.h + * + * Date Name Description + * 12/11/2000 Ram Creation. + */ + +/* This file contains documentation for Doxygen and doesnot have + * any significance with respect to C or C++ API + */ + +/*! \mainpage + * + * \section API API Reference Usage + * + *

C++ Programmers:

+ *

Use Class Hierarchy or Alphabetical List + * or Compound List + * to find the class you are interested in. For example, to find BreakIterator, + * you can go to the Alphabetical List, then click on + * "BreakIterator". Once you are at the class, you will find an inheritance + * chart, a list of the public members, a detailed description of the class, + * then detailed member descriptions.

+ * + *

C Programmers:

+ *

Use Module List or File Members + * to find a list of all the functions and constants. + * For example, to find BreakIterator functions you would click on + * File List, + * then find "ubrk.h" and click on it. You will find descriptions of Defines, + * Typedefs, Enumerations, and Functions, with detailed descriptions below. + * If you want to find a specific function, such as ubrk_next(), then click + * first on File Members, then use your browser + * Find dialog to search for "ubrk_next()".

+ * + * + *

API References for Previous Releases

+ *

The API References for each release of ICU are also available as + * a zip file from the ICU + * download page.

+ * + *
+ * + *

Architecture (User's Guide)

+ * + * + *
+ *\htmlonly

Module List

\endhtmlonly + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Module NameCC++
Basic Types and Constantsutypes.hutypes.h
Strings and Character Iterationustring.h, utf.hUnicodeString, CharacterIterator
Unicode Character
Properties and Names
uchar.huchar.h C API
Codepage Conversionucnv.hucnv.h C API
Unicode Text Compressionucnv.h
(encoding name "SCSU" or "BOCU-1")
ucnv.h C API
Locales uloc.hLocale
Resource Bundlesures.hResourceBundle
Normalizationunorm.hNormalizer
Calendarsucal.hCalendar
Date and Time Formattingudat.hDateFormat
Message Formattingumsg.hMessageFormat
Number Formattingunum.hNumberFormat
Number Spellout
(Rule Based Number Formatting)
unum.h
(use UNUM_SPELLOUT)
RuleBasedNumberFormat
Text Transformation
(Transliteration)
utrans.hTransliterator
Bidirectional Algorithmubidi.hubidi.h C API
Arabic Shapingushape.hushape.h C API
Collationucol.hCollator
String Searchingusearch.hStringSearch
Text Boundary Analysis
(Break Iteration)
ubrk.hBreakIterator
Unicode Setuset.hUnicodeSet
Regular Expressionsuregex.hRegexPattern, RegexMatcher
StringPrepusprep.husprep.h C API
International Domain Names in Applicationsuidna.huidna.h C API
Universal Time Scaleutmscale.hutmscale.h C API
Basic Layout Engine Types and Constants(no C API)LETypes.h
Complex Text Layout(no C API)LayoutEngine, ParagraphLayout
ICU I/Oustdio.hustream.h
+ */ diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/locid.h b/Build/source/libs/icu/icu-xetex/common/unicode/locid.h new file mode 100644 index 00000000000..a3cc23b31fa --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/locid.h @@ -0,0 +1,765 @@ +/* +****************************************************************************** +* +* Copyright (C) 1996-2006, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* +* File locid.h +* +* Created by: Helena Shih +* +* Modification History: +* +* Date Name Description +* 02/11/97 aliu Changed gLocPath to fgLocPath and added methods to +* get and set it. +* 04/02/97 aliu Made operator!= inline; fixed return value of getName(). +* 04/15/97 aliu Cleanup for AIX/Win32. +* 04/24/97 aliu Numerous changes per code review. +* 08/18/98 stephen Added tokenizeString(),changed getDisplayName() +* 09/08/98 stephen Moved definition of kEmptyString for Mac Port +* 11/09/99 weiv Added const char * getName() const; +* 04/12/00 srl removing unicodestring api's and cached hash code +* 08/10/01 grhoten Change the static Locales to accessor functions +****************************************************************************** +*/ + +#ifndef LOCID_H +#define LOCID_H + +#include "unicode/utypes.h" +#include "unicode/uobject.h" +#include "unicode/unistr.h" +#include "unicode/putil.h" +#include "unicode/uloc.h" +#include "unicode/strenum.h" + +/** + * \file + * \brief C++ API: Locale ID object. + */ + +/** + * A Locale object represents a specific geographical, political, + * or cultural region. An operation that requires a Locale to perform + * its task is called locale-sensitive and uses the Locale + * to tailor information for the user. For example, displaying a number + * is a locale-sensitive operation--the number should be formatted + * according to the customs/conventions of the user's native country, + * region, or culture. + * + * The Locale class is not suitable for subclassing. + * + *

+ * You can create a Locale object using the constructor in + * this class: + * \htmlonly

\endhtmlonly + *
+ *       Locale( const   char*  language,
+ *               const   char*  country,
+ *               const   char*  variant);
+ * 
+ * \htmlonly
\endhtmlonly + * The first argument to the constructors is a valid ISO + * Language Code. These codes are the lower-case two-letter + * codes as defined by ISO-639. + * You can find a full list of these codes at: + *
+ * http://www.loc.gov/standards/iso639-2/ + * + *

+ * The second argument to the constructors is a valid ISO Country + * Code. These codes are the upper-case two-letter codes + * as defined by ISO-3166. + * You can find a full list of these codes at a number of sites, such as: + *
+ * http://www.iso.org/iso/en/prods-services/iso3166ma/index.html + * + *

+ * The third constructor requires a third argument--the Variant. + * The Variant codes are vendor and browser-specific. + * For example, use REVISED for a langauge's revised script orthography, and POSIX for POSIX. + * Where there are two variants, separate them with an underscore, and + * put the most important one first. For + * example, a Traditional Spanish collation might be referenced, with + * "ES", "ES", "Traditional_POSIX". + * + *

+ * Because a Locale object is just an identifier for a region, + * no validity check is performed when you construct a Locale. + * If you want to see whether particular resources are available for the + * Locale you construct, you must query those resources. For + * example, ask the NumberFormat for the locales it supports + * using its getAvailableLocales method. + *
Note: When you ask for a resource for a particular + * locale, you get back the best available match, not necessarily + * precisely what you asked for. For more information, look at + * ResourceBundle. + * + *

+ * The Locale class provides a number of convenient constants + * that you can use to create Locale objects for commonly used + * locales. For example, the following refers to a Locale object + * for the United States: + * \htmlonly

\endhtmlonly + *
+ *       Locale::getUS()
+ * 
+ * \htmlonly
\endhtmlonly + * + *

+ * Once you've created a Locale you can query it for information about + * itself. Use getCountry to get the ISO Country Code and + * getLanguage to get the ISO Language Code. You can + * use getDisplayCountry to get the + * name of the country suitable for displaying to the user. Similarly, + * you can use getDisplayLanguage to get the name of + * the language suitable for displaying to the user. Interestingly, + * the getDisplayXXX methods are themselves locale-sensitive + * and have two versions: one that uses the default locale and one + * that takes a locale as an argument and displays the name or country in + * a language appropriate to that locale. + * + *

+ * ICU provides a number of classes that perform locale-sensitive + * operations. For example, the NumberFormat class formats + * numbers, currency, or percentages in a locale-sensitive manner. Classes + * such as NumberFormat have a number of convenience methods + * for creating a default object of that type. For example, the + * NumberFormat class provides these three convenience methods + * for creating a default NumberFormat object: + * \htmlonly

\endhtmlonly + *
+ *     UErrorCode success = U_ZERO_ERROR;
+ *     Locale myLocale;
+ *     NumberFormat *nf;
+ *
+ *     nf = NumberFormat::createInstance( success );          delete nf;
+ *     nf = NumberFormat::createCurrencyInstance( success );  delete nf;
+ *     nf = NumberFormat::createPercentInstance( success );   delete nf;
+ * 
+ * \htmlonly
\endhtmlonly + * Each of these methods has two variants; one with an explicit locale + * and one without; the latter using the default locale. + * \htmlonly
\endhtmlonly + *
+ *     nf = NumberFormat::createInstance( myLocale, success );          delete nf;
+ *     nf = NumberFormat::createCurrencyInstance( myLocale, success );  delete nf;
+ *     nf = NumberFormat::createPercentInstance( myLocale, success );   delete nf;
+ * 
+ * \htmlonly
\endhtmlonly + * A Locale is the mechanism for identifying the kind of object + * (NumberFormat) that you would like to get. The locale is + * just a mechanism for identifying objects, + * not a container for the objects themselves. + * + *

+ * Each class that performs locale-sensitive operations allows you + * to get all the available objects of that type. You can sift + * through these objects by language, country, or variant, + * and use the display names to present a menu to the user. + * For example, you can create a menu of all the collation objects + * suitable for a given language. Such classes implement these + * three class methods: + * \htmlonly

\endhtmlonly + *
+ *       static Locale* getAvailableLocales(int32_t& numLocales)
+ *       static UnicodeString& getDisplayName(const Locale&  objectLocale,
+ *                                            const Locale&  displayLocale,
+ *                                            UnicodeString& displayName)
+ *       static UnicodeString& getDisplayName(const Locale&  objectLocale,
+ *                                            UnicodeString& displayName)
+ * 
+ * \htmlonly
\endhtmlonly + * + * @stable ICU 2.0 + * @see ResourceBundle + */ +U_NAMESPACE_BEGIN +class U_COMMON_API Locale : public UObject { +public: + /** Useful constant for this language. @stable ICU 2.0 */ + static const Locale &U_EXPORT2 getEnglish(void); + /** Useful constant for this language. @stable ICU 2.0 */ + static const Locale &U_EXPORT2 getFrench(void); + /** Useful constant for this language. @stable ICU 2.0 */ + static const Locale &U_EXPORT2 getGerman(void); + /** Useful constant for this language. @stable ICU 2.0 */ + static const Locale &U_EXPORT2 getItalian(void); + /** Useful constant for this language. @stable ICU 2.0 */ + static const Locale &U_EXPORT2 getJapanese(void); + /** Useful constant for this language. @stable ICU 2.0 */ + static const Locale &U_EXPORT2 getKorean(void); + /** Useful constant for this language. @stable ICU 2.0 */ + static const Locale &U_EXPORT2 getChinese(void); + /** Useful constant for this language. @stable ICU 2.0 */ + static const Locale &U_EXPORT2 getSimplifiedChinese(void); + /** Useful constant for this language. @stable ICU 2.0 */ + static const Locale &U_EXPORT2 getTraditionalChinese(void); + + /** Useful constant for this country/region. @stable ICU 2.0 */ + static const Locale &U_EXPORT2 getFrance(void); + /** Useful constant for this country/region. @stable ICU 2.0 */ + static const Locale &U_EXPORT2 getGermany(void); + /** Useful constant for this country/region. @stable ICU 2.0 */ + static const Locale &U_EXPORT2 getItaly(void); + /** Useful constant for this country/region. @stable ICU 2.0 */ + static const Locale &U_EXPORT2 getJapan(void); + /** Useful constant for this country/region. @stable ICU 2.0 */ + static const Locale &U_EXPORT2 getKorea(void); + /** Useful constant for this country/region. @stable ICU 2.0 */ + static const Locale &U_EXPORT2 getChina(void); + /** Useful constant for this country/region. @stable ICU 2.0 */ + static const Locale &U_EXPORT2 getPRC(void); + /** Useful constant for this country/region. @stable ICU 2.0 */ + static const Locale &U_EXPORT2 getTaiwan(void); + /** Useful constant for this country/region. @stable ICU 2.0 */ + static const Locale &U_EXPORT2 getUK(void); + /** Useful constant for this country/region. @stable ICU 2.0 */ + static const Locale &U_EXPORT2 getUS(void); + /** Useful constant for this country/region. @stable ICU 2.0 */ + static const Locale &U_EXPORT2 getCanada(void); + /** Useful constant for this country/region. @stable ICU 2.0 */ + static const Locale &U_EXPORT2 getCanadaFrench(void); + + + /** + * Construct a default locale object, a Locale for the default locale ID. + * + * @see getDefault + * @see uloc_getDefault + * @stable ICU 2.0 + */ + Locale(); + + /** + * Construct a locale from language, country, variant. + * If an error occurs, then the constructed object will be "bogus" + * (isBogus() will return TRUE). + * + * @param language Lowercase two-letter or three-letter ISO-639 code. + * This parameter can instead be an ICU style C locale (e.g. "en_US"), + * but the other parameters must not be used. + * This parameter can be NULL; if so, + * the locale is initialized to match the current default locale. + * (This is the same as using the default constructor.) + * Please note: The Java Locale class does NOT accept the form + * 'new Locale("en_US")' but only 'new Locale("en","US")' + * + * @param country Uppercase two-letter ISO-3166 code. (optional) + * @param variant Uppercase vendor and browser specific code. See class + * description. (optional) + * @param keywordsAndValues A string consisting of keyword/values pairs, such as + * "collation=phonebook;currency=euro" + * + * @see getDefault + * @see uloc_getDefault + * @stable ICU 2.0 + */ + Locale( const char * language, + const char * country = 0, + const char * variant = 0, + const char * keywordsAndValues = 0); + + /** + * Initializes a Locale object from another Locale object. + * + * @param other The Locale object being copied in. + * @stable ICU 2.0 + */ + Locale(const Locale& other); + + + /** + * Destructor + * @stable ICU 2.0 + */ + virtual ~Locale() ; + + /** + * Replaces the entire contents of *this with the specified value. + * + * @param other The Locale object being copied in. + * @return *this + * @stable ICU 2.0 + */ + Locale& operator=(const Locale& other); + + /** + * Checks if two locale keys are the same. + * + * @param other The locale key object to be compared with this. + * @return True if the two locale keys are the same, false otherwise. + * @stable ICU 2.0 + */ + UBool operator==(const Locale& other) const; + + /** + * Checks if two locale keys are not the same. + * + * @param other The locale key object to be compared with this. + * @return True if the two locale keys are not the same, false + * otherwise. + * @stable ICU 2.0 + */ + UBool operator!=(const Locale& other) const; + + /** + * Clone this object. + * Clones can be used concurrently in multiple threads. + * If an error occurs, then NULL is returned. + * The caller must delete the clone. + * + * @return a clone of this object + * + * @see getDynamicClassID + * @stable ICU 2.8 + */ + Locale *clone() const; + + /** + * Common methods of getting the current default Locale. Used for the + * presentation: menus, dialogs, etc. Generally set once when your applet or + * application is initialized, then never reset. (If you do reset the + * default locale, you probably want to reload your GUI, so that the change + * is reflected in your interface.) + * + * More advanced programs will allow users to use different locales for + * different fields, e.g. in a spreadsheet. + * + * Note that the initial setting will match the host system. + * @return a reference to the Locale object for the default locale ID + * @system + * @stable ICU 2.0 + */ + static const Locale& U_EXPORT2 getDefault(void); + + /** + * Sets the default. Normally set once at the beginning of a process, + * then never reset. + * setDefault() only changes ICU's default locale ID, not + * the default locale ID of the runtime environment. + * + * @param newLocale Locale to set to. If NULL, set to the value obtained + * from the runtime environement. + * @param success The error code. + * @system + * @stable ICU 2.0 + */ + static void U_EXPORT2 setDefault(const Locale& newLocale, + UErrorCode& success); + + /** + * Creates a locale which has had minimal canonicalization + * as per uloc_getName(). + * @param name The name to create from. If name is null, + * the default Locale is used. + * @return new locale object + * @stable ICU 2.0 + * @see uloc_getName + */ + static Locale U_EXPORT2 createFromName(const char *name); + + /** + * Creates a locale from the given string after canonicalizing + * the string by calling uloc_canonicalize(). + * @param name the locale ID to create from. Must not be NULL. + * @return a new locale object corresponding to the given name + * @stable ICU 3.0 + * @see uloc_canonicalize + */ + static Locale U_EXPORT2 createCanonical(const char* name); + + /** + * Returns the locale's ISO-639 language code. + * @return An alias to the code + * @stable ICU 2.0 + */ + inline const char * getLanguage( ) const; + + /** + * Returns the locale's ISO-15924 abbreviation script code. + * @return An alias to the code + * @see uscript_getShortName + * @see uscript_getCode + * @stable ICU 2.8 + */ + inline const char * getScript( ) const; + + /** + * Returns the locale's ISO-3166 country code. + * @return An alias to the code + * @stable ICU 2.0 + */ + inline const char * getCountry( ) const; + + /** + * Returns the locale's variant code. + * @return An alias to the code + * @stable ICU 2.0 + */ + inline const char * getVariant( ) const; + + /** + * Returns the programmatic name of the entire locale, with the language, + * country and variant separated by underbars. If a field is missing, up + * to two leading underbars will occur. Example: "en", "de_DE", "en_US_WIN", + * "de__POSIX", "fr__MAC", "__MAC", "_MT", "_FR_EURO" + * @return A pointer to "name". + * @stable ICU 2.0 + */ + inline const char * getName() const; + + /** + * Returns the programmatic name of the entire locale as getName would return, + * but without keywords. + * @return A pointer to "name". + * @see getName + * @stable ICU 2.8 + */ + const char * getBaseName() const; + + + /** + * Gets the list of keywords for the specified locale. + * + * @return pointer to StringEnumeration class. Client must dispose of it by calling delete. + * @param status Returns any error information while performing this operation. + * @stable ICU 2.8 + */ + StringEnumeration * createKeywords(UErrorCode &status) const; + + /** + * Get the value for a keyword. + * + * @param keywordName name of the keyword for which we want the value. Case insensitive. + * @param status Returns any error information while performing this operation. + * @param buffer The buffer to receive the keyword value. + * @param bufferCapacity The capacity of receiving buffer + * @return the length of keyword value + * + * @stable ICU 2.8 + */ + int32_t getKeywordValue(const char* keywordName, char *buffer, int32_t bufferCapacity, UErrorCode &status) const; + + /** + * returns the locale's three-letter language code, as specified + * in ISO draft standard ISO-639-2. + * @return An alias to the code, or NULL + * @stable ICU 2.0 + */ + const char * getISO3Language() const; + + /** + * Fills in "name" with the locale's three-letter ISO-3166 country code. + * @return An alias to the code, or NULL + * @stable ICU 2.0 + */ + const char * getISO3Country() const; + + /** + * Returns the Windows LCID value corresponding to this locale. + * This value is stored in the resource data for the locale as a one-to-four-digit + * hexadecimal number. If the resource is missing, in the wrong format, or + * there is no Windows LCID value that corresponds to this locale, returns 0. + * @stable ICU 2.0 + */ + uint32_t getLCID(void) const; + + /** + * Fills in "dispLang" with the name of this locale's language in a format suitable for + * user display in the default locale. For example, if the locale's language code is + * "fr" and the default locale's language code is "en", this function would set + * dispLang to "French". + * @param dispLang Receives the language's display name. + * @return A reference to "dispLang". + * @stable ICU 2.0 + */ + UnicodeString& getDisplayLanguage(UnicodeString& dispLang) const; + + /** + * Fills in "dispLang" with the name of this locale's language in a format suitable for + * user display in the locale specified by "displayLocale". For example, if the locale's + * language code is "en" and displayLocale's language code is "fr", this function would set + * dispLang to "Anglais". + * @param displayLocale Specifies the locale to be used to display the name. In other words, + * if the locale's language code is "en", passing Locale::getFrench() for + * displayLocale would result in "Anglais", while passing Locale::getGerman() + * for displayLocale would result in "Englisch". + * @param dispLang Receives the language's display name. + * @return A reference to "dispLang". + * @stable ICU 2.0 + */ + UnicodeString& getDisplayLanguage( const Locale& displayLocale, + UnicodeString& dispLang) const; + + /** + * Fills in "dispScript" with the name of this locale's script in a format suitable + * for user display in the default locale. For example, if the locale's script code + * is "LATN" and the default locale's language code is "en", this function would set + * dispScript to "Latin". + * @param dispScript Receives the scripts's display name. + * @return A reference to "dispScript". + * @stable ICU 2.8 + */ + UnicodeString& getDisplayScript( UnicodeString& dispScript) const; + + /** + * Fills in "dispScript" with the name of this locale's country in a format suitable + * for user display in the locale specified by "displayLocale". For example, if the locale's + * script code is "LATN" and displayLocale's language code is "en", this function would set + * dispScript to "Latin". + * @param displayLocale Specifies the locale to be used to display the name. In other + * words, if the locale's script code is "LATN", passing + * Locale::getFrench() for displayLocale would result in "", while + * passing Locale::getGerman() for displayLocale would result in + * "". + * @param dispScript Receives the scripts's display name. + * @return A reference to "dispScript". + * @stable ICU 2.8 + */ + UnicodeString& getDisplayScript( const Locale& displayLocale, + UnicodeString& dispScript) const; + + /** + * Fills in "dispCountry" with the name of this locale's country in a format suitable + * for user display in the default locale. For example, if the locale's country code + * is "FR" and the default locale's language code is "en", this function would set + * dispCountry to "France". + * @param dispCountry Receives the country's display name. + * @return A reference to "dispCountry". + * @stable ICU 2.0 + */ + UnicodeString& getDisplayCountry( UnicodeString& dispCountry) const; + + /** + * Fills in "dispCountry" with the name of this locale's country in a format suitable + * for user display in the locale specified by "displayLocale". For example, if the locale's + * country code is "US" and displayLocale's language code is "fr", this function would set + * dispCountry to "États-Unis". + * @param displayLocale Specifies the locale to be used to display the name. In other + * words, if the locale's country code is "US", passing + * Locale::getFrench() for displayLocale would result in "États-Unis", while + * passing Locale::getGerman() for displayLocale would result in + * "Vereinigte Staaten". + * @param dispCountry Receives the country's display name. + * @return A reference to "dispCountry". + * @stable ICU 2.0 + */ + UnicodeString& getDisplayCountry( const Locale& displayLocale, + UnicodeString& dispCountry) const; + + /** + * Fills in "dispVar" with the name of this locale's variant code in a format suitable + * for user display in the default locale. + * @param dispVar Receives the variant's name. + * @return A reference to "dispVar". + * @stable ICU 2.0 + */ + UnicodeString& getDisplayVariant( UnicodeString& dispVar) const; + + /** + * Fills in "dispVar" with the name of this locale's variant code in a format + * suitable for user display in the locale specified by "displayLocale". + * @param displayLocale Specifies the locale to be used to display the name. + * @param dispVar Receives the variant's display name. + * @return A reference to "dispVar". + * @stable ICU 2.0 + */ + UnicodeString& getDisplayVariant( const Locale& displayLocale, + UnicodeString& dispVar) const; + + /** + * Fills in "name" with the name of this locale in a format suitable for user display + * in the default locale. This function uses getDisplayLanguage(), getDisplayCountry(), + * and getDisplayVariant() to do its work, and outputs the display name in the format + * "language (country[,variant])". For example, if the default locale is en_US, then + * fr_FR's display name would be "French (France)", and es_MX_Traditional's display name + * would be "Spanish (Mexico,Traditional)". + * @param name Receives the locale's display name. + * @return A reference to "name". + * @stable ICU 2.0 + */ + UnicodeString& getDisplayName( UnicodeString& name) const; + + /** + * Fills in "name" with the name of this locale in a format suitable for user display + * in the locale specfied by "displayLocale". This function uses getDisplayLanguage(), + * getDisplayCountry(), and getDisplayVariant() to do its work, and outputs the display + * name in the format "language (country[,variant])". For example, if displayLocale is + * fr_FR, then en_US's display name would be "Anglais (États-Unis)", and no_NO_NY's + * display name would be "norvégien (Norvège,NY)". + * @param displayLocale Specifies the locale to be used to display the name. + * @param name Receives the locale's display name. + * @return A reference to "name". + * @stable ICU 2.0 + */ + UnicodeString& getDisplayName( const Locale& displayLocale, + UnicodeString& name) const; + + /** + * Generates a hash code for the locale. + * @stable ICU 2.0 + */ + int32_t hashCode(void) const; + + /** + * Sets the locale to bogus + * A bogus locale represents a non-existing locale associated + * with services that can be instantiated from non-locale data + * in addition to locale (for example, collation can be + * instantiated from a locale and from a rule set). + * @stable ICU 2.1 + */ + void setToBogus(); + + /** + * Gets the bogus state. Locale object can be bogus if it doesn't exist + * @return FALSE if it is a real locale, TRUE if it is a bogus locale + * @stable ICU 2.1 + */ + UBool isBogus(void) const; + + /** + * Returns a list of all installed locales. + * @param count Receives the number of locales in the list. + * @return A pointer to an array of Locale objects. This array is the list + * of all locales with installed resource files. The called does NOT + * get ownership of this list, and must NOT delete it. + * @stable ICU 2.0 + */ + static const Locale* U_EXPORT2 getAvailableLocales(int32_t& count); + + /** + * Gets a list of all available 2-letter country codes defined in ISO 639. This is a + * pointer to an array of pointers to arrays of char. All of these pointers are + * owned by ICU-- do not delete them, and do not write through them. The array is + * terminated with a null pointer. + * @return a list of all available country codes + * @stable ICU 2.0 + */ + static const char* const* U_EXPORT2 getISOCountries(); + + /** + * Gets a list of all available language codes defined in ISO 639. This is a pointer + * to an array of pointers to arrays of char. All of these pointers are owned + * by ICU-- do not delete them, and do not write through them. The array is + * terminated with a null pointer. + * @return a list of all available language codes + * @stable ICU 2.0 + */ + static const char* const* U_EXPORT2 getISOLanguages(); + + /** + * ICU "poor man's RTTI", returns a UClassID for this class. + * + * @stable ICU 2.2 + */ + static UClassID U_EXPORT2 getStaticClassID(); + + /** + * ICU "poor man's RTTI", returns a UClassID for the actual class. + * + * @stable ICU 2.2 + */ + virtual UClassID getDynamicClassID() const; + +protected: /* only protected for testing purposes. DO NOT USE. */ + /** + * Set this from a single POSIX style locale string. + * @internal + */ + void setFromPOSIXID(const char *posixID); + +private: + /** + * Initialize the locale object with a new name. + * Was deprecated - used in implementation - moved internal + * + * @param cLocaleID The new locale name. + */ + Locale& init(const char* cLocaleID, UBool canonicalize); + + /* + * Internal constructor to allow construction of a locale object with + * NO side effects. (Default constructor tries to get + * the default locale.) + */ + enum ELocaleType { + eBOGUS + }; + Locale(ELocaleType); + + /** + * Initialize the locale cache for commonly used locales + */ + static Locale *getLocaleCache(void); + + char language[ULOC_LANG_CAPACITY]; + char script[ULOC_SCRIPT_CAPACITY]; + char country[ULOC_COUNTRY_CAPACITY]; + int32_t variantBegin; + char* fullName; + char fullNameBuffer[ULOC_FULLNAME_CAPACITY]; + // name without keywords + char* baseName; + char baseNameBuffer[ULOC_FULLNAME_CAPACITY]; + + UBool fIsBogus; + + static const Locale &getLocale(int locid); + + /** + * A friend to allow the default locale to be set by either the C or C++ API. + * @internal + */ + friend void locale_set_default_internal(const char *); +}; + +inline UBool +Locale::operator!=(const Locale& other) const +{ + return !operator==(other); +} + +inline const char * +Locale::getCountry() const +{ + return country; +} + +inline const char * +Locale::getLanguage() const +{ + return language; +} + +inline const char * +Locale::getScript() const +{ + return script; +} + +inline const char * +Locale::getVariant() const +{ + return &fullName[variantBegin]; +} + +inline const char * +Locale::getName() const +{ + return fullName; +} + +inline UBool +Locale::isBogus(void) const { + return fIsBogus; +} + +U_NAMESPACE_END + +#endif + diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/normlzr.h b/Build/source/libs/icu/icu-xetex/common/unicode/normlzr.h new file mode 100644 index 00000000000..7974f1ac4dd --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/normlzr.h @@ -0,0 +1,823 @@ +/* + ******************************************************************** + * COPYRIGHT: + * Copyright (c) 1996-2006, International Business Machines Corporation and + * others. All Rights Reserved. + ******************************************************************** + */ + +#ifndef NORMLZR_H +#define NORMLZR_H + +#include "unicode/utypes.h" + +/** + * \file + * \brief C++ API: Unicode Normalization + */ + +#if !UCONFIG_NO_NORMALIZATION + +#include "unicode/uobject.h" +#include "unicode/unistr.h" +#include "unicode/chariter.h" +#include "unicode/unorm.h" + + +struct UCharIterator; +typedef struct UCharIterator UCharIterator; /**< C typedef for struct UCharIterator. @stable ICU 2.1 */ + +U_NAMESPACE_BEGIN +/** + * The Normalizer class supports the standard normalization forms described in + * + * Unicode Standard Annex #15: Unicode Normalization Forms. + * + * The Normalizer class consists of two parts: + * - static functions that normalize strings or test if strings are normalized + * - a Normalizer object is an iterator that takes any kind of text and + * provides iteration over its normalized form + * + * The Normalizer class is not suitable for subclassing. + * + * The static functions are basically wrappers around the C implementation, + * using UnicodeString instead of UChar*. + * For basic information about normalization forms and details about the C API + * please see the documentation in unorm.h. + * + * The iterator API with the Normalizer constructors and the non-static functions + * uses a CharacterIterator as input. It is possible to pass a string which + * is then internally wrapped in a CharacterIterator. + * The input text is not normalized all at once, but incrementally where needed + * (providing efficient random access). + * This allows to pass in a large text but spend only a small amount of time + * normalizing a small part of that text. + * However, if the entire text is normalized, then the iterator will be + * slower than normalizing the entire text at once and iterating over the result. + * A possible use of the Normalizer iterator is also to report an index into the + * original text that is close to where the normalized characters come from. + * + * Important: The iterator API was cleaned up significantly for ICU 2.0. + * The earlier implementation reported the getIndex() inconsistently, + * and previous() could not be used after setIndex(), next(), first(), and current(). + * + * Normalizer allows to start normalizing from anywhere in the input text by + * calling setIndexOnly(), first(), or last(). + * Without calling any of these, the iterator will start at the beginning of the text. + * + * At any time, next() returns the next normalized code point (UChar32), + * with post-increment semantics (like CharacterIterator::next32PostInc()). + * previous() returns the previous normalized code point (UChar32), + * with pre-decrement semantics (like CharacterIterator::previous32()). + * + * current() returns the current code point + * (respectively the one at the newly set index) without moving + * the getIndex(). Note that if the text at the current position + * needs to be normalized, then these functions will do that. + * (This is why current() is not const.) + * It is more efficient to call setIndexOnly() instead, which does not + * normalize. + * + * getIndex() always refers to the position in the input text where the normalized + * code points are returned from. It does not always change with each returned + * code point. + * The code point that is returned from any of the functions + * corresponds to text at or after getIndex(), according to the + * function's iteration semantics (post-increment or pre-decrement). + * + * next() returns a code point from at or after the getIndex() + * from before the next() call. After the next() call, the getIndex() + * might have moved to where the next code point will be returned from + * (from a next() or current() call). + * This is semantically equivalent to array access with array[index++] + * (post-increment semantics). + * + * previous() returns a code point from at or after the getIndex() + * from after the previous() call. + * This is semantically equivalent to array access with array[--index] + * (pre-decrement semantics). + * + * Internally, the Normalizer iterator normalizes a small piece of text + * starting at the getIndex() and ending at a following "safe" index. + * The normalized results is stored in an internal string buffer, and + * the code points are iterated from there. + * With multiple iteration calls, this is repeated until the next piece + * of text needs to be normalized, and the getIndex() needs to be moved. + * + * The following "safe" index, the internal buffer, and the secondary + * iteration index into that buffer are not exposed on the API. + * This also means that it is currently not practical to return to + * a particular, arbitrary position in the text because one would need to + * know, and be able to set, in addition to the getIndex(), at least also the + * current index into the internal buffer. + * It is currently only possible to observe when getIndex() changes + * (with careful consideration of the iteration semantics), + * at which time the internal index will be 0. + * For example, if getIndex() is different after next() than before it, + * then the internal index is 0 and one can return to this getIndex() + * later with setIndexOnly(). + * + * @author Laura Werner, Mark Davis, Markus Scherer + * @stable ICU 2.0 + */ +class U_COMMON_API Normalizer : public UObject { +public: + /** + * If DONE is returned from an iteration function that returns a code point, + * then there are no more normalization results available. + * @stable ICU 2.0 + */ + enum { + DONE=0xffff + }; + + // Constructors + + /** + * Creates a new Normalizer object for iterating over the + * normalized form of a given string. + *

+ * @param str The string to be normalized. The normalization + * will start at the beginning of the string. + * + * @param mode The normalization mode. + * @stable ICU 2.0 + */ + Normalizer(const UnicodeString& str, UNormalizationMode mode); + + /** + * Creates a new Normalizer object for iterating over the + * normalized form of a given string. + *

+ * @param str The string to be normalized. The normalization + * will start at the beginning of the string. + * + * @param length Length of the string, or -1 if NUL-terminated. + * @param mode The normalization mode. + * @stable ICU 2.0 + */ + Normalizer(const UChar* str, int32_t length, UNormalizationMode mode); + + /** + * Creates a new Normalizer object for iterating over the + * normalized form of the given text. + *

+ * @param iter The input text to be normalized. The normalization + * will start at the beginning of the string. + * + * @param mode The normalization mode. + * @stable ICU 2.0 + */ + Normalizer(const CharacterIterator& iter, UNormalizationMode mode); + + /** + * Copy constructor. + * @param copy The object to be copied. + * @stable ICU 2.0 + */ + Normalizer(const Normalizer& copy); + + /** + * Destructor + * @stable ICU 2.0 + */ + virtual ~Normalizer(); + + + //------------------------------------------------------------------------- + // Static utility methods + //------------------------------------------------------------------------- + + /** + * Normalizes a UnicodeString according to the specified normalization mode. + * This is a wrapper for unorm_normalize(), using UnicodeString's. + * + * The options parameter specifies which optional + * Normalizer features are to be enabled for this operation. + * + * @param source the input string to be normalized. + * @param mode the normalization mode + * @param options the optional features to be enabled (0 for no options) + * @param result The normalized string (on output). + * @param status The error code. + * @stable ICU 2.0 + */ + static void U_EXPORT2 normalize(const UnicodeString& source, + UNormalizationMode mode, int32_t options, + UnicodeString& result, + UErrorCode &status); + + /** + * Compose a UnicodeString. + * This is equivalent to normalize() with mode UNORM_NFC or UNORM_NFKC. + * This is a wrapper for unorm_normalize(), using UnicodeString's. + * + * The options parameter specifies which optional + * Normalizer features are to be enabled for this operation. + * + * @param source the string to be composed. + * @param compat Perform compatibility decomposition before composition. + * If this argument is FALSE, only canonical + * decomposition will be performed. + * @param options the optional features to be enabled (0 for no options) + * @param result The composed string (on output). + * @param status The error code. + * @stable ICU 2.0 + */ + static void U_EXPORT2 compose(const UnicodeString& source, + UBool compat, int32_t options, + UnicodeString& result, + UErrorCode &status); + + /** + * Static method to decompose a UnicodeString. + * This is equivalent to normalize() with mode UNORM_NFD or UNORM_NFKD. + * This is a wrapper for unorm_normalize(), using UnicodeString's. + * + * The options parameter specifies which optional + * Normalizer features are to be enabled for this operation. + * + * @param source the string to be decomposed. + * @param compat Perform compatibility decomposition. + * If this argument is FALSE, only canonical + * decomposition will be performed. + * @param options the optional features to be enabled (0 for no options) + * @param result The decomposed string (on output). + * @param status The error code. + * @stable ICU 2.0 + */ + static void U_EXPORT2 decompose(const UnicodeString& source, + UBool compat, int32_t options, + UnicodeString& result, + UErrorCode &status); + + /** + * Performing quick check on a string, to quickly determine if the string is + * in a particular normalization format. + * This is a wrapper for unorm_quickCheck(), using a UnicodeString. + * + * Three types of result can be returned UNORM_YES, UNORM_NO or + * UNORM_MAYBE. Result UNORM_YES indicates that the argument + * string is in the desired normalized format, UNORM_NO determines that + * argument string is not in the desired normalized format. A + * UNORM_MAYBE result indicates that a more thorough check is required, + * the user may have to put the string in its normalized form and compare the + * results. + * @param source string for determining if it is in a normalized format + * @param mode normalization format + * @param status A reference to a UErrorCode to receive any errors + * @return UNORM_YES, UNORM_NO or UNORM_MAYBE + * + * @see isNormalized + * @stable ICU 2.0 + */ + static inline UNormalizationCheckResult + quickCheck(const UnicodeString &source, UNormalizationMode mode, UErrorCode &status); + + /** + * Performing quick check on a string; same as the other version of quickCheck + * but takes an extra options parameter like most normalization functions. + * + * @param source string for determining if it is in a normalized format + * @param mode normalization format + * @param options the optional features to be enabled (0 for no options) + * @param status A reference to a UErrorCode to receive any errors + * @return UNORM_YES, UNORM_NO or UNORM_MAYBE + * + * @see isNormalized + * @stable ICU 2.6 + */ + static inline UNormalizationCheckResult + quickCheck(const UnicodeString &source, UNormalizationMode mode, int32_t options, UErrorCode &status); + + /** + * Test if a string is in a given normalization form. + * This is semantically equivalent to source.equals(normalize(source, mode)) . + * + * Unlike unorm_quickCheck(), this function returns a definitive result, + * never a "maybe". + * For NFD, NFKD, and FCD, both functions work exactly the same. + * For NFC and NFKC where quickCheck may return "maybe", this function will + * perform further tests to arrive at a TRUE/FALSE result. + * + * @param src String that is to be tested if it is in a normalization format. + * @param mode Which normalization form to test for. + * @param errorCode ICU error code in/out parameter. + * Must fulfill U_SUCCESS before the function call. + * @return Boolean value indicating whether the source string is in the + * "mode" normalization form. + * + * @see quickCheck + * @stable ICU 2.2 + */ + static inline UBool + isNormalized(const UnicodeString &src, UNormalizationMode mode, UErrorCode &errorCode); + + /** + * Test if a string is in a given normalization form; same as the other version of isNormalized + * but takes an extra options parameter like most normalization functions. + * + * @param src String that is to be tested if it is in a normalization format. + * @param mode Which normalization form to test for. + * @param options the optional features to be enabled (0 for no options) + * @param errorCode ICU error code in/out parameter. + * Must fulfill U_SUCCESS before the function call. + * @return Boolean value indicating whether the source string is in the + * "mode" normalization form. + * + * @see quickCheck + * @stable ICU 2.6 + */ + static inline UBool + isNormalized(const UnicodeString &src, UNormalizationMode mode, int32_t options, UErrorCode &errorCode); + + /** + * Concatenate normalized strings, making sure that the result is normalized as well. + * + * If both the left and the right strings are in + * the normalization form according to "mode/options", + * then the result will be + * + * \code + * dest=normalize(left+right, mode, options) + * \endcode + * + * For details see unorm_concatenate in unorm.h. + * + * @param left Left source string. + * @param right Right source string. + * @param result The output string. + * @param mode The normalization mode. + * @param options A bit set of normalization options. + * @param errorCode ICU error code in/out parameter. + * Must fulfill U_SUCCESS before the function call. + * @return result + * + * @see unorm_concatenate + * @see normalize + * @see unorm_next + * @see unorm_previous + * + * @stable ICU 2.1 + */ + static UnicodeString & + U_EXPORT2 concatenate(UnicodeString &left, UnicodeString &right, + UnicodeString &result, + UNormalizationMode mode, int32_t options, + UErrorCode &errorCode); + + /** + * Compare two strings for canonical equivalence. + * Further options include case-insensitive comparison and + * code point order (as opposed to code unit order). + * + * Canonical equivalence between two strings is defined as their normalized + * forms (NFD or NFC) being identical. + * This function compares strings incrementally instead of normalizing + * (and optionally case-folding) both strings entirely, + * improving performance significantly. + * + * Bulk normalization is only necessary if the strings do not fulfill the FCD + * conditions. Only in this case, and only if the strings are relatively long, + * is memory allocated temporarily. + * For FCD strings and short non-FCD strings there is no memory allocation. + * + * Semantically, this is equivalent to + * strcmp[CodePointOrder](NFD(foldCase(s1)), NFD(foldCase(s2))) + * where code point order and foldCase are all optional. + * + * UAX 21 2.5 Caseless Matching specifies that for a canonical caseless match + * the case folding must be performed first, then the normalization. + * + * @param s1 First source string. + * @param s2 Second source string. + * + * @param options A bit set of options: + * - U_FOLD_CASE_DEFAULT or 0 is used for default options: + * Case-sensitive comparison in code unit order, and the input strings + * are quick-checked for FCD. + * + * - UNORM_INPUT_IS_FCD + * Set if the caller knows that both s1 and s2 fulfill the FCD conditions. + * If not set, the function will quickCheck for FCD + * and normalize if necessary. + * + * - U_COMPARE_CODE_POINT_ORDER + * Set to choose code point order instead of code unit order + * (see u_strCompare for details). + * + * - U_COMPARE_IGNORE_CASE + * Set to compare strings case-insensitively using case folding, + * instead of case-sensitively. + * If set, then the following case folding options are used. + * + * - Options as used with case-insensitive comparisons, currently: + * + * - U_FOLD_CASE_EXCLUDE_SPECIAL_I + * (see u_strCaseCompare for details) + * + * - regular normalization options shifted left by UNORM_COMPARE_NORM_OPTIONS_SHIFT + * + * @param errorCode ICU error code in/out parameter. + * Must fulfill U_SUCCESS before the function call. + * @return <0 or 0 or >0 as usual for string comparisons + * + * @see unorm_compare + * @see normalize + * @see UNORM_FCD + * @see u_strCompare + * @see u_strCaseCompare + * + * @stable ICU 2.2 + */ + static inline int32_t + compare(const UnicodeString &s1, const UnicodeString &s2, + uint32_t options, + UErrorCode &errorCode); + + //------------------------------------------------------------------------- + // Iteration API + //------------------------------------------------------------------------- + + /** + * Return the current character in the normalized text. + * current() may need to normalize some text at getIndex(). + * The getIndex() is not changed. + * + * @return the current normalized code point + * @stable ICU 2.0 + */ + UChar32 current(void); + + /** + * Return the first character in the normalized text. + * This is equivalent to setIndexOnly(startIndex()) followed by next(). + * (Post-increment semantics.) + * + * @return the first normalized code point + * @stable ICU 2.0 + */ + UChar32 first(void); + + /** + * Return the last character in the normalized text. + * This is equivalent to setIndexOnly(endIndex()) followed by previous(). + * (Pre-decrement semantics.) + * + * @return the last normalized code point + * @stable ICU 2.0 + */ + UChar32 last(void); + + /** + * Return the next character in the normalized text. + * (Post-increment semantics.) + * If the end of the text has already been reached, DONE is returned. + * The DONE value could be confused with a U+FFFF non-character code point + * in the text. If this is possible, you can test getIndex()startIndex() || first()!=DONE). (Calling first() will change + * the iterator state!) + * + * The C API unorm_previous() is more efficient and does not have this ambiguity. + * + * @return the previous normalized code point + * @stable ICU 2.0 + */ + UChar32 previous(void); + + /** + * Set the iteration position in the input text that is being normalized, + * without any immediate normalization. + * After setIndexOnly(), getIndex() will return the same index that is + * specified here. + * + * @param index the desired index in the input text. + * @stable ICU 2.0 + */ + void setIndexOnly(int32_t index); + + /** + * Reset the index to the beginning of the text. + * This is equivalent to setIndexOnly(startIndex)). + * @stable ICU 2.0 + */ + void reset(void); + + /** + * Retrieve the current iteration position in the input text that is + * being normalized. + * + * A following call to next() will return a normalized code point from + * the input text at or after this index. + * + * After a call to previous(), getIndex() will point at or before the + * position in the input text where the normalized code point + * was returned from with previous(). + * + * @return the current index in the input text + * @stable ICU 2.0 + */ + int32_t getIndex(void) const; + + /** + * Retrieve the index of the start of the input text. This is the begin index + * of the CharacterIterator or the start (i.e. index 0) of the string + * over which this Normalizer is iterating. + * + * @return the smallest index in the input text where the Normalizer operates + * @stable ICU 2.0 + */ + int32_t startIndex(void) const; + + /** + * Retrieve the index of the end of the input text. This is the end index + * of the CharacterIterator or the length of the string + * over which this Normalizer is iterating. + * This end index is exclusive, i.e., the Normalizer operates only on characters + * before this index. + * + * @return the first index in the input text where the Normalizer does not operate + * @stable ICU 2.0 + */ + int32_t endIndex(void) const; + + /** + * Returns TRUE when both iterators refer to the same character in the same + * input text. + * + * @param that a Normalizer object to compare this one to + * @return comparison result + * @stable ICU 2.0 + */ + UBool operator==(const Normalizer& that) const; + + /** + * Returns FALSE when both iterators refer to the same character in the same + * input text. + * + * @param that a Normalizer object to compare this one to + * @return comparison result + * @stable ICU 2.0 + */ + inline UBool operator!=(const Normalizer& that) const; + + /** + * Returns a pointer to a new Normalizer that is a clone of this one. + * The caller is responsible for deleting the new clone. + * @return a pointer to a new Normalizer + * @stable ICU 2.0 + */ + Normalizer* clone(void) const; + + /** + * Generates a hash code for this iterator. + * + * @return the hash code + * @stable ICU 2.0 + */ + int32_t hashCode(void) const; + + //------------------------------------------------------------------------- + // Property access methods + //------------------------------------------------------------------------- + + /** + * Set the normalization mode for this object. + *

+ * Note:If the normalization mode is changed while iterating + * over a string, calls to {@link #next() } and {@link #previous() } may + * return previously buffers characters in the old normalization mode + * until the iteration is able to re-sync at the next base character. + * It is safest to call {@link #setIndexOnly }, {@link #reset() }, + * {@link #setText }, {@link #first() }, + * {@link #last() }, etc. after calling setMode. + *

+ * @param newMode the new mode for this Normalizer. + * @see #getUMode + * @stable ICU 2.0 + */ + void setMode(UNormalizationMode newMode); + + /** + * Return the normalization mode for this object. + * + * This is an unusual name because there used to be a getMode() that + * returned a different type. + * + * @return the mode for this Normalizer + * @see #setMode + * @stable ICU 2.0 + */ + UNormalizationMode getUMode(void) const; + + /** + * Set options that affect this Normalizer's operation. + * Options do not change the basic composition or decomposition operation + * that is being performed, but they control whether + * certain optional portions of the operation are done. + * Currently the only available option is obsolete. + * + * It is possible to specify multiple options that are all turned on or off. + * + * @param option the option(s) whose value is/are to be set. + * @param value the new setting for the option. Use TRUE to + * turn the option(s) on and FALSE to turn it/them off. + * + * @see #getOption + * @stable ICU 2.0 + */ + void setOption(int32_t option, + UBool value); + + /** + * Determine whether an option is turned on or off. + * If multiple options are specified, then the result is TRUE if any + * of them are set. + *

+ * @param option the option(s) that are to be checked + * @return TRUE if any of the option(s) are set + * @see #setOption + * @stable ICU 2.0 + */ + UBool getOption(int32_t option) const; + + /** + * Set the input text over which this Normalizer will iterate. + * The iteration position is set to the beginning. + * + * @param newText a string that replaces the current input text + * @param status a UErrorCode + * @stable ICU 2.0 + */ + void setText(const UnicodeString& newText, + UErrorCode &status); + + /** + * Set the input text over which this Normalizer will iterate. + * The iteration position is set to the beginning. + * + * @param newText a CharacterIterator object that replaces the current input text + * @param status a UErrorCode + * @stable ICU 2.0 + */ + void setText(const CharacterIterator& newText, + UErrorCode &status); + + /** + * Set the input text over which this Normalizer will iterate. + * The iteration position is set to the beginning. + * + * @param newText a string that replaces the current input text + * @param length the length of the string, or -1 if NUL-terminated + * @param status a UErrorCode + * @stable ICU 2.0 + */ + void setText(const UChar* newText, + int32_t length, + UErrorCode &status); + /** + * Copies the input text into the UnicodeString argument. + * + * @param result Receives a copy of the text under iteration. + * @stable ICU 2.0 + */ + void getText(UnicodeString& result); + + /** + * ICU "poor man's RTTI", returns a UClassID for this class. + * @returns a UClassID for this class. + * @stable ICU 2.2 + */ + static UClassID U_EXPORT2 getStaticClassID(); + + /** + * ICU "poor man's RTTI", returns a UClassID for the actual class. + * @return a UClassID for the actual class. + * @stable ICU 2.2 + */ + virtual UClassID getDynamicClassID() const; + +private: + //------------------------------------------------------------------------- + // Private functions + //------------------------------------------------------------------------- + + Normalizer(); // default constructor not implemented + Normalizer &operator=(const Normalizer &that); // assignment operator not implemented + + // Private utility methods for iteration + // For documentation, see the source code + UBool nextNormalize(); + UBool previousNormalize(); + + void init(CharacterIterator *iter); + void clearBuffer(void); + + //------------------------------------------------------------------------- + // Private data + //------------------------------------------------------------------------- + + UNormalizationMode fUMode; + int32_t fOptions; + + // The input text and our position in it + UCharIterator *text; + + // The normalization buffer is the result of normalization + // of the source in [currentIndex..nextIndex[ . + int32_t currentIndex, nextIndex; + + // A buffer for holding intermediate results + UnicodeString buffer; + int32_t bufferPos; + +}; + +//------------------------------------------------------------------------- +// Inline implementations +//------------------------------------------------------------------------- + +inline UBool +Normalizer::operator!= (const Normalizer& other) const +{ return ! operator==(other); } + +inline UNormalizationCheckResult +Normalizer::quickCheck(const UnicodeString& source, + UNormalizationMode mode, + UErrorCode &status) { + if(U_FAILURE(status)) { + return UNORM_MAYBE; + } + + return unorm_quickCheck(source.getBuffer(), source.length(), + mode, &status); +} + +inline UNormalizationCheckResult +Normalizer::quickCheck(const UnicodeString& source, + UNormalizationMode mode, int32_t options, + UErrorCode &status) { + if(U_FAILURE(status)) { + return UNORM_MAYBE; + } + + return unorm_quickCheckWithOptions(source.getBuffer(), source.length(), + mode, options, &status); +} + +inline UBool +Normalizer::isNormalized(const UnicodeString& source, + UNormalizationMode mode, + UErrorCode &status) { + if(U_FAILURE(status)) { + return FALSE; + } + + return unorm_isNormalized(source.getBuffer(), source.length(), + mode, &status); +} + +inline UBool +Normalizer::isNormalized(const UnicodeString& source, + UNormalizationMode mode, int32_t options, + UErrorCode &status) { + if(U_FAILURE(status)) { + return FALSE; + } + + return unorm_isNormalizedWithOptions(source.getBuffer(), source.length(), + mode, options, &status); +} + +inline int32_t +Normalizer::compare(const UnicodeString &s1, const UnicodeString &s2, + uint32_t options, + UErrorCode &errorCode) { + // all argument checking is done in unorm_compare + return unorm_compare(s1.getBuffer(), s1.length(), + s2.getBuffer(), s2.length(), + options, + &errorCode); +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_NORMALIZATION */ + +#endif // NORMLZR_H diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/parseerr.h b/Build/source/libs/icu/icu-xetex/common/unicode/parseerr.h new file mode 100644 index 00000000000..44ff00811de --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/parseerr.h @@ -0,0 +1,92 @@ +/* +********************************************************************** +* Copyright (C) 1999-2005, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 03/14/00 aliu Creation. +* 06/27/00 aliu Change from C++ class to C struct +********************************************************************** +*/ +#ifndef PARSEERR_H +#define PARSEERR_H + +#include "unicode/utypes.h" + + +/** + * \file + * \brief C API: Parse Error Information + */ +/** + * The capacity of the context strings in UParseError. + * @stable ICU 2.0 + */ +enum { U_PARSE_CONTEXT_LEN = 16 }; + +/** + * A UParseError struct is used to returned detailed information about + * parsing errors. It is used by ICU parsing engines that parse long + * rules, patterns, or programs, where the text being parsed is long + * enough that more information than a UErrorCode is needed to + * localize the error. + * + *

The line, offset, and context fields are optional; parsing + * engines may choose not to use to use them. + * + *

The preContext and postContext strings include some part of the + * context surrounding the error. If the source text is "let for=7" + * and "for" is the error (e.g., because it is a reserved word), then + * some examples of what a parser might produce are the following: + * + *

+ * preContext   postContext
+ * ""           ""            The parser does not support context
+ * "let "       "=7"          Pre- and post-context only
+ * "let "       "for=7"       Pre- and post-context and error text
+ * ""           "for"         Error text only
+ * 
+ * + *

Examples of engines which use UParseError (or may use it in the + * future) are Transliterator, RuleBasedBreakIterator, and + * RegexPattern. + * + * @stable ICU 2.0 + */ +typedef struct UParseError { + + /** + * The line on which the error occured. If the parser uses this + * field, it sets it to the line number of the source text line on + * which the error appears, which will be be a value >= 1. If the + * parse does not support line numbers, the value will be <= 0. + * @stable ICU 2.0 + */ + int32_t line; + + /** + * The character offset to the error. If the line field is >= 1, + * then this is the offset from the start of the line. Otherwise, + * this is the offset from the start of the text. If the parser + * does not support this field, it will have a value < 0. + * @stable ICU 2.0 + */ + int32_t offset; + + /** + * Textual context before the error. Null-terminated. The empty + * string if not supported by parser. + * @stable ICU 2.0 + */ + UChar preContext[U_PARSE_CONTEXT_LEN]; + + /** + * The error itself and/or textual context after the error. + * Null-terminated. The empty string if not supported by parser. + * @stable ICU 2.0 + */ + UChar postContext[U_PARSE_CONTEXT_LEN]; + +} UParseError; + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/parsepos.h b/Build/source/libs/icu/icu-xetex/common/unicode/parsepos.h new file mode 100644 index 00000000000..cdf49e04ec2 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/parsepos.h @@ -0,0 +1,230 @@ +/* +* Copyright (C) 1997-2005, International Business Machines Corporation and others. All Rights Reserved. +******************************************************************************* +* +* File PARSEPOS.H +* +* Modification History: +* +* Date Name Description +* 07/09/97 helena Converted from java. +* 07/17/98 stephen Added errorIndex support. +* 05/11/99 stephen Cleaned up. +******************************************************************************* +*/ + +#ifndef PARSEPOS_H +#define PARSEPOS_H + +#include "unicode/utypes.h" +#include "unicode/uobject.h" + + +U_NAMESPACE_BEGIN + +/** + * \file + * \brief C++ API: Canonical Iterator + */ +/** + * ParsePosition is a simple class used by Format + * and its subclasses to keep track of the current position during parsing. + * The parseObject method in the various Format + * classes requires a ParsePosition object as an argument. + * + *

+ * By design, as you parse through a string with different formats, + * you can use the same ParsePosition, since the index parameter + * records the current position. + * + * The ParsePosition class is not suitable for subclassing. + * + * @version 1.3 10/30/97 + * @author Mark Davis, Helena Shih + * @see java.text.Format + */ + +class U_COMMON_API ParsePosition : public UObject { +public: + /** + * Default constructor, the index starts with 0 as default. + * @stable ICU 2.0 + */ + ParsePosition() + : UObject(), + index(0), + errorIndex(-1) + {} + + /** + * Create a new ParsePosition with the given initial index. + * @param newIndex the new text offset. + * @stable ICU 2.0 + */ + ParsePosition(int32_t newIndex) + : UObject(), + index(newIndex), + errorIndex(-1) + {} + + /** + * Copy constructor + * @param copy the object to be copied from. + * @stable ICU 2.0 + */ + ParsePosition(const ParsePosition& copy) + : UObject(copy), + index(copy.index), + errorIndex(copy.errorIndex) + {} + + /** + * Destructor + * @stable ICU 2.0 + */ + virtual ~ParsePosition(); + + /** + * Assignment operator + * @stable ICU 2.0 + */ + ParsePosition& operator=(const ParsePosition& copy); + + /** + * Equality operator. + * @return TRUE if the two parse positions are equal, FALSE otherwise. + * @stable ICU 2.0 + */ + UBool operator==(const ParsePosition& that) const; + + /** + * Equality operator. + * @return TRUE if the two parse positions are not equal, FALSE otherwise. + * @stable ICU 2.0 + */ + UBool operator!=(const ParsePosition& that) const; + + /** + * Clone this object. + * Clones can be used concurrently in multiple threads. + * If an error occurs, then NULL is returned. + * The caller must delete the clone. + * + * @return a clone of this object + * + * @see getDynamicClassID + * @stable ICU 2.8 + */ + ParsePosition *clone() const; + + /** + * Retrieve the current parse position. On input to a parse method, this + * is the index of the character at which parsing will begin; on output, it + * is the index of the character following the last character parsed. + * @return the current index. + * @stable ICU 2.0 + */ + int32_t getIndex(void) const; + + /** + * Set the current parse position. + * @param index the new index. + * @stable ICU 2.0 + */ + void setIndex(int32_t index); + + /** + * Set the index at which a parse error occurred. Formatters + * should set this before returning an error code from their + * parseObject method. The default value is -1 if this is not + * set. + * @stable ICU 2.0 + */ + void setErrorIndex(int32_t ei); + + /** + * Retrieve the index at which an error occurred, or -1 if the + * error index has not been set. + * @stable ICU 2.0 + */ + int32_t getErrorIndex(void) const; + + /** + * ICU "poor man's RTTI", returns a UClassID for this class. + * + * @stable ICU 2.2 + */ + static UClassID U_EXPORT2 getStaticClassID(); + + /** + * ICU "poor man's RTTI", returns a UClassID for the actual class. + * + * @stable ICU 2.2 + */ + virtual UClassID getDynamicClassID() const; + +private: + /** + * Input: the place you start parsing. + *
Output: position where the parse stopped. + * This is designed to be used serially, + * with each call setting index up for the next one. + */ + int32_t index; + + /** + * The index at which a parse error occurred. + */ + int32_t errorIndex; + +}; + +inline ParsePosition& +ParsePosition::operator=(const ParsePosition& copy) +{ + index = copy.index; + errorIndex = copy.errorIndex; + return *this; +} + +inline UBool +ParsePosition::operator==(const ParsePosition& copy) const +{ + if(index != copy.index || errorIndex != copy.errorIndex) + return FALSE; + else + return TRUE; +} + +inline UBool +ParsePosition::operator!=(const ParsePosition& copy) const +{ + return !operator==(copy); +} + +inline int32_t +ParsePosition::getIndex() const +{ + return index; +} + +inline void +ParsePosition::setIndex(int32_t offset) +{ + this->index = offset; +} + +inline int32_t +ParsePosition::getErrorIndex() const +{ + return errorIndex; +} + +inline void +ParsePosition::setErrorIndex(int32_t ei) +{ + this->errorIndex = ei; +} +U_NAMESPACE_END + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/platform.h.in b/Build/source/libs/icu/icu-xetex/common/unicode/platform.h.in new file mode 100644 index 00000000000..7abd7ca5e62 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/platform.h.in @@ -0,0 +1,313 @@ +/* +****************************************************************************** +* +* Copyright (C) 1997-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* +* FILE NAME : platform.h +* +* Date Name Description +* 05/13/98 nos Creation (content moved here from ptypes.h). +* 03/02/99 stephen Added AS400 support. +* 03/30/99 stephen Added Linux support. +* 04/13/99 stephen Reworked for autoconf. +****************************************************************************** +*/ + +/** + * \file + * \brief Basic types for the platform + */ + +/* Define the platform we're on. */ +#ifndef @platform@ +#define @platform@ +#endif + +/* Define whether inttypes.h is available */ +#ifndef U_HAVE_INTTYPES_H +#define U_HAVE_INTTYPES_H @U_HAVE_INTTYPES_H@ +#endif + +/* + * Define what support for C++ streams is available. + * If U_IOSTREAM_SOURCE is set to 199711, then is available + * (1997711 is the date the ISO/IEC C++ FDIS was published), and then + * one should qualify streams using the std namespace in ICU header + * files. + * If U_IOSTREAM_SOURCE is set to 198506, then is + * available instead (198506 is the date when Stroustrup published + * "An Extensible I/O Facility for C++" at the summer USENIX conference). + * If U_IOSTREAM_SOURCE is 0, then C++ streams are not available and + * support for them will be silently suppressed in ICU. + * + */ + +#ifndef U_IOSTREAM_SOURCE +#define U_IOSTREAM_SOURCE @U_IOSTREAM_SOURCE@ +#endif + +/* Determines whether specific types are available */ +#ifndef U_HAVE_INT8_T +#define U_HAVE_INT8_T @HAVE_INT8_T@ +#endif + +#ifndef U_HAVE_UINT8_T +#define U_HAVE_UINT8_T @HAVE_UINT8_T@ +#endif + +#ifndef U_HAVE_INT16_T +#define U_HAVE_INT16_T @HAVE_INT16_T@ +#endif + +#ifndef U_HAVE_UINT16_T +#define U_HAVE_UINT16_T @HAVE_UINT16_T@ +#endif + +#ifndef U_HAVE_INT32_T +#define U_HAVE_INT32_T @HAVE_INT32_T@ +#endif + +#ifndef U_HAVE_UINT32_T +#define U_HAVE_UINT32_T @HAVE_UINT32_T@ +#endif + +#ifndef U_HAVE_INT64_T +#define U_HAVE_INT64_T @HAVE_INT64_T@ +#endif + +#ifndef U_HAVE_UINT64_T +#define U_HAVE_UINT64_T @HAVE_UINT64_T@ +#endif + +/*===========================================================================*/ +/* Generic data types */ +/*===========================================================================*/ + +#include + +/* If your platform does not have the header, you may + need to edit the typedefs below. */ +#if U_HAVE_INTTYPES_H + +/* autoconf 2.13 sometimes can't properly find the data types in */ +/* os/390 needs , but it doesn't have int8_t, and it sometimes */ +/* doesn't have uint8_t depending on the OS version. */ +/* So we have this work around. */ +#ifdef OS390 +/* The features header is needed to get (u)int64_t sometimes. */ +#include +#if ! U_HAVE_INT8_T +typedef signed char int8_t; +#endif +#if !defined(__uint8_t) +#define __uint8_t 1 +typedef unsigned char uint8_t; +#endif +#endif /* OS390 */ + +#include + +#else /* U_HAVE_INTTYPES_H */ + +#if ! U_HAVE_INT8_T +typedef signed char int8_t; +#endif + +#if ! U_HAVE_UINT8_T +typedef unsigned char uint8_t; +#endif + +#if ! U_HAVE_INT16_T +typedef signed short int16_t; +#endif + +#if ! U_HAVE_UINT16_T +typedef unsigned short uint16_t; +#endif + +#if ! U_HAVE_INT32_T +typedef signed int int32_t; +#endif + +#if ! U_HAVE_UINT32_T +typedef unsigned int uint32_t; +#endif + +#if ! U_HAVE_INT64_T + typedef signed long long int64_t; +/* else we may not have a 64-bit type */ +#endif + +#if ! U_HAVE_UINT64_T + typedef unsigned long long uint64_t; +/* else we may not have a 64-bit type */ +#endif + +#endif + +/*===========================================================================*/ +/* Compiler and environment features */ +/*===========================================================================*/ + +/* Define whether namespace is supported */ +#ifndef U_HAVE_NAMESPACE +#define U_HAVE_NAMESPACE @U_HAVE_NAMESPACE@ +#endif + +/* Determines the endianness of the platform + It's done this way in case multiple architectures are being built at once. + For example, Darwin supports fat binaries, which can be both PPC and x86 based. */ +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) +#define U_IS_BIG_ENDIAN (BYTE_ORDER == BIG_ENDIAN) +#else +#define U_IS_BIG_ENDIAN @U_IS_BIG_ENDIAN@ +#endif + +/* 1 or 0 to enable or disable threads. If undefined, default is: enable threads. */ +#define ICU_USE_THREADS @ICU_USE_THREADS@ + +/* On strong memory model CPUs (e.g. x86 CPUs), we use a safe & quick double check lock. */ +#if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) +#define UMTX_STRONG_MEMORY_MODEL 1 +#endif + +#ifndef U_DEBUG +#define U_DEBUG @ENABLE_DEBUG@ +#endif + +#ifndef U_RELEASE +#define U_RELEASE @ENABLE_RELEASE@ +#endif + +/* Determine whether to disable renaming or not. This overrides the + setting in umachine.h which is for all platforms. */ +#ifndef U_DISABLE_RENAMING +#define U_DISABLE_RENAMING @U_DISABLE_RENAMING@ +#endif + +/* Determine whether to override new and delete. */ +#ifndef U_OVERRIDE_CXX_ALLOCATION +#define U_OVERRIDE_CXX_ALLOCATION @U_OVERRIDE_CXX_ALLOCATION@ +#endif +/* Determine whether to override placement new and delete for STL. */ +#ifndef U_HAVE_PLACEMENT_NEW +#define U_HAVE_PLACEMENT_NEW @U_HAVE_PLACEMENT_NEW@ +#endif + +/* Determine whether to enable tracing. */ +#ifndef U_ENABLE_TRACING +#define U_ENABLE_TRACING @U_ENABLE_TRACING@ +#endif + +/* Do we allow ICU users to use the draft APIs by default? */ +#ifndef U_DEFAULT_SHOW_DRAFT +#define U_DEFAULT_SHOW_DRAFT @U_DEFAULT_SHOW_DRAFT@ +#endif + +/* Define the library suffix in a C syntax. */ +#define U_HAVE_LIB_SUFFIX @U_HAVE_LIB_SUFFIX@ +#define U_LIB_SUFFIX_C_NAME @ICULIBSUFFIXCNAME@ +#define U_LIB_SUFFIX_C_NAME_STRING "@ICULIBSUFFIXCNAME@" + +/*===========================================================================*/ +/* Character data types */ +/*===========================================================================*/ + +#if ((defined(OS390) && (!defined(__CHARSET_LIB) || !__CHARSET_LIB))) || defined(OS400) +# define U_CHARSET_FAMILY 1 +#endif + +/*===========================================================================*/ +/* Information about wchar support */ +/*===========================================================================*/ + +#define U_HAVE_WCHAR_H @U_HAVE_WCHAR_H@ +#define U_SIZEOF_WCHAR_T @U_SIZEOF_WCHAR_T@ + +#define U_HAVE_WCSCPY @U_HAVE_WCSCPY@ + +/** + * \def U_DECLARE_UTF16 + * Do not use this macro. Use the UNICODE_STRING or U_STRING_DECL macros + * instead. + * @internal + */ +#if @U_CHECK_UTF16_STRING@ || defined(U_CHECK_UTF16_STRING) +#if (defined(__xlC__) && defined(__IBM_UTF_LITERAL) && U_SIZEOF_WCHAR_T != 2) \ + || (defined(__HP_aCC) && __HP_aCC >= 035000) \ + || (defined(__HP_cc) && __HP_cc >= 111106) +#define U_DECLARE_UTF16(string) u ## string +#elif (defined(__SUNPRO_CC) && __SUNPRO_CC >= 0x550) +/* || (defined(__SUNPRO_C) && __SUNPRO_C >= 0x580) */ +/* Sun's C compiler has issues with this notation, and it's unreliable. */ +#define U_DECLARE_UTF16(string) U ## string +#elif U_SIZEOF_WCHAR_T == 2 \ + && (U_CHARSET_FAMILY == 0 || ((defined(OS390) || defined(OS400)) && defined(__UCS2__))) +#define U_DECLARE_UTF16(string) L ## string +#endif +#endif + +/*===========================================================================*/ +/* Information about POSIX support */ +/*===========================================================================*/ + +#define U_HAVE_NL_LANGINFO_CODESET @U_HAVE_NL_LANGINFO_CODESET@ +#define U_NL_LANGINFO_CODESET @U_NL_LANGINFO_CODESET@ + +#if @U_HAVE_TZSET@ +#define U_TZSET @U_TZSET@ +#endif +#if @U_HAVE_TIMEZONE@ +#define U_TIMEZONE @U_TIMEZONE@ +#endif +#if @U_HAVE_TZNAME@ +#define U_TZNAME @U_TZNAME@ +#endif + +#define U_HAVE_MMAP @HAVE_MMAP@ +#define U_HAVE_POPEN @U_HAVE_POPEN@ + +/*===========================================================================*/ +/* Symbol import-export control */ +/*===========================================================================*/ + +#if @U_USE_GCC_VISIBILITY_ATTRIBUTE@ +#define U_EXPORT __attribute__((visibility("default"))) +/*#elif defined(__HP_aCC) || defined(__HP_cc) +#define U_EXPORT __declspec(dllexport)*/ +#else +#define U_EXPORT +#endif + +/* U_CALLCONV is releated to U_EXPORT2 */ +#define U_EXPORT2 + +/* cygwin needs to export/import data */ +#ifdef U_CYGWIN +#define U_IMPORT __declspec(dllimport) +#else +#define U_IMPORT +#endif + +/*===========================================================================*/ +/* Code alignment and C function inlining */ +/*===========================================================================*/ + +#ifndef U_INLINE +# ifdef __cplusplus +# define U_INLINE inline +# else +# define U_INLINE @U_INLINE@ +# endif +#endif + +#define U_ALIGN_CODE(n) + +/*===========================================================================*/ +/* Programs used by ICU code */ +/*===========================================================================*/ + +#define U_MAKE "@U_MAKE@" diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/ppalmos.h b/Build/source/libs/icu/icu-xetex/common/unicode/ppalmos.h new file mode 100644 index 00000000000..c15b2ceae32 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/ppalmos.h @@ -0,0 +1,273 @@ +/* +****************************************************************************** +* +* Copyright (C) 1997-2006, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* +* FILE NAME : ppalmos.h +* +* Date Name Description +* 05/10/04 Ken Krugler Creation (copied from pwin32.h & modified). +****************************************************************************** +*/ + +#ifndef U_PPALMOS_H +#define U_PPALMOS_H + + /** + * \file + * \brief Configuration constants for the Palm OS platform + */ + +/* Define the platform we're on. */ +#ifndef U_PALMOS +#define U_PALMOS +#endif + +/* _MSC_VER is used to detect the Microsoft compiler. */ +#if defined(_MSC_VER) +#define U_INT64_IS_LONG_LONG 0 +#else +#define U_INT64_IS_LONG_LONG 1 +#endif + +/* Define whether inttypes.h is available */ +#ifndef U_HAVE_INTTYPES_H +#define U_HAVE_INTTYPES_H 1 +#endif + +/* + * Define what support for C++ streams is available. + * If U_IOSTREAM_SOURCE is set to 199711, then is available + * (1997711 is the date the ISO/IEC C++ FDIS was published), and then + * one should qualify streams using the std namespace in ICU header + * files. + * If U_IOSTREAM_SOURCE is set to 198506, then is + * available instead (198506 is the date when Stroustrup published + * "An Extensible I/O Facility for C++" at the summer USENIX conference). + * If U_IOSTREAM_SOURCE is 0, then C++ streams are not available and + * support for them will be silently suppressed in ICU. + * + */ + +#ifndef U_IOSTREAM_SOURCE +#define U_IOSTREAM_SOURCE 199711 +#endif + +/* Determines whether specific types are available */ +#ifndef U_HAVE_INT8_T +#define U_HAVE_INT8_T U_HAVE_INTTYPES_H +#endif + +#ifndef U_HAVE_UINT8_T +#define U_HAVE_UINT8_T U_HAVE_INTTYPES_H +#endif + +#ifndef U_HAVE_INT16_T +#define U_HAVE_INT16_T U_HAVE_INTTYPES_H +#endif + +#ifndef U_HAVE_UINT16_T +#define U_HAVE_UINT16_T U_HAVE_INTTYPES_H +#endif + +#ifndef U_HAVE_INT32_T +#define U_HAVE_INT32_T U_HAVE_INTTYPES_H +#endif + +#ifndef U_HAVE_UINT32_T +#define U_HAVE_UINT32_T U_HAVE_INTTYPES_H +#endif + +#ifndef U_HAVE_INT64_T +#define U_HAVE_INT64_T U_HAVE_INTTYPES_H +#endif + +#ifndef U_HAVE_UINT64_T +#define U_HAVE_UINT64_T U_HAVE_INTTYPES_H +#endif + + +/*===========================================================================*/ +/* Generic data types */ +/*===========================================================================*/ + +/* If your platform does not have the header, you may + need to edit the typedefs below. */ +#if U_HAVE_INTTYPES_H +#include +#else /* U_HAVE_INTTYPES_H */ + +#if ! U_HAVE_INT8_T +typedef signed char int8_t; +#endif + +#if ! U_HAVE_UINT8_T +typedef unsigned char uint8_t; +#endif + +#if ! U_HAVE_INT16_T +typedef signed short int16_t; +#endif + +#if ! U_HAVE_UINT16_T +typedef unsigned short uint16_t; +#endif + +#if ! U_HAVE_INT32_T +typedef signed int int32_t; +#endif + +#if ! U_HAVE_UINT32_T +typedef unsigned int uint32_t; +#endif + +#if ! U_HAVE_INT64_T +#if U_INT64_IS_LONG_LONG + typedef signed long long int64_t; +#else + typedef signed __int64 int64_t; +#endif +#endif + +#if ! U_HAVE_UINT64_T +#if U_INT64_IS_LONG_LONG + typedef unsigned long long uint64_t; +#else + typedef unsigned __int64 uint64_t; +#endif +#endif +#endif + +/*===========================================================================*/ +/* Compiler and environment features */ +/*===========================================================================*/ + +/* Define whether namespace is supported */ +#ifndef U_HAVE_NAMESPACE +#define U_HAVE_NAMESPACE 1 +#endif + +/* Determines the endianness of the platform */ +#define U_IS_BIG_ENDIAN 0 + +/* 1 or 0 to enable or disable threads. If undefined, default is: enable threads. */ +#define ICU_USE_THREADS 1 + +#ifndef U_DEBUG +#ifdef _DEBUG +#define U_DEBUG 1 +#else +#define U_DEBUG 0 +#endif +#endif + +#ifndef U_RELEASE +#ifdef NDEBUG +#define U_RELEASE 1 +#else +#define U_RELEASE 0 +#endif +#endif + +/* Determine whether to disable renaming or not. This overrides the + setting in umachine.h which is for all platforms. */ +#ifndef U_DISABLE_RENAMING +#define U_DISABLE_RENAMING 0 +#endif + +/* Determine whether to override new and delete. */ +#ifndef U_OVERRIDE_CXX_ALLOCATION +#define U_OVERRIDE_CXX_ALLOCATION 1 +#endif +/* Determine whether to override placement new and delete for STL. */ +#ifndef U_HAVE_PLACEMENT_NEW +#define U_HAVE_PLACEMENT_NEW 0 +#endif +/* Determine whether to override new and delete for MFC. */ +#if !defined(U_HAVE_DEBUG_LOCATION_NEW) && defined(_MSC_VER) +#define U_HAVE_DEBUG_LOCATION_NEW 0 +#endif + +/* Determine whether to enable tracing. */ +#ifndef U_ENABLE_TRACING +#define U_ENABLE_TRACING 1 +#endif + +/* Do we allow ICU users to use the draft APIs by default? */ +#ifndef U_DEFAULT_SHOW_DRAFT +#define U_DEFAULT_SHOW_DRAFT 1 +#endif + +/* Define the library suffix in a C syntax. */ +#define U_HAVE_LIB_SUFFIX 0 +#define U_LIB_SUFFIX_C_NAME +#define U_LIB_SUFFIX_C_NAME_STRING "" + +/*===========================================================================*/ +/* Information about wchar support */ +/*===========================================================================*/ + +#define U_HAVE_WCHAR_H 1 +#define U_SIZEOF_WCHAR_T 2 + +#define U_HAVE_WCSCPY 0 + +/*===========================================================================*/ +/* Information about POSIX support */ +/*===========================================================================*/ + + +/* TODO: Fix Palm OS's determination of a timezone */ +#if 0 +#define U_TZSET _tzset +#endif +#if 0 +#define U_TIMEZONE _timezone +#endif +#if 0 +#define U_TZNAME _tzname +#endif + +#define U_HAVE_MMAP 0 +#define U_HAVE_POPEN 0 + +/*===========================================================================*/ +/* Symbol import-export control */ +/*===========================================================================*/ + +#define U_EXPORT +#define U_EXPORT2 +#define U_IMPORT + +/*===========================================================================*/ +/* Code alignment and C function inlining */ +/*===========================================================================*/ + +#ifndef U_INLINE +# ifdef __cplusplus +# define U_INLINE inline +# else +# define U_INLINE __inline +# endif +#endif + +#if defined(_MSC_VER) && defined(_M_IX86) +#define U_ALIGN_CODE(val) __asm align val +#else +#define U_ALIGN_CODE(val) +#endif + + +/*===========================================================================*/ +/* Programs used by ICU code */ +/*===========================================================================*/ + +#ifndef U_MAKE +#define U_MAKE "nmake" +#define U_MAKE_IS_NMAKE 1 +#endif + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/putil.h b/Build/source/libs/icu/icu-xetex/common/unicode/putil.h new file mode 100644 index 00000000000..bdb385c7f2f --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/putil.h @@ -0,0 +1,184 @@ +/* +****************************************************************************** +* +* Copyright (C) 1997-2005, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* +* FILE NAME : putil.h +* +* Date Name Description +* 05/14/98 nos Creation (content moved here from utypes.h). +* 06/17/99 erm Added IEEE_754 +* 07/22/98 stephen Added IEEEremainder, max, min, trunc +* 08/13/98 stephen Added isNegativeInfinity, isPositiveInfinity +* 08/24/98 stephen Added longBitsFromDouble +* 03/02/99 stephen Removed openFile(). Added AS400 support. +* 04/15/99 stephen Converted to C +* 11/15/99 helena Integrated S/390 changes for IEEE support. +* 01/11/00 helena Added u_getVersion. +****************************************************************************** +*/ + +#ifndef PUTIL_H +#define PUTIL_H + +#include "unicode/utypes.h" + /** + * \file + * \brief C API: Platform Utilities + */ + +/* Define this to 1 if your platform supports IEEE 754 floating point, + to 0 if it does not. */ +#ifndef IEEE_754 +# define IEEE_754 1 +#endif + +/*==========================================================================*/ +/* Platform utilities */ +/*==========================================================================*/ + +/** + * Platform utilities isolates the platform dependencies of the + * libarary. For each platform which this code is ported to, these + * functions may have to be re-implemented. + */ + +/** + * Return the ICU data directory. + * The data directory is where common format ICU data files (.dat files) + * are loaded from. Note that normal use of the built-in ICU + * facilities does not require loading of an external data file; + * unless you are adding custom data to ICU, the data directory + * does not need to be set. + * + * The data directory is determined as follows: + * If u_setDataDirectory() has been called, that is it, otherwise + * if the ICU_DATA environment variable is set, use that, otherwise + * If a data directory was specifed at ICU build time + * (#define ICU_DATA_DIR "path"), use that, + * otherwise no data directory is available. + * + * @return the data directory, or an empty string ("") if no data directory has + * been specified. + * + * @stable ICU 2.0 + */ +U_STABLE const char* U_EXPORT2 u_getDataDirectory(void); + +/** + * Set the ICU data directory. + * The data directory is where common format ICU data files (.dat files) + * are loaded from. Note that normal use of the built-in ICU + * facilities does not require loading of an external data file; + * unless you are adding custom data to ICU, the data directory + * does not need to be set. + * + * This function should be called at most once in a process, before the + * first ICU operation (e.g., u_init()) that will require the loading of an + * ICU data file. + * This function is not thread-safe. Use it before calling ICU APIs from + * multiple threads. + * + * @param directory The directory to be set. + * + * @see u_init + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 u_setDataDirectory(const char *directory); + +/** + * Please use ucnv_getDefaultName() instead. + * Return the default codepage for this platform and locale. + * This function can call setlocale() on Unix platforms. Please read the + * platform documentation on setlocale() before calling this function. + * @return the default codepage for this platform + * @internal + */ +U_INTERNAL const char* U_EXPORT2 uprv_getDefaultCodepage(void); + +/** + * Please use uloc_getDefault() instead. + * Return the default locale ID string by querying ths system, or + * zero if one cannot be found. + * This function can call setlocale() on Unix platforms. Please read the + * platform documentation on setlocale() before calling this function. + * @return the default locale ID string + * @internal + */ +U_INTERNAL const char* U_EXPORT2 uprv_getDefaultLocaleID(void); + +/** + * Filesystem file and path separator characters. + * Example: '/' and ':' on Unix, '\\' and ';' on Windows. + * @stable ICU 2.0 + */ +#ifdef XP_MAC +# define U_FILE_SEP_CHAR ':' +# define U_FILE_ALT_SEP_CHAR ':' +# define U_PATH_SEP_CHAR ';' +# define U_FILE_SEP_STRING ":" +# define U_FILE_ALT_SEP_STRING ":" +# define U_PATH_SEP_STRING ";" +#elif defined(U_WINDOWS) +# define U_FILE_SEP_CHAR '\\' +# define U_FILE_ALT_SEP_CHAR '/' +# define U_PATH_SEP_CHAR ';' +# define U_FILE_SEP_STRING "\\" +# define U_FILE_ALT_SEP_STRING "/" +# define U_PATH_SEP_STRING ";" +#else +# define U_FILE_SEP_CHAR '/' +# define U_FILE_ALT_SEP_CHAR '/' +# define U_PATH_SEP_CHAR ':' +# define U_FILE_SEP_STRING "/" +# define U_FILE_ALT_SEP_STRING "/" +# define U_PATH_SEP_STRING ":" +#endif + +/** + * Convert char characters to UChar characters. + * This utility function is useful only for "invariant characters" + * that are encoded in the platform default encoding. + * They are a small, constant subset of the encoding and include + * just the latin letters, digits, and some punctuation. + * For details, see U_CHARSET_FAMILY. + * + * @param cs Input string, points to length + * character bytes from a subset of the platform encoding. + * @param us Output string, points to memory for length + * Unicode characters. + * @param length The number of characters to convert; this may + * include the terminating NUL. + * + * @see U_CHARSET_FAMILY + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +u_charsToUChars(const char *cs, UChar *us, int32_t length); + +/** + * Convert UChar characters to char characters. + * This utility function is useful only for "invariant characters" + * that can be encoded in the platform default encoding. + * They are a small, constant subset of the encoding and include + * just the latin letters, digits, and some punctuation. + * For details, see U_CHARSET_FAMILY. + * + * @param us Input string, points to length + * Unicode characters that can be encoded with the + * codepage-invariant subset of the platform encoding. + * @param cs Output string, points to memory for length + * character bytes. + * @param length The number of characters to convert; this may + * include the terminating NUL. + * + * @see U_CHARSET_FAMILY + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +u_UCharsToChars(const UChar *us, char *cs, int32_t length); + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/pwin32.h b/Build/source/libs/icu/icu-xetex/common/unicode/pwin32.h new file mode 100644 index 00000000000..9aad3530f0a --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/pwin32.h @@ -0,0 +1,311 @@ +/* + ****************************************************************************** + * + * Copyright (C) 1997-2007, International Business Machines + * Corporation and others. All Rights Reserved. + * + ****************************************************************************** + * + * FILE NAME : platform.h + * + * Date Name Description + * 05/13/98 nos Creation (content moved here from ptypes.h). + * 03/02/99 stephen Added AS400 support. + * 03/30/99 stephen Added Linux support. + * 04/13/99 stephen Reworked for autoconf. + ****************************************************************************** + */ + + /** + * \file + * \brief Configuration constants for the Windows platform + */ + +/* Define the platform we're on. */ +#ifndef U_WINDOWS +#define U_WINDOWS +#endif + +#if defined(__BORLANDC__) +#define U_HAVE_PLACEMENT_NEW 0 +#define U_HAVE_INTTYPES_H 1 +#define __STDC_CONSTANT_MACROS +#endif + +/* _MSC_VER is used to detect the Microsoft compiler. */ +#if defined(_MSC_VER) +#define U_INT64_IS_LONG_LONG 0 +#else +#define U_INT64_IS_LONG_LONG 1 +#endif + +/* Define whether inttypes.h is available */ +#ifndef U_HAVE_INTTYPES_H +#define U_HAVE_INTTYPES_H 0 +#endif + +/* + * Define what support for C++ streams is available. + * If U_IOSTREAM_SOURCE is set to 199711, then is available + * (1997711 is the date the ISO/IEC C++ FDIS was published), and then + * one should qualify streams using the std namespace in ICU header + * files. + * If U_IOSTREAM_SOURCE is set to 198506, then is + * available instead (198506 is the date when Stroustrup published + * "An Extensible I/O Facility for C++" at the summer USENIX conference). + * If U_IOSTREAM_SOURCE is 0, then C++ streams are not available and + * support for them will be silently suppressed in ICU. + * + */ + +#ifndef U_IOSTREAM_SOURCE +#define U_IOSTREAM_SOURCE 199711 +#endif + +/* Determines whether specific types are available */ +#ifndef U_HAVE_INT8_T +#define U_HAVE_INT8_T U_HAVE_INTTYPES_H +#endif + +#ifndef U_HAVE_UINT8_T +#define U_HAVE_UINT8_T U_HAVE_INTTYPES_H +#endif + +#ifndef U_HAVE_INT16_T +#define U_HAVE_INT16_T U_HAVE_INTTYPES_H +#endif + +#ifndef U_HAVE_UINT16_T +#define U_HAVE_UINT16_T U_HAVE_INTTYPES_H +#endif + +#ifndef U_HAVE_INT32_T +#define U_HAVE_INT32_T U_HAVE_INTTYPES_H +#endif + +#ifndef U_HAVE_UINT32_T +#define U_HAVE_UINT32_T U_HAVE_INTTYPES_H +#endif + +#ifndef U_HAVE_INT64_T +#define U_HAVE_INT64_T U_HAVE_INTTYPES_H +#endif + +#ifndef U_HAVE_UINT64_T +#define U_HAVE_UINT64_T U_HAVE_INTTYPES_H +#endif + +/* Define 64 bit limits */ +#if !U_INT64_IS_LONG_LONG +# ifndef INT64_C +# define INT64_C(x) ((int64_t)x) +# endif +# ifndef UINT64_C +# define UINT64_C(x) ((uint64_t)x) +# endif +/* else use the umachine.h definition */ +#endif + +/*===========================================================================*/ +/* Generic data types */ +/*===========================================================================*/ + +/* If your platform does not have the header, you may + need to edit the typedefs below. */ +#if U_HAVE_INTTYPES_H +#include +#else /* U_HAVE_INTTYPES_H */ + +#if ! U_HAVE_INT8_T +typedef signed char int8_t; +#endif + +#if ! U_HAVE_UINT8_T +typedef unsigned char uint8_t; +#endif + +#if ! U_HAVE_INT16_T +typedef signed short int16_t; +#endif + +#if ! U_HAVE_UINT16_T +typedef unsigned short uint16_t; +#endif + +#if ! U_HAVE_INT32_T +typedef signed int int32_t; +#endif + +#if ! U_HAVE_UINT32_T +typedef unsigned int uint32_t; +#endif + +#if ! U_HAVE_INT64_T +#if U_INT64_IS_LONG_LONG + typedef signed long long int64_t; +#else + typedef signed __int64 int64_t; +#endif +#endif + +#if ! U_HAVE_UINT64_T +#if U_INT64_IS_LONG_LONG + typedef unsigned long long uint64_t; +#else + typedef unsigned __int64 uint64_t; +#endif +#endif +#endif + +/*===========================================================================*/ +/* Compiler and environment features */ +/*===========================================================================*/ + +/* Define whether namespace is supported */ +#ifndef U_HAVE_NAMESPACE +#define U_HAVE_NAMESPACE 1 +#endif + +/* Determines the endianness of the platform */ +#define U_IS_BIG_ENDIAN 0 + +/* 1 or 0 to enable or disable threads. If undefined, default is: enable threads. */ +#define ICU_USE_THREADS 1 + +/* On strong memory model CPUs (e.g. x86 CPUs), we use a safe & quick double check mutex lock. */ +/* +Microsoft can define _M_IX86, _M_AMD64 (before Visual Studio 8) or _M_X64 (starting in Visual Studio 8). +Intel can define _M_IX86 or _M_X64 +*/ +#if defined(_M_IX86) || defined(_M_AMD64) || defined(_M_X64) || (defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))) +#define UMTX_STRONG_MEMORY_MODEL 1 +#endif + +#ifndef U_DEBUG +#ifdef _DEBUG +#define U_DEBUG 1 +#else +#define U_DEBUG 0 +#endif +#endif + +#ifndef U_RELEASE +#ifdef NDEBUG +#define U_RELEASE 1 +#else +#define U_RELEASE 0 +#endif +#endif + +/* Determine whether to disable renaming or not. This overrides the + setting in umachine.h which is for all platforms. */ +#ifndef U_DISABLE_RENAMING +#define U_DISABLE_RENAMING 0 +#endif + +/* Determine whether to override new and delete. */ +#ifndef U_OVERRIDE_CXX_ALLOCATION +#define U_OVERRIDE_CXX_ALLOCATION 1 +#endif +/* Determine whether to override placement new and delete for STL. */ +#ifndef U_HAVE_PLACEMENT_NEW +#define U_HAVE_PLACEMENT_NEW 1 +#endif +/* Determine whether to override new and delete for MFC. */ +#if !defined(U_HAVE_DEBUG_LOCATION_NEW) && defined(_MSC_VER) +#define U_HAVE_DEBUG_LOCATION_NEW 1 +#endif + +/* Determine whether to enable tracing. */ +#ifndef U_ENABLE_TRACING +#define U_ENABLE_TRACING 0 +#endif + +/* Do we allow ICU users to use the draft APIs by default? */ +#ifndef U_DEFAULT_SHOW_DRAFT +#define U_DEFAULT_SHOW_DRAFT 1 +#endif + +/* Define the library suffix in a C syntax. */ +#define U_HAVE_LIB_SUFFIX 0 +#define U_LIB_SUFFIX_C_NAME +#define U_LIB_SUFFIX_C_NAME_STRING "" + +/*===========================================================================*/ +/* Information about wchar support */ +/*===========================================================================*/ + +#define U_HAVE_WCHAR_H 1 +#define U_SIZEOF_WCHAR_T 2 + +#define U_HAVE_WCSCPY 1 + +/** + * \def U_DECLARE_UTF16 + * Do not use this macro. Use the UNICODE_STRING or U_STRING_DECL macros + * instead. + * @internal + */ +#if 1 +#define U_DECLARE_UTF16(string) L ## string +#endif + +/*===========================================================================*/ +/* Information about POSIX support */ +/*===========================================================================*/ + +#if 1 +#define U_TZSET _tzset +#endif +#if 1 +#define U_TIMEZONE _timezone +#endif +#if 1 +#define U_TZNAME _tzname +#endif +#if 1 +#define U_DAYLIGHT _daylight +#endif + +#define U_HAVE_MMAP 0 +#define U_HAVE_POPEN 0 + +/*===========================================================================*/ +/* Symbol import-export control */ +/*===========================================================================*/ + +#ifdef U_STATIC_IMPLEMENTATION +#define U_EXPORT +#else +#define U_EXPORT __declspec(dllexport) +#endif +#define U_EXPORT2 __cdecl +#define U_IMPORT __declspec(dllimport) + +/*===========================================================================*/ +/* Code alignment and C function inlining */ +/*===========================================================================*/ + +#ifndef U_INLINE +# ifdef __cplusplus +# define U_INLINE inline +# else +# define U_INLINE __inline +# endif +#endif + +#if defined(_MSC_VER) && defined(_M_IX86) && !defined(_MANAGED) +#define U_ALIGN_CODE(val) __asm align val +#else +#define U_ALIGN_CODE(val) +#endif + + +/*===========================================================================*/ +/* Programs used by ICU code */ +/*===========================================================================*/ + +#ifndef U_MAKE +#define U_MAKE "nmake" +#define U_MAKE_IS_NMAKE 1 +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/rbbi.h b/Build/source/libs/icu/icu-xetex/common/unicode/rbbi.h new file mode 100644 index 00000000000..747ea44ce1c --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/rbbi.h @@ -0,0 +1,700 @@ +/* +*************************************************************************** +* Copyright (C) 1999-2007 International Business Machines Corporation * +* and others. All rights reserved. * +*************************************************************************** + +********************************************************************** +* Date Name Description +* 10/22/99 alan Creation. +* 11/11/99 rgillam Complete port from Java. +********************************************************************** +*/ + +#ifndef RBBI_H +#define RBBI_H + +#include "unicode/utypes.h" + +/** + * \file + * \brief C++ API: Rule Based Break Iterator + */ + +#if !UCONFIG_NO_BREAK_ITERATION + +#include "unicode/brkiter.h" +#include "unicode/udata.h" +#include "unicode/parseerr.h" +#include "unicode/schriter.h" +#include "unicode/uchriter.h" + + +struct UTrie; + +U_NAMESPACE_BEGIN + +/** @internal */ +struct RBBIDataHeader; +class RuleBasedBreakIteratorTables; +class BreakIterator; +class RBBIDataWrapper; +class UStack; +class LanguageBreakEngine; +class UnhandledEngine; +struct RBBIStateTable; + + + + +/** + * + * A subclass of BreakIterator whose behavior is specified using a list of rules. + *

Instances of this class are most commonly created by the factory methods of + * BreakIterator::createWordInstance(), BreakIterator::createLineInstance(), etc., + * and then used via the abstract API in class BreakIterator

+ * + *

See the ICU User Guide for information on Break Iterator Rules.

+ * + *

This class is not intended to be subclassed. (Class DictionaryBasedBreakIterator + * is a subclass, but that relationship is effectively internal to the ICU + * implementation. The subclassing interface to RulesBasedBreakIterator is + * not part of the ICU API, and may not remain stable.

+ * + */ +class U_COMMON_API RuleBasedBreakIterator : public BreakIterator { + +protected: + /** + * The UText through which this BreakIterator accesses the text + * @internal + */ + UText *fText; + + /** + * A character iterator that refers to the same text as the UText, above. + * Only included for compatibility with old API, which was based on CharacterIterators. + * Value may be adopted from outside, or one of fSCharIter or fDCharIter, below. + */ + CharacterIterator *fCharIter; + + /** + * When the input text is provided by a UnicodeString, this will point to + * a characterIterator that wraps that data. Needed only for the + * implementation of getText(), a backwards compatibility issue. + */ + StringCharacterIterator *fSCharIter; + + /** + * When the input text is provided by a UText, this + * dummy CharacterIterator over an empty string will + * be returned from getText() + */ + UCharCharacterIterator *fDCharIter; + + /** + * The rule data for this BreakIterator instance + * @internal + */ + RBBIDataWrapper *fData; + + /** Index of the Rule {tag} values for the most recent match. + * @internal + */ + int32_t fLastRuleStatusIndex; + + /** + * Rule tag value valid flag. + * Some iterator operations don't intrinsically set the correct tag value. + * This flag lets us lazily compute the value if we are ever asked for it. + * @internal + */ + UBool fLastStatusIndexValid; + + /** + * Counter for the number of characters encountered with the "dictionary" + * flag set. + * @internal + */ + uint32_t fDictionaryCharCount; + + /** + * When a range of characters is divided up using the dictionary, the break + * positions that are discovered are stored here, preventing us from having + * to use either the dictionary or the state table again until the iterator + * leaves this range of text. Has the most impact for line breaking. + * @internal + */ + int32_t* fCachedBreakPositions; + + /** + * The number of elements in fCachedBreakPositions + * @internal + */ + int32_t fNumCachedBreakPositions; + + /** + * if fCachedBreakPositions is not null, this indicates which item in the + * cache the current iteration position refers to + * @internal + */ + int32_t fPositionInCache; + + /** + * + * If present, UStack of LanguageBreakEngine objects that might handle + * dictionary characters. Searched from top to bottom to find an object to + * handle a given character. + * @internal + */ + UStack *fLanguageBreakEngines; + + /** + * + * If present, the special LanguageBreakEngine used for handling + * characters that are in the dictionary set, but not handled by any + * LangugageBreakEngine. + * @internal + */ + UnhandledEngine *fUnhandledBreakEngine; + + /** + * + * The type of the break iterator, or -1 if it has not been set. + * @internal + */ + int32_t fBreakType; + +protected: + //======================================================================= + // constructors + //======================================================================= + + /** + * Constructor from a flattened set of RBBI data in malloced memory. + * RulesBasedBreakIterators built from a custom set of rules + * are created via this constructor; the rules are compiled + * into memory, then the break iterator is constructed here. + * + * The break iterator adopts the memory, and will + * free it when done. + * @internal + */ + RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status); + + + friend class RBBIRuleBuilder; + /** @internal */ + friend class BreakIterator; + + + +public: + + /** Default constructor. Creates an empty shell of an iterator, with no + * rules or text to iterate over. Object can subsequently be assigned to. + * @stable ICU 2.2 + */ + RuleBasedBreakIterator(); + + /** + * Copy constructor. Will produce a break iterator with the same behavior, + * and which iterates over the same text, as the one passed in. + * @param that The RuleBasedBreakIterator passed to be copied + * @stable ICU 2.0 + */ + RuleBasedBreakIterator(const RuleBasedBreakIterator& that); + + /** + * Construct a RuleBasedBreakIterator from a set of rules supplied as a string. + * @param rules The break rules to be used. + * @param parseError In the event of a syntax error in the rules, provides the location + * within the rules of the problem. + * @param status Information on any errors encountered. + * @stable ICU 2.2 + */ + RuleBasedBreakIterator( const UnicodeString &rules, + UParseError &parseError, + UErrorCode &status); + + + /** + * This constructor uses the udata interface to create a BreakIterator + * whose internal tables live in a memory-mapped file. "image" is an + * ICU UDataMemory handle for the pre-compiled break iterator tables. + * @param image handle to the memory image for the break iterator data. + * Ownership of the UDataMemory handle passes to the Break Iterator, + * which will be responsible for closing it when it is no longer needed. + * @param status Information on any errors encountered. + * @see udata_open + * @see #getBinaryRules + * @stable ICU 2.8 + */ + RuleBasedBreakIterator(UDataMemory* image, UErrorCode &status); + + /** + * Destructor + * @stable ICU 2.0 + */ + virtual ~RuleBasedBreakIterator(); + + /** + * Assignment operator. Sets this iterator to have the same behavior, + * and iterate over the same text, as the one passed in. + * @param that The RuleBasedBreakItertor passed in + * @return the newly created RuleBasedBreakIterator + * @stable ICU 2.0 + */ + RuleBasedBreakIterator& operator=(const RuleBasedBreakIterator& that); + + /** + * Equality operator. Returns TRUE if both BreakIterators are of the + * same class, have the same behavior, and iterate over the same text. + * @param that The BreakIterator to be compared for equality + * @return TRUE if both BreakIterators are of the + * same class, have the same behavior, and iterate over the same text. + * @stable ICU 2.0 + */ + virtual UBool operator==(const BreakIterator& that) const; + + /** + * Not-equal operator. If operator== returns TRUE, this returns FALSE, + * and vice versa. + * @param that The BreakIterator to be compared for inequality + * @return TRUE if both BreakIterators are not same. + * @stable ICU 2.0 + */ + UBool operator!=(const BreakIterator& that) const; + + /** + * Returns a newly-constructed RuleBasedBreakIterator with the same + * behavior, and iterating over the same text, as this one. + * Differs from the copy constructor in that it is polymorphic, and + * will correctly clone (copy) a derived class. + * clone() is thread safe. Multiple threads may simultaeneously + * clone the same source break iterator. + * @return a newly-constructed RuleBasedBreakIterator + * @stable ICU 2.0 + */ + virtual BreakIterator* clone() const; + + /** + * Compute a hash code for this BreakIterator + * @return A hash code + * @stable ICU 2.0 + */ + virtual int32_t hashCode(void) const; + + /** + * Returns the description used to create this iterator + * @return the description used to create this iterator + * @stable ICU 2.0 + */ + virtual const UnicodeString& getRules(void) const; + + //======================================================================= + // BreakIterator overrides + //======================================================================= + + /** + *

+ * Return a CharacterIterator over the text being analyzed. + * The returned character iterator is owned by the break iterator, and must + * not be deleted by the caller. Repeated calls to this function may + * return the same CharacterIterator. + *

+ *

+ * The returned character iterator must not be used concurrently with + * the break iterator. If concurrent operation is needed, clone the + * returned character iterator first and operate on the clone. + *

+ *

+ * When the break iterator is operating on text supplied via a UText, + * this function will fail. Lacking any way to signal failures, it + * returns an CharacterIterator containing no text. + * The function getUText() provides similar functionality, + * is reliable, and is more efficient. + *

+ * + * TODO: deprecate this function? + * + * @return An iterator over the text being analyzed. + * @stable ICU 2.0 + */ + virtual CharacterIterator& getText(void) const; + + + /** + * Get a UText for the text being analyzed. + * The returned UText is a shallow clone of the UText used internally + * by the break iterator implementation. It can safely be used to + * access the text without impacting any break iterator operations, + * but the underlying text itself must not be altered. + * + * @param fillIn A UText to be filled in. If NULL, a new UText will be + * allocated to hold the result. + * @param status receives any error codes. + * @return The current UText for this break iterator. If an input + * UText was provided, it will always be returned. + * @stable ICU 3.4 + */ + virtual UText *getUText(UText *fillIn, UErrorCode &status) const; + + /** + * Set the iterator to analyze a new piece of text. This function resets + * the current iteration position to the beginning of the text. + * @param newText An iterator over the text to analyze. The BreakIterator + * takes ownership of the character iterator. The caller MUST NOT delete it! + * @stable ICU 2.0 + */ + virtual void adoptText(CharacterIterator* newText); + + /** + * Set the iterator to analyze a new piece of text. This function resets + * the current iteration position to the beginning of the text. + * @param newText The text to analyze. + * @stable ICU 2.0 + */ + virtual void setText(const UnicodeString& newText); + + /** + * Reset the break iterator to operate over the text represented by + * the UText. The iterator position is reset to the start. + * + * This function makes a shallow clone of the supplied UText. This means + * that the caller is free to immediately close or otherwise reuse the + * Utext that was passed as a parameter, but that the underlying text itself + * must not be altered while being referenced by the break iterator. + * + * @param text The UText used to change the text. + * @param status Receives any error codes. + * @stable ICU 3.4 + */ + virtual void setText(UText *text, UErrorCode &status); + + /** + * Sets the current iteration position to the beginning of the text. + * @return The offset of the beginning of the text. + * @stable ICU 2.0 + */ + virtual int32_t first(void); + + /** + * Sets the current iteration position to the end of the text. + * @return The text's past-the-end offset. + * @stable ICU 2.0 + */ + virtual int32_t last(void); + + /** + * Advances the iterator either forward or backward the specified number of steps. + * Negative values move backward, and positive values move forward. This is + * equivalent to repeatedly calling next() or previous(). + * @param n The number of steps to move. The sign indicates the direction + * (negative is backwards, and positive is forwards). + * @return The character offset of the boundary position n boundaries away from + * the current one. + * @stable ICU 2.0 + */ + virtual int32_t next(int32_t n); + + /** + * Advances the iterator to the next boundary position. + * @return The position of the first boundary after this one. + * @stable ICU 2.0 + */ + virtual int32_t next(void); + + /** + * Moves the iterator backwards, to the last boundary preceding this one. + * @return The position of the last boundary position preceding this one. + * @stable ICU 2.0 + */ + virtual int32_t previous(void); + + /** + * Sets the iterator to refer to the first boundary position following + * the specified position. + * @param offset The position from which to begin searching for a break position. + * @return The position of the first break after the current position. + * @stable ICU 2.0 + */ + virtual int32_t following(int32_t offset); + + /** + * Sets the iterator to refer to the last boundary position before the + * specified position. + * @param offset The position to begin searching for a break from. + * @return The position of the last boundary before the starting position. + * @stable ICU 2.0 + */ + virtual int32_t preceding(int32_t offset); + + /** + * Returns true if the specfied position is a boundary position. As a side + * effect, leaves the iterator pointing to the first boundary position at + * or after "offset". + * @param offset the offset to check. + * @return True if "offset" is a boundary position. + * @stable ICU 2.0 + */ + virtual UBool isBoundary(int32_t offset); + + /** + * Returns the current iteration position. + * @return The current iteration position. + * @stable ICU 2.0 + */ + virtual int32_t current(void) const; + + + /** + * Return the status tag from the break rule that determined the most recently + * returned break position. For break rules that do not specify a + * status, a default value of 0 is returned. If more than one break rule + * would cause a boundary to be located at some position in the text, + * the numerically largest of the applicable status values is returned. + *

+ * Of the standard types of ICU break iterators, only word break and + * line break provide status values. The values are defined in + * the header file ubrk.h. For Word breaks, the status allows distinguishing between words + * that contain alphabetic letters, "words" that appear to be numbers, + * punctuation and spaces, words containing ideographic characters, and + * more. For Line Break, the status distinguishes between hard (mandatory) breaks + * and soft (potential) break positions. + *

+ * getRuleStatus() can be called after obtaining a boundary + * position from next(), previous(), or + * any other break iterator functions that returns a boundary position. + *

+ * When creating custom break rules, one is free to define whatever + * status values may be convenient for the application. + *

+ * Note: this function is not thread safe. It should not have been + * declared const, and the const remains only for compatibility + * reasons. (The function is logically const, but not bit-wise const). + *

+ * @return the status from the break rule that determined the most recently + * returned break position. + * + * @see UWordBreak + * @stable ICU 2.2 + */ + virtual int32_t getRuleStatus() const; + + /** + * Get the status (tag) values from the break rule(s) that determined the most + * recently returned break position. + *

+ * The returned status value(s) are stored into an array provided by the caller. + * The values are stored in sorted (ascending) order. + * If the capacity of the output array is insufficient to hold the data, + * the output will be truncated to the available length, and a + * U_BUFFER_OVERFLOW_ERROR will be signaled. + * + * @param fillInVec an array to be filled in with the status values. + * @param capacity the length of the supplied vector. A length of zero causes + * the function to return the number of status values, in the + * normal way, without attemtping to store any values. + * @param status receives error codes. + * @return The number of rule status values from rules that determined + * the most recent boundary returned by the break iterator. + * In the event of a U_BUFFER_OVERFLOW_ERROR, the return value + * is the total number of status values that were available, + * not the reduced number that were actually returned. + * @see getRuleStatus + * @stable ICU 3.0 + */ + virtual int32_t getRuleStatusVec(int32_t *fillInVec, int32_t capacity, UErrorCode &status); + + /** + * Returns a unique class ID POLYMORPHICALLY. Pure virtual override. + * This method is to implement a simple version of RTTI, since not all + * C++ compilers support genuine RTTI. Polymorphic operator==() and + * clone() methods call this method. + * + * @return The class ID for this object. All objects of a + * given class have the same class ID. Objects of + * other classes have different class IDs. + * @stable ICU 2.0 + */ + virtual UClassID getDynamicClassID(void) const; + + /** + * Returns the class ID for this class. This is useful only for + * comparing to a return value from getDynamicClassID(). For example: + * + * Base* polymorphic_pointer = createPolymorphicObject(); + * if (polymorphic_pointer->getDynamicClassID() == + * Derived::getStaticClassID()) ... + * + * @return The class ID for all objects of this class. + * @stable ICU 2.0 + */ + static UClassID U_EXPORT2 getStaticClassID(void); + + /* + * Create a clone (copy) of this break iterator in memory provided + * by the caller. The idea is to increase performance by avoiding + * a storage allocation. Use of this functoin is NOT RECOMMENDED. + * Performance gains are minimal, and correct buffer management is + * tricky. Use clone() instead. + * + * @param stackBuffer The pointer to the memory into which the cloned object + * should be placed. If NULL, allocate heap memory + * for the cloned object. + * @param BufferSize The size of the buffer. If zero, return the required + * buffer size, but do not clone the object. If the + * size was too small (but not zero), allocate heap + * storage for the cloned object. + * + * @param status Error status. U_SAFECLONE_ALLOCATED_WARNING will be + * returned if the the provided buffer was too small, and + * the clone was therefore put on the heap. + * + * @return Pointer to the clone object. This may differ from the stackBuffer + * address if the byte alignment of the stack buffer was not suitable + * or if the stackBuffer was too small to hold the clone. + * @stable ICU 2.0 + */ + virtual BreakIterator * createBufferClone(void *stackBuffer, + int32_t &BufferSize, + UErrorCode &status); + + + /** + * Return the binary form of compiled break rules, + * which can then be used to create a new break iterator at some + * time in the future. Creating a break iterator from pre-compiled rules + * is much faster than building one from the source form of the + * break rules. + * + * The binary data can only be used with the same version of ICU + * and on the same platform type (processor endian-ness) + * + * @param length Returns the length of the binary data. (Out paramter.) + * + * @return A pointer to the binary (compiled) rule data. The storage + * belongs to the RulesBasedBreakIterator object, not the + * caller, and must not be modified or deleted. + * @internal + */ + virtual const uint8_t *getBinaryRules(uint32_t &length); + + +protected: + //======================================================================= + // implementation + //======================================================================= + /** + * Dumps caches and performs other actions associated with a complete change + * in text or iteration position. + * @internal + */ + virtual void reset(void); + +#if 0 + /** + * Return true if the category lookup for this char + * indicates that it is in the set of dictionary lookup chars. + * This function is intended for use by dictionary based break iterators. + * @return true if the category lookup for this char + * indicates that it is in the set of dictionary lookup chars. + * @internal + */ + virtual UBool isDictionaryChar(UChar32); + + /** + * Get the type of the break iterator. + * @internal + */ + virtual int32_t getBreakType() const; +#endif + + /** + * Set the type of the break iterator. + * @internal + */ + virtual void setBreakType(int32_t type); + + /** + * Common initialization function, used by constructors and bufferClone. + * (Also used by DictionaryBasedBreakIterator::createBufferClone().) + * @internal + */ + void init(); + +private: + + /** + * This method backs the iterator back up to a "safe position" in the text. + * This is a position that we know, without any context, must be a break position. + * The various calling methods then iterate forward from this safe position to + * the appropriate position to return. (For more information, see the description + * of buildBackwardsStateTable() in RuleBasedBreakIterator.Builder.) + * @param statetable state table used of moving backwards + * @internal + */ + int32_t handlePrevious(const RBBIStateTable *statetable); + + /** + * This method is the actual implementation of the next() method. All iteration + * vectors through here. This method initializes the state machine to state 1 + * and advances through the text character by character until we reach the end + * of the text or the state machine transitions to state 0. We update our return + * value every time the state machine passes through a possible end state. + * @param statetable state table used of moving forwards + * @internal + */ + int32_t handleNext(const RBBIStateTable *statetable); + +protected: + + /** + * This is the function that actually implements dictionary-based + * breaking. Covering at least the range from startPos to endPos, + * it checks for dictionary characters, and if it finds them determines + * the appropriate object to deal with them. It may cache found breaks in + * fCachedBreakPositions as it goes. It may well also look at text outside + * the range startPos to endPos. + * If going forward, endPos is the normal Unicode break result, and + * if goind in reverse, startPos is the normal Unicode break result + * @param startPos The start position of a range of text + * @param endPos The end position of a range of text + * @param reverse The call is for the reverse direction + * @internal + */ + int32_t checkDictionary(int32_t startPos, int32_t endPos, UBool reverse); + +private: + + /** + * This function returns the appropriate LanguageBreakEngine for a + * given character c. + * @param c A character in the dictionary set + * @internal + */ + const LanguageBreakEngine *getLanguageBreakEngine(UChar32 c); + + /** + * @internal + */ + void makeRuleStatusValid(); + +}; + +//------------------------------------------------------------------------------ +// +// Inline Functions Definitions ... +// +//------------------------------------------------------------------------------ + +inline UBool RuleBasedBreakIterator::operator!=(const BreakIterator& that) const { + return !operator==(that); +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_BREAK_ITERATION */ + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/rep.h b/Build/source/libs/icu/icu-xetex/common/unicode/rep.h new file mode 100644 index 00000000000..3fab6fa1272 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/rep.h @@ -0,0 +1,259 @@ +/* +************************************************************************** +* Copyright (C) 1999-2005, International Business Machines Corporation and +* others. All Rights Reserved. +************************************************************************** +* Date Name Description +* 11/17/99 aliu Creation. Ported from java. Modified to +* match current UnicodeString API. Forced +* to use name "handleReplaceBetween" because +* of existing methods in UnicodeString. +************************************************************************** +*/ + +#ifndef REP_H +#define REP_H + +#include "unicode/uobject.h" + +/** + * \file + * \brief C++ API: Replaceable String + */ + +U_NAMESPACE_BEGIN + +class UnicodeString; + +/** + * Replaceable is an abstract base class representing a + * string of characters that supports the replacement of a range of + * itself with a new string of characters. It is used by APIs that + * change a piece of text while retaining metadata. Metadata is data + * other than the Unicode characters returned by char32At(). One + * example of metadata is style attributes; another is an edit + * history, marking each character with an author and revision number. + * + *

An implicit aspect of the Replaceable API is that + * during a replace operation, new characters take on the metadata of + * the old characters. For example, if the string "the bold + * font" has range (4, 8) replaced with "strong", then it becomes "the + * strong font". + * + *

Replaceable specifies ranges using a start + * offset and a limit offset. The range of characters thus specified + * includes the characters at offset start..limit-1. That is, the + * start offset is inclusive, and the limit offset is exclusive. + * + *

Replaceable also includes API to access characters + * in the string: length(), charAt(), + * char32At(), and extractBetween(). + * + *

For a subclass to support metadata, typical behavior of + * replace() is the following: + *

    + *
  • Set the metadata of the new text to the metadata of the first + * character replaced
  • + *
  • If no characters are replaced, use the metadata of the + * previous character
  • + *
  • If there is no previous character (i.e. start == 0), use the + * following character
  • + *
  • If there is no following character (i.e. the replaceable was + * empty), use default metadata.
    + *
  • If the code point U+FFFF is seen, it should be interpreted as + * a special marker having no metadata
  • + *
  • + *
+ * If this is not the behavior, the subclass should document any differences. + * @author Alan Liu + * @stable ICU 2.0 + */ +class U_COMMON_API Replaceable : public UObject { + +public: + /** + * Destructor. + * @stable ICU 2.0 + */ + virtual ~Replaceable(); + + /** + * Returns the number of 16-bit code units in the text. + * @return number of 16-bit code units in text + * @stable ICU 1.8 + */ + inline int32_t length() const; + + /** + * Returns the 16-bit code unit at the given offset into the text. + * @param offset an integer between 0 and length()-1 + * inclusive + * @return 16-bit code unit of text at given offset + * @stable ICU 1.8 + */ + inline UChar charAt(int32_t offset) const; + + /** + * Returns the 32-bit code point at the given 16-bit offset into + * the text. This assumes the text is stored as 16-bit code units + * with surrogate pairs intermixed. If the offset of a leading or + * trailing code unit of a surrogate pair is given, return the + * code point of the surrogate pair. + * + * @param offset an integer between 0 and length()-1 + * inclusive + * @return 32-bit code point of text at given offset + * @stable ICU 1.8 + */ + inline UChar32 char32At(int32_t offset) const; + + /** + * Copies characters in the range [start, limit) + * into the UnicodeString target. + * @param start offset of first character which will be copied + * @param limit offset immediately following the last character to + * be copied + * @param target UnicodeString into which to copy characters. + * @return A reference to target + * @stable ICU 2.1 + */ + virtual void extractBetween(int32_t start, + int32_t limit, + UnicodeString& target) const = 0; + + /** + * Replaces a substring of this object with the given text. If the + * characters being replaced have metadata, the new characters + * that replace them should be given the same metadata. + * + *

Subclasses must ensure that if the text between start and + * limit is equal to the replacement text, that replace has no + * effect. That is, any metadata + * should be unaffected. In addition, subclasses are encouraged to + * check for initial and trailing identical characters, and make a + * smaller replacement if possible. This will preserve as much + * metadata as possible. + * @param start the beginning index, inclusive; 0 <= start + * <= limit. + * @param limit the ending index, exclusive; start <= limit + * <= length(). + * @param text the text to replace characters start + * to limit - 1 + * @stable ICU 2.0 + */ + virtual void handleReplaceBetween(int32_t start, + int32_t limit, + const UnicodeString& text) = 0; + // Note: All other methods in this class take the names of + // existing UnicodeString methods. This method is the exception. + // It is named differently because all replace methods of + // UnicodeString return a UnicodeString&. The 'between' is + // required in order to conform to the UnicodeString naming + // convention; API taking start/length are named , and + // those taking start/limit are named . The + // 'handle' is added because 'replaceBetween' and + // 'doReplaceBetween' are already taken. + + /** + * Copies a substring of this object, retaining metadata. + * This method is used to duplicate or reorder substrings. + * The destination index must not overlap the source range. + * + * @param start the beginning index, inclusive; 0 <= start <= + * limit. + * @param limit the ending index, exclusive; start <= limit <= + * length(). + * @param dest the destination index. The characters from + * start..limit-1 will be copied to dest. + * Implementations of this method may assume that dest <= start || + * dest >= limit. + * @stable ICU 2.0 + */ + virtual void copy(int32_t start, int32_t limit, int32_t dest) = 0; + + /** + * Returns true if this object contains metadata. If a + * Replaceable object has metadata, calls to the Replaceable API + * must be made so as to preserve metadata. If it does not, calls + * to the Replaceable API may be optimized to improve performance. + * The default implementation returns true. + * @return true if this object contains metadata + * @stable ICU 2.2 + */ + virtual UBool hasMetaData() const; + + /** + * Clone this object, an instance of a subclass of Replaceable. + * Clones can be used concurrently in multiple threads. + * If a subclass does not implement clone(), or if an error occurs, + * then NULL is returned. + * The clone functions in all subclasses return a pointer to a Replaceable + * because some compilers do not support covariant (same-as-this) + * return types; cast to the appropriate subclass if necessary. + * The caller must delete the clone. + * + * @return a clone of this object + * + * @see getDynamicClassID + * @stable ICU 2.6 + */ + virtual Replaceable *clone() const; + +protected: + + /** + * Default constructor. + * @stable ICU 2.4 + */ + Replaceable(); + + /* + * Assignment operator not declared. The compiler will provide one + * which does nothing since this class does not contain any data members. + * API/code coverage may show the assignment operator as present and + * untested - ignore. + * Subclasses need this assignment operator if they use compiler-provided + * assignment operators of their own. An alternative to not declaring one + * here would be to declare and empty-implement a protected or public one. + Replaceable &Replaceable::operator=(const Replaceable &); + */ + + /** + * Virtual version of length(). + * @stable ICU 2.4 + */ + virtual int32_t getLength() const = 0; + + /** + * Virtual version of charAt(). + * @stable ICU 2.4 + */ + virtual UChar getCharAt(int32_t offset) const = 0; + + /** + * Virtual version of char32At(). + * @stable ICU 2.4 + */ + virtual UChar32 getChar32At(int32_t offset) const = 0; +}; + +inline int32_t +Replaceable::length() const { + return getLength(); +} + +inline UChar +Replaceable::charAt(int32_t offset) const { + return getCharAt(offset); +} + +inline UChar32 +Replaceable::char32At(int32_t offset) const { + return getChar32At(offset); +} + +// There is no rep.cpp, see unistr.cpp for Replaceable function implementations. + +U_NAMESPACE_END + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/resbund.h b/Build/source/libs/icu/icu-xetex/common/unicode/resbund.h new file mode 100644 index 00000000000..6d6b991361b --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/resbund.h @@ -0,0 +1,485 @@ +/* +****************************************************************************** +* +* Copyright (C) 1996-2007, International Business Machines Corporation +* and others. All Rights Reserved. +* +****************************************************************************** +* +* File resbund.h +* +* CREATED BY +* Richard Gillam +* +* Modification History: +* +* Date Name Description +* 2/5/97 aliu Added scanForLocaleInFile. Added +* constructor which attempts to read resource bundle +* from a specific file, without searching other files. +* 2/11/97 aliu Added UErrorCode return values to constructors. Fixed +* infinite loops in scanForFile and scanForLocale. +* Modified getRawResourceData to not delete storage +* in localeData and resourceData which it doesn't own. +* Added Mac compatibility #ifdefs for tellp() and +* ios::nocreate. +* 2/18/97 helena Updated with 100% documentation coverage. +* 3/13/97 aliu Rewrote to load in entire resource bundle and store +* it as a Hashtable of ResourceBundleData objects. +* Added state table to govern parsing of files. +* Modified to load locale index out of new file +* distinct from default.txt. +* 3/25/97 aliu Modified to support 2-d arrays, needed for timezone +* data. Added support for custom file suffixes. Again, +* needed to support timezone data. +* 4/7/97 aliu Cleaned up. +* 03/02/99 stephen Removed dependency on FILE*. +* 03/29/99 helena Merged Bertrand and Stephen's changes. +* 06/11/99 stephen Removed parsing of .txt files. +* Reworked to use new binary format. +* Cleaned up. +* 06/14/99 stephen Removed methods taking a filename suffix. +* 11/09/99 weiv Added getLocale(), fRealLocale, removed fRealLocaleID +****************************************************************************** +*/ + +#ifndef RESBUND_H +#define RESBUND_H + +#include "unicode/utypes.h" +#include "unicode/uobject.h" +#include "unicode/ures.h" +#include "unicode/unistr.h" +#include "unicode/locid.h" + +/** + * \file + * \brief C++ API: Resource Bundle + */ + +U_NAMESPACE_BEGIN + +/** + * A class representing a collection of resource information pertaining to a given + * locale. A resource bundle provides a way of accessing locale- specfic information in + * a data file. You create a resource bundle that manages the resources for a given + * locale and then ask it for individual resources. + *

+ * Resource bundles in ICU4C are currently defined using text files which conform to the following + * BNF definition. + * More on resource bundle concepts and syntax can be found in the + * Users Guide. + *

+ * + * The ResourceBundle class is not suitable for subclassing. + * + * @stable ICU 2.0 + */ +class U_COMMON_API ResourceBundle : public UObject { +public: + /** + * Constructor + * + * @param packageName The packageName and locale together point to an ICU udata object, + * as defined by udata_open( packageName, "res", locale, err) + * or equivalent. Typically, packageName will refer to a (.dat) file, or to + * a package registered with udata_setAppData(). Using a full file or directory + * pathname for packageName is deprecated. + * @param locale This is the locale this resource bundle is for. To get resources + * for the French locale, for example, you would create a + * ResourceBundle passing Locale::FRENCH for the "locale" parameter, + * and all subsequent calls to that resource bundle will return + * resources that pertain to the French locale. If the caller doesn't + * pass a locale parameter, the default locale for the system (as + * returned by Locale::getDefault()) will be used. + * @param err The Error Code. + * The UErrorCode& err parameter is used to return status information to the user. To + * check whether the construction succeeded or not, you should check the value of + * U_SUCCESS(err). If you wish more detailed information, you can check for + * informational error results which still indicate success. U_USING_FALLBACK_WARNING + * indicates that a fall back locale was used. For example, 'de_CH' was requested, + * but nothing was found there, so 'de' was used. U_USING_DEFAULT_WARNING indicates that + * the default locale data was used; neither the requested locale nor any of its + * fall back locales could be found. + * @stable ICU 2.0 + */ + ResourceBundle(const UnicodeString& packageName, + const Locale& locale, + UErrorCode& err); + + /** + * Construct a resource bundle for the default bundle in the specified package. + * + * @param packageName The packageName and locale together point to an ICU udata object, + * as defined by udata_open( packageName, "res", locale, err) + * or equivalent. Typically, packageName will refer to a (.dat) file, or to + * a package registered with udata_setAppData(). Using a full file or directory + * pathname for packageName is deprecated. + * @param err A UErrorCode value + * @stable ICU 2.0 + */ + ResourceBundle(const UnicodeString& packageName, + UErrorCode& err); + + /** + * Construct a resource bundle for the ICU default bundle. + * + * @param err A UErrorCode value + * @stable ICU 2.0 + */ + ResourceBundle(UErrorCode &err); + + /** + * Standard constructor, onstructs a resource bundle for the locale-specific + * bundle in the specified package. + * + * @param packageName The packageName and locale together point to an ICU udata object, + * as defined by udata_open( packageName, "res", locale, err) + * or equivalent. Typically, packageName will refer to a (.dat) file, or to + * a package registered with udata_setAppData(). Using a full file or directory + * pathname for packageName is deprecated. + * NULL is used to refer to ICU data. + * @param locale The locale for which to open a resource bundle. + * @param err A UErrorCode value + * @stable ICU 2.0 + */ + ResourceBundle(const char* packageName, + const Locale& locale, + UErrorCode& err); + + /** + * Copy constructor. + * + * @param original The resource bundle to copy. + * @stable ICU 2.0 + */ + ResourceBundle(const ResourceBundle &original); + + /** + * Constructor from a C UResourceBundle. The resource bundle is + * copied and not adopted. ures_close will still need to be used on the + * original resource bundle. + * + * @param res A pointer to the C resource bundle. + * @param status A UErrorCode value. + * @stable ICU 2.0 + */ + ResourceBundle(UResourceBundle *res, + UErrorCode &status); + + /** + * Assignment operator. + * + * @param other The resource bundle to copy. + * @stable ICU 2.0 + */ + ResourceBundle& + operator=(const ResourceBundle& other); + + /** Destructor. + * @stable ICU 2.0 + */ + virtual ~ResourceBundle(); + + /** + * Clone this object. + * Clones can be used concurrently in multiple threads. + * If an error occurs, then NULL is returned. + * The caller must delete the clone. + * + * @return a clone of this object + * + * @see getDynamicClassID + * @stable ICU 2.8 + */ + ResourceBundle *clone() const; + + /** + * Returns the size of a resource. Size for scalar types is always 1, and for vector/table types is + * the number of child resources. + * @warning Integer array is treated as a scalar type. There are no + * APIs to access individual members of an integer array. It + * is always returned as a whole. + * + * @return number of resources in a given resource. + * @stable ICU 2.0 + */ + int32_t + getSize(void) const; + + /** + * returns a string from a string resource type + * + * @param status fills in the outgoing error code + * could be U_MISSING_RESOURCE_ERROR if the key is not found + * could be a warning + * e.g.: U_USING_FALLBACK_WARNING,U_USING_DEFAULT_WARNING + * @return a pointer to a zero-terminated UChar array which lives in a memory mapped/DLL file. + * @stable ICU 2.0 + */ + UnicodeString + getString(UErrorCode& status) const; + + /** + * returns a binary data from a resource. Can be used at most primitive resource types (binaries, + * strings, ints) + * + * @param len fills in the length of resulting byte chunk + * @param status fills in the outgoing error code + * could be U_MISSING_RESOURCE_ERROR if the key is not found + * could be a warning + * e.g.: U_USING_FALLBACK_WARNING,U_USING_DEFAULT_WARNING + * @return a pointer to a chunk of unsigned bytes which live in a memory mapped/DLL file. + * @stable ICU 2.0 + */ + const uint8_t* + getBinary(int32_t& len, UErrorCode& status) const; + + + /** + * returns an integer vector from a resource. + * + * @param len fills in the length of resulting integer vector + * @param status fills in the outgoing error code + * could be U_MISSING_RESOURCE_ERROR if the key is not found + * could be a warning + * e.g.: U_USING_FALLBACK_WARNING,U_USING_DEFAULT_WARNING + * @return a pointer to a vector of integers that lives in a memory mapped/DLL file. + * @stable ICU 2.0 + */ + const int32_t* + getIntVector(int32_t& len, UErrorCode& status) const; + + /** + * returns an unsigned integer from a resource. + * This integer is originally 28 bits. + * + * @param status fills in the outgoing error code + * could be U_MISSING_RESOURCE_ERROR if the key is not found + * could be a warning + * e.g.: U_USING_FALLBACK_WARNING,U_USING_DEFAULT_WARNING + * @return an unsigned integer value + * @stable ICU 2.0 + */ + uint32_t + getUInt(UErrorCode& status) const; + + /** + * returns a signed integer from a resource. + * This integer is originally 28 bit and the sign gets propagated. + * + * @param status fills in the outgoing error code + * could be U_MISSING_RESOURCE_ERROR if the key is not found + * could be a warning + * e.g.: U_USING_FALLBACK_WARNING,U_USING_DEFAULT_WARNING + * @return a signed integer value + * @stable ICU 2.0 + */ + int32_t + getInt(UErrorCode& status) const; + + /** + * Checks whether the resource has another element to iterate over. + * + * @return TRUE if there are more elements, FALSE if there is no more elements + * @stable ICU 2.0 + */ + UBool + hasNext(void) const; + + /** + * Resets the internal context of a resource so that iteration starts from the first element. + * + * @stable ICU 2.0 + */ + void + resetIterator(void); + + /** + * Returns the key associated with this resource. Not all the resources have a key - only + * those that are members of a table. + * + * @return a key associated to this resource, or NULL if it doesn't have a key + * @stable ICU 2.0 + */ + const char* + getKey(void) const; + + /** + * Gets the locale ID of the resource bundle as a string. + * Same as getLocale().getName() . + * + * @return the locale ID of the resource bundle as a string + * @stable ICU 2.0 + */ + const char* + getName(void) const; + + + /** + * Returns the type of a resource. Available types are defined in enum UResType + * + * @return type of the given resource. + * @stable ICU 2.0 + */ + UResType + getType(void) const; + + /** + * Returns the next resource in a given resource or NULL if there are no more resources + * + * @param status fills in the outgoing error code + * @return ResourceBundle object. + * @stable ICU 2.0 + */ + ResourceBundle + getNext(UErrorCode& status); + + /** + * Returns the next string in a resource or NULL if there are no more resources + * to iterate over. + * + * @param status fills in the outgoing error code + * @return an UnicodeString object. + * @stable ICU 2.0 + */ + UnicodeString + getNextString(UErrorCode& status); + + /** + * Returns the next string in a resource or NULL if there are no more resources + * to iterate over. + * + * @param key fill in for key associated with this string + * @param status fills in the outgoing error code + * @return an UnicodeString object. + * @stable ICU 2.0 + */ + UnicodeString + getNextString(const char ** key, + UErrorCode& status); + + /** + * Returns the resource in a resource at the specified index. + * + * @param index an index to the wanted resource. + * @param status fills in the outgoing error code + * @return ResourceBundle object. If there is an error, resource is invalid. + * @stable ICU 2.0 + */ + ResourceBundle + get(int32_t index, + UErrorCode& status) const; + + /** + * Returns the string in a given resource at the specified index. + * + * @param index an index to the wanted string. + * @param status fills in the outgoing error code + * @return an UnicodeString object. If there is an error, string is bogus + * @stable ICU 2.0 + */ + UnicodeString + getStringEx(int32_t index, + UErrorCode& status) const; + + /** + * Returns a resource in a resource that has a given key. This procedure works only with table + * resources. + * + * @param key a key associated with the wanted resource + * @param status fills in the outgoing error code. + * @return ResourceBundle object. If there is an error, resource is invalid. + * @stable ICU 2.0 + */ + ResourceBundle + get(const char* key, + UErrorCode& status) const; + + /** + * Returns a string in a resource that has a given key. This procedure works only with table + * resources. + * + * @param key a key associated with the wanted string + * @param status fills in the outgoing error code + * @return an UnicodeString object. If there is an error, string is bogus + * @stable ICU 2.0 + */ + UnicodeString + getStringEx(const char* key, + UErrorCode& status) const; + + /** + * Return the version number associated with this ResourceBundle as a string. Please + * use getVersion, as this method is going to be deprecated. + * + * @return A version number string as specified in the resource bundle or its parent. + * The caller does not own this string. + * @see getVersion + * @deprecated ICU 2.8 Use getVersion instead. + */ + const char* + getVersionNumber(void) const; + + /** + * Return the version number associated with this ResourceBundle as a UVersionInfo array. + * + * @param versionInfo A UVersionInfo array that is filled with the version number + * as specified in the resource bundle or its parent. + * @stable ICU 2.0 + */ + void + getVersion(UVersionInfo versionInfo) const; + + /** + * Return the Locale associated with this ResourceBundle. + * + * @return a Locale object + * @deprecated ICU 2.8 Use getLocale(ULocDataLocaleType type, UErrorCode &status) overload instead. + */ + const Locale& + getLocale(void) const; + + /** + * Return the Locale associated with this ResourceBundle. + * @param type You can choose between requested, valid and actual + * locale. For description see the definition of + * ULocDataLocaleType in uloc.h + * @param status just for catching illegal arguments + * + * @return a Locale object + * @stable ICU 2.8 + */ + const Locale + getLocale(ULocDataLocaleType type, UErrorCode &status) const; + /** + * This API implements multilevel fallback + * @internal + */ + ResourceBundle + getWithFallback(const char* key, UErrorCode& status); + /** + * ICU "poor man's RTTI", returns a UClassID for the actual class. + * + * @stable ICU 2.2 + */ + virtual UClassID getDynamicClassID() const; + + /** + * ICU "poor man's RTTI", returns a UClassID for this class. + * + * @stable ICU 2.2 + */ + static UClassID U_EXPORT2 getStaticClassID(); + +private: + ResourceBundle(); // default constructor not implemented + + UResourceBundle *fResource; + void constructForLocale(const UnicodeString& path, const Locale& locale, UErrorCode& error); + Locale *fLocale; + +}; + +U_NAMESPACE_END +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/schriter.h b/Build/source/libs/icu/icu-xetex/common/unicode/schriter.h new file mode 100644 index 00000000000..d0b5e22503c --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/schriter.h @@ -0,0 +1,187 @@ +/* +****************************************************************************** +* +* Copyright (C) 1998-2005, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* +* File schriter.h +* +* Modification History: +* +* Date Name Description +* 05/05/99 stephen Cleaned up. +****************************************************************************** +*/ + +#ifndef SCHRITER_H +#define SCHRITER_H + +#include "unicode/utypes.h" +#include "unicode/chariter.h" +#include "unicode/uchriter.h" + +/** + * \file + * \brief C++ API: String Character Iterator + */ + +U_NAMESPACE_BEGIN +/** + * A concrete subclass of CharacterIterator that iterates over the + * characters (code units or code points) in a UnicodeString. + * It's possible not only to create an + * iterator that iterates over an entire UnicodeString, but also to + * create one that iterates over only a subrange of a UnicodeString + * (iterators over different subranges of the same UnicodeString don't + * compare equal). + * @see CharacterIterator + * @see ForwardCharacterIterator + * @stable ICU 2.0 + */ +class U_COMMON_API StringCharacterIterator : public UCharCharacterIterator { +public: + /** + * Create an iterator over the UnicodeString referred to by "textStr". + * The UnicodeString object is copied. + * The iteration range is the whole string, and the starting position is 0. + * @param textStr The unicode string used to create an iterator + * @stable ICU 2.0 + */ + StringCharacterIterator(const UnicodeString& textStr); + + /** + * Create an iterator over the UnicodeString referred to by "textStr". + * The iteration range is the whole string, and the starting + * position is specified by "textPos". If "textPos" is outside the valid + * iteration range, the behavior of this object is undefined. + * @param textStr The unicode string used to create an iterator + * @param textPos The starting position of the iteration + * @stable ICU 2.0 + */ + StringCharacterIterator(const UnicodeString& textStr, + int32_t textPos); + + /** + * Create an iterator over the UnicodeString referred to by "textStr". + * The UnicodeString object is copied. + * The iteration range begins with the code unit specified by + * "textBegin" and ends with the code unit BEFORE the code unit specfied + * by "textEnd". The starting position is specified by "textPos". If + * "textBegin" and "textEnd" don't form a valid range on "text" (i.e., + * textBegin >= textEnd or either is negative or greater than text.size()), + * or "textPos" is outside the range defined by "textBegin" and "textEnd", + * the behavior of this iterator is undefined. + * @param textStr The unicode string used to create the StringCharacterIterator + * @param textBegin The begin position of the iteration range + * @param textEnd The end position of the iteration range + * @param textPos The starting position of the iteration + * @stable ICU 2.0 + */ + StringCharacterIterator(const UnicodeString& textStr, + int32_t textBegin, + int32_t textEnd, + int32_t textPos); + + /** + * Copy constructor. The new iterator iterates over the same range + * of the same string as "that", and its initial position is the + * same as "that"'s current position. + * The UnicodeString object in "that" is copied. + * @param that The StringCharacterIterator to be copied + * @stable ICU 2.0 + */ + StringCharacterIterator(const StringCharacterIterator& that); + + /** + * Destructor. + * @stable ICU 2.0 + */ + virtual ~StringCharacterIterator(); + + /** + * Assignment operator. *this is altered to iterate over the same + * range of the same string as "that", and refers to the same + * character within that string as "that" does. + * @param that The object to be copied. + * @return the newly created object. + * @stable ICU 2.0 + */ + StringCharacterIterator& + operator=(const StringCharacterIterator& that); + + /** + * Returns true if the iterators iterate over the same range of the + * same string and are pointing at the same character. + * @param that The ForwardCharacterIterator to be compared for equality + * @return true if the iterators iterate over the same range of the + * same string and are pointing at the same character. + * @stable ICU 2.0 + */ + virtual UBool operator==(const ForwardCharacterIterator& that) const; + + /** + * Returns a new StringCharacterIterator referring to the same + * character in the same range of the same string as this one. The + * caller must delete the new iterator. + * @return the newly cloned object. + * @stable ICU 2.0 + */ + virtual CharacterIterator* clone(void) const; + + /** + * Sets the iterator to iterate over the provided string. + * @param newText The string to be iterated over + * @stable ICU 2.0 + */ + void setText(const UnicodeString& newText); + + /** + * Copies the UnicodeString under iteration into the UnicodeString + * referred to by "result". Even if this iterator iterates across + * only a part of this string, the whole string is copied. + * @param result Receives a copy of the text under iteration. + * @stable ICU 2.0 + */ + virtual void getText(UnicodeString& result); + + /** + * Return a class ID for this object (not really public) + * @return a class ID for this object. + * @stable ICU 2.0 + */ + virtual UClassID getDynamicClassID(void) const; + + /** + * Return a class ID for this class (not really public) + * @return a class ID for this class + * @stable ICU 2.0 + */ + static UClassID U_EXPORT2 getStaticClassID(void); + +protected: + /** + * Default constructor, iteration over empty string. + * @stable ICU 2.0 + */ + StringCharacterIterator(); + + /** + * Sets the iterator to iterate over the provided string. + * @param newText The string to be iterated over + * @param newTextLength The length of the String + * @stable ICU 2.0 + */ + void setText(const UChar* newText, int32_t newTextLength); + + /** + * Copy of the iterated string object. + * @stable ICU 2.0 + */ + UnicodeString text; + +}; + +U_NAMESPACE_END +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/strenum.h b/Build/source/libs/icu/icu-xetex/common/unicode/strenum.h new file mode 100644 index 00000000000..ce42195a4f2 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/strenum.h @@ -0,0 +1,271 @@ +/* +******************************************************************************* +* +* Copyright (C) 2002-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +*/ + +#ifndef STRENUM_H +#define STRENUM_H + +#include "unicode/uobject.h" +#include "unicode/unistr.h" + +/** + * \file + * \brief C++ API: String Enumeration + */ + +U_NAMESPACE_BEGIN + +/** + * Base class for 'pure' C++ implementations of uenum api. Adds a + * method that returns the next UnicodeString since in C++ this can + * be a common storage format for strings. + * + *

The model is that the enumeration is over strings maintained by + * a 'service.' At any point, the service might change, invalidating + * the enumerator (though this is expected to be rare). The iterator + * returns an error if this has occurred. Lack of the error is no + * guarantee that the service didn't change immediately after the + * call, so the returned string still might not be 'valid' on + * subsequent use.

+ * + *

Strings may take the form of const char*, const UChar*, or const + * UnicodeString*. The type you get is determine by the variant of + * 'next' that you call. In general the StringEnumeration is + * optimized for one of these types, but all StringEnumerations can + * return all types. Returned strings are each terminated with a NUL. + * Depending on the service data, they might also include embedded NUL + * characters, so API is provided to optionally return the true + * length, counting the embedded NULs but not counting the terminating + * NUL.

+ * + *

The pointers returned by next, unext, and snext become invalid + * upon any subsequent call to the enumeration's destructor, next, + * unext, snext, or reset.

+ * + * ICU 2.8 adds some default implementations and helper functions + * for subclasses. + * + * @stable ICU 2.4 + */ +class U_COMMON_API StringEnumeration : public UObject { +public: + /** + * Destructor. + * @stable ICU 2.4 + */ + virtual ~StringEnumeration(); + + /** + * Clone this object, an instance of a subclass of StringEnumeration. + * Clones can be used concurrently in multiple threads. + * If a subclass does not implement clone(), or if an error occurs, + * then NULL is returned. + * The clone functions in all subclasses return a base class pointer + * because some compilers do not support covariant (same-as-this) + * return types; cast to the appropriate subclass if necessary. + * The caller must delete the clone. + * + * @return a clone of this object + * + * @see getDynamicClassID + * @stable ICU 2.8 + */ + virtual StringEnumeration *clone() const; + + /** + *

Return the number of elements that the iterator traverses. If + * the iterator is out of sync with its service, status is set to + * U_ENUM_OUT_OF_SYNC_ERROR, and the return value is zero.

+ * + *

The return value will not change except possibly as a result of + * a subsequent call to reset, or if the iterator becomes out of sync.

+ * + *

This is a convenience function. It can end up being very + * expensive as all the items might have to be pre-fetched + * (depending on the storage format of the data being + * traversed).

+ * + * @param status the error code. + * @return number of elements in the iterator. + * + * @stable ICU 2.4 */ + virtual int32_t count(UErrorCode& status) const = 0; + + /** + *

Returns the next element as a NUL-terminated char*. If there + * are no more elements, returns NULL. If the resultLength pointer + * is not NULL, the length of the string (not counting the + * terminating NUL) is returned at that address. If an error + * status is returned, the value at resultLength is undefined.

+ * + *

The returned pointer is owned by this iterator and must not be + * deleted by the caller. The pointer is valid until the next call + * to next, unext, snext, reset, or the enumerator's destructor.

+ * + *

If the iterator is out of sync with its service, status is set + * to U_ENUM_OUT_OF_SYNC_ERROR and NULL is returned.

+ * + *

If the native service string is a UChar* string, it is + * converted to char* with the invariant converter. If the + * conversion fails (because a character cannot be converted) then + * status is set to U_INVARIANT_CONVERSION_ERROR and the return + * value is undefined (though not NULL).

+ * + * Starting with ICU 2.8, the default implementation calls snext() + * and handles the conversion. + * + * @param status the error code. + * @param resultLength a pointer to receive the length, can be NULL. + * @return a pointer to the string, or NULL. + * + * @stable ICU 2.4 + */ + virtual const char* next(int32_t *resultLength, UErrorCode& status); + + /** + *

Returns the next element as a NUL-terminated UChar*. If there + * are no more elements, returns NULL. If the resultLength pointer + * is not NULL, the length of the string (not counting the + * terminating NUL) is returned at that address. If an error + * status is returned, the value at resultLength is undefined.

+ * + *

The returned pointer is owned by this iterator and must not be + * deleted by the caller. The pointer is valid until the next call + * to next, unext, snext, reset, or the enumerator's destructor.

+ * + *

If the iterator is out of sync with its service, status is set + * to U_ENUM_OUT_OF_SYNC_ERROR and NULL is returned.

+ * + * Starting with ICU 2.8, the default implementation calls snext() + * and handles the conversion. + * + * @param status the error code. + * @param resultLength a ponter to receive the length, can be NULL. + * @return a pointer to the string, or NULL. + * + * @stable ICU 2.4 + */ + virtual const UChar* unext(int32_t *resultLength, UErrorCode& status); + + /** + *

Returns the next element a UnicodeString*. If there are no + * more elements, returns NULL.

+ * + *

The returned pointer is owned by this iterator and must not be + * deleted by the caller. The pointer is valid until the next call + * to next, unext, snext, reset, or the enumerator's destructor.

+ * + *

If the iterator is out of sync with its service, status is set + * to U_ENUM_OUT_OF_SYNC_ERROR and NULL is returned.

+ * + * @param status the error code. + * @return a pointer to the string, or NULL. + * + * @stable ICU 2.4 + */ + virtual const UnicodeString* snext(UErrorCode& status) = 0; + + /** + *

Resets the iterator. This re-establishes sync with the + * service and rewinds the iterator to start at the first + * element.

+ * + *

Previous pointers returned by next, unext, or snext become + * invalid, and the value returned by count might change.

+ * + * @param status the error code. + * + * @stable ICU 2.4 + */ + virtual void reset(UErrorCode& status) = 0; + + /** + * Compares this enumeration to other to check if both are equal + * + * @param that The other string enumeration to compare this object to + * @return TRUE if the enumerations are equal. FALSE if not. + * @stable ICU 3.6 + */ + virtual UBool operator==(const StringEnumeration& that)const; + /** + * Compares this enumeration to other to check if both are not equal + * + * @param that The other string enumeration to compare this object to + * @return TRUE if the enumerations are equal. FALSE if not. + * @stable ICU 3.6 + */ + virtual UBool operator!=(const StringEnumeration& that)const; + +protected: + /** + * UnicodeString field for use with default implementations and subclasses. + * @stable ICU 2.8 + */ + UnicodeString unistr; + /** + * char * default buffer for use with default implementations and subclasses. + * @stable ICU 2.8 + */ + char charsBuffer[32]; + /** + * char * buffer for use with default implementations and subclasses. + * Allocated in constructor and in ensureCharsCapacity(). + * @stable ICU 2.8 + */ + char *chars; + /** + * Capacity of chars, for use with default implementations and subclasses. + * @stable ICU 2.8 + */ + int32_t charsCapacity; + + /** + * Default constructor for use with default implementations and subclasses. + * @stable ICU 2.8 + */ + StringEnumeration(); + + /** + * Ensures that chars is at least as large as the requested capacity. + * For use with default implementations and subclasses. + * + * @param capacity Requested capacity. + * @param status ICU in/out error code. + * @stable ICU 2.8 + */ + void ensureCharsCapacity(int32_t capacity, UErrorCode &status); + + /** + * Converts s to Unicode and sets unistr to the result. + * For use with default implementations and subclasses, + * especially for implementations of snext() in terms of next(). + * This is provided with a helper function instead of a default implementation + * of snext() to avoid potential infinite loops between next() and snext(). + * + * For example: + * \code + * const UnicodeString* snext(UErrorCode& status) { + * int32_t resultLength=0; + * const char *s=next(&resultLength, status); + * return setChars(s, resultLength, status); + * } + * \endcode + * + * @param s String to be converted to Unicode. + * @param length Length of the string. + * @param status ICU in/out error code. + * @return A pointer to unistr. + * @stable ICU 2.8 + */ + UnicodeString *setChars(const char *s, int32_t length, UErrorCode &status); +}; + +U_NAMESPACE_END + +/* STRENUM_H */ +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/symtable.h b/Build/source/libs/icu/icu-xetex/common/unicode/symtable.h new file mode 100644 index 00000000000..428f8bff23e --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/symtable.h @@ -0,0 +1,112 @@ +/* +********************************************************************** +* Copyright (c) 2000-2005, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 02/04/00 aliu Creation. +********************************************************************** +*/ +#ifndef SYMTABLE_H +#define SYMTABLE_H + +#include "unicode/utypes.h" +#include "unicode/uobject.h" + +/** + * \file + * \brief C++ API: An interface that defines both lookup protocol and parsing of + * symbolic names. + */ + +U_NAMESPACE_BEGIN + +class ParsePosition; +class UnicodeFunctor; +class UnicodeSet; +class UnicodeString; + +/** + * An interface that defines both lookup protocol and parsing of + * symbolic names. + * + *

A symbol table maintains two kinds of mappings. The first is + * between symbolic names and their values. For example, if the + * variable with the name "start" is set to the value "alpha" + * (perhaps, though not necessarily, through an expression such as + * "$start=alpha"), then the call lookup("start") will return the + * char[] array ['a', 'l', 'p', 'h', 'a']. + * + *

The second kind of mapping is between character values and + * UnicodeMatcher objects. This is used by RuleBasedTransliterator, + * which uses characters in the private use area to represent objects + * such as UnicodeSets. If U+E015 is mapped to the UnicodeSet [a-z], + * then lookupMatcher(0xE015) will return the UnicodeSet [a-z]. + * + *

Finally, a symbol table defines parsing behavior for symbolic + * names. All symbolic names start with the SYMBOL_REF character. + * When a parser encounters this character, it calls parseReference() + * with the position immediately following the SYMBOL_REF. The symbol + * table parses the name, if there is one, and returns it. + * + * @stable ICU 2.8 + */ +class U_COMMON_API SymbolTable /* not : public UObject because this is an interface/mixin class */ { +public: + + /** + * The character preceding a symbol reference name. + * @stable ICU 2.8 + */ + enum { SYMBOL_REF = 0x0024 /*$*/ }; + + /** + * Destructor. + * @stable ICU 2.8 + */ + virtual ~SymbolTable(); + + /** + * Lookup the characters associated with this string and return it. + * Return NULL if no such name exists. The resultant + * string may have length zero. + * @param s the symbolic name to lookup + * @return a string containing the name's value, or NULL if + * there is no mapping for s. + * @stable ICU 2.8 + */ + virtual const UnicodeString* lookup(const UnicodeString& s) const = 0; + + /** + * Lookup the UnicodeMatcher associated with the given character, and + * return it. Return NULL if not found. + * @param ch a 32-bit code point from 0 to 0x10FFFF inclusive. + * @return the UnicodeMatcher object represented by the given + * character, or NULL if there is no mapping for ch. + * @stable ICU 2.8 + */ + virtual const UnicodeFunctor* lookupMatcher(UChar32 ch) const = 0; + + /** + * Parse a symbol reference name from the given string, starting + * at the given position. If no valid symbol reference name is + * found, return the empty string and leave pos unchanged. That is, if the + * character at pos cannot start a name, or if pos is at or after + * text.length(), then return an empty string. This indicates an + * isolated SYMBOL_REF character. + * @param text the text to parse for the name + * @param pos on entry, the index of the first character to parse. + * This is the character following the SYMBOL_REF character. On + * exit, the index after the last parsed character. If the parse + * failed, pos is unchanged on exit. + * @param limit the index after the last character to be parsed. + * @return the parsed name, or an empty string if there is no + * valid symbolic name at the given position. + * @stable ICU 2.8 + */ + virtual UnicodeString parseReference(const UnicodeString& text, + ParsePosition& pos, int32_t limit) const = 0; +}; +U_NAMESPACE_END + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/ubidi.h b/Build/source/libs/icu/icu-xetex/common/unicode/ubidi.h new file mode 100644 index 00000000000..2a487e0c262 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/ubidi.h @@ -0,0 +1,1999 @@ +/* +****************************************************************************** +* +* Copyright (C) 1999-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: ubidi.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 1999jul27 +* created by: Markus W. Scherer, updated by Matitiahu Allouche +*/ + +#ifndef UBIDI_H +#define UBIDI_H + +#include "unicode/utypes.h" +#include "unicode/uchar.h" + +/** + *\file + * \brief C API: Bidi algorithm + * + *

Bidi algorithm for ICU

+ * + * This is an implementation of the Unicode Bidirectional algorithm. + * The algorithm is defined in the + * Unicode Standard Annex #9, + * version 13, also described in The Unicode Standard, Version 4.0 .

+ * + * Note: Libraries that perform a bidirectional algorithm and + * reorder strings accordingly are sometimes called "Storage Layout Engines". + * ICU's Bidi and shaping (u_shapeArabic()) APIs can be used at the core of such + * "Storage Layout Engines". + * + *

General remarks about the API:

+ * + * In functions with an error code parameter, + * the pErrorCode pointer must be valid + * and the value that it points to must not indicate a failure before + * the function call. Otherwise, the function returns immediately. + * After the function call, the value indicates success or failure.

+ * + * The "limit" of a sequence of characters is the position just after their + * last character, i.e., one more than that position.

+ * + * Some of the API functions provide access to "runs". + * Such a "run" is defined as a sequence of characters + * that are at the same embedding level + * after performing the Bidi algorithm.

+ * + * @author Markus W. Scherer + * @version 1.0 + * + * + *

Sample code for the ICU Bidi API

+ * + *
Rendering a paragraph with the ICU Bidi API
+ * + * This is (hypothetical) sample code that illustrates + * how the ICU Bidi API could be used to render a paragraph of text. + * Rendering code depends highly on the graphics system, + * therefore this sample code must make a lot of assumptions, + * which may or may not match any existing graphics system's properties. + * + *

The basic assumptions are:

+ *
    + *
  • Rendering is done from left to right on a horizontal line.
  • + *
  • A run of single-style, unidirectional text can be rendered at once.
  • + *
  • Such a run of text is passed to the graphics system with + * characters (code units) in logical order.
  • + *
  • The line-breaking algorithm is very complicated + * and Locale-dependent - + * and therefore its implementation omitted from this sample code.
  • + *
+ * + *
+ * \code
+ *#include "unicode/ubidi.h"
+ *
+ *typedef enum {
+ *     styleNormal=0, styleSelected=1,
+ *     styleBold=2, styleItalics=4,
+ *     styleSuper=8, styleSub=16
+ *} Style;
+ *
+ *typedef struct { int32_t limit; Style style; } StyleRun;
+ *
+ *int getTextWidth(const UChar *text, int32_t start, int32_t limit,
+ *                  const StyleRun *styleRuns, int styleRunCount);
+ *
+ * // set *pLimit and *pStyleRunLimit for a line
+ * // from text[start] and from styleRuns[styleRunStart]
+ * // using ubidi_getLogicalRun(para, ...)
+ *void getLineBreak(const UChar *text, int32_t start, int32_t *pLimit,
+ *                  UBiDi *para,
+ *                  const StyleRun *styleRuns, int styleRunStart, int *pStyleRunLimit,
+ *                  int *pLineWidth);
+ *
+ * // render runs on a line sequentially, always from left to right
+ *
+ * // prepare rendering a new line
+ * void startLine(UBiDiDirection textDirection, int lineWidth);
+ *
+ * // render a run of text and advance to the right by the run width
+ * // the text[start..limit-1] is always in logical order
+ * void renderRun(const UChar *text, int32_t start, int32_t limit,
+ *               UBiDiDirection textDirection, Style style);
+ *
+ * // We could compute a cross-product
+ * // from the style runs with the directional runs
+ * // and then reorder it.
+ * // Instead, here we iterate over each run type
+ * // and render the intersections -
+ * // with shortcuts in simple (and common) cases.
+ * // renderParagraph() is the main function.
+ *
+ * // render a directional run with
+ * // (possibly) multiple style runs intersecting with it
+ * void renderDirectionalRun(const UChar *text,
+ *                           int32_t start, int32_t limit,
+ *                           UBiDiDirection direction,
+ *                           const StyleRun *styleRuns, int styleRunCount) {
+ *     int i;
+ *
+ *     // iterate over style runs
+ *     if(direction==UBIDI_LTR) {
+ *         int styleLimit;
+ *
+ *         for(i=0; ilimit) { styleLimit=limit; }
+ *                 renderRun(text, start, styleLimit,
+ *                           direction, styleRun[i].style);
+ *                 if(styleLimit==limit) { break; }
+ *                 start=styleLimit;
+ *             }
+ *         }
+ *     } else {
+ *         int styleStart;
+ *
+ *         for(i=styleRunCount-1; i>=0; --i) {
+ *             if(i>0) {
+ *                 styleStart=styleRun[i-1].limit;
+ *             } else {
+ *                 styleStart=0;
+ *             }
+ *             if(limit>=styleStart) {
+ *                 if(styleStart=length
+ *
+ *         width=getTextWidth(text, 0, length, styleRuns, styleRunCount);
+ *         if(width<=lineWidth) {
+ *             // everything fits onto one line
+ *
+ *            // prepare rendering a new line from either left or right
+ *             startLine(paraLevel, width);
+ *
+ *             renderLine(para, text, 0, length,
+ *                        styleRuns, styleRunCount);
+ *         } else {
+ *             UBiDi *line;
+ *
+ *             // we need to render several lines
+ *             line=ubidi_openSized(length, 0, pErrorCode);
+ *             if(line!=NULL) {
+ *                 int32_t start=0, limit;
+ *                 int styleRunStart=0, styleRunLimit;
+ *
+ *                 for(;;) {
+ *                     limit=length;
+ *                     styleRunLimit=styleRunCount;
+ *                     getLineBreak(text, start, &limit, para,
+ *                                  styleRuns, styleRunStart, &styleRunLimit,
+ *                                 &width);
+ *                     ubidi_setLine(para, start, limit, line, pErrorCode);
+ *                     if(U_SUCCESS(*pErrorCode)) {
+ *                         // prepare rendering a new line
+ *                         // from either left or right
+ *                         startLine(paraLevel, width);
+ *
+ *                         renderLine(line, text, start, limit,
+ *                                    styleRuns+styleRunStart,
+ *                                    styleRunLimit-styleRunStart);
+ *                     }
+ *                     if(limit==length) { break; }
+ *                     start=limit;
+ *                     styleRunStart=styleRunLimit-1;
+ *                     if(start>=styleRuns[styleRunStart].limit) {
+ *                         ++styleRunStart;
+ *                     }
+ *                 }
+ *
+ *                 ubidi_close(line);
+ *             }
+ *        }
+ *    }
+ *
+ *     ubidi_close(para);
+ *}
+ *\endcode
+ * 
+ */ + +/*DOCXX_TAG*/ +/*@{*/ + +/** + * UBiDiLevel is the type of the level values in this + * Bidi implementation. + * It holds an embedding level and indicates the visual direction + * by its bit 0 (even/odd value).

+ * + * It can also hold non-level values for the + * paraLevel and embeddingLevels + * arguments of ubidi_setPara(); there: + *

    + *
  • bit 7 of an embeddingLevels[] + * value indicates whether the using application is + * specifying the level of a character to override whatever the + * Bidi implementation would resolve it to.
  • + *
  • paraLevel can be set to the + * pseudo-level values UBIDI_DEFAULT_LTR + * and UBIDI_DEFAULT_RTL.
  • + *
+ * + * @see ubidi_setPara + * + *

The related constants are not real, valid level values. + * UBIDI_DEFAULT_XXX can be used to specify + * a default for the paragraph level for + * when the ubidi_setPara() function + * shall determine it but there is no + * strongly typed character in the input.

+ * + * Note that the value for UBIDI_DEFAULT_LTR is even + * and the one for UBIDI_DEFAULT_RTL is odd, + * just like with normal LTR and RTL level values - + * these special values are designed that way. Also, the implementation + * assumes that UBIDI_MAX_EXPLICIT_LEVEL is odd. + * + * @see UBIDI_DEFAULT_LTR + * @see UBIDI_DEFAULT_RTL + * @see UBIDI_LEVEL_OVERRIDE + * @see UBIDI_MAX_EXPLICIT_LEVEL + * @stable ICU 2.0 + */ +typedef uint8_t UBiDiLevel; + +/** Paragraph level setting.

+ * + * Constant indicating that the base direction depends on the first strong + * directional character in the text according to the Unicode Bidirectional + * Algorithm. If no strong directional character is present, + * then set the paragraph level to 0 (left-to-right).

+ * + * If this value is used in conjunction with reordering modes + * UBIDI_REORDER_INVERSE_LIKE_DIRECT or + * UBIDI_REORDER_INVERSE_FOR_NUMBERS_SPECIAL, the text to reorder + * is assumed to be visual LTR, and the text after reordering is required + * to be the corresponding logical string with appropriate contextual + * direction. The direction of the result string will be RTL if either + * the righmost or leftmost strong character of the source text is RTL + * or Arabic Letter, the direction will be LTR otherwise.

+ * + * If reordering option UBIDI_OPTION_INSERT_MARKS is set, an RLM may + * be added at the beginning of the result string to ensure round trip + * (that the result string, when reordered back to visual, will produce + * the original source text). + * @see UBIDI_REORDER_INVERSE_LIKE_DIRECT + * @see UBIDI_REORDER_INVERSE_FOR_NUMBERS_SPECIAL + * @stable ICU 2.0 + */ +#define UBIDI_DEFAULT_LTR 0xfe + +/** Paragraph level setting.

+ * + * Constant indicating that the base direction depends on the first strong + * directional character in the text according to the Unicode Bidirectional + * Algorithm. If no strong directional character is present, + * then set the paragraph level to 1 (right-to-left).

+ * + * If this value is used in conjunction with reordering modes + * UBIDI_REORDER_INVERSE_LIKE_DIRECT or + * UBIDI_REORDER_INVERSE_FOR_NUMBERS_SPECIAL, the text to reorder + * is assumed to be visual LTR, and the text after reordering is required + * to be the corresponding logical string with appropriate contextual + * direction. The direction of the result string will be RTL if either + * the righmost or leftmost strong character of the source text is RTL + * or Arabic Letter, or if the text contains no strong character; + * the direction will be LTR otherwise.

+ * + * If reordering option UBIDI_OPTION_INSERT_MARKS is set, an RLM may + * be added at the beginning of the result string to ensure round trip + * (that the result string, when reordered back to visual, will produce + * the original source text). + * @see UBIDI_REORDER_INVERSE_LIKE_DIRECT + * @see UBIDI_REORDER_INVERSE_FOR_NUMBERS_SPECIAL + * @stable ICU 2.0 + */ +#define UBIDI_DEFAULT_RTL 0xff + +/** + * Maximum explicit embedding level. + * (The maximum resolved level can be up to UBIDI_MAX_EXPLICIT_LEVEL+1). + * @stable ICU 2.0 + */ +#define UBIDI_MAX_EXPLICIT_LEVEL 61 + +/** Bit flag for level input. + * Overrides directional properties. + * @stable ICU 2.0 + */ +#define UBIDI_LEVEL_OVERRIDE 0x80 + +/** + * Special value which can be returned by the mapping functions when a logical + * index has no corresponding visual index or vice-versa. This may happen + * for the logical-to-visual mapping of a Bidi control when option + * #UBIDI_OPTION_REMOVE_CONTROLS is specified. This can also happen + * for the visual-to-logical mapping of a Bidi mark (LRM or RLM) inserted + * by option #UBIDI_OPTION_INSERT_MARKS. + * @see ubidi_getVisualIndex + * @see ubidi_getVisualMap + * @see ubidi_getLogicalIndex + * @see ubidi_getLogicalMap + * @stable ICU 3.6 + */ +#define UBIDI_MAP_NOWHERE (-1) + +/** + * UBiDiDirection values indicate the text direction. + * @stable ICU 2.0 + */ +enum UBiDiDirection { + /** All left-to-right text. This is a 0 value. @stable ICU 2.0 */ + UBIDI_LTR, + /** All right-to-left text. This is a 1 value. @stable ICU 2.0 */ + UBIDI_RTL, + /** Mixed-directional text. @stable ICU 2.0 */ + UBIDI_MIXED +}; + +/** @stable ICU 2.0 */ +typedef enum UBiDiDirection UBiDiDirection; + +/** + * Forward declaration of the UBiDi structure for the declaration of + * the API functions. Its fields are implementation-specific.

+ * This structure holds information about a paragraph (or multiple paragraphs) + * of text with Bidi-algorithm-related details, or about one line of + * such a paragraph.

+ * Reordering can be done on a line, or on one or more paragraphs which are + * then interpreted each as one single line. + * @stable ICU 2.0 + */ +struct UBiDi; + +/** @stable ICU 2.0 */ +typedef struct UBiDi UBiDi; + +/** + * Allocate a UBiDi structure. + * Such an object is initially empty. It is assigned + * the Bidi properties of a piece of text containing one or more paragraphs + * by ubidi_setPara() + * or the Bidi properties of a line within a paragraph by + * ubidi_setLine().

+ * This object can be reused for as long as it is not deallocated + * by calling ubidi_close().

+ * ubidi_setPara() and ubidi_setLine() will allocate + * additional memory for internal structures as necessary. + * + * @return An empty UBiDi object. + * @stable ICU 2.0 + */ +U_STABLE UBiDi * U_EXPORT2 +ubidi_open(void); + +/** + * Allocate a UBiDi structure with preallocated memory + * for internal structures. + * This function provides a UBiDi object like ubidi_open() + * with no arguments, but it also preallocates memory for internal structures + * according to the sizings supplied by the caller.

+ * Subsequent functions will not allocate any more memory, and are thus + * guaranteed not to fail because of lack of memory.

+ * The preallocation can be limited to some of the internal memory + * by setting some values to 0 here. That means that if, e.g., + * maxRunCount cannot be reasonably predetermined and should not + * be set to maxLength (the only failproof value) to avoid + * wasting memory, then maxRunCount could be set to 0 here + * and the internal structures that are associated with it will be allocated + * on demand, just like with ubidi_open(). + * + * @param maxLength is the maximum text or line length that internal memory + * will be preallocated for. An attempt to associate this object with a + * longer text will fail, unless this value is 0, which leaves the allocation + * up to the implementation. + * + * @param maxRunCount is the maximum anticipated number of same-level runs + * that internal memory will be preallocated for. An attempt to access + * visual runs on an object that was not preallocated for as many runs + * as the text was actually resolved to will fail, + * unless this value is 0, which leaves the allocation up to the implementation.

+ * The number of runs depends on the actual text and maybe anywhere between + * 1 and maxLength. It is typically small. + * + * @param pErrorCode must be a valid pointer to an error code value. + * + * @return An empty UBiDi object with preallocated memory. + * @stable ICU 2.0 + */ +U_STABLE UBiDi * U_EXPORT2 +ubidi_openSized(int32_t maxLength, int32_t maxRunCount, UErrorCode *pErrorCode); + +/** + * ubidi_close() must be called to free the memory + * associated with a UBiDi object.

+ * + * Important: + * A parent UBiDi object must not be destroyed or reused if + * it still has children. + * If a UBiDi object has become the child + * of another one (its parent) by calling + * ubidi_setLine(), then the child object must + * be destroyed (closed) or reused (by calling + * ubidi_setPara() or ubidi_setLine()) + * before the parent object. + * + * @param pBiDi is a UBiDi object. + * + * @see ubidi_setPara + * @see ubidi_setLine + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ubidi_close(UBiDi *pBiDi); + +/** + * Modify the operation of the Bidi algorithm such that it + * approximates an "inverse Bidi" algorithm. This function + * must be called before ubidi_setPara(). + * + *

The normal operation of the Bidi algorithm as described + * in the Unicode Technical Report is to take text stored in logical + * (keyboard, typing) order and to determine the reordering of it for visual + * rendering. + * Some legacy systems store text in visual order, and for operations + * with standard, Unicode-based algorithms, the text needs to be transformed + * to logical order. This is effectively the inverse algorithm of the + * described Bidi algorithm. Note that there is no standard algorithm for + * this "inverse Bidi" and that the current implementation provides only an + * approximation of "inverse Bidi".

+ * + *

With isInverse set to TRUE, + * this function changes the behavior of some of the subsequent functions + * in a way that they can be used for the inverse Bidi algorithm. + * Specifically, runs of text with numeric characters will be treated in a + * special way and may need to be surrounded with LRM characters when they are + * written in reordered sequence.

+ * + *

Output runs should be retrieved using ubidi_getVisualRun(). + * Since the actual input for "inverse Bidi" is visually ordered text and + * ubidi_getVisualRun() gets the reordered runs, these are actually + * the runs of the logically ordered output.

+ * + *

Calling this function with argument isInverse set to + * TRUE is equivalent to calling + * ubidi_setReorderingMode with argument + * reorderingMode + * set to #UBIDI_REORDER_INVERSE_NUMBERS_AS_L.
+ * Calling this function with argument isInverse set to + * FALSE is equivalent to calling + * ubidi_setReorderingMode with argument + * reorderingMode + * set to #UBIDI_REORDER_DEFAULT. + * + * @param pBiDi is a UBiDi object. + * + * @param isInverse specifies "forward" or "inverse" Bidi operation. + * + * @see ubidi_setPara + * @see ubidi_writeReordered + * @see ubidi_setReorderingMode + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ubidi_setInverse(UBiDi *pBiDi, UBool isInverse); + +/** + * Is this Bidi object set to perform the inverse Bidi algorithm? + *

Note: calling this function after setting the reordering mode with + * ubidi_setReorderingMode will return TRUE if the + * reordering mode was set to #UBIDI_REORDER_INVERSE_NUMBERS_AS_L, + * FALSE for all other values.

+ * + * @param pBiDi is a UBiDi object. + * @return TRUE if the Bidi object is set to perform the inverse Bidi algorithm + * by handling numbers as L. + * + * @see ubidi_setInverse + * @see ubidi_setReorderingMode + * @stable ICU 2.0 + */ + +U_STABLE UBool U_EXPORT2 +ubidi_isInverse(UBiDi *pBiDi); + +/** + * Specify whether block separators must be allocated level zero, + * so that successive paragraphs will progress from left to right. + * This function must be called before ubidi_setPara(). + * Paragraph separators (B) may appear in the text. Setting them to level zero + * means that all paragraph separators (including one possibly appearing + * in the last text position) are kept in the reordered text after the text + * that they follow in the source text. + * When this feature is not enabled, a paragraph separator at the last + * position of the text before reordering will go to the first position + * of the reordered text when the paragraph level is odd. + * + * @param pBiDi is a UBiDi object. + * + * @param orderParagraphsLTR specifies whether paragraph separators (B) must + * receive level 0, so that successive paragraphs progress from left to right. + * + * @see ubidi_setPara + * @stable ICU 3.4 + */ +U_STABLE void U_EXPORT2 +ubidi_orderParagraphsLTR(UBiDi *pBiDi, UBool orderParagraphsLTR); + +/** + * Is this Bidi object set to allocate level 0 to block separators so that + * successive paragraphs progress from left to right? + * + * @param pBiDi is a UBiDi object. + * @return TRUE if the Bidi object is set to allocate level 0 to block + * separators. + * + * @see ubidi_orderParagraphsLTR + * @stable ICU 3.4 + */ +U_STABLE UBool U_EXPORT2 +ubidi_isOrderParagraphsLTR(UBiDi *pBiDi); + +/** + * UBiDiReorderingMode values indicate which variant of the Bidi + * algorithm to use. + * + * @see ubidi_setReorderingMode + * @stable ICU 3.6 + */ +typedef enum UBiDiReorderingMode { + /** Regular Logical to Visual Bidi algorithm according to Unicode. + * This is a 0 value. + * @stable ICU 3.6 */ + UBIDI_REORDER_DEFAULT = 0, + /** Logical to Visual algorithm which handles numbers in a way which + * mimicks the behavior of Windows XP. + * @stable ICU 3.6 */ + UBIDI_REORDER_NUMBERS_SPECIAL, + /** Logical to Visual algorithm grouping numbers with adjacent R characters + * (reversible algorithm). + * @stable ICU 3.6 */ + UBIDI_REORDER_GROUP_NUMBERS_WITH_R, + /** Reorder runs only to transform a Logical LTR string to the Logical RTL + * string with the same display, or vice-versa.
+ * If this mode is set together with option + * #UBIDI_OPTION_INSERT_MARKS, some Bidi controls in the source + * text may be removed and other controls may be added to produce the + * minimum combination which has the required display. + * @stable ICU 3.6 */ + UBIDI_REORDER_RUNS_ONLY, + /** Visual to Logical algorithm which handles numbers like L + * (same algorithm as selected by ubidi_setInverse(TRUE). + * @see ubidi_setInverse + * @stable ICU 3.6 */ + UBIDI_REORDER_INVERSE_NUMBERS_AS_L, + /** Visual to Logical algorithm equivalent to the regular Logical to Visual + * algorithm. + * @stable ICU 3.6 */ + UBIDI_REORDER_INVERSE_LIKE_DIRECT, + /** Inverse Bidi (Visual to Logical) algorithm for the + * UBIDI_REORDER_NUMBERS_SPECIAL Bidi algorithm. + * @stable ICU 3.6 */ + UBIDI_REORDER_INVERSE_FOR_NUMBERS_SPECIAL, + /** Number of values for reordering mode. + * @stable ICU 3.6 */ + UBIDI_REORDER_COUNT +} UBiDiReorderingMode; + +/** + * Modify the operation of the Bidi algorithm such that it implements some + * variant to the basic Bidi algorithm or approximates an "inverse Bidi" + * algorithm, depending on different values of the "reordering mode". + * This function must be called before ubidi_setPara(), and stays + * in effect until called again with a different argument. + * + *

The normal operation of the Bidi algorithm as described + * in the Unicode Standard Annex #9 is to take text stored in logical + * (keyboard, typing) order and to determine how to reorder it for visual + * rendering.

+ * + *

With the reordering mode set to a value other than + * #UBIDI_REORDER_DEFAULT, this function changes the behavior of + * some of the subsequent functions in a way such that they implement an + * inverse Bidi algorithm or some other algorithm variants.

+ * + *

Some legacy systems store text in visual order, and for operations + * with standard, Unicode-based algorithms, the text needs to be transformed + * into logical order. This is effectively the inverse algorithm of the + * described Bidi algorithm. Note that there is no standard algorithm for + * this "inverse Bidi", so a number of variants are implemented here.

+ * + *

In other cases, it may be desirable to emulate some variant of the + * Logical to Visual algorithm (e.g. one used in MS Windows), or perform a + * Logical to Logical transformation.

+ * + *
    + *
  • When the reordering mode is set to #UBIDI_REORDER_DEFAULT, + * the standard Bidi Logical to Visual algorithm is applied.
  • + * + *
  • When the reordering mode is set to + * #UBIDI_REORDER_NUMBERS_SPECIAL, + * the algorithm used to perform Bidi transformations when calling + * ubidi_setPara should approximate the algorithm used in + * Microsoft Windows XP rather than strictly conform to the Unicode Bidi + * algorithm. + *
    + * The differences between the basic algorithm and the algorithm addressed + * by this option are as follows: + *
      + *
    • Within text at an even embedding level, the sequence "123AB" + * (where AB represent R or AL letters) is transformed to "123BA" by the + * Unicode algorithm and to "BA123" by the Windows algorithm.
    • + *
    • Arabic-Indic numbers (AN) are handled by the Windows algorithm just + * like regular numbers (EN).
    • + *
  • + * + *
  • When the reordering mode is set to + * #UBIDI_REORDER_GROUP_NUMBERS_WITH_R, + * numbers located between LTR text and RTL text are associated with the RTL + * text. For instance, an LTR paragraph with content "abc 123 DEF" (where + * upper case letters represent RTL characters) will be transformed to + * "abc FED 123" (and not "abc 123 FED"), "DEF 123 abc" will be transformed + * to "123 FED abc" and "123 FED abc" will be transformed to "DEF 123 abc". + * This makes the algorithm reversible and makes it useful when round trip + * (from visual to logical and back to visual) must be achieved without + * adding LRM characters. However, this is a variation from the standard + * Unicode Bidi algorithm.
    + * The source text should not contain Bidi control characters other than LRM + * or RLM.
  • + * + *
  • When the reordering mode is set to + * #UBIDI_REORDER_RUNS_ONLY, + * a "Logical to Logical" transformation must be performed: + *
      + *
    • If the default text level of the source text (argument paraLevel + * in ubidi_setPara) is even, the source text will be handled as + * LTR logical text and will be transformed to the RTL logical text which has + * the same LTR visual display.
    • + *
    • If the default level of the source text is odd, the source text + * will be handled as RTL logical text and will be transformed to the + * LTR logical text which has the same LTR visual display.
    • + *
    + * This mode may be needed when logical text which is basically Arabic or + * Hebrew, with possible included numbers or phrases in English, has to be + * displayed as if it had an even embedding level (this can happen if the + * displaying application treats all text as if it was basically LTR). + *
    + * This mode may also be needed in the reverse case, when logical text which is + * basically English, with possible included phrases in Arabic or Hebrew, has to + * be displayed as if it had an odd embedding level. + *
    + * Both cases could be handled by adding LRE or RLE at the head of the text, + * if the display subsystem supports these formatting controls. If it does not, + * the problem may be handled by transforming the source text in this mode + * before displaying it, so that it will be displayed properly.
    + * The source text should not contain Bidi control characters other than LRM + * or RLM.
  • + * + *
  • When the reordering mode is set to + * #UBIDI_REORDER_INVERSE_NUMBERS_AS_L, an "inverse Bidi" algorithm + * is applied. + * Runs of text with numeric characters will be treated like LTR letters and + * may need to be surrounded with LRM characters when they are written in + * reordered sequence (the option #UBIDI_INSERT_LRM_FOR_NUMERIC can + * be used with function ubidi_writeReordered to this end. This + * mode is equivalent to calling ubidi_setInverse() with + * argument isInverse set to TRUE.
  • + * + *
  • When the reordering mode is set to + * #UBIDI_REORDER_INVERSE_LIKE_DIRECT, the "direct" Logical to Visual + * Bidi algorithm is used as an approximation of an "inverse Bidi" algorithm. + * This mode is similar to mode #UBIDI_REORDER_INVERSE_NUMBERS_AS_L + * but is closer to the regular Bidi algorithm. + *
    + * For example, an LTR paragraph with the content "FED 123 456 CBA" (where + * upper case represents RTL characters) will be transformed to + * "ABC 456 123 DEF", as opposed to "DEF 123 456 ABC" + * with mode UBIDI_REORDER_INVERSE_NUMBERS_AS_L.
    + * When used in conjunction with option + * #UBIDI_OPTION_INSERT_MARKS, this mode generally + * adds Bidi marks to the output significantly more sparingly than mode + * #UBIDI_REORDER_INVERSE_NUMBERS_AS_L with option + * #UBIDI_INSERT_LRM_FOR_NUMERIC in calls to + * ubidi_writeReordered.
  • + * + *
  • When the reordering mode is set to + * #UBIDI_REORDER_INVERSE_FOR_NUMBERS_SPECIAL, the Logical to Visual + * Bidi algorithm used in Windows XP is used as an approximation of an + * "inverse Bidi" algorithm. + *
    + * For example, an LTR paragraph with the content "abc FED123" (where + * upper case represents RTL characters) will be transformed to + * "abc 123DEF.
  • + *
+ * + *

In all the reordering modes specifying an "inverse Bidi" algorithm + * (i.e. those with a name starting with UBIDI_REORDER_INVERSE), + * output runs should be retrieved using + * ubidi_getVisualRun(), and the output text with + * ubidi_writeReordered(). The caller should keep in mind that in + * "inverse Bidi" modes the input is actually visually ordered text and + * reordered output returned by ubidi_getVisualRun() or + * ubidi_writeReordered() are actually runs or character string + * of logically ordered output.
+ * For all the "inverse Bidi" modes, the source text should not contain + * Bidi control characters other than LRM or RLM.

+ * + *

Note that option #UBIDI_OUTPUT_REVERSE of + * ubidi_writeReordered has no useful meaning and should not be + * used in conjunction with any value of the reordering mode specifying + * "inverse Bidi" or with value UBIDI_REORDER_RUNS_ONLY. + * + * @param pBiDi is a UBiDi object. + * @param reorderingMode specifies the required variant of the Bidi algorithm. + * + * @see UBiDiReorderingMode + * @see ubidi_setInverse + * @see ubidi_setPara + * @see ubidi_writeReordered + * @stable ICU 3.6 + */ +U_STABLE void U_EXPORT2 +ubidi_setReorderingMode(UBiDi *pBiDi, UBiDiReorderingMode reorderingMode); + +/** + * What is the requested reordering mode for a given Bidi object? + * + * @param pBiDi is a UBiDi object. + * @return the current reordering mode of the Bidi object + * @see ubidi_setReorderingMode + * @stable ICU 3.6 + */ +U_STABLE UBiDiReorderingMode U_EXPORT2 +ubidi_getReorderingMode(UBiDi *pBiDi); + +/** + * UBiDiReorderingOption values indicate which options are + * specified to affect the Bidi algorithm. + * + * @see ubidi_setReorderingOptions + * @stable ICU 3.6 + */ +typedef enum UBiDiReorderingOption { + /** + * option value for ubidi_setReorderingOptions: + * disable all the options which can be set with this function + * @see ubidi_setReorderingOptions + * @stable ICU 3.6 + */ + UBIDI_OPTION_DEFAULT = 0, + + /** + * option bit for ubidi_setReorderingOptions: + * insert Bidi marks (LRM or RLM) when needed to ensure correct result of + * a reordering to a Logical order + * + *

This option must be set or reset before calling + * ubidi_setPara.

+ * + *

This option is significant only with reordering modes which generate + * a result with Logical order, specifically:

+ *
    + *
  • #UBIDI_REORDER_RUNS_ONLY
  • + *
  • #UBIDI_REORDER_INVERSE_NUMBERS_AS_L
  • + *
  • #UBIDI_REORDER_INVERSE_LIKE_DIRECT
  • + *
  • #UBIDI_REORDER_INVERSE_FOR_NUMBERS_SPECIAL
  • + *
+ * + *

If this option is set in conjunction with reordering mode + * #UBIDI_REORDER_INVERSE_NUMBERS_AS_L or with calling + * ubidi_setInverse(TRUE), it implies + * option #UBIDI_INSERT_LRM_FOR_NUMERIC + * in calls to function ubidi_writeReordered().

+ * + *

For other reordering modes, a minimum number of LRM or RLM characters + * will be added to the source text after reordering it so as to ensure + * round trip, i.e. when applying the inverse reordering mode on the + * resulting logical text with removal of Bidi marks + * (option #UBIDI_OPTION_REMOVE_CONTROLS set before calling + * ubidi_setPara() or option #UBIDI_REMOVE_BIDI_CONTROLS + * in ubidi_writeReordered), the result will be identical to the + * source text in the first transformation. + * + *

This option will be ignored if specified together with option + * #UBIDI_OPTION_REMOVE_CONTROLS. It inhibits option + * UBIDI_REMOVE_BIDI_CONTROLS in calls to function + * ubidi_writeReordered() and it implies option + * #UBIDI_INSERT_LRM_FOR_NUMERIC in calls to function + * ubidi_writeReordered() if the reordering mode is + * #UBIDI_REORDER_INVERSE_NUMBERS_AS_L.

+ * + * @see ubidi_setReorderingMode + * @see ubidi_setReorderingOptions + * @stable ICU 3.6 + */ + UBIDI_OPTION_INSERT_MARKS = 1, + + /** + * option bit for ubidi_setReorderingOptions: + * remove Bidi control characters + * + *

This option must be set or reset before calling + * ubidi_setPara.

+ * + *

This option nullifies option #UBIDI_OPTION_INSERT_MARKS. + * It inhibits option #UBIDI_INSERT_LRM_FOR_NUMERIC in calls + * to function ubidi_writeReordered() and it implies option + * #UBIDI_REMOVE_BIDI_CONTROLS in calls to that function.

+ * + * @see ubidi_setReorderingMode + * @see ubidi_setReorderingOptions + * @stable ICU 3.6 + */ + UBIDI_OPTION_REMOVE_CONTROLS = 2, + + /** + * option bit for ubidi_setReorderingOptions: + * process the output as part of a stream to be continued + * + *

This option must be set or reset before calling + * ubidi_setPara.

+ * + *

This option specifies that the caller is interested in processing large + * text object in parts. + * The results of the successive calls are expected to be concatenated by the + * caller. Only the call for the last part will have this option bit off.

+ * + *

When this option bit is on, ubidi_setPara() may process + * less than the full source text in order to truncate the text at a meaningful + * boundary. The caller should call ubidi_getProcessedLength() + * immediately after calling ubidi_setPara() in order to + * determine how much of the source text has been processed. + * Source text beyond that length should be resubmitted in following calls to + * ubidi_setPara. The processed length may be less than + * the length of the source text if a character preceding the last character of + * the source text constitutes a reasonable boundary (like a block separator) + * for text to be continued.
+ * If the last character of the source text constitutes a reasonable + * boundary, the whole text will be processed at once.
+ * If nowhere in the source text there exists + * such a reasonable boundary, the processed length will be zero.
+ * The caller should check for such an occurrence and do one of the following: + *

  • submit a larger amount of text with a better chance to include + * a reasonable boundary.
  • + *
  • resubmit the same text after turning off option + * UBIDI_OPTION_STREAMING.
+ * In all cases, this option should be turned off before processing the last + * part of the text.

+ * + *

When the UBIDI_OPTION_STREAMING option is used, + * it is recommended to call ubidi_orderParagraphsLTR() with + * argument orderParagraphsLTR set to TRUE before + * calling ubidi_setPara so that later paragraphs may be + * concatenated to previous paragraphs on the right.

+ * + * @see ubidi_setReorderingMode + * @see ubidi_setReorderingOptions + * @see ubidi_getProcessedLength + * @see ubidi_orderParagraphsLTR + * @stable ICU 3.6 + */ + UBIDI_OPTION_STREAMING = 4 +} UBiDiReorderingOption; + +/** + * Specify which of the reordering options + * should be applied during Bidi transformations. + * + * @param pBiDi is a UBiDi object. + * @param reorderingOptions is a combination of zero or more of the following + * options: + * #UBIDI_OPTION_DEFAULT, #UBIDI_OPTION_INSERT_MARKS, + * #UBIDI_OPTION_REMOVE_CONTROLS, #UBIDI_OPTION_STREAMING. + * + * @see ubidi_getReorderingOptions + * @stable ICU 3.6 + */ +U_STABLE void U_EXPORT2 +ubidi_setReorderingOptions(UBiDi *pBiDi, uint32_t reorderingOptions); + +/** + * What are the reordering options applied to a given Bidi object? + * + * @param pBiDi is a UBiDi object. + * @return the current reordering options of the Bidi object + * @see ubidi_setReorderingOptions + * @stable ICU 3.6 + */ +U_STABLE uint32_t U_EXPORT2 +ubidi_getReorderingOptions(UBiDi *pBiDi); + +/** + * Perform the Unicode Bidi algorithm. It is defined in the + * Unicode Standard Anned #9, + * version 13, + * also described in The Unicode Standard, Version 4.0 .

+ * + * This function takes a piece of plain text containing one or more paragraphs, + * with or without externally specified embedding levels from styled + * text and computes the left-right-directionality of each character.

+ * + * If the entire text is all of the same directionality, then + * the function may not perform all the steps described by the algorithm, + * i.e., some levels may not be the same as if all steps were performed. + * This is not relevant for unidirectional text.
+ * For example, in pure LTR text with numbers the numbers would get + * a resolved level of 2 higher than the surrounding text according to + * the algorithm. This implementation may set all resolved levels to + * the same value in such a case.

+ * + * The text can be composed of multiple paragraphs. Occurrence of a block + * separator in the text terminates a paragraph, and whatever comes next starts + * a new paragraph. The exception to this rule is when a Carriage Return (CR) + * is followed by a Line Feed (LF). Both CR and LF are block separators, but + * in that case, the pair of characters is considered as terminating the + * preceding paragraph, and a new paragraph will be started by a character + * coming after the LF. + * + * @param pBiDi A UBiDi object allocated with ubidi_open() + * which will be set to contain the reordering information, + * especially the resolved levels for all the characters in text. + * + * @param text is a pointer to the text that the Bidi algorithm will be performed on. + * This pointer is stored in the UBiDi object and can be retrieved + * with ubidi_getText().
+ * Note: the text must be (at least) length long. + * + * @param length is the length of the text; if length==-1 then + * the text must be zero-terminated. + * + * @param paraLevel specifies the default level for the text; + * it is typically 0 (LTR) or 1 (RTL). + * If the function shall determine the paragraph level from the text, + * then paraLevel can be set to + * either #UBIDI_DEFAULT_LTR + * or #UBIDI_DEFAULT_RTL; if the text contains multiple + * paragraphs, the paragraph level shall be determined separately for + * each paragraph; if a paragraph does not include any strongly typed + * character, then the desired default is used (0 for LTR or 1 for RTL). + * Any other value between 0 and #UBIDI_MAX_EXPLICIT_LEVEL + * is also valid, with odd levels indicating RTL. + * + * @param embeddingLevels (in) may be used to preset the embedding and override levels, + * ignoring characters like LRE and PDF in the text. + * A level overrides the directional property of its corresponding + * (same index) character if the level has the + * #UBIDI_LEVEL_OVERRIDE bit set.

+ * Except for that bit, it must be + * paraLevel<=embeddingLevels[]<=UBIDI_MAX_EXPLICIT_LEVEL, + * with one exception: a level of zero may be specified for a paragraph + * separator even if paraLevel>0 when multiple paragraphs + * are submitted in the same call to ubidi_setPara().

+ * Caution: A copy of this pointer, not of the levels, + * will be stored in the UBiDi object; + * the embeddingLevels array must not be + * deallocated before the UBiDi structure is destroyed or reused, + * and the embeddingLevels + * should not be modified to avoid unexpected results on subsequent Bidi operations. + * However, the ubidi_setPara() and + * ubidi_setLine() functions may modify some or all of the levels.

+ * After the UBiDi object is reused or destroyed, the caller + * must take care of the deallocation of the embeddingLevels array.

+ * Note: the embeddingLevels array must be + * at least length long. + * + * @param pErrorCode must be a valid pointer to an error code value. + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ubidi_setPara(UBiDi *pBiDi, const UChar *text, int32_t length, + UBiDiLevel paraLevel, UBiDiLevel *embeddingLevels, + UErrorCode *pErrorCode); + +/** + * ubidi_setLine() sets a UBiDi to + * contain the reordering information, especially the resolved levels, + * for all the characters in a line of text. This line of text is + * specified by referring to a UBiDi object representing + * this information for a piece of text containing one or more paragraphs, + * and by specifying a range of indexes in this text.

+ * In the new line object, the indexes will range from 0 to limit-start-1.

+ * + * This is used after calling ubidi_setPara() + * for a piece of text, and after line-breaking on that text. + * It is not necessary if each paragraph is treated as a single line.

+ * + * After line-breaking, rules (L1) and (L2) for the treatment of + * trailing WS and for reordering are performed on + * a UBiDi object that represents a line.

+ * + * Important: pLineBiDi shares data with + * pParaBiDi. + * You must destroy or reuse pLineBiDi before pParaBiDi. + * In other words, you must destroy or reuse the UBiDi object for a line + * before the object for its parent paragraph.

+ * + * The text pointer that was stored in pParaBiDi is also copied, + * and start is added to it so that it points to the beginning of the + * line for this object. + * + * @param pParaBiDi is the parent paragraph object. It must have been set + * by a successful call to ubidi_setPara. + * + * @param start is the line's first index into the text. + * + * @param limit is just behind the line's last index into the text + * (its last index +1).
+ * It must be 0<=startcontaining paragraph limit. + * If the specified line crosses a paragraph boundary, the function + * will terminate with error code U_ILLEGAL_ARGUMENT_ERROR. + * + * @param pLineBiDi is the object that will now represent a line of the text. + * + * @param pErrorCode must be a valid pointer to an error code value. + * + * @see ubidi_setPara + * @see ubidi_getProcessedLength + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ubidi_setLine(const UBiDi *pParaBiDi, + int32_t start, int32_t limit, + UBiDi *pLineBiDi, + UErrorCode *pErrorCode); + +/** + * Get the directionality of the text. + * + * @param pBiDi is the paragraph or line UBiDi object. + * + * @return a value of UBIDI_LTR, UBIDI_RTL + * or UBIDI_MIXED + * that indicates if the entire text + * represented by this object is unidirectional, + * and which direction, or if it is mixed-directional. + * + * @see UBiDiDirection + * @stable ICU 2.0 + */ +U_STABLE UBiDiDirection U_EXPORT2 +ubidi_getDirection(const UBiDi *pBiDi); + +/** + * Get the pointer to the text. + * + * @param pBiDi is the paragraph or line UBiDi object. + * + * @return The pointer to the text that the UBiDi object was created for. + * + * @see ubidi_setPara + * @see ubidi_setLine + * @stable ICU 2.0 + */ +U_STABLE const UChar * U_EXPORT2 +ubidi_getText(const UBiDi *pBiDi); + +/** + * Get the length of the text. + * + * @param pBiDi is the paragraph or line UBiDi object. + * + * @return The length of the text that the UBiDi object was created for. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +ubidi_getLength(const UBiDi *pBiDi); + +/** + * Get the paragraph level of the text. + * + * @param pBiDi is the paragraph or line UBiDi object. + * + * @return The paragraph level. If there are multiple paragraphs, their + * level may vary if the required paraLevel is UBIDI_DEFAULT_LTR or + * UBIDI_DEFAULT_RTL. In that case, the level of the first paragraph + * is returned. + * + * @see UBiDiLevel + * @see ubidi_getParagraph + * @see ubidi_getParagraphByIndex + * @stable ICU 2.0 + */ +U_STABLE UBiDiLevel U_EXPORT2 +ubidi_getParaLevel(const UBiDi *pBiDi); + +/** + * Get the number of paragraphs. + * + * @param pBiDi is the paragraph or line UBiDi object. + * + * @return The number of paragraphs. + * @stable ICU 3.4 + */ +U_STABLE int32_t U_EXPORT2 +ubidi_countParagraphs(UBiDi *pBiDi); + +/** + * Get a paragraph, given a position within the text. + * This function returns information about a paragraph.
+ * Note: if the paragraph index is known, it is more efficient to + * retrieve the paragraph information using ubidi_getParagraphByIndex().

+ * + * @param pBiDi is the paragraph or line UBiDi object. + * + * @param charIndex is the index of a character within the text, in the + * range [0..ubidi_getProcessedLength(pBiDi)-1]. + * + * @param pParaStart will receive the index of the first character of the + * paragraph in the text. + * This pointer can be NULL if this + * value is not necessary. + * + * @param pParaLimit will receive the limit of the paragraph. + * The l-value that you point to here may be the + * same expression (variable) as the one for + * charIndex. + * This pointer can be NULL if this + * value is not necessary. + * + * @param pParaLevel will receive the level of the paragraph. + * This pointer can be NULL if this + * value is not necessary. + * + * @param pErrorCode must be a valid pointer to an error code value. + * + * @return The index of the paragraph containing the specified position. + * + * @see ubidi_getProcessedLength + * @stable ICU 3.4 + */ +U_STABLE int32_t U_EXPORT2 +ubidi_getParagraph(const UBiDi *pBiDi, int32_t charIndex, int32_t *pParaStart, + int32_t *pParaLimit, UBiDiLevel *pParaLevel, + UErrorCode *pErrorCode); + +/** + * Get a paragraph, given the index of this paragraph. + * + * This function returns information about a paragraph.

+ * + * @param pBiDi is the paragraph UBiDi object. + * + * @param paraIndex is the number of the paragraph, in the + * range [0..ubidi_countParagraphs(pBiDi)-1]. + * + * @param pParaStart will receive the index of the first character of the + * paragraph in the text. + * This pointer can be NULL if this + * value is not necessary. + * + * @param pParaLimit will receive the limit of the paragraph. + * This pointer can be NULL if this + * value is not necessary. + * + * @param pParaLevel will receive the level of the paragraph. + * This pointer can be NULL if this + * value is not necessary. + * + * @param pErrorCode must be a valid pointer to an error code value. + * + * @stable ICU 3.4 + */ +U_STABLE void U_EXPORT2 +ubidi_getParagraphByIndex(const UBiDi *pBiDi, int32_t paraIndex, + int32_t *pParaStart, int32_t *pParaLimit, + UBiDiLevel *pParaLevel, UErrorCode *pErrorCode); + +/** + * Get the level for one character. + * + * @param pBiDi is the paragraph or line UBiDi object. + * + * @param charIndex the index of a character. It must be in the range + * [0..ubidi_getProcessedLength(pBiDi)]. + * + * @return The level for the character at charIndex (0 if charIndex is not + * in the valid range). + * + * @see UBiDiLevel + * @see ubidi_getProcessedLength + * @stable ICU 2.0 + */ +U_STABLE UBiDiLevel U_EXPORT2 +ubidi_getLevelAt(const UBiDi *pBiDi, int32_t charIndex); + +/** + * Get an array of levels for each character.

+ * + * Note that this function may allocate memory under some + * circumstances, unlike ubidi_getLevelAt(). + * + * @param pBiDi is the paragraph or line UBiDi object, whose + * text length must be strictly positive. + * + * @param pErrorCode must be a valid pointer to an error code value. + * + * @return The levels array for the text, + * or NULL if an error occurs. + * + * @see UBiDiLevel + * @see ubidi_getProcessedLength + * @stable ICU 2.0 + */ +U_STABLE const UBiDiLevel * U_EXPORT2 +ubidi_getLevels(UBiDi *pBiDi, UErrorCode *pErrorCode); + +/** + * Get a logical run. + * This function returns information about a run and is used + * to retrieve runs in logical order.

+ * This is especially useful for line-breaking on a paragraph. + * + * @param pBiDi is the paragraph or line UBiDi object. + * + * @param logicalPosition is a logical position within the source text. + * + * @param pLogicalLimit will receive the limit of the corresponding run. + * The l-value that you point to here may be the + * same expression (variable) as the one for + * logicalPosition. + * This pointer can be NULL if this + * value is not necessary. + * + * @param pLevel will receive the level of the corresponding run. + * This pointer can be NULL if this + * value is not necessary. + * + * @see ubidi_getProcessedLength + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ubidi_getLogicalRun(const UBiDi *pBiDi, int32_t logicalPosition, + int32_t *pLogicalLimit, UBiDiLevel *pLevel); + +/** + * Get the number of runs. + * This function may invoke the actual reordering on the + * UBiDi object, after ubidi_setPara() + * may have resolved only the levels of the text. Therefore, + * ubidi_countRuns() may have to allocate memory, + * and may fail doing so. + * + * @param pBiDi is the paragraph or line UBiDi object. + * + * @param pErrorCode must be a valid pointer to an error code value. + * + * @return The number of runs. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +ubidi_countRuns(UBiDi *pBiDi, UErrorCode *pErrorCode); + +/** + * Get one run's logical start, length, and directionality, + * which can be 0 for LTR or 1 for RTL. + * In an RTL run, the character at the logical start is + * visually on the right of the displayed run. + * The length is the number of characters in the run.

+ * ubidi_countRuns() should be called + * before the runs are retrieved. + * + * @param pBiDi is the paragraph or line UBiDi object. + * + * @param runIndex is the number of the run in visual order, in the + * range [0..ubidi_countRuns(pBiDi)-1]. + * + * @param pLogicalStart is the first logical character index in the text. + * The pointer may be NULL if this index is not needed. + * + * @param pLength is the number of characters (at least one) in the run. + * The pointer may be NULL if this is not needed. + * + * @return the directionality of the run, + * UBIDI_LTR==0 or UBIDI_RTL==1, + * never UBIDI_MIXED. + * + * @see ubidi_countRuns + * + * Example: + *

+ * \code
+ * int32_t i, count=ubidi_countRuns(pBiDi),
+ *         logicalStart, visualIndex=0, length;
+ * for(i=0; i0);
+ *     } else {
+ *         logicalStart+=length;  // logicalLimit
+ *         do { // RTL
+ *             show_char(text[--logicalStart], visualIndex++);
+ *         } while(--length>0);
+ *     }
+ * }
+ *\endcode
+ * 
+ * + * Note that in right-to-left runs, code like this places + * modifier letters before base characters and second surrogates + * before first ones. + * @stable ICU 2.0 + */ +U_STABLE UBiDiDirection U_EXPORT2 +ubidi_getVisualRun(UBiDi *pBiDi, int32_t runIndex, + int32_t *pLogicalStart, int32_t *pLength); + +/** + * Get the visual position from a logical text position. + * If such a mapping is used many times on the same + * UBiDi object, then calling + * ubidi_getLogicalMap() is more efficient.

+ * + * The value returned may be #UBIDI_MAP_NOWHERE if there is no + * visual position because the corresponding text character is a Bidi control + * removed from output by the option #UBIDI_OPTION_REMOVE_CONTROLS. + *

+ * When the visual output is altered by using options of + * ubidi_writeReordered() such as UBIDI_INSERT_LRM_FOR_NUMERIC, + * UBIDI_KEEP_BASE_COMBINING, UBIDI_OUTPUT_REVERSE, + * UBIDI_REMOVE_BIDI_CONTROLS, the visual position returned may not + * be correct. It is advised to use, when possible, reordering options + * such as UBIDI_OPTION_INSERT_MARKS and UBIDI_OPTION_REMOVE_CONTROLS. + *

+ * Note that in right-to-left runs, this mapping places + * modifier letters before base characters and second surrogates + * before first ones. + * + * @param pBiDi is the paragraph or line UBiDi object. + * + * @param logicalIndex is the index of a character in the text. + * + * @param pErrorCode must be a valid pointer to an error code value. + * + * @return The visual position of this character. + * + * @see ubidi_getLogicalMap + * @see ubidi_getLogicalIndex + * @see ubidi_getProcessedLength + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +ubidi_getVisualIndex(UBiDi *pBiDi, int32_t logicalIndex, UErrorCode *pErrorCode); + +/** + * Get the logical text position from a visual position. + * If such a mapping is used many times on the same + * UBiDi object, then calling + * ubidi_getVisualMap() is more efficient.

+ * + * The value returned may be #UBIDI_MAP_NOWHERE if there is no + * logical position because the corresponding text character is a Bidi mark + * inserted in the output by option #UBIDI_OPTION_INSERT_MARKS. + *

+ * This is the inverse function to ubidi_getVisualIndex(). + *

+ * When the visual output is altered by using options of + * ubidi_writeReordered() such as UBIDI_INSERT_LRM_FOR_NUMERIC, + * UBIDI_KEEP_BASE_COMBINING, UBIDI_OUTPUT_REVERSE, + * UBIDI_REMOVE_BIDI_CONTROLS, the logical position returned may not + * be correct. It is advised to use, when possible, reordering options + * such as UBIDI_OPTION_INSERT_MARKS and UBIDI_OPTION_REMOVE_CONTROLS. + * + * @param pBiDi is the paragraph or line UBiDi object. + * + * @param visualIndex is the visual position of a character. + * + * @param pErrorCode must be a valid pointer to an error code value. + * + * @return The index of this character in the text. + * + * @see ubidi_getVisualMap + * @see ubidi_getVisualIndex + * @see ubidi_getResultLength + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +ubidi_getLogicalIndex(UBiDi *pBiDi, int32_t visualIndex, UErrorCode *pErrorCode); + +/** + * Get a logical-to-visual index map (array) for the characters in the UBiDi + * (paragraph or line) object. + *

+ * Some values in the map may be #UBIDI_MAP_NOWHERE if the + * corresponding text characters are Bidi controls removed from the visual + * output by the option #UBIDI_OPTION_REMOVE_CONTROLS. + *

+ * When the visual output is altered by using options of + * ubidi_writeReordered() such as UBIDI_INSERT_LRM_FOR_NUMERIC, + * UBIDI_KEEP_BASE_COMBINING, UBIDI_OUTPUT_REVERSE, + * UBIDI_REMOVE_BIDI_CONTROLS, the visual positions returned may not + * be correct. It is advised to use, when possible, reordering options + * such as UBIDI_OPTION_INSERT_MARKS and UBIDI_OPTION_REMOVE_CONTROLS. + * + * @param pBiDi is the paragraph or line UBiDi object. + * + * @param indexMap is a pointer to an array of ubidi_getProcessedLength() + * indexes which will reflect the reordering of the characters. + * If option #UBIDI_OPTION_INSERT_MARKS is set, the number + * of elements allocated in indexMap must be no less than + * ubidi_getResultLength(). + * The array does not need to be initialized.

+ * The index map will result in indexMap[logicalIndex]==visualIndex. + * + * @param pErrorCode must be a valid pointer to an error code value. + * + * @see ubidi_getVisualMap + * @see ubidi_getVisualIndex + * @see ubidi_getProcessedLength + * @see ubidi_getResultLength + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ubidi_getLogicalMap(UBiDi *pBiDi, int32_t *indexMap, UErrorCode *pErrorCode); + +/** + * Get a visual-to-logical index map (array) for the characters in the UBiDi + * (paragraph or line) object. + *

+ * Some values in the map may be #UBIDI_MAP_NOWHERE if the + * corresponding text characters are Bidi marks inserted in the visual output + * by the option #UBIDI_OPTION_INSERT_MARKS. + *

+ * When the visual output is altered by using options of + * ubidi_writeReordered() such as UBIDI_INSERT_LRM_FOR_NUMERIC, + * UBIDI_KEEP_BASE_COMBINING, UBIDI_OUTPUT_REVERSE, + * UBIDI_REMOVE_BIDI_CONTROLS, the logical positions returned may not + * be correct. It is advised to use, when possible, reordering options + * such as UBIDI_OPTION_INSERT_MARKS and UBIDI_OPTION_REMOVE_CONTROLS. + * + * @param pBiDi is the paragraph or line UBiDi object. + * + * @param indexMap is a pointer to an array of ubidi_getResultLength() + * indexes which will reflect the reordering of the characters. + * If option #UBIDI_OPTION_REMOVE_CONTROLS is set, the number + * of elements allocated in indexMap must be no less than + * ubidi_getProcessedLength(). + * The array does not need to be initialized.

+ * The index map will result in indexMap[visualIndex]==logicalIndex. + * + * @param pErrorCode must be a valid pointer to an error code value. + * + * @see ubidi_getLogicalMap + * @see ubidi_getLogicalIndex + * @see ubidi_getProcessedLength + * @see ubidi_getResultLength + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ubidi_getVisualMap(UBiDi *pBiDi, int32_t *indexMap, UErrorCode *pErrorCode); + +/** + * This is a convenience function that does not use a UBiDi object. + * It is intended to be used for when an application has determined the levels + * of objects (character sequences) and just needs to have them reordered (L2). + * This is equivalent to using ubidi_getLogicalMap() on a + * UBiDi object. + * + * @param levels is an array with length levels that have been determined by + * the application. + * + * @param length is the number of levels in the array, or, semantically, + * the number of objects to be reordered. + * It must be length>0. + * + * @param indexMap is a pointer to an array of length + * indexes which will reflect the reordering of the characters. + * The array does not need to be initialized.

+ * The index map will result in indexMap[logicalIndex]==visualIndex. + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ubidi_reorderLogical(const UBiDiLevel *levels, int32_t length, int32_t *indexMap); + +/** + * This is a convenience function that does not use a UBiDi object. + * It is intended to be used for when an application has determined the levels + * of objects (character sequences) and just needs to have them reordered (L2). + * This is equivalent to using ubidi_getVisualMap() on a + * UBiDi object. + * + * @param levels is an array with length levels that have been determined by + * the application. + * + * @param length is the number of levels in the array, or, semantically, + * the number of objects to be reordered. + * It must be length>0. + * + * @param indexMap is a pointer to an array of length + * indexes which will reflect the reordering of the characters. + * The array does not need to be initialized.

+ * The index map will result in indexMap[visualIndex]==logicalIndex. + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ubidi_reorderVisual(const UBiDiLevel *levels, int32_t length, int32_t *indexMap); + +/** + * Invert an index map. + * The index mapping of the first map is inverted and written to + * the second one. + * + * @param srcMap is an array with length elements + * which defines the original mapping from a source array containing + * length elements to a destination array. + * Some elements of the source array may have no mapping in the + * destination array. In that case, their value will be + * the special value UBIDI_MAP_NOWHERE. + * All elements must be >=0 or equal to UBIDI_MAP_NOWHERE. + * Some elements may have a value >= length, if the + * destination array has more elements than the source array. + * There must be no duplicate indexes (two or more elements with the + * same value except UBIDI_MAP_NOWHERE). + * + * @param destMap is an array with a number of elements equal to 1 + the highest + * value in srcMap. + * destMap will be filled with the inverse mapping. + * If element with index i in srcMap has a value k different + * from UBIDI_MAP_NOWHERE, this means that element i of + * the source array maps to element k in the destination array. + * The inverse map will have value i in its k-th element. + * For all elements of the destination array which do not map to + * an element in the source array, the corresponding element in the + * inverse map will have a value equal to UBIDI_MAP_NOWHERE. + * + * @param length is the length of each array. + * @See UBIDI_MAP_NOWHERE + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ubidi_invertMap(const int32_t *srcMap, int32_t *destMap, int32_t length); + +/** option flags for ubidi_writeReordered() */ + +/** + * option bit for ubidi_writeReordered(): + * keep combining characters after their base characters in RTL runs + * + * @see ubidi_writeReordered + * @stable ICU 2.0 + */ +#define UBIDI_KEEP_BASE_COMBINING 1 + +/** + * option bit for ubidi_writeReordered(): + * replace characters with the "mirrored" property in RTL runs + * by their mirror-image mappings + * + * @see ubidi_writeReordered + * @stable ICU 2.0 + */ +#define UBIDI_DO_MIRRORING 2 + +/** + * option bit for ubidi_writeReordered(): + * surround the run with LRMs if necessary; + * this is part of the approximate "inverse Bidi" algorithm + * + *

This option does not imply corresponding adjustment of the index + * mappings.

+ * + * @see ubidi_setInverse + * @see ubidi_writeReordered + * @stable ICU 2.0 + */ +#define UBIDI_INSERT_LRM_FOR_NUMERIC 4 + +/** + * option bit for ubidi_writeReordered(): + * remove Bidi control characters + * (this does not affect #UBIDI_INSERT_LRM_FOR_NUMERIC) + * + *

This option does not imply corresponding adjustment of the index + * mappings.

+ * + * @see ubidi_writeReordered + * @stable ICU 2.0 + */ +#define UBIDI_REMOVE_BIDI_CONTROLS 8 + +/** + * option bit for ubidi_writeReordered(): + * write the output in reverse order + * + *

This has the same effect as calling ubidi_writeReordered() + * first without this option, and then calling + * ubidi_writeReverse() without mirroring. + * Doing this in the same step is faster and avoids a temporary buffer. + * An example for using this option is output to a character terminal that + * is designed for RTL scripts and stores text in reverse order.

+ * + * @see ubidi_writeReordered + * @stable ICU 2.0 + */ +#define UBIDI_OUTPUT_REVERSE 16 + +/** + * Get the length of the source text processed by the last call to + * ubidi_setPara(). This length may be different from the length + * of the source text if option #UBIDI_OPTION_STREAMING + * has been set. + *
+ * Note that whenever the length of the text affects the execution or the + * result of a function, it is the processed length which must be considered, + * except for ubidi_setPara (which receives unprocessed source + * text) and ubidi_getLength (which returns the original length + * of the source text).
+ * In particular, the processed length is the one to consider in the following + * cases: + *
    + *
  • maximum value of the limit argument of + * ubidi_setLine
  • + *
  • maximum value of the charIndex argument of + * ubidi_getParagraph
  • + *
  • maximum value of the charIndex argument of + * ubidi_getLevelAt
  • + *
  • number of elements in the array returned by ubidi_getLevels
  • + *
  • maximum value of the logicalStart argument of + * ubidi_getLogicalRun
  • + *
  • maximum value of the logicalIndex argument of + * ubidi_getVisualIndex
  • + *
  • number of elements filled in the *indexMap argument of + * ubidi_getLogicalMap
  • + *
  • length of text processed by ubidi_writeReordered
  • + *
+ * + * @param pBiDi is the paragraph UBiDi object. + * + * @return The length of the part of the source text processed by + * the last call to ubidi_setPara. + * @see ubidi_setPara + * @see UBIDI_OPTION_STREAMING + * @stable ICU 3.6 + */ +U_STABLE int32_t U_EXPORT2 +ubidi_getProcessedLength(const UBiDi *pBiDi); + +/** + * Get the length of the reordered text resulting from the last call to + * ubidi_setPara(). This length may be different from the length + * of the source text if option #UBIDI_OPTION_INSERT_MARKS + * or option #UBIDI_OPTION_REMOVE_CONTROLS has been set. + *
+ * This resulting length is the one to consider in the following cases: + *
    + *
  • maximum value of the visualIndex argument of + * ubidi_getLogicalIndex
  • + *
  • number of elements of the *indexMap argument of + * ubidi_getVisualMap
  • + *
+ * Note that this length stays identical to the source text length if + * Bidi marks are inserted or removed using option bits of + * ubidi_writeReordered, or if option + * #UBIDI_REORDER_INVERSE_NUMBERS_AS_L has been set. + * + * @param pBiDi is the paragraph UBiDi object. + * + * @return The length of the reordered text resulting from + * the last call to ubidi_setPara. + * @see ubidi_setPara + * @see UBIDI_OPTION_INSERT_MARKS + * @see UBIDI_OPTION_REMOVE_CONTROLS + * @stable ICU 3.6 + */ +U_STABLE int32_t U_EXPORT2 +ubidi_getResultLength(const UBiDi *pBiDi); + +U_CDECL_BEGIN +/** + * value returned by UBiDiClassCallback callbacks when + * there is no need to override the standard Bidi class for a given code point. + * @see UBiDiClassCallback + * @stable ICU 3.6 + */ +#define U_BIDI_CLASS_DEFAULT U_CHAR_DIRECTION_COUNT + +/** + * Callback type declaration for overriding default Bidi class values with + * custom ones. + *

Usually, the function pointer will be propagated to a UBiDi + * object by calling the ubidi_setClassCallback() function; + * then the callback will be invoked by the UBA implementation any time the + * class of a character is to be determined.

+ * + * @param context is a pointer to the callback private data. + * + * @param c is the code point to get a Bidi class for. + * + * @return The directional property / Bidi class for the given code point + * c if the default class has been overridden, or + * #U_BIDI_CLASS_DEFAULT if the standard Bidi class value + * for c is to be used. + * @see ubidi_setClassCallback + * @see ubidi_getClassCallback + * @stable ICU 3.6 + */ +typedef UCharDirection U_CALLCONV +UBiDiClassCallback(const void *context, UChar32 c); + +U_CDECL_END + +/** + * Retrieve the Bidi class for a given code point. + *

If a #UBiDiClassCallback callback is defined and returns a + * value other than #U_BIDI_CLASS_DEFAULT, that value is used; + * otherwise the default class determination mechanism is invoked.

+ * + * @param pBiDi is the paragraph UBiDi object. + * + * @param c is the code point whose Bidi class must be retrieved. + * + * @return The Bidi class for character c based + * on the given pBiDi instance. + * @see UBiDiClassCallback + * @stable ICU 3.6 + */ +U_STABLE UCharDirection U_EXPORT2 +ubidi_getCustomizedClass(UBiDi *pBiDi, UChar32 c); + +/** + * Set the callback function and callback data used by the UBA + * implementation for Bidi class determination. + *

This may be useful for assigning Bidi classes to PUA characters, or + * for special application needs. For instance, an application may want to + * handle all spaces like L or R characters (according to the base direction) + * when creating the visual ordering of logical lines which are part of a report + * organized in columns: there should not be interaction between adjacent + * cells.

+ * + * @param pBiDi is the paragraph UBiDi object. + * + * @param newFn is the new callback function pointer. + * + * @param newContext is the new callback context pointer. This can be NULL. + * + * @param oldFn fillin: Returns the old callback function pointer. This can be + * NULL. + * + * @param oldContext fillin: Returns the old callback's context. This can be + * NULL. + * + * @param pErrorCode must be a valid pointer to an error code value. + * + * @see ubidi_getClassCallback + * @stable ICU 3.6 + */ +U_STABLE void U_EXPORT2 +ubidi_setClassCallback(UBiDi *pBiDi, UBiDiClassCallback *newFn, + const void *newContext, UBiDiClassCallback **oldFn, + const void **oldContext, UErrorCode *pErrorCode); + +/** + * Get the current callback function used for Bidi class determination. + * + * @param pBiDi is the paragraph UBiDi object. + * + * @param fn fillin: Returns the callback function pointer. + * + * @param context fillin: Returns the callback's private context. + * + * @see ubidi_setClassCallback + * @stable ICU 3.6 + */ +U_STABLE void U_EXPORT2 +ubidi_getClassCallback(UBiDi *pBiDi, UBiDiClassCallback **fn, const void **context); + +/** + * Take a UBiDi object containing the reordering + * information for a piece of text (one or more paragraphs) set by + * ubidi_setPara() or for a line of text set by + * ubidi_setLine() and write a reordered string to the + * destination buffer. + * + * This function preserves the integrity of characters with multiple + * code units and (optionally) modifier letters. + * Characters in RTL runs can be replaced by mirror-image characters + * in the destination buffer. Note that "real" mirroring has + * to be done in a rendering engine by glyph selection + * and that for many "mirrored" characters there are no + * Unicode characters as mirror-image equivalents. + * There are also options to insert or remove Bidi control + * characters; see the description of the destSize + * and options parameters and of the option bit flags. + * + * @param pBiDi A pointer to a UBiDi object that + * is set by ubidi_setPara() or + * ubidi_setLine() and contains the reordering + * information for the text that it was defined for, + * as well as a pointer to that text.

+ * The text was aliased (only the pointer was stored + * without copying the contents) and must not have been modified + * since the ubidi_setPara() call. + * + * @param dest A pointer to where the reordered text is to be copied. + * The source text and dest[destSize] + * must not overlap. + * + * @param destSize The size of the dest buffer, + * in number of UChars. + * If the UBIDI_INSERT_LRM_FOR_NUMERIC + * option is set, then the destination length could be + * as large as + * ubidi_getLength(pBiDi)+2*ubidi_countRuns(pBiDi). + * If the UBIDI_REMOVE_BIDI_CONTROLS option + * is set, then the destination length may be less than + * ubidi_getLength(pBiDi). + * If none of these options is set, then the destination length + * will be exactly ubidi_getProcessedLength(pBiDi). + * + * @param options A bit set of options for the reordering that control + * how the reordered text is written. + * The options include mirroring the characters on a code + * point basis and inserting LRM characters, which is used + * especially for transforming visually stored text + * to logically stored text (although this is still an + * imperfect implementation of an "inverse Bidi" algorithm + * because it uses the "forward Bidi" algorithm at its core). + * The available options are: + * #UBIDI_DO_MIRRORING, + * #UBIDI_INSERT_LRM_FOR_NUMERIC, + * #UBIDI_KEEP_BASE_COMBINING, + * #UBIDI_OUTPUT_REVERSE, + * #UBIDI_REMOVE_BIDI_CONTROLS + * + * @param pErrorCode must be a valid pointer to an error code value. + * + * @return The length of the output string. + * + * @see ubidi_getProcessedLength + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +ubidi_writeReordered(UBiDi *pBiDi, + UChar *dest, int32_t destSize, + uint16_t options, + UErrorCode *pErrorCode); + +/** + * Reverse a Right-To-Left run of Unicode text. + * + * This function preserves the integrity of characters with multiple + * code units and (optionally) modifier letters. + * Characters can be replaced by mirror-image characters + * in the destination buffer. Note that "real" mirroring has + * to be done in a rendering engine by glyph selection + * and that for many "mirrored" characters there are no + * Unicode characters as mirror-image equivalents. + * There are also options to insert or remove Bidi control + * characters. + * + * This function is the implementation for reversing RTL runs as part + * of ubidi_writeReordered(). For detailed descriptions + * of the parameters, see there. + * Since no Bidi controls are inserted here, the output string length + * will never exceed srcLength. + * + * @see ubidi_writeReordered + * + * @param src A pointer to the RTL run text. + * + * @param srcLength The length of the RTL run. + * + * @param dest A pointer to where the reordered text is to be copied. + * src[srcLength] and dest[destSize] + * must not overlap. + * + * @param destSize The size of the dest buffer, + * in number of UChars. + * If the UBIDI_REMOVE_BIDI_CONTROLS option + * is set, then the destination length may be less than + * srcLength. + * If this option is not set, then the destination length + * will be exactly srcLength. + * + * @param options A bit set of options for the reordering that control + * how the reordered text is written. + * See the options parameter in ubidi_writeReordered(). + * + * @param pErrorCode must be a valid pointer to an error code value. + * + * @return The length of the output string. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +ubidi_writeReverse(const UChar *src, int32_t srcLength, + UChar *dest, int32_t destSize, + uint16_t options, + UErrorCode *pErrorCode); + +/*#define BIDI_SAMPLE_CODE*/ +/*@}*/ + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/ubrk.h b/Build/source/libs/icu/icu-xetex/common/unicode/ubrk.h new file mode 100644 index 00000000000..d57ba37a1a2 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/ubrk.h @@ -0,0 +1,482 @@ +/* +****************************************************************************** +* Copyright (C) 1996-2007, International Business Machines Corporation and others. +* All Rights Reserved. +****************************************************************************** +*/ + +#ifndef UBRK_H +#define UBRK_H + +#include "unicode/utypes.h" +#include "unicode/uloc.h" +#include "unicode/utext.h" + +/** + * A text-break iterator. + * For usage in C programs. + */ +#ifndef UBRK_TYPEDEF_UBREAK_ITERATOR +# define UBRK_TYPEDEF_UBREAK_ITERATOR + /** + * Opaque type representing an ICU Break iterator object. + * @stable ICU 2.0 + */ + typedef void UBreakIterator; +#endif + +#if !UCONFIG_NO_BREAK_ITERATION + +#include "unicode/parseerr.h" + +/** + * \file + * \brief C API: BreakIterator + * + *

BreakIterator C API

+ * + * The BreakIterator C API defines methods for finding the location + * of boundaries in text. Pointer to a UBreakIterator maintain a + * current position and scan over text returning the index of characters + * where boundaries occur. + *

+ * Line boundary analysis determines where a text string can be broken + * when line-wrapping. The mechanism correctly handles punctuation and + * hyphenated words. + *

+ * Sentence boundary analysis allows selection with correct + * interpretation of periods within numbers and abbreviations, and + * trailing punctuation marks such as quotation marks and parentheses. + *

+ * Word boundary analysis is used by search and replace functions, as + * well as within text editing applications that allow the user to + * select words with a double click. Word selection provides correct + * interpretation of punctuation marks within and following + * words. Characters that are not part of a word, such as symbols or + * punctuation marks, have word-breaks on both sides. + *

+ * Character boundary analysis allows users to interact with + * characters as they expect to, for example, when moving the cursor + * through a text string. Character boundary analysis provides correct + * navigation of through character strings, regardless of how the + * character is stored. For example, an accented character might be + * stored as a base character and a diacritical mark. What users + * consider to be a character can differ between languages. + *

+ * Title boundary analysis locates all positions, + * typically starts of words, that should be set to Title Case + * when title casing the text. + *

+ * The text boundary positions are found according to the rules + * described in Unicode Standard Annex #29, Text Boundaries, and + * Unicode Standard Annex #14, Line Breaking Properties. These + * are available at http://www.unicode.org/reports/tr14/ and + * http://www.unicode.org/reports/tr29/. + *

+ * In addition to the plain C API defined in this header file, an + * object oriented C++ API with equivalent functionality is defined in the + * file brkiter.h. + *

+ * Code snippits illustrating the use of the Break Iterator APIs + * are available in the ICU User Guide, + * http://icu-project.org/userguide/boundaryAnalysis.html + * and in the sample program icu/source/samples/break/break.cpp" + */ + +/** The possible types of text boundaries. @stable ICU 2.0 */ +typedef enum UBreakIteratorType { + /** Character breaks @stable ICU 2.0 */ + UBRK_CHARACTER = 0, + /** Word breaks @stable ICU 2.0 */ + UBRK_WORD = 1, + /** Line breaks @stable ICU 2.0 */ + UBRK_LINE = 2, + /** Sentence breaks @stable ICU 2.0 */ + UBRK_SENTENCE = 3, + +#ifndef U_HIDE_DEPRECATED_API + /** + * Title Case breaks + * The iterator created using this type locates title boundaries as described for + * Unicode 3.2 only. For Unicode 4.0 and above title boundary iteration, + * please use Word Boundary iterator. + * + * @deprecated ICU 2.8 Use the word break iterator for titlecasing for Unicode 4 and later. + */ + UBRK_TITLE = 4, +#endif /* U_HIDE_DEPRECATED_API */ + UBRK_COUNT = 5 +} UBreakIteratorType; + +/** Value indicating all text boundaries have been returned. + * @stable ICU 2.0 + */ +#define UBRK_DONE ((int32_t) -1) + + +/** + * Enum constants for the word break tags returned by + * getRuleStatus(). A range of values is defined for each category of + * word, to allow for further subdivisions of a category in future releases. + * Applications should check for tag values falling within the range, rather + * than for single individual values. + * @stable ICU 2.2 +*/ +typedef enum UWordBreak { + /** Tag value for "words" that do not fit into any of other categories. + * Includes spaces and most punctuation. */ + UBRK_WORD_NONE = 0, + /** Upper bound for tags for uncategorized words. */ + UBRK_WORD_NONE_LIMIT = 100, + /** Tag value for words that appear to be numbers, lower limit. */ + UBRK_WORD_NUMBER = 100, + /** Tag value for words that appear to be numbers, upper limit. */ + UBRK_WORD_NUMBER_LIMIT = 200, + /** Tag value for words that contain letters, excluding + * hiragana, katakana or ideographic characters, lower limit. */ + UBRK_WORD_LETTER = 200, + /** Tag value for words containing letters, upper limit */ + UBRK_WORD_LETTER_LIMIT = 300, + /** Tag value for words containing kana characters, lower limit */ + UBRK_WORD_KANA = 300, + /** Tag value for words containing kana characters, upper limit */ + UBRK_WORD_KANA_LIMIT = 400, + /** Tag value for words containing ideographic characters, lower limit */ + UBRK_WORD_IDEO = 400, + /** Tag value for words containing ideographic characters, upper limit */ + UBRK_WORD_IDEO_LIMIT = 500 +} UWordBreak; + +/** + * Enum constants for the line break tags returned by getRuleStatus(). + * A range of values is defined for each category of + * word, to allow for further subdivisions of a category in future releases. + * Applications should check for tag values falling within the range, rather + * than for single individual values. + * @stable ICU 2.8 +*/ +typedef enum ULineBreakTag { + /** Tag value for soft line breaks, positions at which a line break + * is acceptable but not required */ + UBRK_LINE_SOFT = 0, + /** Upper bound for soft line breaks. */ + UBRK_LINE_SOFT_LIMIT = 100, + /** Tag value for a hard, or mandatory line break */ + UBRK_LINE_HARD = 100, + /** Upper bound for hard line breaks. */ + UBRK_LINE_HARD_LIMIT = 200 +} ULineBreakTag; + + + +/** + * Enum constants for the sentence break tags returned by getRuleStatus(). + * A range of values is defined for each category of + * sentence, to allow for further subdivisions of a category in future releases. + * Applications should check for tag values falling within the range, rather + * than for single individual values. + * @stable ICU 2.8 +*/ +typedef enum USentenceBreakTag { + /** Tag value for for sentences ending with a sentence terminator + * ('.', '?', '!', etc.) character, possibly followed by a + * hard separator (CR, LF, PS, etc.) + */ + UBRK_SENTENCE_TERM = 0, + /** Upper bound for tags for sentences ended by sentence terminators. */ + UBRK_SENTENCE_TERM_LIMIT = 100, + /** Tag value for for sentences that do not contain an ending + * sentence terminator ('.', '?', '!', etc.) character, but + * are ended only by a hard separator (CR, LF, PS, etc.) or end of input. + */ + UBRK_SENTENCE_SEP = 100, + /** Upper bound for tags for sentences ended by a separator. */ + UBRK_SENTENCE_SEP_LIMIT = 200 + /** Tag value for a hard, or mandatory line break */ +} USentenceBreakTag; + + +/** + * Open a new UBreakIterator for locating text boundaries for a specified locale. + * A UBreakIterator may be used for detecting character, line, word, + * and sentence breaks in text. + * @param type The type of UBreakIterator to open: one of UBRK_CHARACTER, UBRK_WORD, + * UBRK_LINE, UBRK_SENTENCE + * @param locale The locale specifying the text-breaking conventions. + * @param text The text to be iterated over. + * @param textLength The number of characters in text, or -1 if null-terminated. + * @param status A UErrorCode to receive any errors. + * @return A UBreakIterator for the specified locale. + * @see ubrk_openRules + * @stable ICU 2.0 + */ +U_STABLE UBreakIterator* U_EXPORT2 +ubrk_open(UBreakIteratorType type, + const char *locale, + const UChar *text, + int32_t textLength, + UErrorCode *status); + +/** + * Open a new UBreakIterator for locating text boundaries using specified breaking rules. + * The rule syntax is ... (TBD) + * @param rules A set of rules specifying the text breaking conventions. + * @param rulesLength The number of characters in rules, or -1 if null-terminated. + * @param text The text to be iterated over. May be null, in which case ubrk_setText() is + * used to specify the text to be iterated. + * @param textLength The number of characters in text, or -1 if null-terminated. + * @param parseErr Receives position and context information for any syntax errors + * detected while parsing the rules. + * @param status A UErrorCode to receive any errors. + * @return A UBreakIterator for the specified rules. + * @see ubrk_open + * @stable ICU 2.2 + */ +U_STABLE UBreakIterator* U_EXPORT2 +ubrk_openRules(const UChar *rules, + int32_t rulesLength, + const UChar *text, + int32_t textLength, + UParseError *parseErr, + UErrorCode *status); + +/** + * Thread safe cloning operation + * @param bi iterator to be cloned + * @param stackBuffer user allocated space for the new clone. If NULL new memory will be allocated. + * If buffer is not large enough, new memory will be allocated. + * Clients can use the U_BRK_SAFECLONE_BUFFERSIZE. This will probably be enough to avoid memory allocations. + * @param pBufferSize pointer to size of allocated space. + * If *pBufferSize == 0, a sufficient size for use in cloning will + * be returned ('pre-flighting') + * If *pBufferSize is not enough for a stack-based safe clone, + * new memory will be allocated. + * @param status to indicate whether the operation went on smoothly or there were errors + * An informational status value, U_SAFECLONE_ALLOCATED_ERROR, is used if any allocations were necessary. + * @return pointer to the new clone + * @stable ICU 2.0 + */ +U_STABLE UBreakIterator * U_EXPORT2 +ubrk_safeClone( + const UBreakIterator *bi, + void *stackBuffer, + int32_t *pBufferSize, + UErrorCode *status); + +/** + * A recommended size (in bytes) for the memory buffer to be passed to ubrk_saveClone(). + * @stable ICU 2.0 + */ +#define U_BRK_SAFECLONE_BUFFERSIZE 512 + +/** +* Close a UBreakIterator. +* Once closed, a UBreakIterator may no longer be used. +* @param bi The break iterator to close. + * @stable ICU 2.0 +*/ +U_STABLE void U_EXPORT2 +ubrk_close(UBreakIterator *bi); + +/** + * Sets an existing iterator to point to a new piece of text + * @param bi The iterator to use + * @param text The text to be set + * @param textLength The length of the text + * @param status The error code + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ubrk_setText(UBreakIterator* bi, + const UChar* text, + int32_t textLength, + UErrorCode* status); + + +/** + * Sets an existing iterator to point to a new piece of text + * @param bi The iterator to use + * @param text The text to be set. + * This function makes a shallow clone of the supplied UText. This means + * that the caller is free to immediately close or otherwise reuse the + * UText that was passed as a parameter, but that the underlying text itself + * must not be altered while being referenced by the break iterator. + * @param status The error code + * @stable ICU 3.4 + */ +U_STABLE void U_EXPORT2 +ubrk_setUText(UBreakIterator* bi, + UText* text, + UErrorCode* status); + + + +/** + * Determine the most recently-returned text boundary. + * + * @param bi The break iterator to use. + * @return The character index most recently returned by \ref ubrk_next, \ref ubrk_previous, + * \ref ubrk_first, or \ref ubrk_last. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +ubrk_current(const UBreakIterator *bi); + +/** + * Determine the text boundary following the current text boundary. + * + * @param bi The break iterator to use. + * @return The character index of the next text boundary, or UBRK_DONE + * if all text boundaries have been returned. + * @see ubrk_previous + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +ubrk_next(UBreakIterator *bi); + +/** + * Determine the text boundary preceding the current text boundary. + * + * @param bi The break iterator to use. + * @return The character index of the preceding text boundary, or UBRK_DONE + * if all text boundaries have been returned. + * @see ubrk_next + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +ubrk_previous(UBreakIterator *bi); + +/** + * Determine the index of the first character in the text being scanned. + * This is not always the same as index 0 of the text. + * @param bi The break iterator to use. + * @return The character index of the first character in the text being scanned. + * @see ubrk_last + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +ubrk_first(UBreakIterator *bi); + +/** + * Determine the index immediately beyond the last character in the text being + * scanned. + * This is not the same as the last character. + * @param bi The break iterator to use. + * @return The character offset immediately beyond the last character in the + * text being scanned. + * @see ubrk_first + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +ubrk_last(UBreakIterator *bi); + +/** + * Determine the text boundary preceding the specified offset. + * The value returned is always smaller than offset, or UBRK_DONE. + * @param bi The break iterator to use. + * @param offset The offset to begin scanning. + * @return The text boundary preceding offset, or UBRK_DONE. + * @see ubrk_following + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +ubrk_preceding(UBreakIterator *bi, + int32_t offset); + +/** + * Determine the text boundary following the specified offset. + * The value returned is always greater than offset, or UBRK_DONE. + * @param bi The break iterator to use. + * @param offset The offset to begin scanning. + * @return The text boundary following offset, or UBRK_DONE. + * @see ubrk_preceding + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +ubrk_following(UBreakIterator *bi, + int32_t offset); + +/** +* Get a locale for which text breaking information is available. +* A UBreakIterator in a locale returned by this function will perform the correct +* text breaking for the locale. +* @param index The index of the desired locale. +* @return A locale for which number text breaking information is available, or 0 if none. +* @see ubrk_countAvailable +* @stable ICU 2.0 +*/ +U_STABLE const char* U_EXPORT2 +ubrk_getAvailable(int32_t index); + +/** +* Determine how many locales have text breaking information available. +* This function is most useful as determining the loop ending condition for +* calls to \ref ubrk_getAvailable. +* @return The number of locales for which text breaking information is available. +* @see ubrk_getAvailable +* @stable ICU 2.0 +*/ +U_STABLE int32_t U_EXPORT2 +ubrk_countAvailable(void); + + +/** +* Returns true if the specfied position is a boundary position. As a side +* effect, leaves the iterator pointing to the first boundary position at +* or after "offset". +* @param bi The break iterator to use. +* @param offset the offset to check. +* @return True if "offset" is a boundary position. +* @stable ICU 2.0 +*/ +U_STABLE UBool U_EXPORT2 +ubrk_isBoundary(UBreakIterator *bi, int32_t offset); + +/** + * Return the status from the break rule that determined the most recently + * returned break position. The values appear in the rule source + * within brackets, {123}, for example. For rules that do not specify a + * status, a default value of 0 is returned. + *

+ * For word break iterators, the possible values are defined in enum UWordBreak. + * @stable ICU 2.2 + */ +U_STABLE int32_t U_EXPORT2 +ubrk_getRuleStatus(UBreakIterator *bi); + +/** + * Get the statuses from the break rules that determined the most recently + * returned break position. The values appear in the rule source + * within brackets, {123}, for example. The default status value for rules + * that do not explicitly provide one is zero. + *

+ * For word break iterators, the possible values are defined in enum UWordBreak. + * @param bi The break iterator to use + * @param fillInVec an array to be filled in with the status values. + * @param capacity the length of the supplied vector. A length of zero causes + * the function to return the number of status values, in the + * normal way, without attemtping to store any values. + * @param status receives error codes. + * @return The number of rule status values from rules that determined + * the most recent boundary returned by the break iterator. + * @stable ICU 3.0 + */ +U_STABLE int32_t U_EXPORT2 +ubrk_getRuleStatusVec(UBreakIterator *bi, int32_t *fillInVec, int32_t capacity, UErrorCode *status); + +/** + * Return the locale of the break iterator. You can choose between the valid and + * the actual locale. + * @param bi break iterator + * @param type locale type (valid or actual) + * @param status error code + * @return locale string + * @stable ICU 2.8 + */ +U_STABLE const char* U_EXPORT2 +ubrk_getLocaleByType(const UBreakIterator *bi, ULocDataLocaleType type, UErrorCode* status); + + +#endif /* #if !UCONFIG_NO_BREAK_ITERATION */ + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/ucasemap.h b/Build/source/libs/icu/icu-xetex/common/unicode/ucasemap.h new file mode 100644 index 00000000000..9f5880cd6a7 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/ucasemap.h @@ -0,0 +1,395 @@ +/* +******************************************************************************* +* +* Copyright (C) 2005-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: ucasemap.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2005may06 +* created by: Markus W. Scherer +* +* Case mapping service object and functions using it. +*/ + +#ifndef __UCASEMAP_H__ +#define __UCASEMAP_H__ + +#include "unicode/utypes.h" +#include "unicode/ustring.h" + +/** + * \file + * \brief C API: Unicode case mapping functions using a UCaseMap service object. + * + * The service object takes care of memory allocations, data loading, and setup + * for the attributes, as usual. + * + * Currently, the functionality provided here does not overlap with uchar.h + * and ustring.h, except for ucasemap_toTitle(). + * + * ucasemap_utf8XYZ() functions operate directly on UTF-8 strings. + */ + +/** + * UCaseMap is an opaque service object for newer ICU case mapping functions. + * Older functions did not use a service object. + * @stable ICU 3.4 + */ +struct UCaseMap; +typedef struct UCaseMap UCaseMap; /**< C typedef for struct UCaseMap. @stable ICU 3.4 */ + +/** + * Open a UCaseMap service object for a locale and a set of options. + * The locale ID and options are preprocessed so that functions using the + * service object need not process them in each call. + * + * @param locale ICU locale ID, used for language-dependent + * upper-/lower-/title-casing according to the Unicode standard. + * Usual semantics: ""=root, NULL=default locale, etc. + * @param options Options bit set, used for case folding and string comparisons. + * Same flags as for u_foldCase(), u_strFoldCase(), + * u_strCaseCompare(), etc. + * Use 0 or U_FOLD_CASE_DEFAULT for default behavior. + * @param pErrorCode Must be a valid pointer to an error code value, + * which must not indicate a failure before the function call. + * @return Pointer to a UCaseMap service object, if successful. + * + * @see U_FOLD_CASE_DEFAULT + * @see U_FOLD_CASE_EXCLUDE_SPECIAL_I + * @see U_TITLECASE_NO_LOWERCASE + * @see U_TITLECASE_NO_BREAK_ADJUSTMENT + * @stable ICU 3.4 + */ +U_STABLE UCaseMap * U_EXPORT2 +ucasemap_open(const char *locale, uint32_t options, UErrorCode *pErrorCode); + +/** + * Close a UCaseMap service object. + * @param csm Object to be closed. + * @stable ICU 3.4 + */ +U_STABLE void U_EXPORT2 +ucasemap_close(UCaseMap *csm); + +/** + * Get the locale ID that is used for language-dependent case mappings. + * @param csm UCaseMap service object. + * @return locale ID + * @stable ICU 3.4 + */ +U_STABLE const char * U_EXPORT2 +ucasemap_getLocale(const UCaseMap *csm); + +/** + * Get the options bit set that is used for case folding and string comparisons. + * @param csm UCaseMap service object. + * @return options bit set + * @stable ICU 3.4 + */ +U_STABLE uint32_t U_EXPORT2 +ucasemap_getOptions(const UCaseMap *csm); + +/** + * Set the locale ID that is used for language-dependent case mappings. + * + * @param csm UCaseMap service object. + * @param locale Locale ID, see ucasemap_open(). + * @param pErrorCode Must be a valid pointer to an error code value, + * which must not indicate a failure before the function call. + * + * @see ucasemap_open + * @stable ICU 3.4 + */ +U_STABLE void U_EXPORT2 +ucasemap_setLocale(UCaseMap *csm, const char *locale, UErrorCode *pErrorCode); + +/** + * Set the options bit set that is used for case folding and string comparisons. + * + * @param csm UCaseMap service object. + * @param options Options bit set, see ucasemap_open(). + * @param pErrorCode Must be a valid pointer to an error code value, + * which must not indicate a failure before the function call. + * + * @see ucasemap_open + * @stable ICU 3.4 + */ +U_STABLE void U_EXPORT2 +ucasemap_setOptions(UCaseMap *csm, uint32_t options, UErrorCode *pErrorCode); + +#ifndef U_HIDE_DRAFT_API + +/** + * Do not lowercase non-initial parts of words when titlecasing. + * Option bit for titlecasing APIs that take an options bit set. + * + * By default, titlecasing will titlecase the first cased character + * of a word and lowercase all other characters. + * With this option, the other characters will not be modified. + * + * @see ucasemap_setOptions + * @see ucasemap_toTitle + * @see ucasemap_utf8ToTitle + * @see UnicodeString::toTitle + * @draft ICU 3.8 + */ +#define U_TITLECASE_NO_LOWERCASE 0x100 + +/** + * Do not adjust the titlecasing indexes from BreakIterator::next() indexes; + * titlecase exactly the characters at breaks from the iterator. + * Option bit for titlecasing APIs that take an options bit set. + * + * By default, titlecasing will take each break iterator index, + * adjust it by looking for the next cased character, and titlecase that one. + * Other characters are lowercased. + * + * This follows Unicode 4 & 5 section 3.13 Default Case Operations: + * + * R3 toTitlecase(X): Find the word boundaries based on Unicode Standard Annex + * #29, "Text Boundaries." Between each pair of word boundaries, find the first + * cased character F. If F exists, map F to default_title(F); then map each + * subsequent character C to default_lower(C). + * + * @see ucasemap_setOptions + * @see ucasemap_toTitle + * @see ucasemap_utf8ToTitle + * @see UnicodeString::toTitle + * @see U_TITLECASE_NO_LOWERCASE + * @draft ICU 3.8 + */ +#define U_TITLECASE_NO_BREAK_ADJUSTMENT 0x200 + +#endif + +#if !UCONFIG_NO_BREAK_ITERATION + +/** + * Get the break iterator that is used for titlecasing. + * Do not modify the returned break iterator. + * @param csm UCaseMap service object. + * @return titlecasing break iterator + * @draft ICU 3.8 + */ +U_DRAFT const UBreakIterator * U_EXPORT2 +ucasemap_getBreakIterator(const UCaseMap *csm); + +/** + * Set the break iterator that is used for titlecasing. + * The UCaseMap service object releases a previously set break iterator + * and "adopts" this new one, taking ownership of it. + * It will be released in a subsequent call to ucasemap_setBreakIterator() + * or ucasemap_close(). + * + * Break iterator operations are not thread-safe. Therefore, titlecasing + * functions use non-const UCaseMap objects. It is not possible to titlecase + * strings concurrently using the same UCaseMap. + * + * @param csm UCaseMap service object. + * @param iterToAdopt Break iterator to be adopted for titlecasing. + * @param pErrorCode Must be a valid pointer to an error code value, + * which must not indicate a failure before the function call. + * + * @see ucasemap_toTitle + * @see ucasemap_utf8ToTitle + * @draft ICU 3.8 + */ +U_DRAFT void U_EXPORT2 +ucasemap_setBreakIterator(UCaseMap *csm, UBreakIterator *iterToAdopt, UErrorCode *pErrorCode); + +/** + * Titlecase a UTF-16 string. This function is almost a duplicate of u_strToTitle(), + * except that it takes ucasemap_setOptions() into account and has performance + * advantages from being able to use a UCaseMap object for multiple case mapping + * operations, saving setup time. + * + * Casing is locale-dependent and context-sensitive. + * Titlecasing uses a break iterator to find the first characters of words + * that are to be titlecased. It titlecases those characters and lowercases + * all others. (This can be modified with ucasemap_setOptions().) + * + * The titlecase break iterator can be provided to customize for arbitrary + * styles, using rules and dictionaries beyond the standard iterators. + * It may be more efficient to always provide an iterator to avoid + * opening and closing one for each string. + * The standard titlecase iterator for the root locale implements the + * algorithm of Unicode TR 21. + * + * This function uses only the setText(), first() and next() methods of the + * provided break iterator. + * + * The result may be longer or shorter than the original. + * The source string and the destination buffer must not overlap. + * + * @param csm UCaseMap service object. + * @param dest A buffer for the result string. The result will be NUL-terminated if + * the buffer is large enough. + * The contents is undefined in case of failure. + * @param destCapacity The size of the buffer (number of bytes). If it is 0, then + * dest may be NULL and the function will only return the length of the result + * without writing any of the result string. + * @param src The original string. + * @param srcLength The length of the original string. If -1, then src must be NUL-terminated. + * @param pErrorCode Must be a valid pointer to an error code value, + * which must not indicate a failure before the function call. + * @return The length of the result string, if successful - or in case of a buffer overflow, + * in which case it will be greater than destCapacity. + * + * @see u_strToTitle + * @draft ICU 3.8 + */ +U_DRAFT int32_t U_EXPORT2 +ucasemap_toTitle(UCaseMap *csm, + UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + UErrorCode *pErrorCode); + +#endif + +/** + * Lowercase the characters in a UTF-8 string. + * Casing is locale-dependent and context-sensitive. + * The result may be longer or shorter than the original. + * The source string and the destination buffer must not overlap. + * + * @param csm UCaseMap service object. + * @param dest A buffer for the result string. The result will be NUL-terminated if + * the buffer is large enough. + * The contents is undefined in case of failure. + * @param destCapacity The size of the buffer (number of bytes). If it is 0, then + * dest may be NULL and the function will only return the length of the result + * without writing any of the result string. + * @param src The original string. + * @param srcLength The length of the original string. If -1, then src must be NUL-terminated. + * @param pErrorCode Must be a valid pointer to an error code value, + * which must not indicate a failure before the function call. + * @return The length of the result string, if successful - or in case of a buffer overflow, + * in which case it will be greater than destCapacity. + * + * @see u_strToLower + * @stable ICU 3.4 + */ +U_STABLE int32_t U_EXPORT2 +ucasemap_utf8ToLower(const UCaseMap *csm, + char *dest, int32_t destCapacity, + const char *src, int32_t srcLength, + UErrorCode *pErrorCode); + +/** + * Uppercase the characters in a UTF-8 string. + * Casing is locale-dependent and context-sensitive. + * The result may be longer or shorter than the original. + * The source string and the destination buffer must not overlap. + * + * @param csm UCaseMap service object. + * @param dest A buffer for the result string. The result will be NUL-terminated if + * the buffer is large enough. + * The contents is undefined in case of failure. + * @param destCapacity The size of the buffer (number of bytes). If it is 0, then + * dest may be NULL and the function will only return the length of the result + * without writing any of the result string. + * @param src The original string. + * @param srcLength The length of the original string. If -1, then src must be NUL-terminated. + * @param pErrorCode Must be a valid pointer to an error code value, + * which must not indicate a failure before the function call. + * @return The length of the result string, if successful - or in case of a buffer overflow, + * in which case it will be greater than destCapacity. + * + * @see u_strToUpper + * @stable ICU 3.4 + */ +U_STABLE int32_t U_EXPORT2 +ucasemap_utf8ToUpper(const UCaseMap *csm, + char *dest, int32_t destCapacity, + const char *src, int32_t srcLength, + UErrorCode *pErrorCode); + +#if !UCONFIG_NO_BREAK_ITERATION + +/** + * Titlecase a UTF-8 string. + * Casing is locale-dependent and context-sensitive. + * Titlecasing uses a break iterator to find the first characters of words + * that are to be titlecased. It titlecases those characters and lowercases + * all others. (This can be modified with ucasemap_setOptions().) + * + * The titlecase break iterator can be provided to customize for arbitrary + * styles, using rules and dictionaries beyond the standard iterators. + * It may be more efficient to always provide an iterator to avoid + * opening and closing one for each string. + * The standard titlecase iterator for the root locale implements the + * algorithm of Unicode TR 21. + * + * This function uses only the setText(), first() and next() methods of the + * provided break iterator. + * + * The result may be longer or shorter than the original. + * The source string and the destination buffer must not overlap. + * + * @param csm UCaseMap service object. + * @param dest A buffer for the result string. The result will be NUL-terminated if + * the buffer is large enough. + * The contents is undefined in case of failure. + * @param destCapacity The size of the buffer (number of bytes). If it is 0, then + * dest may be NULL and the function will only return the length of the result + * without writing any of the result string. + * @param src The original string. + * @param srcLength The length of the original string. If -1, then src must be NUL-terminated. + * @param pErrorCode Must be a valid pointer to an error code value, + * which must not indicate a failure before the function call. + * @return The length of the result string, if successful - or in case of a buffer overflow, + * in which case it will be greater than destCapacity. + * + * @see u_strToTitle + * @see U_TITLECASE_NO_LOWERCASE + * @see U_TITLECASE_NO_BREAK_ADJUSTMENT + * @draft ICU 3.8 + */ +U_DRAFT int32_t U_EXPORT2 +ucasemap_utf8ToTitle(UCaseMap *csm, + char *dest, int32_t destCapacity, + const char *src, int32_t srcLength, + UErrorCode *pErrorCode); + +#endif + +/** + * Case-fold the characters in a UTF-8 string. + * Case-folding is locale-independent and not context-sensitive, + * but there is an option for whether to include or exclude mappings for dotted I + * and dotless i that are marked with 'I' in CaseFolding.txt. + * The result may be longer or shorter than the original. + * The source string and the destination buffer must not overlap. + * + * @param csm UCaseMap service object. + * @param dest A buffer for the result string. The result will be NUL-terminated if + * the buffer is large enough. + * The contents is undefined in case of failure. + * @param destCapacity The size of the buffer (number of bytes). If it is 0, then + * dest may be NULL and the function will only return the length of the result + * without writing any of the result string. + * @param src The original string. + * @param srcLength The length of the original string. If -1, then src must be NUL-terminated. + * @param pErrorCode Must be a valid pointer to an error code value, + * which must not indicate a failure before the function call. + * @return The length of the result string, if successful - or in case of a buffer overflow, + * in which case it will be greater than destCapacity. + * + * @see u_strFoldCase + * @see ucasemap_setOptions + * @see U_FOLD_CASE_DEFAULT + * @see U_FOLD_CASE_EXCLUDE_SPECIAL_I + * @draft ICU 3.8 + */ +U_DRAFT int32_t U_EXPORT2 +ucasemap_utf8FoldCase(const UCaseMap *csm, + char *dest, int32_t destCapacity, + const char *src, int32_t srcLength, + UErrorCode *pErrorCode); + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/ucat.h b/Build/source/libs/icu/icu-xetex/common/unicode/ucat.h new file mode 100644 index 00000000000..ad9f0373a30 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/ucat.h @@ -0,0 +1,158 @@ +/* +********************************************************************** +* Copyright (c) 2003-2004, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Author: Alan Liu +* Created: March 19 2003 +* Since: ICU 2.6 +********************************************************************** +*/ +#ifndef UCAT_H +#define UCAT_H + +#include "unicode/utypes.h" +#include "unicode/ures.h" + +/** + * \file + * \brief C API: Message Catalog Wrappers + * + * This C API provides look-alike functions that deliberately resemble + * the POSIX catopen, catclose, and catgets functions. The underlying + * implementation is in terms of ICU resource bundles, rather than + * POSIX message catalogs. + * + * The ICU resource bundles obey standard ICU inheritance policies. + * To facilitate this, sets and messages are flattened into one tier. + * This is done by creating resource bundle keys of the form + * <set_num>%<msg_num> where set_num is the set number and msg_num is + * the message number, formatted as decimal strings. + * + * Example: Consider a message catalog containing two sets: + * + * Set 1: Message 4 = "Good morning." + * Message 5 = "Good afternoon." + * Message 7 = "Good evening." + * Message 8 = "Good night." + * Set 4: Message 14 = "Please " + * Message 19 = "Thank you." + * Message 20 = "Sincerely," + * + * The ICU resource bundle source file would, assuming it is named + * "greet.txt", would look like this: + * + * greet + * { + * 1%4 { "Good morning." } + * 1%5 { "Good afternoon." } + * 1%7 { "Good evening." } + * 1%8 { "Good night." } + * + * 4%14 { "Please " } + * 4%19 { "Thank you." } + * 4%20 { "Sincerely," } + * } + * + * The catgets function is commonly used in combination with functions + * like printf and strftime. ICU components like message format can + * be used instead, although they use a different format syntax. + * There is an ICU package, icuio, that provides some of + * the POSIX-style formatting API. + */ + +U_CDECL_BEGIN + +/** + * An ICU message catalog descriptor, analogous to nl_catd. + * + * @stable ICU 2.6 + */ +typedef UResourceBundle* u_nl_catd; + +/** + * Open and return an ICU message catalog descriptor. The descriptor + * may be passed to u_catgets() to retrieve localized strings. + * + * @param name string containing the full path pointing to the + * directory where the resources reside followed by the package name + * e.g. "/usr/resource/my_app/resources/guimessages" on a Unix system. + * If NULL, ICU default data files will be used. + * + * Unlike POSIX, environment variables are not interpolated within the + * name. + * + * @param locale the locale for which we want to open the resource. If + * NULL, the default ICU locale will be used (see uloc_getDefault). If + * strlen(locale) == 0, the root locale will be used. + * + * @param ec input/output error code. Upon output, + * U_USING_FALLBACK_WARNING indicates that a fallback locale was + * used. For example, 'de_CH' was requested, but nothing was found + * there, so 'de' was used. U_USING_DEFAULT_WARNING indicates that the + * default locale data or root locale data was used; neither the + * requested locale nor any of its fallback locales were found. + * + * @return a message catalog descriptor that may be passed to + * u_catgets(). If the ec parameter indicates success, then the caller + * is responsible for calling u_catclose() to close the message + * catalog. If the ec parameter indicates failure, then NULL will be + * returned. + * + * @stable ICU 2.6 + */ +U_STABLE u_nl_catd U_EXPORT2 +u_catopen(const char* name, const char* locale, UErrorCode* ec); + +/** + * Close an ICU message catalog, given its descriptor. + * + * @param catd a message catalog descriptor to be closed. May be NULL, + * in which case no action is taken. + * + * @stable ICU 2.6 + */ +U_STABLE void U_EXPORT2 +u_catclose(u_nl_catd catd); + +/** + * Retrieve a localized string from an ICU message catalog. + * + * @param catd a message catalog descriptor returned by u_catopen. + * + * @param set_num the message catalog set number. Sets need not be + * numbered consecutively. + * + * @param msg_num the message catalog message number within the + * set. Messages need not be numbered consecutively. + * + * @param s the default string. This is returned if the string + * specified by the set_num and msg_num is not found. It must be + * zero-terminated. + * + * @param len fill-in parameter to receive the length of the result. + * May be NULL, in which case it is ignored. + * + * @param ec input/output error code. May be U_USING_FALLBACK_WARNING + * or U_USING_DEFAULT_WARNING. U_MISSING_RESOURCE_ERROR indicates that + * the set_num/msg_num tuple does not specify a valid message string + * in this catalog. + * + * @return a pointer to a zero-terminated UChar array which lives in + * an internal buffer area, typically a memory mapped/DLL file. The + * caller must NOT delete this pointer. If the call is unsuccessful + * for any reason, then s is returned. This includes the situation in + * which ec indicates a failing error code upon entry to this + * function. + * + * @stable ICU 2.6 + */ +U_STABLE const UChar* U_EXPORT2 +u_catgets(u_nl_catd catd, int32_t set_num, int32_t msg_num, + const UChar* s, + int32_t* len, UErrorCode* ec); + +U_CDECL_END + +#endif /*UCAT_H*/ +/*eof*/ diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/uchar.h b/Build/source/libs/icu/icu-xetex/common/unicode/uchar.h new file mode 100644 index 00000000000..b702ec636c2 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/uchar.h @@ -0,0 +1,3013 @@ +/* +********************************************************************** +* Copyright (C) 1997-2007, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* +* File UCHAR.H +* +* Modification History: +* +* Date Name Description +* 04/02/97 aliu Creation. +* 03/29/99 helena Updated for C APIs. +* 4/15/99 Madhu Updated for C Implementation and Javadoc +* 5/20/99 Madhu Added the function u_getVersion() +* 8/19/1999 srl Upgraded scripts to Unicode 3.0 +* 8/27/1999 schererm UCharDirection constants: U_... +* 11/11/1999 weiv added u_isalnum(), cleaned comments +* 01/11/2000 helena Renamed u_getVersion to u_getUnicodeVersion(). +****************************************************************************** +*/ + +#ifndef UCHAR_H +#define UCHAR_H + +#include "unicode/utypes.h" + +U_CDECL_BEGIN + +/*==========================================================================*/ +/* Unicode version number */ +/*==========================================================================*/ +/** + * Unicode version number, default for the current ICU version. + * The actual Unicode Character Database (UCD) data is stored in uprops.dat + * and may be generated from UCD files from a different Unicode version. + * Call u_getUnicodeVersion to get the actual Unicode version of the data. + * + * @see u_getUnicodeVersion + * @stable ICU 2.0 + */ +#define U_UNICODE_VERSION "5.0" + +/** + * \file + * \brief C API: Unicode Properties + * + * This C API provides low-level access to the Unicode Character Database. + * In addition to raw property values, some convenience functions calculate + * derived properties, for example for Java-style programming. + * + * Unicode assigns each code point (not just assigned character) values for + * many properties. + * Most of them are simple boolean flags, or constants from a small enumerated list. + * For some properties, values are strings or other relatively more complex types. + * + * For more information see + * "About the Unicode Character Database" (http://www.unicode.org/ucd/) + * and the ICU User Guide chapter on Properties (http://icu-project.org/userguide/properties.html). + * + * Many functions are designed to match java.lang.Character functions. + * See the individual function documentation, + * and see the JDK 1.4 java.lang.Character documentation + * at http://java.sun.com/j2se/1.4/docs/api/java/lang/Character.html + * + * There are also functions that provide easy migration from C/POSIX functions + * like isblank(). Their use is generally discouraged because the C/POSIX + * standards do not define their semantics beyond the ASCII range, which means + * that different implementations exhibit very different behavior. + * Instead, Unicode properties should be used directly. + * + * There are also only a few, broad C/POSIX character classes, and they tend + * to be used for conflicting purposes. For example, the "isalpha()" class + * is sometimes used to determine word boundaries, while a more sophisticated + * approach would at least distinguish initial letters from continuation + * characters (the latter including combining marks). + * (In ICU, BreakIterator is the most sophisticated API for word boundaries.) + * Another example: There is no "istitle()" class for titlecase characters. + * + * ICU 3.4 and later provides API access for all twelve C/POSIX character classes. + * ICU implements them according to the Standard Recommendations in + * Annex C: Compatibility Properties of UTS #18 Unicode Regular Expressions + * (http://www.unicode.org/reports/tr18/#Compatibility_Properties). + * + * API access for C/POSIX character classes is as follows: + * - alpha: u_isUAlphabetic(c) or u_hasBinaryProperty(c, UCHAR_ALPHABETIC) + * - lower: u_isULowercase(c) or u_hasBinaryProperty(c, UCHAR_LOWERCASE) + * - upper: u_isUUppercase(c) or u_hasBinaryProperty(c, UCHAR_UPPERCASE) + * - punct: u_ispunct(c) + * - digit: u_isdigit(c) or u_charType(c)==U_DECIMAL_DIGIT_NUMBER + * - xdigit: u_isxdigit(c) or u_hasBinaryProperty(c, UCHAR_POSIX_XDIGIT) + * - alnum: u_hasBinaryProperty(c, UCHAR_POSIX_ALNUM) + * - space: u_isUWhiteSpace(c) or u_hasBinaryProperty(c, UCHAR_WHITE_SPACE) + * - blank: u_isblank(c) or u_hasBinaryProperty(c, UCHAR_POSIX_BLANK) + * - cntrl: u_charType(c)==U_CONTROL_CHAR + * - graph: u_hasBinaryProperty(c, UCHAR_POSIX_GRAPH) + * - print: u_hasBinaryProperty(c, UCHAR_POSIX_PRINT) + * + * Note: Some of the u_isxyz() functions in uchar.h predate, and do not match, + * the Standard Recommendations in UTS #18. Instead, they match Java + * functions according to their API documentation. + * + * \htmlonly + * The C/POSIX character classes are also available in UnicodeSet patterns, + * using patterns like [:graph:] or \p{graph}. + * \endhtmlonly + * + * Note: There are several ICU whitespace functions. + * Comparison: + * - u_isUWhiteSpace=UCHAR_WHITE_SPACE: Unicode White_Space property; + * most of general categories "Z" (separators) + most whitespace ISO controls + * (including no-break spaces, but excluding IS1..IS4 and ZWSP) + * - u_isWhitespace: Java isWhitespace; Z + whitespace ISO controls but excluding no-break spaces + * - u_isJavaSpaceChar: Java isSpaceChar; just Z (including no-break spaces) + * - u_isspace: Z + whitespace ISO controls (including no-break spaces) + * - u_isblank: "horizontal spaces" = TAB + Zs - ZWSP + */ + +/** + * Constants. + */ + +/** The lowest Unicode code point value. Code points are non-negative. @stable ICU 2.0 */ +#define UCHAR_MIN_VALUE 0 + +/** + * The highest Unicode code point value (scalar value) according to + * The Unicode Standard. This is a 21-bit value (20.1 bits, rounded up). + * For a single character, UChar32 is a simple type that can hold any code point value. + * + * @see UChar32 + * @stable ICU 2.0 + */ +#define UCHAR_MAX_VALUE 0x10ffff + +/** + * Get a single-bit bit set (a flag) from a bit number 0..31. + * @stable ICU 2.1 + */ +#define U_MASK(x) ((uint32_t)1<<(x)) + +/* + * !! Note: Several comments in this file are machine-read by the + * genpname tool. These comments describe the correspondence between + * icu enum constants and UCD entities. Do not delete them. Update + * these comments as needed. + * + * Any comment of the form "/ *[name]* /" (spaces added) is such + * a comment. + * + * The U_JG_* and U_GC_*_MASK constants are matched by their symbolic + * name, which must match PropertyValueAliases.txt. + */ + +/** + * Selection constants for Unicode properties. + * These constants are used in functions like u_hasBinaryProperty to select + * one of the Unicode properties. + * + * The properties APIs are intended to reflect Unicode properties as defined + * in the Unicode Character Database (UCD) and Unicode Technical Reports (UTR). + * For details about the properties see http://www.unicode.org/ucd/ . + * For names of Unicode properties see the UCD file PropertyAliases.txt. + * + * Important: If ICU is built with UCD files from Unicode versions below, e.g., 3.2, + * then properties marked with "new in Unicode 3.2" are not or not fully available. + * Check u_getUnicodeVersion to be sure. + * + * @see u_hasBinaryProperty + * @see u_getIntPropertyValue + * @see u_getUnicodeVersion + * @stable ICU 2.1 + */ +typedef enum UProperty { + /* See note !!. Comments of the form "Binary property Dash", + "Enumerated property Script", "Double property Numeric_Value", + and "String property Age" are read by genpname. */ + + /* Note: Place UCHAR_ALPHABETIC before UCHAR_BINARY_START so that + debuggers display UCHAR_ALPHABETIC as the symbolic name for 0, + rather than UCHAR_BINARY_START. Likewise for other *_START + identifiers. */ + + /** Binary property Alphabetic. Same as u_isUAlphabetic, different from u_isalpha. + Lu+Ll+Lt+Lm+Lo+Nl+Other_Alphabetic @stable ICU 2.1 */ + UCHAR_ALPHABETIC=0, + /** First constant for binary Unicode properties. @stable ICU 2.1 */ + UCHAR_BINARY_START=UCHAR_ALPHABETIC, + /** Binary property ASCII_Hex_Digit. 0-9 A-F a-f @stable ICU 2.1 */ + UCHAR_ASCII_HEX_DIGIT=1, + /** Binary property Bidi_Control. + Format controls which have specific functions + in the Bidi Algorithm. @stable ICU 2.1 */ + UCHAR_BIDI_CONTROL=2, + /** Binary property Bidi_Mirrored. + Characters that may change display in RTL text. + Same as u_isMirrored. + See Bidi Algorithm, UTR 9. @stable ICU 2.1 */ + UCHAR_BIDI_MIRRORED=3, + /** Binary property Dash. Variations of dashes. @stable ICU 2.1 */ + UCHAR_DASH=4, + /** Binary property Default_Ignorable_Code_Point (new in Unicode 3.2). + Ignorable in most processing. + <2060..206F, FFF0..FFFB, E0000..E0FFF>+Other_Default_Ignorable_Code_Point+(Cf+Cc+Cs-White_Space) @stable ICU 2.1 */ + UCHAR_DEFAULT_IGNORABLE_CODE_POINT=5, + /** Binary property Deprecated (new in Unicode 3.2). + The usage of deprecated characters is strongly discouraged. @stable ICU 2.1 */ + UCHAR_DEPRECATED=6, + /** Binary property Diacritic. Characters that linguistically modify + the meaning of another character to which they apply. @stable ICU 2.1 */ + UCHAR_DIACRITIC=7, + /** Binary property Extender. + Extend the value or shape of a preceding alphabetic character, + e.g., length and iteration marks. @stable ICU 2.1 */ + UCHAR_EXTENDER=8, + /** Binary property Full_Composition_Exclusion. + CompositionExclusions.txt+Singleton Decompositions+ + Non-Starter Decompositions. @stable ICU 2.1 */ + UCHAR_FULL_COMPOSITION_EXCLUSION=9, + /** Binary property Grapheme_Base (new in Unicode 3.2). + For programmatic determination of grapheme cluster boundaries. + [0..10FFFF]-Cc-Cf-Cs-Co-Cn-Zl-Zp-Grapheme_Link-Grapheme_Extend-CGJ @stable ICU 2.1 */ + UCHAR_GRAPHEME_BASE=10, + /** Binary property Grapheme_Extend (new in Unicode 3.2). + For programmatic determination of grapheme cluster boundaries. + Me+Mn+Mc+Other_Grapheme_Extend-Grapheme_Link-CGJ @stable ICU 2.1 */ + UCHAR_GRAPHEME_EXTEND=11, + /** Binary property Grapheme_Link (new in Unicode 3.2). + For programmatic determination of grapheme cluster boundaries. @stable ICU 2.1 */ + UCHAR_GRAPHEME_LINK=12, + /** Binary property Hex_Digit. + Characters commonly used for hexadecimal numbers. @stable ICU 2.1 */ + UCHAR_HEX_DIGIT=13, + /** Binary property Hyphen. Dashes used to mark connections + between pieces of words, plus the Katakana middle dot. @stable ICU 2.1 */ + UCHAR_HYPHEN=14, + /** Binary property ID_Continue. + Characters that can continue an identifier. + DerivedCoreProperties.txt also says "NOTE: Cf characters should be filtered out." + ID_Start+Mn+Mc+Nd+Pc @stable ICU 2.1 */ + UCHAR_ID_CONTINUE=15, + /** Binary property ID_Start. + Characters that can start an identifier. + Lu+Ll+Lt+Lm+Lo+Nl @stable ICU 2.1 */ + UCHAR_ID_START=16, + /** Binary property Ideographic. + CJKV ideographs. @stable ICU 2.1 */ + UCHAR_IDEOGRAPHIC=17, + /** Binary property IDS_Binary_Operator (new in Unicode 3.2). + For programmatic determination of + Ideographic Description Sequences. @stable ICU 2.1 */ + UCHAR_IDS_BINARY_OPERATOR=18, + /** Binary property IDS_Trinary_Operator (new in Unicode 3.2). + For programmatic determination of + Ideographic Description Sequences. @stable ICU 2.1 */ + UCHAR_IDS_TRINARY_OPERATOR=19, + /** Binary property Join_Control. + Format controls for cursive joining and ligation. @stable ICU 2.1 */ + UCHAR_JOIN_CONTROL=20, + /** Binary property Logical_Order_Exception (new in Unicode 3.2). + Characters that do not use logical order and + require special handling in most processing. @stable ICU 2.1 */ + UCHAR_LOGICAL_ORDER_EXCEPTION=21, + /** Binary property Lowercase. Same as u_isULowercase, different from u_islower. + Ll+Other_Lowercase @stable ICU 2.1 */ + UCHAR_LOWERCASE=22, + /** Binary property Math. Sm+Other_Math @stable ICU 2.1 */ + UCHAR_MATH=23, + /** Binary property Noncharacter_Code_Point. + Code points that are explicitly defined as illegal + for the encoding of characters. @stable ICU 2.1 */ + UCHAR_NONCHARACTER_CODE_POINT=24, + /** Binary property Quotation_Mark. @stable ICU 2.1 */ + UCHAR_QUOTATION_MARK=25, + /** Binary property Radical (new in Unicode 3.2). + For programmatic determination of + Ideographic Description Sequences. @stable ICU 2.1 */ + UCHAR_RADICAL=26, + /** Binary property Soft_Dotted (new in Unicode 3.2). + Characters with a "soft dot", like i or j. + An accent placed on these characters causes + the dot to disappear. @stable ICU 2.1 */ + UCHAR_SOFT_DOTTED=27, + /** Binary property Terminal_Punctuation. + Punctuation characters that generally mark + the end of textual units. @stable ICU 2.1 */ + UCHAR_TERMINAL_PUNCTUATION=28, + /** Binary property Unified_Ideograph (new in Unicode 3.2). + For programmatic determination of + Ideographic Description Sequences. @stable ICU 2.1 */ + UCHAR_UNIFIED_IDEOGRAPH=29, + /** Binary property Uppercase. Same as u_isUUppercase, different from u_isupper. + Lu+Other_Uppercase @stable ICU 2.1 */ + UCHAR_UPPERCASE=30, + /** Binary property White_Space. + Same as u_isUWhiteSpace, different from u_isspace and u_isWhitespace. + Space characters+TAB+CR+LF-ZWSP-ZWNBSP @stable ICU 2.1 */ + UCHAR_WHITE_SPACE=31, + /** Binary property XID_Continue. + ID_Continue modified to allow closure under + normalization forms NFKC and NFKD. @stable ICU 2.1 */ + UCHAR_XID_CONTINUE=32, + /** Binary property XID_Start. ID_Start modified to allow + closure under normalization forms NFKC and NFKD. @stable ICU 2.1 */ + UCHAR_XID_START=33, + /** Binary property Case_Sensitive. Either the source of a case + mapping or _in_ the target of a case mapping. Not the same as + the general category Cased_Letter. @stable ICU 2.6 */ + UCHAR_CASE_SENSITIVE=34, + /** Binary property STerm (new in Unicode 4.0.1). + Sentence Terminal. Used in UAX #29: Text Boundaries + (http://www.unicode.org/reports/tr29/) + @stable ICU 3.0 */ + UCHAR_S_TERM=35, + /** Binary property Variation_Selector (new in Unicode 4.0.1). + Indicates all those characters that qualify as Variation Selectors. + For details on the behavior of these characters, + see StandardizedVariants.html and 15.6 Variation Selectors. + @stable ICU 3.0 */ + UCHAR_VARIATION_SELECTOR=36, + /** Binary property NFD_Inert. + ICU-specific property for characters that are inert under NFD, + i.e., they do not interact with adjacent characters. + Used for example in normalizing transforms in incremental mode + to find the boundary of safely normalizable text despite possible + text additions. + + There is one such property per normalization form. + These properties are computed as follows - an inert character is: + a) unassigned, or ALL of the following: + b) of combining class 0. + c) not decomposed by this normalization form. + AND if NFC or NFKC, + d) can never compose with a previous character. + e) can never compose with a following character. + f) can never change if another character is added. + Example: a-breve might satisfy all but f, but if you + add an ogonek it changes to a-ogonek + breve + + See also com.ibm.text.UCD.NFSkippable in the ICU4J repository, + and icu/source/common/unormimp.h . + @stable ICU 3.0 */ + UCHAR_NFD_INERT=37, + /** Binary property NFKD_Inert. + ICU-specific property for characters that are inert under NFKD, + i.e., they do not interact with adjacent characters. + Used for example in normalizing transforms in incremental mode + to find the boundary of safely normalizable text despite possible + text additions. + @see UCHAR_NFD_INERT + @stable ICU 3.0 */ + UCHAR_NFKD_INERT=38, + /** Binary property NFC_Inert. + ICU-specific property for characters that are inert under NFC, + i.e., they do not interact with adjacent characters. + Used for example in normalizing transforms in incremental mode + to find the boundary of safely normalizable text despite possible + text additions. + @see UCHAR_NFD_INERT + @stable ICU 3.0 */ + UCHAR_NFC_INERT=39, + /** Binary property NFKC_Inert. + ICU-specific property for characters that are inert under NFKC, + i.e., they do not interact with adjacent characters. + Used for example in normalizing transforms in incremental mode + to find the boundary of safely normalizable text despite possible + text additions. + @see UCHAR_NFD_INERT + @stable ICU 3.0 */ + UCHAR_NFKC_INERT=40, + /** Binary Property Segment_Starter. + ICU-specific property for characters that are starters in terms of + Unicode normalization and combining character sequences. + They have ccc=0 and do not occur in non-initial position of the + canonical decomposition of any character + (like " in NFD(a-umlaut) and a Jamo T in an NFD(Hangul LVT)). + ICU uses this property for segmenting a string for generating a set of + canonically equivalent strings, e.g. for canonical closure while + processing collation tailoring rules. + @stable ICU 3.0 */ + UCHAR_SEGMENT_STARTER=41, + /** Binary property Pattern_Syntax (new in Unicode 4.1). + See UAX #31 Identifier and Pattern Syntax + (http://www.unicode.org/reports/tr31/) + @stable ICU 3.4 */ + UCHAR_PATTERN_SYNTAX=42, + /** Binary property Pattern_White_Space (new in Unicode 4.1). + See UAX #31 Identifier and Pattern Syntax + (http://www.unicode.org/reports/tr31/) + @stable ICU 3.4 */ + UCHAR_PATTERN_WHITE_SPACE=43, + /** Binary property alnum (a C/POSIX character class). + Implemented according to the UTS #18 Annex C Standard Recommendation. + See the uchar.h file documentation. + @stable ICU 3.4 */ + UCHAR_POSIX_ALNUM=44, + /** Binary property blank (a C/POSIX character class). + Implemented according to the UTS #18 Annex C Standard Recommendation. + See the uchar.h file documentation. + @stable ICU 3.4 */ + UCHAR_POSIX_BLANK=45, + /** Binary property graph (a C/POSIX character class). + Implemented according to the UTS #18 Annex C Standard Recommendation. + See the uchar.h file documentation. + @stable ICU 3.4 */ + UCHAR_POSIX_GRAPH=46, + /** Binary property print (a C/POSIX character class). + Implemented according to the UTS #18 Annex C Standard Recommendation. + See the uchar.h file documentation. + @stable ICU 3.4 */ + UCHAR_POSIX_PRINT=47, + /** Binary property xdigit (a C/POSIX character class). + Implemented according to the UTS #18 Annex C Standard Recommendation. + See the uchar.h file documentation. + @stable ICU 3.4 */ + UCHAR_POSIX_XDIGIT=48, + /** One more than the last constant for binary Unicode properties. @stable ICU 2.1 */ + UCHAR_BINARY_LIMIT=49, + + /** Enumerated property Bidi_Class. + Same as u_charDirection, returns UCharDirection values. @stable ICU 2.2 */ + UCHAR_BIDI_CLASS=0x1000, + /** First constant for enumerated/integer Unicode properties. @stable ICU 2.2 */ + UCHAR_INT_START=UCHAR_BIDI_CLASS, + /** Enumerated property Block. + Same as ublock_getCode, returns UBlockCode values. @stable ICU 2.2 */ + UCHAR_BLOCK=0x1001, + /** Enumerated property Canonical_Combining_Class. + Same as u_getCombiningClass, returns 8-bit numeric values. @stable ICU 2.2 */ + UCHAR_CANONICAL_COMBINING_CLASS=0x1002, + /** Enumerated property Decomposition_Type. + Returns UDecompositionType values. @stable ICU 2.2 */ + UCHAR_DECOMPOSITION_TYPE=0x1003, + /** Enumerated property East_Asian_Width. + See http://www.unicode.org/reports/tr11/ + Returns UEastAsianWidth values. @stable ICU 2.2 */ + UCHAR_EAST_ASIAN_WIDTH=0x1004, + /** Enumerated property General_Category. + Same as u_charType, returns UCharCategory values. @stable ICU 2.2 */ + UCHAR_GENERAL_CATEGORY=0x1005, + /** Enumerated property Joining_Group. + Returns UJoiningGroup values. @stable ICU 2.2 */ + UCHAR_JOINING_GROUP=0x1006, + /** Enumerated property Joining_Type. + Returns UJoiningType values. @stable ICU 2.2 */ + UCHAR_JOINING_TYPE=0x1007, + /** Enumerated property Line_Break. + Returns ULineBreak values. @stable ICU 2.2 */ + UCHAR_LINE_BREAK=0x1008, + /** Enumerated property Numeric_Type. + Returns UNumericType values. @stable ICU 2.2 */ + UCHAR_NUMERIC_TYPE=0x1009, + /** Enumerated property Script. + Same as uscript_getScript, returns UScriptCode values. @stable ICU 2.2 */ + UCHAR_SCRIPT=0x100A, + /** Enumerated property Hangul_Syllable_Type, new in Unicode 4. + Returns UHangulSyllableType values. @stable ICU 2.6 */ + UCHAR_HANGUL_SYLLABLE_TYPE=0x100B, + /** Enumerated property NFD_Quick_Check. + Returns UNormalizationCheckResult values. @stable ICU 3.0 */ + UCHAR_NFD_QUICK_CHECK=0x100C, + /** Enumerated property NFKD_Quick_Check. + Returns UNormalizationCheckResult values. @stable ICU 3.0 */ + UCHAR_NFKD_QUICK_CHECK=0x100D, + /** Enumerated property NFC_Quick_Check. + Returns UNormalizationCheckResult values. @stable ICU 3.0 */ + UCHAR_NFC_QUICK_CHECK=0x100E, + /** Enumerated property NFKC_Quick_Check. + Returns UNormalizationCheckResult values. @stable ICU 3.0 */ + UCHAR_NFKC_QUICK_CHECK=0x100F, + /** Enumerated property Lead_Canonical_Combining_Class. + ICU-specific property for the ccc of the first code point + of the decomposition, or lccc(c)=ccc(NFD(c)[0]). + Useful for checking for canonically ordered text; + see UNORM_FCD and http://www.unicode.org/notes/tn5/#FCD . + Returns 8-bit numeric values like UCHAR_CANONICAL_COMBINING_CLASS. @stable ICU 3.0 */ + UCHAR_LEAD_CANONICAL_COMBINING_CLASS=0x1010, + /** Enumerated property Trail_Canonical_Combining_Class. + ICU-specific property for the ccc of the last code point + of the decomposition, or tccc(c)=ccc(NFD(c)[last]). + Useful for checking for canonically ordered text; + see UNORM_FCD and http://www.unicode.org/notes/tn5/#FCD . + Returns 8-bit numeric values like UCHAR_CANONICAL_COMBINING_CLASS. @stable ICU 3.0 */ + UCHAR_TRAIL_CANONICAL_COMBINING_CLASS=0x1011, + /** Enumerated property Grapheme_Cluster_Break (new in Unicode 4.1). + Used in UAX #29: Text Boundaries + (http://www.unicode.org/reports/tr29/) + Returns UGraphemeClusterBreak values. @stable ICU 3.4 */ + UCHAR_GRAPHEME_CLUSTER_BREAK=0x1012, + /** Enumerated property Sentence_Break (new in Unicode 4.1). + Used in UAX #29: Text Boundaries + (http://www.unicode.org/reports/tr29/) + Returns USentenceBreak values. @stable ICU 3.4 */ + UCHAR_SENTENCE_BREAK=0x1013, + /** Enumerated property Word_Break (new in Unicode 4.1). + Used in UAX #29: Text Boundaries + (http://www.unicode.org/reports/tr29/) + Returns UWordBreakValues values. @stable ICU 3.4 */ + UCHAR_WORD_BREAK=0x1014, + /** One more than the last constant for enumerated/integer Unicode properties. @stable ICU 2.2 */ + UCHAR_INT_LIMIT=0x1015, + + /** Bitmask property General_Category_Mask. + This is the General_Category property returned as a bit mask. + When used in u_getIntPropertyValue(c), same as U_MASK(u_charType(c)), + returns bit masks for UCharCategory values where exactly one bit is set. + When used with u_getPropertyValueName() and u_getPropertyValueEnum(), + a multi-bit mask is used for sets of categories like "Letters". + Mask values should be cast to uint32_t. + @stable ICU 2.4 */ + UCHAR_GENERAL_CATEGORY_MASK=0x2000, + /** First constant for bit-mask Unicode properties. @stable ICU 2.4 */ + UCHAR_MASK_START=UCHAR_GENERAL_CATEGORY_MASK, + /** One more than the last constant for bit-mask Unicode properties. @stable ICU 2.4 */ + UCHAR_MASK_LIMIT=0x2001, + + /** Double property Numeric_Value. + Corresponds to u_getNumericValue. @stable ICU 2.4 */ + UCHAR_NUMERIC_VALUE=0x3000, + /** First constant for double Unicode properties. @stable ICU 2.4 */ + UCHAR_DOUBLE_START=UCHAR_NUMERIC_VALUE, + /** One more than the last constant for double Unicode properties. @stable ICU 2.4 */ + UCHAR_DOUBLE_LIMIT=0x3001, + + /** String property Age. + Corresponds to u_charAge. @stable ICU 2.4 */ + UCHAR_AGE=0x4000, + /** First constant for string Unicode properties. @stable ICU 2.4 */ + UCHAR_STRING_START=UCHAR_AGE, + /** String property Bidi_Mirroring_Glyph. + Corresponds to u_charMirror. @stable ICU 2.4 */ + UCHAR_BIDI_MIRRORING_GLYPH=0x4001, + /** String property Case_Folding. + Corresponds to u_strFoldCase in ustring.h. @stable ICU 2.4 */ + UCHAR_CASE_FOLDING=0x4002, + /** String property ISO_Comment. + Corresponds to u_getISOComment. @stable ICU 2.4 */ + UCHAR_ISO_COMMENT=0x4003, + /** String property Lowercase_Mapping. + Corresponds to u_strToLower in ustring.h. @stable ICU 2.4 */ + UCHAR_LOWERCASE_MAPPING=0x4004, + /** String property Name. + Corresponds to u_charName. @stable ICU 2.4 */ + UCHAR_NAME=0x4005, + /** String property Simple_Case_Folding. + Corresponds to u_foldCase. @stable ICU 2.4 */ + UCHAR_SIMPLE_CASE_FOLDING=0x4006, + /** String property Simple_Lowercase_Mapping. + Corresponds to u_tolower. @stable ICU 2.4 */ + UCHAR_SIMPLE_LOWERCASE_MAPPING=0x4007, + /** String property Simple_Titlecase_Mapping. + Corresponds to u_totitle. @stable ICU 2.4 */ + UCHAR_SIMPLE_TITLECASE_MAPPING=0x4008, + /** String property Simple_Uppercase_Mapping. + Corresponds to u_toupper. @stable ICU 2.4 */ + UCHAR_SIMPLE_UPPERCASE_MAPPING=0x4009, + /** String property Titlecase_Mapping. + Corresponds to u_strToTitle in ustring.h. @stable ICU 2.4 */ + UCHAR_TITLECASE_MAPPING=0x400A, + /** String property Unicode_1_Name. + Corresponds to u_charName. @stable ICU 2.4 */ + UCHAR_UNICODE_1_NAME=0x400B, + /** String property Uppercase_Mapping. + Corresponds to u_strToUpper in ustring.h. @stable ICU 2.4 */ + UCHAR_UPPERCASE_MAPPING=0x400C, + /** One more than the last constant for string Unicode properties. @stable ICU 2.4 */ + UCHAR_STRING_LIMIT=0x400D, + + /** Represents a nonexistent or invalid property or property value. @stable ICU 2.4 */ + UCHAR_INVALID_CODE = -1 +} UProperty; + +/** + * Data for enumerated Unicode general category types. + * See http://www.unicode.org/Public/UNIDATA/UnicodeData.html . + * @stable ICU 2.0 + */ +typedef enum UCharCategory +{ + /** See note !!. Comments of the form "Cn" are read by genpname. */ + + /** Non-category for unassigned and non-character code points. @stable ICU 2.0 */ + U_UNASSIGNED = 0, + /** Cn "Other, Not Assigned (no characters in [UnicodeData.txt] have this property)" (same as U_UNASSIGNED!) @stable ICU 2.0 */ + U_GENERAL_OTHER_TYPES = 0, + /** Lu @stable ICU 2.0 */ + U_UPPERCASE_LETTER = 1, + /** Ll @stable ICU 2.0 */ + U_LOWERCASE_LETTER = 2, + /** Lt @stable ICU 2.0 */ + U_TITLECASE_LETTER = 3, + /** Lm @stable ICU 2.0 */ + U_MODIFIER_LETTER = 4, + /** Lo @stable ICU 2.0 */ + U_OTHER_LETTER = 5, + /** Mn @stable ICU 2.0 */ + U_NON_SPACING_MARK = 6, + /** Me @stable ICU 2.0 */ + U_ENCLOSING_MARK = 7, + /** Mc @stable ICU 2.0 */ + U_COMBINING_SPACING_MARK = 8, + /** Nd @stable ICU 2.0 */ + U_DECIMAL_DIGIT_NUMBER = 9, + /** Nl @stable ICU 2.0 */ + U_LETTER_NUMBER = 10, + /** No @stable ICU 2.0 */ + U_OTHER_NUMBER = 11, + /** Zs @stable ICU 2.0 */ + U_SPACE_SEPARATOR = 12, + /** Zl @stable ICU 2.0 */ + U_LINE_SEPARATOR = 13, + /** Zp @stable ICU 2.0 */ + U_PARAGRAPH_SEPARATOR = 14, + /** Cc @stable ICU 2.0 */ + U_CONTROL_CHAR = 15, + /** Cf @stable ICU 2.0 */ + U_FORMAT_CHAR = 16, + /** Co @stable ICU 2.0 */ + U_PRIVATE_USE_CHAR = 17, + /** Cs @stable ICU 2.0 */ + U_SURROGATE = 18, + /** Pd @stable ICU 2.0 */ + U_DASH_PUNCTUATION = 19, + /** Ps @stable ICU 2.0 */ + U_START_PUNCTUATION = 20, + /** Pe @stable ICU 2.0 */ + U_END_PUNCTUATION = 21, + /** Pc @stable ICU 2.0 */ + U_CONNECTOR_PUNCTUATION = 22, + /** Po @stable ICU 2.0 */ + U_OTHER_PUNCTUATION = 23, + /** Sm @stable ICU 2.0 */ + U_MATH_SYMBOL = 24, + /** Sc @stable ICU 2.0 */ + U_CURRENCY_SYMBOL = 25, + /** Sk @stable ICU 2.0 */ + U_MODIFIER_SYMBOL = 26, + /** So @stable ICU 2.0 */ + U_OTHER_SYMBOL = 27, + /** Pi @stable ICU 2.0 */ + U_INITIAL_PUNCTUATION = 28, + /** Pf @stable ICU 2.0 */ + U_FINAL_PUNCTUATION = 29, + /** One higher than the last enum UCharCategory constant. @stable ICU 2.0 */ + U_CHAR_CATEGORY_COUNT +} UCharCategory; + +/** + * U_GC_XX_MASK constants are bit flags corresponding to Unicode + * general category values. + * For each category, the nth bit is set if the numeric value of the + * corresponding UCharCategory constant is n. + * + * There are also some U_GC_Y_MASK constants for groups of general categories + * like L for all letter categories. + * + * @see u_charType + * @see U_GET_GC_MASK + * @see UCharCategory + * @stable ICU 2.1 + */ +#define U_GC_CN_MASK U_MASK(U_GENERAL_OTHER_TYPES) + +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_LU_MASK U_MASK(U_UPPERCASE_LETTER) +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_LL_MASK U_MASK(U_LOWERCASE_LETTER) +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_LT_MASK U_MASK(U_TITLECASE_LETTER) +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_LM_MASK U_MASK(U_MODIFIER_LETTER) +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_LO_MASK U_MASK(U_OTHER_LETTER) + +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_MN_MASK U_MASK(U_NON_SPACING_MARK) +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_ME_MASK U_MASK(U_ENCLOSING_MARK) +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_MC_MASK U_MASK(U_COMBINING_SPACING_MARK) + +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_ND_MASK U_MASK(U_DECIMAL_DIGIT_NUMBER) +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_NL_MASK U_MASK(U_LETTER_NUMBER) +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_NO_MASK U_MASK(U_OTHER_NUMBER) + +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_ZS_MASK U_MASK(U_SPACE_SEPARATOR) +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_ZL_MASK U_MASK(U_LINE_SEPARATOR) +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_ZP_MASK U_MASK(U_PARAGRAPH_SEPARATOR) + +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_CC_MASK U_MASK(U_CONTROL_CHAR) +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_CF_MASK U_MASK(U_FORMAT_CHAR) +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_CO_MASK U_MASK(U_PRIVATE_USE_CHAR) +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_CS_MASK U_MASK(U_SURROGATE) + +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_PD_MASK U_MASK(U_DASH_PUNCTUATION) +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_PS_MASK U_MASK(U_START_PUNCTUATION) +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_PE_MASK U_MASK(U_END_PUNCTUATION) +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_PC_MASK U_MASK(U_CONNECTOR_PUNCTUATION) +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_PO_MASK U_MASK(U_OTHER_PUNCTUATION) + +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_SM_MASK U_MASK(U_MATH_SYMBOL) +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_SC_MASK U_MASK(U_CURRENCY_SYMBOL) +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_SK_MASK U_MASK(U_MODIFIER_SYMBOL) +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_SO_MASK U_MASK(U_OTHER_SYMBOL) + +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_PI_MASK U_MASK(U_INITIAL_PUNCTUATION) +/** Mask constant for a UCharCategory. @stable ICU 2.1 */ +#define U_GC_PF_MASK U_MASK(U_FINAL_PUNCTUATION) + + +/** Mask constant for multiple UCharCategory bits (L Letters). @stable ICU 2.1 */ +#define U_GC_L_MASK \ + (U_GC_LU_MASK|U_GC_LL_MASK|U_GC_LT_MASK|U_GC_LM_MASK|U_GC_LO_MASK) + +/** Mask constant for multiple UCharCategory bits (LC Cased Letters). @stable ICU 2.1 */ +#define U_GC_LC_MASK \ + (U_GC_LU_MASK|U_GC_LL_MASK|U_GC_LT_MASK) + +/** Mask constant for multiple UCharCategory bits (M Marks). @stable ICU 2.1 */ +#define U_GC_M_MASK (U_GC_MN_MASK|U_GC_ME_MASK|U_GC_MC_MASK) + +/** Mask constant for multiple UCharCategory bits (N Numbers). @stable ICU 2.1 */ +#define U_GC_N_MASK (U_GC_ND_MASK|U_GC_NL_MASK|U_GC_NO_MASK) + +/** Mask constant for multiple UCharCategory bits (Z Separators). @stable ICU 2.1 */ +#define U_GC_Z_MASK (U_GC_ZS_MASK|U_GC_ZL_MASK|U_GC_ZP_MASK) + +/** Mask constant for multiple UCharCategory bits (C Others). @stable ICU 2.1 */ +#define U_GC_C_MASK \ + (U_GC_CN_MASK|U_GC_CC_MASK|U_GC_CF_MASK|U_GC_CO_MASK|U_GC_CS_MASK) + +/** Mask constant for multiple UCharCategory bits (P Punctuation). @stable ICU 2.1 */ +#define U_GC_P_MASK \ + (U_GC_PD_MASK|U_GC_PS_MASK|U_GC_PE_MASK|U_GC_PC_MASK|U_GC_PO_MASK| \ + U_GC_PI_MASK|U_GC_PF_MASK) + +/** Mask constant for multiple UCharCategory bits (S Symbols). @stable ICU 2.1 */ +#define U_GC_S_MASK (U_GC_SM_MASK|U_GC_SC_MASK|U_GC_SK_MASK|U_GC_SO_MASK) + +/** + * This specifies the language directional property of a character set. + * @stable ICU 2.0 + */ +typedef enum UCharDirection { + /** See note !!. Comments of the form "EN" are read by genpname. */ + + /** L @stable ICU 2.0 */ + U_LEFT_TO_RIGHT = 0, + /** R @stable ICU 2.0 */ + U_RIGHT_TO_LEFT = 1, + /** EN @stable ICU 2.0 */ + U_EUROPEAN_NUMBER = 2, + /** ES @stable ICU 2.0 */ + U_EUROPEAN_NUMBER_SEPARATOR = 3, + /** ET @stable ICU 2.0 */ + U_EUROPEAN_NUMBER_TERMINATOR = 4, + /** AN @stable ICU 2.0 */ + U_ARABIC_NUMBER = 5, + /** CS @stable ICU 2.0 */ + U_COMMON_NUMBER_SEPARATOR = 6, + /** B @stable ICU 2.0 */ + U_BLOCK_SEPARATOR = 7, + /** S @stable ICU 2.0 */ + U_SEGMENT_SEPARATOR = 8, + /** WS @stable ICU 2.0 */ + U_WHITE_SPACE_NEUTRAL = 9, + /** ON @stable ICU 2.0 */ + U_OTHER_NEUTRAL = 10, + /** LRE @stable ICU 2.0 */ + U_LEFT_TO_RIGHT_EMBEDDING = 11, + /** LRO @stable ICU 2.0 */ + U_LEFT_TO_RIGHT_OVERRIDE = 12, + /** AL @stable ICU 2.0 */ + U_RIGHT_TO_LEFT_ARABIC = 13, + /** RLE @stable ICU 2.0 */ + U_RIGHT_TO_LEFT_EMBEDDING = 14, + /** RLO @stable ICU 2.0 */ + U_RIGHT_TO_LEFT_OVERRIDE = 15, + /** PDF @stable ICU 2.0 */ + U_POP_DIRECTIONAL_FORMAT = 16, + /** NSM @stable ICU 2.0 */ + U_DIR_NON_SPACING_MARK = 17, + /** BN @stable ICU 2.0 */ + U_BOUNDARY_NEUTRAL = 18, + /** @stable ICU 2.0 */ + U_CHAR_DIRECTION_COUNT +} UCharDirection; + +/** + * Constants for Unicode blocks, see the Unicode Data file Blocks.txt + * @stable ICU 2.0 + */ +enum UBlockCode { + + /** New No_Block value in Unicode 4. @stable ICU 2.6 */ + UBLOCK_NO_BLOCK = 0, /*[none]*/ /* Special range indicating No_Block */ + + /** @stable ICU 2.0 */ + UBLOCK_BASIC_LATIN = 1, /*[0000]*/ /*See note !!*/ + + /** @stable ICU 2.0 */ + UBLOCK_LATIN_1_SUPPLEMENT=2, /*[0080]*/ + + /** @stable ICU 2.0 */ + UBLOCK_LATIN_EXTENDED_A =3, /*[0100]*/ + + /** @stable ICU 2.0 */ + UBLOCK_LATIN_EXTENDED_B =4, /*[0180]*/ + + /** @stable ICU 2.0 */ + UBLOCK_IPA_EXTENSIONS =5, /*[0250]*/ + + /** @stable ICU 2.0 */ + UBLOCK_SPACING_MODIFIER_LETTERS =6, /*[02B0]*/ + + /** @stable ICU 2.0 */ + UBLOCK_COMBINING_DIACRITICAL_MARKS =7, /*[0300]*/ + + /** + * Unicode 3.2 renames this block to "Greek and Coptic". + * @stable ICU 2.0 + */ + UBLOCK_GREEK =8, /*[0370]*/ + + /** @stable ICU 2.0 */ + UBLOCK_CYRILLIC =9, /*[0400]*/ + + /** @stable ICU 2.0 */ + UBLOCK_ARMENIAN =10, /*[0530]*/ + + /** @stable ICU 2.0 */ + UBLOCK_HEBREW =11, /*[0590]*/ + + /** @stable ICU 2.0 */ + UBLOCK_ARABIC =12, /*[0600]*/ + + /** @stable ICU 2.0 */ + UBLOCK_SYRIAC =13, /*[0700]*/ + + /** @stable ICU 2.0 */ + UBLOCK_THAANA =14, /*[0780]*/ + + /** @stable ICU 2.0 */ + UBLOCK_DEVANAGARI =15, /*[0900]*/ + + /** @stable ICU 2.0 */ + UBLOCK_BENGALI =16, /*[0980]*/ + + /** @stable ICU 2.0 */ + UBLOCK_GURMUKHI =17, /*[0A00]*/ + + /** @stable ICU 2.0 */ + UBLOCK_GUJARATI =18, /*[0A80]*/ + + /** @stable ICU 2.0 */ + UBLOCK_ORIYA =19, /*[0B00]*/ + + /** @stable ICU 2.0 */ + UBLOCK_TAMIL =20, /*[0B80]*/ + + /** @stable ICU 2.0 */ + UBLOCK_TELUGU =21, /*[0C00]*/ + + /** @stable ICU 2.0 */ + UBLOCK_KANNADA =22, /*[0C80]*/ + + /** @stable ICU 2.0 */ + UBLOCK_MALAYALAM =23, /*[0D00]*/ + + /** @stable ICU 2.0 */ + UBLOCK_SINHALA =24, /*[0D80]*/ + + /** @stable ICU 2.0 */ + UBLOCK_THAI =25, /*[0E00]*/ + + /** @stable ICU 2.0 */ + UBLOCK_LAO =26, /*[0E80]*/ + + /** @stable ICU 2.0 */ + UBLOCK_TIBETAN =27, /*[0F00]*/ + + /** @stable ICU 2.0 */ + UBLOCK_MYANMAR =28, /*[1000]*/ + + /** @stable ICU 2.0 */ + UBLOCK_GEORGIAN =29, /*[10A0]*/ + + /** @stable ICU 2.0 */ + UBLOCK_HANGUL_JAMO =30, /*[1100]*/ + + /** @stable ICU 2.0 */ + UBLOCK_ETHIOPIC =31, /*[1200]*/ + + /** @stable ICU 2.0 */ + UBLOCK_CHEROKEE =32, /*[13A0]*/ + + /** @stable ICU 2.0 */ + UBLOCK_UNIFIED_CANADIAN_ABORIGINAL_SYLLABICS =33, /*[1400]*/ + + /** @stable ICU 2.0 */ + UBLOCK_OGHAM =34, /*[1680]*/ + + /** @stable ICU 2.0 */ + UBLOCK_RUNIC =35, /*[16A0]*/ + + /** @stable ICU 2.0 */ + UBLOCK_KHMER =36, /*[1780]*/ + + /** @stable ICU 2.0 */ + UBLOCK_MONGOLIAN =37, /*[1800]*/ + + /** @stable ICU 2.0 */ + UBLOCK_LATIN_EXTENDED_ADDITIONAL =38, /*[1E00]*/ + + /** @stable ICU 2.0 */ + UBLOCK_GREEK_EXTENDED =39, /*[1F00]*/ + + /** @stable ICU 2.0 */ + UBLOCK_GENERAL_PUNCTUATION =40, /*[2000]*/ + + /** @stable ICU 2.0 */ + UBLOCK_SUPERSCRIPTS_AND_SUBSCRIPTS =41, /*[2070]*/ + + /** @stable ICU 2.0 */ + UBLOCK_CURRENCY_SYMBOLS =42, /*[20A0]*/ + + /** + * Unicode 3.2 renames this block to "Combining Diacritical Marks for Symbols". + * @stable ICU 2.0 + */ + UBLOCK_COMBINING_MARKS_FOR_SYMBOLS =43, /*[20D0]*/ + + /** @stable ICU 2.0 */ + UBLOCK_LETTERLIKE_SYMBOLS =44, /*[2100]*/ + + /** @stable ICU 2.0 */ + UBLOCK_NUMBER_FORMS =45, /*[2150]*/ + + /** @stable ICU 2.0 */ + UBLOCK_ARROWS =46, /*[2190]*/ + + /** @stable ICU 2.0 */ + UBLOCK_MATHEMATICAL_OPERATORS =47, /*[2200]*/ + + /** @stable ICU 2.0 */ + UBLOCK_MISCELLANEOUS_TECHNICAL =48, /*[2300]*/ + + /** @stable ICU 2.0 */ + UBLOCK_CONTROL_PICTURES =49, /*[2400]*/ + + /** @stable ICU 2.0 */ + UBLOCK_OPTICAL_CHARACTER_RECOGNITION =50, /*[2440]*/ + + /** @stable ICU 2.0 */ + UBLOCK_ENCLOSED_ALPHANUMERICS =51, /*[2460]*/ + + /** @stable ICU 2.0 */ + UBLOCK_BOX_DRAWING =52, /*[2500]*/ + + /** @stable ICU 2.0 */ + UBLOCK_BLOCK_ELEMENTS =53, /*[2580]*/ + + /** @stable ICU 2.0 */ + UBLOCK_GEOMETRIC_SHAPES =54, /*[25A0]*/ + + /** @stable ICU 2.0 */ + UBLOCK_MISCELLANEOUS_SYMBOLS =55, /*[2600]*/ + + /** @stable ICU 2.0 */ + UBLOCK_DINGBATS =56, /*[2700]*/ + + /** @stable ICU 2.0 */ + UBLOCK_BRAILLE_PATTERNS =57, /*[2800]*/ + + /** @stable ICU 2.0 */ + UBLOCK_CJK_RADICALS_SUPPLEMENT =58, /*[2E80]*/ + + /** @stable ICU 2.0 */ + UBLOCK_KANGXI_RADICALS =59, /*[2F00]*/ + + /** @stable ICU 2.0 */ + UBLOCK_IDEOGRAPHIC_DESCRIPTION_CHARACTERS =60, /*[2FF0]*/ + + /** @stable ICU 2.0 */ + UBLOCK_CJK_SYMBOLS_AND_PUNCTUATION =61, /*[3000]*/ + + /** @stable ICU 2.0 */ + UBLOCK_HIRAGANA =62, /*[3040]*/ + + /** @stable ICU 2.0 */ + UBLOCK_KATAKANA =63, /*[30A0]*/ + + /** @stable ICU 2.0 */ + UBLOCK_BOPOMOFO =64, /*[3100]*/ + + /** @stable ICU 2.0 */ + UBLOCK_HANGUL_COMPATIBILITY_JAMO =65, /*[3130]*/ + + /** @stable ICU 2.0 */ + UBLOCK_KANBUN =66, /*[3190]*/ + + /** @stable ICU 2.0 */ + UBLOCK_BOPOMOFO_EXTENDED =67, /*[31A0]*/ + + /** @stable ICU 2.0 */ + UBLOCK_ENCLOSED_CJK_LETTERS_AND_MONTHS =68, /*[3200]*/ + + /** @stable ICU 2.0 */ + UBLOCK_CJK_COMPATIBILITY =69, /*[3300]*/ + + /** @stable ICU 2.0 */ + UBLOCK_CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A =70, /*[3400]*/ + + /** @stable ICU 2.0 */ + UBLOCK_CJK_UNIFIED_IDEOGRAPHS =71, /*[4E00]*/ + + /** @stable ICU 2.0 */ + UBLOCK_YI_SYLLABLES =72, /*[A000]*/ + + /** @stable ICU 2.0 */ + UBLOCK_YI_RADICALS =73, /*[A490]*/ + + /** @stable ICU 2.0 */ + UBLOCK_HANGUL_SYLLABLES =74, /*[AC00]*/ + + /** @stable ICU 2.0 */ + UBLOCK_HIGH_SURROGATES =75, /*[D800]*/ + + /** @stable ICU 2.0 */ + UBLOCK_HIGH_PRIVATE_USE_SURROGATES =76, /*[DB80]*/ + + /** @stable ICU 2.0 */ + UBLOCK_LOW_SURROGATES =77, /*[DC00]*/ + + /** + * Same as UBLOCK_PRIVATE_USE_AREA. + * Until Unicode 3.1.1, the corresponding block name was "Private Use", + * and multiple code point ranges had this block. + * Unicode 3.2 renames the block for the BMP PUA to "Private Use Area" and + * adds separate blocks for the supplementary PUAs. + * + * @stable ICU 2.0 + */ + UBLOCK_PRIVATE_USE = 78, + /** + * Same as UBLOCK_PRIVATE_USE. + * Until Unicode 3.1.1, the corresponding block name was "Private Use", + * and multiple code point ranges had this block. + * Unicode 3.2 renames the block for the BMP PUA to "Private Use Area" and + * adds separate blocks for the supplementary PUAs. + * + * @stable ICU 2.0 + */ + UBLOCK_PRIVATE_USE_AREA =UBLOCK_PRIVATE_USE, /*[E000]*/ + + /** @stable ICU 2.0 */ + UBLOCK_CJK_COMPATIBILITY_IDEOGRAPHS =79, /*[F900]*/ + + /** @stable ICU 2.0 */ + UBLOCK_ALPHABETIC_PRESENTATION_FORMS =80, /*[FB00]*/ + + /** @stable ICU 2.0 */ + UBLOCK_ARABIC_PRESENTATION_FORMS_A =81, /*[FB50]*/ + + /** @stable ICU 2.0 */ + UBLOCK_COMBINING_HALF_MARKS =82, /*[FE20]*/ + + /** @stable ICU 2.0 */ + UBLOCK_CJK_COMPATIBILITY_FORMS =83, /*[FE30]*/ + + /** @stable ICU 2.0 */ + UBLOCK_SMALL_FORM_VARIANTS =84, /*[FE50]*/ + + /** @stable ICU 2.0 */ + UBLOCK_ARABIC_PRESENTATION_FORMS_B =85, /*[FE70]*/ + + /** @stable ICU 2.0 */ + UBLOCK_SPECIALS =86, /*[FFF0]*/ + + /** @stable ICU 2.0 */ + UBLOCK_HALFWIDTH_AND_FULLWIDTH_FORMS =87, /*[FF00]*/ + + /* New blocks in Unicode 3.1 */ + + /** @stable ICU 2.0 */ + UBLOCK_OLD_ITALIC = 88 , /*[10300]*/ + /** @stable ICU 2.0 */ + UBLOCK_GOTHIC = 89 , /*[10330]*/ + /** @stable ICU 2.0 */ + UBLOCK_DESERET = 90 , /*[10400]*/ + /** @stable ICU 2.0 */ + UBLOCK_BYZANTINE_MUSICAL_SYMBOLS = 91 , /*[1D000]*/ + /** @stable ICU 2.0 */ + UBLOCK_MUSICAL_SYMBOLS = 92 , /*[1D100]*/ + /** @stable ICU 2.0 */ + UBLOCK_MATHEMATICAL_ALPHANUMERIC_SYMBOLS = 93 , /*[1D400]*/ + /** @stable ICU 2.0 */ + UBLOCK_CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B = 94 , /*[20000]*/ + /** @stable ICU 2.0 */ + UBLOCK_CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT = 95 , /*[2F800]*/ + /** @stable ICU 2.0 */ + UBLOCK_TAGS = 96, /*[E0000]*/ + + /* New blocks in Unicode 3.2 */ + + /** + * Unicode 4.0.1 renames the "Cyrillic Supplementary" block to "Cyrillic Supplement". + * @stable ICU 2.2 + */ + UBLOCK_CYRILLIC_SUPPLEMENTARY = 97, + /** @stable ICU 3.0 */ + UBLOCK_CYRILLIC_SUPPLEMENT = UBLOCK_CYRILLIC_SUPPLEMENTARY, /*[0500]*/ + /** @stable ICU 2.2 */ + UBLOCK_TAGALOG = 98, /*[1700]*/ + /** @stable ICU 2.2 */ + UBLOCK_HANUNOO = 99, /*[1720]*/ + /** @stable ICU 2.2 */ + UBLOCK_BUHID = 100, /*[1740]*/ + /** @stable ICU 2.2 */ + UBLOCK_TAGBANWA = 101, /*[1760]*/ + /** @stable ICU 2.2 */ + UBLOCK_MISCELLANEOUS_MATHEMATICAL_SYMBOLS_A = 102, /*[27C0]*/ + /** @stable ICU 2.2 */ + UBLOCK_SUPPLEMENTAL_ARROWS_A = 103, /*[27F0]*/ + /** @stable ICU 2.2 */ + UBLOCK_SUPPLEMENTAL_ARROWS_B = 104, /*[2900]*/ + /** @stable ICU 2.2 */ + UBLOCK_MISCELLANEOUS_MATHEMATICAL_SYMBOLS_B = 105, /*[2980]*/ + /** @stable ICU 2.2 */ + UBLOCK_SUPPLEMENTAL_MATHEMATICAL_OPERATORS = 106, /*[2A00]*/ + /** @stable ICU 2.2 */ + UBLOCK_KATAKANA_PHONETIC_EXTENSIONS = 107, /*[31F0]*/ + /** @stable ICU 2.2 */ + UBLOCK_VARIATION_SELECTORS = 108, /*[FE00]*/ + /** @stable ICU 2.2 */ + UBLOCK_SUPPLEMENTARY_PRIVATE_USE_AREA_A = 109, /*[F0000]*/ + /** @stable ICU 2.2 */ + UBLOCK_SUPPLEMENTARY_PRIVATE_USE_AREA_B = 110, /*[100000]*/ + + /* New blocks in Unicode 4 */ + + /** @stable ICU 2.6 */ + UBLOCK_LIMBU = 111, /*[1900]*/ + /** @stable ICU 2.6 */ + UBLOCK_TAI_LE = 112, /*[1950]*/ + /** @stable ICU 2.6 */ + UBLOCK_KHMER_SYMBOLS = 113, /*[19E0]*/ + /** @stable ICU 2.6 */ + UBLOCK_PHONETIC_EXTENSIONS = 114, /*[1D00]*/ + /** @stable ICU 2.6 */ + UBLOCK_MISCELLANEOUS_SYMBOLS_AND_ARROWS = 115, /*[2B00]*/ + /** @stable ICU 2.6 */ + UBLOCK_YIJING_HEXAGRAM_SYMBOLS = 116, /*[4DC0]*/ + /** @stable ICU 2.6 */ + UBLOCK_LINEAR_B_SYLLABARY = 117, /*[10000]*/ + /** @stable ICU 2.6 */ + UBLOCK_LINEAR_B_IDEOGRAMS = 118, /*[10080]*/ + /** @stable ICU 2.6 */ + UBLOCK_AEGEAN_NUMBERS = 119, /*[10100]*/ + /** @stable ICU 2.6 */ + UBLOCK_UGARITIC = 120, /*[10380]*/ + /** @stable ICU 2.6 */ + UBLOCK_SHAVIAN = 121, /*[10450]*/ + /** @stable ICU 2.6 */ + UBLOCK_OSMANYA = 122, /*[10480]*/ + /** @stable ICU 2.6 */ + UBLOCK_CYPRIOT_SYLLABARY = 123, /*[10800]*/ + /** @stable ICU 2.6 */ + UBLOCK_TAI_XUAN_JING_SYMBOLS = 124, /*[1D300]*/ + /** @stable ICU 2.6 */ + UBLOCK_VARIATION_SELECTORS_SUPPLEMENT = 125, /*[E0100]*/ + + /* New blocks in Unicode 4.1 */ + + /** @stable ICU 3.4 */ + UBLOCK_ANCIENT_GREEK_MUSICAL_NOTATION = 126, /*[1D200]*/ + /** @stable ICU 3.4 */ + UBLOCK_ANCIENT_GREEK_NUMBERS = 127, /*[10140]*/ + /** @stable ICU 3.4 */ + UBLOCK_ARABIC_SUPPLEMENT = 128, /*[0750]*/ + /** @stable ICU 3.4 */ + UBLOCK_BUGINESE = 129, /*[1A00]*/ + /** @stable ICU 3.4 */ + UBLOCK_CJK_STROKES = 130, /*[31C0]*/ + /** @stable ICU 3.4 */ + UBLOCK_COMBINING_DIACRITICAL_MARKS_SUPPLEMENT = 131, /*[1DC0]*/ + /** @stable ICU 3.4 */ + UBLOCK_COPTIC = 132, /*[2C80]*/ + /** @stable ICU 3.4 */ + UBLOCK_ETHIOPIC_EXTENDED = 133, /*[2D80]*/ + /** @stable ICU 3.4 */ + UBLOCK_ETHIOPIC_SUPPLEMENT = 134, /*[1380]*/ + /** @stable ICU 3.4 */ + UBLOCK_GEORGIAN_SUPPLEMENT = 135, /*[2D00]*/ + /** @stable ICU 3.4 */ + UBLOCK_GLAGOLITIC = 136, /*[2C00]*/ + /** @stable ICU 3.4 */ + UBLOCK_KHAROSHTHI = 137, /*[10A00]*/ + /** @stable ICU 3.4 */ + UBLOCK_MODIFIER_TONE_LETTERS = 138, /*[A700]*/ + /** @stable ICU 3.4 */ + UBLOCK_NEW_TAI_LUE = 139, /*[1980]*/ + /** @stable ICU 3.4 */ + UBLOCK_OLD_PERSIAN = 140, /*[103A0]*/ + /** @stable ICU 3.4 */ + UBLOCK_PHONETIC_EXTENSIONS_SUPPLEMENT = 141, /*[1D80]*/ + /** @stable ICU 3.4 */ + UBLOCK_SUPPLEMENTAL_PUNCTUATION = 142, /*[2E00]*/ + /** @stable ICU 3.4 */ + UBLOCK_SYLOTI_NAGRI = 143, /*[A800]*/ + /** @stable ICU 3.4 */ + UBLOCK_TIFINAGH = 144, /*[2D30]*/ + /** @stable ICU 3.4 */ + UBLOCK_VERTICAL_FORMS = 145, /*[FE10]*/ + + /* New blocks in Unicode 5.0 */ + + /** @stable ICU 3.6 */ + UBLOCK_NKO = 146, /*[07C0]*/ + /** @stable ICU 3.6 */ + UBLOCK_BALINESE = 147, /*[1B00]*/ + /** @stable ICU 3.6 */ + UBLOCK_LATIN_EXTENDED_C = 148, /*[2C60]*/ + /** @stable ICU 3.6 */ + UBLOCK_LATIN_EXTENDED_D = 149, /*[A720]*/ + /** @stable ICU 3.6 */ + UBLOCK_PHAGS_PA = 150, /*[A840]*/ + /** @stable ICU 3.6 */ + UBLOCK_PHOENICIAN = 151, /*[10900]*/ + /** @stable ICU 3.6 */ + UBLOCK_CUNEIFORM = 152, /*[12000]*/ + /** @stable ICU 3.6 */ + UBLOCK_CUNEIFORM_NUMBERS_AND_PUNCTUATION = 153, /*[12400]*/ + /** @stable ICU 3.6 */ + UBLOCK_COUNTING_ROD_NUMERALS = 154, /*[1D360]*/ + + /** @stable ICU 2.0 */ + UBLOCK_COUNT = 155, + + /** @stable ICU 2.0 */ + UBLOCK_INVALID_CODE=-1 +}; + +/** @stable ICU 2.0 */ +typedef enum UBlockCode UBlockCode; + +/** + * East Asian Width constants. + * + * @see UCHAR_EAST_ASIAN_WIDTH + * @see u_getIntPropertyValue + * @stable ICU 2.2 + */ +typedef enum UEastAsianWidth { + U_EA_NEUTRAL, /*[N]*/ /*See note !!*/ + U_EA_AMBIGUOUS, /*[A]*/ + U_EA_HALFWIDTH, /*[H]*/ + U_EA_FULLWIDTH, /*[F]*/ + U_EA_NARROW, /*[Na]*/ + U_EA_WIDE, /*[W]*/ + U_EA_COUNT +} UEastAsianWidth; +/* + * Implementation note: + * Keep UEastAsianWidth constant values in sync with names list in genprops/props2.c. + */ + +/** + * Selector constants for u_charName(). + * u_charName() returns the "modern" name of a + * Unicode character; or the name that was defined in + * Unicode version 1.0, before the Unicode standard merged + * with ISO-10646; or an "extended" name that gives each + * Unicode code point a unique name. + * + * @see u_charName + * @stable ICU 2.0 + */ +typedef enum UCharNameChoice { + U_UNICODE_CHAR_NAME, + U_UNICODE_10_CHAR_NAME, + U_EXTENDED_CHAR_NAME, + U_CHAR_NAME_CHOICE_COUNT +} UCharNameChoice; + +/** + * Selector constants for u_getPropertyName() and + * u_getPropertyValueName(). These selectors are used to choose which + * name is returned for a given property or value. All properties and + * values have a long name. Most have a short name, but some do not. + * Unicode allows for additional names, beyond the long and short + * name, which would be indicated by U_LONG_PROPERTY_NAME + i, where + * i=1, 2,... + * + * @see u_getPropertyName() + * @see u_getPropertyValueName() + * @stable ICU 2.4 + */ +typedef enum UPropertyNameChoice { + U_SHORT_PROPERTY_NAME, + U_LONG_PROPERTY_NAME, + U_PROPERTY_NAME_CHOICE_COUNT +} UPropertyNameChoice; + +/** + * Decomposition Type constants. + * + * @see UCHAR_DECOMPOSITION_TYPE + * @stable ICU 2.2 + */ +typedef enum UDecompositionType { + U_DT_NONE, /*[none]*/ /*See note !!*/ + U_DT_CANONICAL, /*[can]*/ + U_DT_COMPAT, /*[com]*/ + U_DT_CIRCLE, /*[enc]*/ + U_DT_FINAL, /*[fin]*/ + U_DT_FONT, /*[font]*/ + U_DT_FRACTION, /*[fra]*/ + U_DT_INITIAL, /*[init]*/ + U_DT_ISOLATED, /*[iso]*/ + U_DT_MEDIAL, /*[med]*/ + U_DT_NARROW, /*[nar]*/ + U_DT_NOBREAK, /*[nb]*/ + U_DT_SMALL, /*[sml]*/ + U_DT_SQUARE, /*[sqr]*/ + U_DT_SUB, /*[sub]*/ + U_DT_SUPER, /*[sup]*/ + U_DT_VERTICAL, /*[vert]*/ + U_DT_WIDE, /*[wide]*/ + U_DT_COUNT /* 18 */ +} UDecompositionType; + +/** + * Joining Type constants. + * + * @see UCHAR_JOINING_TYPE + * @stable ICU 2.2 + */ +typedef enum UJoiningType { + U_JT_NON_JOINING, /*[U]*/ /*See note !!*/ + U_JT_JOIN_CAUSING, /*[C]*/ + U_JT_DUAL_JOINING, /*[D]*/ + U_JT_LEFT_JOINING, /*[L]*/ + U_JT_RIGHT_JOINING, /*[R]*/ + U_JT_TRANSPARENT, /*[T]*/ + U_JT_COUNT /* 6 */ +} UJoiningType; + +/** + * Joining Group constants. + * + * @see UCHAR_JOINING_GROUP + * @stable ICU 2.2 + */ +typedef enum UJoiningGroup { + U_JG_NO_JOINING_GROUP, + U_JG_AIN, + U_JG_ALAPH, + U_JG_ALEF, + U_JG_BEH, + U_JG_BETH, + U_JG_DAL, + U_JG_DALATH_RISH, + U_JG_E, + U_JG_FEH, + U_JG_FINAL_SEMKATH, + U_JG_GAF, + U_JG_GAMAL, + U_JG_HAH, + U_JG_HAMZA_ON_HEH_GOAL, + U_JG_HE, + U_JG_HEH, + U_JG_HEH_GOAL, + U_JG_HETH, + U_JG_KAF, + U_JG_KAPH, + U_JG_KNOTTED_HEH, + U_JG_LAM, + U_JG_LAMADH, + U_JG_MEEM, + U_JG_MIM, + U_JG_NOON, + U_JG_NUN, + U_JG_PE, + U_JG_QAF, + U_JG_QAPH, + U_JG_REH, + U_JG_REVERSED_PE, + U_JG_SAD, + U_JG_SADHE, + U_JG_SEEN, + U_JG_SEMKATH, + U_JG_SHIN, + U_JG_SWASH_KAF, + U_JG_SYRIAC_WAW, + U_JG_TAH, + U_JG_TAW, + U_JG_TEH_MARBUTA, + U_JG_TETH, + U_JG_WAW, + U_JG_YEH, + U_JG_YEH_BARREE, + U_JG_YEH_WITH_TAIL, + U_JG_YUDH, + U_JG_YUDH_HE, + U_JG_ZAIN, + U_JG_FE, /**< @stable ICU 2.6 */ + U_JG_KHAPH, /**< @stable ICU 2.6 */ + U_JG_ZHAIN, /**< @stable ICU 2.6 */ + U_JG_COUNT +} UJoiningGroup; + +/** + * Grapheme Cluster Break constants. + * + * @see UCHAR_GRAPHEME_CLUSTER_BREAK + * @stable ICU 3.4 + */ +typedef enum UGraphemeClusterBreak { + U_GCB_OTHER = 0, /*[XX]*/ /*See note !!*/ + U_GCB_CONTROL = 1, /*[CN]*/ + U_GCB_CR = 2, /*[CR]*/ + U_GCB_EXTEND = 3, /*[EX]*/ + U_GCB_L = 4, /*[L]*/ + U_GCB_LF = 5, /*[LF]*/ + U_GCB_LV = 6, /*[LV]*/ + U_GCB_LVT = 7, /*[LVT]*/ + U_GCB_T = 8, /*[T]*/ + U_GCB_V = 9, /*[V]*/ + U_GCB_COUNT = 10 +} UGraphemeClusterBreak; + +/** + * Word Break constants. + * (UWordBreak is a pre-existing enum type in ubrk.h for word break status tags.) + * + * @see UCHAR_WORD_BREAK + * @stable ICU 3.4 + */ +typedef enum UWordBreakValues { + U_WB_OTHER = 0, /*[XX]*/ /*See note !!*/ + U_WB_ALETTER = 1, /*[LE]*/ + U_WB_FORMAT = 2, /*[FO]*/ + U_WB_KATAKANA = 3, /*[KA]*/ + U_WB_MIDLETTER = 4, /*[ML]*/ + U_WB_MIDNUM = 5, /*[MN]*/ + U_WB_NUMERIC = 6, /*[NU]*/ + U_WB_EXTENDNUMLET = 7, /*[EX]*/ + U_WB_COUNT = 8 +} UWordBreakValues; + +/** + * Sentence Break constants. + * + * @see UCHAR_SENTENCE_BREAK + * @stable ICU 3.4 + */ +typedef enum USentenceBreak { + U_SB_OTHER = 0, /*[XX]*/ /*See note !!*/ + U_SB_ATERM = 1, /*[AT]*/ + U_SB_CLOSE = 2, /*[CL]*/ + U_SB_FORMAT = 3, /*[FO]*/ + U_SB_LOWER = 4, /*[LO]*/ + U_SB_NUMERIC = 5, /*[NU]*/ + U_SB_OLETTER = 6, /*[LE]*/ + U_SB_SEP = 7, /*[SE]*/ + U_SB_SP = 8, /*[SP]*/ + U_SB_STERM = 9, /*[ST]*/ + U_SB_UPPER = 10, /*[UP]*/ + U_SB_COUNT = 11 +} USentenceBreak; + +/** + * Line Break constants. + * + * @see UCHAR_LINE_BREAK + * @stable ICU 2.2 + */ +typedef enum ULineBreak { + U_LB_UNKNOWN = 0, /*[XX]*/ /*See note !!*/ + U_LB_AMBIGUOUS = 1, /*[AI]*/ + U_LB_ALPHABETIC = 2, /*[AL]*/ + U_LB_BREAK_BOTH = 3, /*[B2]*/ + U_LB_BREAK_AFTER = 4, /*[BA]*/ + U_LB_BREAK_BEFORE = 5, /*[BB]*/ + U_LB_MANDATORY_BREAK = 6, /*[BK]*/ + U_LB_CONTINGENT_BREAK = 7, /*[CB]*/ + U_LB_CLOSE_PUNCTUATION = 8, /*[CL]*/ + U_LB_COMBINING_MARK = 9, /*[CM]*/ + U_LB_CARRIAGE_RETURN = 10, /*[CR]*/ + U_LB_EXCLAMATION = 11, /*[EX]*/ + U_LB_GLUE = 12, /*[GL]*/ + U_LB_HYPHEN = 13, /*[HY]*/ + U_LB_IDEOGRAPHIC = 14, /*[ID]*/ + U_LB_INSEPERABLE = 15, + /** Renamed from the misspelled "inseperable" in Unicode 4.0.1/ICU 3.0 @stable ICU 3.0 */ + U_LB_INSEPARABLE=U_LB_INSEPERABLE,/*[IN]*/ + U_LB_INFIX_NUMERIC = 16, /*[IS]*/ + U_LB_LINE_FEED = 17, /*[LF]*/ + U_LB_NONSTARTER = 18, /*[NS]*/ + U_LB_NUMERIC = 19, /*[NU]*/ + U_LB_OPEN_PUNCTUATION = 20, /*[OP]*/ + U_LB_POSTFIX_NUMERIC = 21, /*[PO]*/ + U_LB_PREFIX_NUMERIC = 22, /*[PR]*/ + U_LB_QUOTATION = 23, /*[QU]*/ + U_LB_COMPLEX_CONTEXT = 24, /*[SA]*/ + U_LB_SURROGATE = 25, /*[SG]*/ + U_LB_SPACE = 26, /*[SP]*/ + U_LB_BREAK_SYMBOLS = 27, /*[SY]*/ + U_LB_ZWSPACE = 28, /*[ZW]*/ + U_LB_NEXT_LINE = 29, /*[NL]*/ /* from here on: new in Unicode 4/ICU 2.6 */ + U_LB_WORD_JOINER = 30, /*[WJ]*/ + U_LB_H2 = 31, /*[H2]*/ /* from here on: new in Unicode 4.1/ICU 3.4 */ + U_LB_H3 = 32, /*[H3]*/ + U_LB_JL = 33, /*[JL]*/ + U_LB_JT = 34, /*[JT]*/ + U_LB_JV = 35, /*[JV]*/ + U_LB_COUNT = 36 +} ULineBreak; + +/** + * Numeric Type constants. + * + * @see UCHAR_NUMERIC_TYPE + * @stable ICU 2.2 + */ +typedef enum UNumericType { + U_NT_NONE, /*[None]*/ /*See note !!*/ + U_NT_DECIMAL, /*[de]*/ + U_NT_DIGIT, /*[di]*/ + U_NT_NUMERIC, /*[nu]*/ + U_NT_COUNT +} UNumericType; + +/** + * Hangul Syllable Type constants. + * + * @see UCHAR_HANGUL_SYLLABLE_TYPE + * @stable ICU 2.6 + */ +typedef enum UHangulSyllableType { + U_HST_NOT_APPLICABLE, /*[NA]*/ /*See note !!*/ + U_HST_LEADING_JAMO, /*[L]*/ + U_HST_VOWEL_JAMO, /*[V]*/ + U_HST_TRAILING_JAMO, /*[T]*/ + U_HST_LV_SYLLABLE, /*[LV]*/ + U_HST_LVT_SYLLABLE, /*[LVT]*/ + U_HST_COUNT +} UHangulSyllableType; + +/** + * Check a binary Unicode property for a code point. + * + * Unicode, especially in version 3.2, defines many more properties than the + * original set in UnicodeData.txt. + * + * The properties APIs are intended to reflect Unicode properties as defined + * in the Unicode Character Database (UCD) and Unicode Technical Reports (UTR). + * For details about the properties see http://www.unicode.org/ucd/ . + * For names of Unicode properties see the UCD file PropertyAliases.txt. + * + * Important: If ICU is built with UCD files from Unicode versions below 3.2, + * then properties marked with "new in Unicode 3.2" are not or not fully available. + * + * @param c Code point to test. + * @param which UProperty selector constant, identifies which binary property to check. + * Must be UCHAR_BINARY_START<=which=0. + * True for characters with general category "Nd" (decimal digit numbers) + * as well as Latin letters a-f and A-F in both ASCII and Fullwidth ASCII. + * (That is, for letters with code points + * 0041..0046, 0061..0066, FF21..FF26, FF41..FF46.) + * + * In order to narrow the definition of hexadecimal digits to only ASCII + * characters, use (c<=0x7f && u_isxdigit(c)). + * + * This is a C/POSIX migration function. + * See the comments about C/POSIX character classification functions in the + * documentation at the top of this header file. + * + * @param c the code point to be tested + * @return TRUE if the code point is a hexadecimal digit + * + * @stable ICU 2.6 + */ +U_STABLE UBool U_EXPORT2 +u_isxdigit(UChar32 c); + +/** + * Determines whether the specified code point is a punctuation character. + * True for characters with general categories "P" (punctuation). + * + * This is a C/POSIX migration function. + * See the comments about C/POSIX character classification functions in the + * documentation at the top of this header file. + * + * @param c the code point to be tested + * @return TRUE if the code point is a punctuation character + * + * @stable ICU 2.6 + */ +U_STABLE UBool U_EXPORT2 +u_ispunct(UChar32 c); + +/** + * Determines whether the specified code point is a "graphic" character + * (printable, excluding spaces). + * TRUE for all characters except those with general categories + * "Cc" (control codes), "Cf" (format controls), "Cs" (surrogates), + * "Cn" (unassigned), and "Z" (separators). + * + * This is a C/POSIX migration function. + * See the comments about C/POSIX character classification functions in the + * documentation at the top of this header file. + * + * @param c the code point to be tested + * @return TRUE if the code point is a "graphic" character + * + * @stable ICU 2.6 + */ +U_STABLE UBool U_EXPORT2 +u_isgraph(UChar32 c); + +/** + * Determines whether the specified code point is a "blank" or "horizontal space", + * a character that visibly separates words on a line. + * The following are equivalent definitions: + * + * TRUE for Unicode White_Space characters except for "vertical space controls" + * where "vertical space controls" are the following characters: + * U+000A (LF) U+000B (VT) U+000C (FF) U+000D (CR) U+0085 (NEL) U+2028 (LS) U+2029 (PS) + * + * same as + * + * TRUE for U+0009 (TAB) and characters with general category "Zs" (space separators) + * except Zero Width Space (ZWSP, U+200B). + * + * Note: There are several ICU whitespace functions; please see the uchar.h + * file documentation for a detailed comparison. + * + * This is a C/POSIX migration function. + * See the comments about C/POSIX character classification functions in the + * documentation at the top of this header file. + * + * @param c the code point to be tested + * @return TRUE if the code point is a "blank" + * + * @stable ICU 2.6 + */ +U_STABLE UBool U_EXPORT2 +u_isblank(UChar32 c); + +/** + * Determines whether the specified code point is "defined", + * which usually means that it is assigned a character. + * True for general categories other than "Cn" (other, not assigned), + * i.e., true for all code points mentioned in UnicodeData.txt. + * + * Note that non-character code points (e.g., U+FDD0) are not "defined" + * (they are Cn), but surrogate code points are "defined" (Cs). + * + * Same as java.lang.Character.isDefined(). + * + * @param c the code point to be tested + * @return TRUE if the code point is assigned a character + * + * @see u_isdigit + * @see u_isalpha + * @see u_isalnum + * @see u_isupper + * @see u_islower + * @see u_istitle + * @stable ICU 2.0 + */ +U_STABLE UBool U_EXPORT2 +u_isdefined(UChar32 c); + +/** + * Determines if the specified character is a space character or not. + * + * Note: There are several ICU whitespace functions; please see the uchar.h + * file documentation for a detailed comparison. + * + * This is a C/POSIX migration function. + * See the comments about C/POSIX character classification functions in the + * documentation at the top of this header file. + * + * @param c the character to be tested + * @return true if the character is a space character; false otherwise. + * + * @see u_isJavaSpaceChar + * @see u_isWhitespace + * @see u_isUWhiteSpace + * @stable ICU 2.0 + */ +U_STABLE UBool U_EXPORT2 +u_isspace(UChar32 c); + +/** + * Determine if the specified code point is a space character according to Java. + * True for characters with general categories "Z" (separators), + * which does not include control codes (e.g., TAB or Line Feed). + * + * Same as java.lang.Character.isSpaceChar(). + * + * Note: There are several ICU whitespace functions; please see the uchar.h + * file documentation for a detailed comparison. + * + * @param c the code point to be tested + * @return TRUE if the code point is a space character according to Character.isSpaceChar() + * + * @see u_isspace + * @see u_isWhitespace + * @see u_isUWhiteSpace + * @stable ICU 2.6 + */ +U_STABLE UBool U_EXPORT2 +u_isJavaSpaceChar(UChar32 c); + +/** + * Determines if the specified code point is a whitespace character according to Java/ICU. + * A character is considered to be a Java whitespace character if and only + * if it satisfies one of the following criteria: + * + * - It is a Unicode separator (categories "Z"), but is not + * a no-break space (U+00A0 NBSP or U+2007 Figure Space or U+202F Narrow NBSP). + * - It is U+0009 HORIZONTAL TABULATION. + * - It is U+000A LINE FEED. + * - It is U+000B VERTICAL TABULATION. + * - It is U+000C FORM FEED. + * - It is U+000D CARRIAGE RETURN. + * - It is U+001C FILE SEPARATOR. + * - It is U+001D GROUP SEPARATOR. + * - It is U+001E RECORD SEPARATOR. + * - It is U+001F UNIT SEPARATOR. + * - It is U+0085 NEXT LINE. + * + * Same as java.lang.Character.isWhitespace() except that Java omits U+0085. + * + * Note: There are several ICU whitespace functions; please see the uchar.h + * file documentation for a detailed comparison. + * + * @param c the code point to be tested + * @return TRUE if the code point is a whitespace character according to Java/ICU + * + * @see u_isspace + * @see u_isJavaSpaceChar + * @see u_isUWhiteSpace + * @stable ICU 2.0 + */ +U_STABLE UBool U_EXPORT2 +u_isWhitespace(UChar32 c); + +/** + * Determines whether the specified code point is a control character + * (as defined by this function). + * A control character is one of the following: + * - ISO 8-bit control character (U+0000..U+001f and U+007f..U+009f) + * - U_CONTROL_CHAR (Cc) + * - U_FORMAT_CHAR (Cf) + * - U_LINE_SEPARATOR (Zl) + * - U_PARAGRAPH_SEPARATOR (Zp) + * + * This is a C/POSIX migration function. + * See the comments about C/POSIX character classification functions in the + * documentation at the top of this header file. + * + * @param c the code point to be tested + * @return TRUE if the code point is a control character + * + * @see UCHAR_DEFAULT_IGNORABLE_CODE_POINT + * @see u_isprint + * @stable ICU 2.0 + */ +U_STABLE UBool U_EXPORT2 +u_iscntrl(UChar32 c); + +/** + * Determines whether the specified code point is an ISO control code. + * True for U+0000..U+001f and U+007f..U+009f (general category "Cc"). + * + * Same as java.lang.Character.isISOControl(). + * + * @param c the code point to be tested + * @return TRUE if the code point is an ISO control code + * + * @see u_iscntrl + * @stable ICU 2.6 + */ +U_STABLE UBool U_EXPORT2 +u_isISOControl(UChar32 c); + +/** + * Determines whether the specified code point is a printable character. + * True for general categories other than "C" (controls). + * + * This is a C/POSIX migration function. + * See the comments about C/POSIX character classification functions in the + * documentation at the top of this header file. + * + * @param c the code point to be tested + * @return TRUE if the code point is a printable character + * + * @see UCHAR_DEFAULT_IGNORABLE_CODE_POINT + * @see u_iscntrl + * @stable ICU 2.0 + */ +U_STABLE UBool U_EXPORT2 +u_isprint(UChar32 c); + +/** + * Determines whether the specified code point is a base character. + * True for general categories "L" (letters), "N" (numbers), + * "Mc" (spacing combining marks), and "Me" (enclosing marks). + * + * Note that this is different from the Unicode definition in + * chapter 3.5, conformance clause D13, + * which defines base characters to be all characters (not Cn) + * that do not graphically combine with preceding characters (M) + * and that are neither control (Cc) or format (Cf) characters. + * + * @param c the code point to be tested + * @return TRUE if the code point is a base character according to this function + * + * @see u_isalpha + * @see u_isdigit + * @stable ICU 2.0 + */ +U_STABLE UBool U_EXPORT2 +u_isbase(UChar32 c); + +/** + * Returns the bidirectional category value for the code point, + * which is used in the Unicode bidirectional algorithm + * (UAX #9 http://www.unicode.org/reports/tr9/). + * Note that some unassigned code points have bidi values + * of R or AL because they are in blocks that are reserved + * for Right-To-Left scripts. + * + * Same as java.lang.Character.getDirectionality() + * + * @param c the code point to be tested + * @return the bidirectional category (UCharDirection) value + * + * @see UCharDirection + * @stable ICU 2.0 + */ +U_STABLE UCharDirection U_EXPORT2 +u_charDirection(UChar32 c); + +/** + * Determines whether the code point has the Bidi_Mirrored property. + * This property is set for characters that are commonly used in + * Right-To-Left contexts and need to be displayed with a "mirrored" + * glyph. + * + * Same as java.lang.Character.isMirrored(). + * Same as UCHAR_BIDI_MIRRORED + * + * @param c the code point to be tested + * @return TRUE if the character has the Bidi_Mirrored property + * + * @see UCHAR_BIDI_MIRRORED + * @stable ICU 2.0 + */ +U_STABLE UBool U_EXPORT2 +u_isMirrored(UChar32 c); + +/** + * Maps the specified character to a "mirror-image" character. + * For characters with the Bidi_Mirrored property, implementations + * sometimes need a "poor man's" mapping to another Unicode + * character (code point) such that the default glyph may serve + * as the mirror-image of the default glyph of the specified + * character. This is useful for text conversion to and from + * codepages with visual order, and for displays without glyph + * selecetion capabilities. + * + * @param c the code point to be mapped + * @return another Unicode code point that may serve as a mirror-image + * substitute, or c itself if there is no such mapping or c + * does not have the Bidi_Mirrored property + * + * @see UCHAR_BIDI_MIRRORED + * @see u_isMirrored + * @stable ICU 2.0 + */ +U_STABLE UChar32 U_EXPORT2 +u_charMirror(UChar32 c); + +/** + * Returns the general category value for the code point. + * + * Same as java.lang.Character.getType(). + * + * @param c the code point to be tested + * @return the general category (UCharCategory) value + * + * @see UCharCategory + * @stable ICU 2.0 + */ +U_STABLE int8_t U_EXPORT2 +u_charType(UChar32 c); + +/** + * Get a single-bit bit set for the general category of a character. + * This bit set can be compared bitwise with U_GC_SM_MASK, U_GC_L_MASK, etc. + * Same as U_MASK(u_charType(c)). + * + * @param c the code point to be tested + * @return a single-bit mask corresponding to the general category (UCharCategory) value + * + * @see u_charType + * @see UCharCategory + * @see U_GC_CN_MASK + * @stable ICU 2.1 + */ +#define U_GET_GC_MASK(c) U_MASK(u_charType(c)) + +/** + * Callback from u_enumCharTypes(), is called for each contiguous range + * of code points c (where start<=cnameChoice, the character name written + * into the buffer is the "modern" name or the name that was defined + * in Unicode version 1.0. + * The name contains only "invariant" characters + * like A-Z, 0-9, space, and '-'. + * Unicode 1.0 names are only retrieved if they are different from the modern + * names and if the data file contains the data for them. gennames may or may + * not be called with a command line option to include 1.0 names in unames.dat. + * + * @param code The character (code point) for which to get the name. + * It must be 0<=code<=0x10ffff. + * @param nameChoice Selector for which name to get. + * @param buffer Destination address for copying the name. + * The name will always be zero-terminated. + * If there is no name, then the buffer will be set to the empty string. + * @param bufferLength ==sizeof(buffer) + * @param pErrorCode Pointer to a UErrorCode variable; + * check for U_SUCCESS() after u_charName() + * returns. + * @return The length of the name, or 0 if there is no name for this character. + * If the bufferLength is less than or equal to the length, then the buffer + * contains the truncated name and the returned length indicates the full + * length of the name. + * The length does not include the zero-termination. + * + * @see UCharNameChoice + * @see u_charFromName + * @see u_enumCharNames + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +u_charName(UChar32 code, UCharNameChoice nameChoice, + char *buffer, int32_t bufferLength, + UErrorCode *pErrorCode); + +/** + * Get the ISO 10646 comment for a character. + * The ISO 10646 comment is an informative field in the Unicode Character + * Database (UnicodeData.txt field 11) and is from the ISO 10646 names list. + * + * @param c The character (code point) for which to get the ISO comment. + * It must be 0<=c<=0x10ffff. + * @param dest Destination address for copying the comment. + * The comment will be zero-terminated if possible. + * If there is no comment, then the buffer will be set to the empty string. + * @param destCapacity ==sizeof(dest) + * @param pErrorCode Pointer to a UErrorCode variable; + * check for U_SUCCESS() after u_getISOComment() + * returns. + * @return The length of the comment, or 0 if there is no comment for this character. + * If the destCapacity is less than or equal to the length, then the buffer + * contains the truncated name and the returned length indicates the full + * length of the name. + * The length does not include the zero-termination. + * + * @stable ICU 2.2 + */ +U_STABLE int32_t U_EXPORT2 +u_getISOComment(UChar32 c, + char *dest, int32_t destCapacity, + UErrorCode *pErrorCode); + +/** + * Find a Unicode character by its name and return its code point value. + * The name is matched exactly and completely. + * If the name does not correspond to a code point, pErrorCode + * is set to U_INVALID_CHAR_FOUND. + * A Unicode 1.0 name is matched only if it differs from the modern name. + * Unicode names are all uppercase. Extended names are lowercase followed + * by an uppercase hexadecimal number, and within angle brackets. + * + * @param nameChoice Selector for which name to match. + * @param name The name to match. + * @param pErrorCode Pointer to a UErrorCode variable + * @return The Unicode value of the code point with the given name, + * or an undefined value if there is no such code point. + * + * @see UCharNameChoice + * @see u_charName + * @see u_enumCharNames + * @stable ICU 1.7 + */ +U_STABLE UChar32 U_EXPORT2 +u_charFromName(UCharNameChoice nameChoice, + const char *name, + UErrorCode *pErrorCode); + +/** + * Type of a callback function for u_enumCharNames() that gets called + * for each Unicode character with the code point value and + * the character name. + * If such a function returns FALSE, then the enumeration is stopped. + * + * @param context The context pointer that was passed to u_enumCharNames(). + * @param code The Unicode code point for the character with this name. + * @param nameChoice Selector for which kind of names is enumerated. + * @param name The character's name, zero-terminated. + * @param length The length of the name. + * @return TRUE if the enumeration should continue, FALSE to stop it. + * + * @see UCharNameChoice + * @see u_enumCharNames + * @stable ICU 1.7 + */ +typedef UBool U_CALLCONV UEnumCharNamesFn(void *context, + UChar32 code, + UCharNameChoice nameChoice, + const char *name, + int32_t length); + +/** + * Enumerate all assigned Unicode characters between the start and limit + * code points (start inclusive, limit exclusive) and call a function + * for each, passing the code point value and the character name. + * For Unicode 1.0 names, only those are enumerated that differ from the + * modern names. + * + * @param start The first code point in the enumeration range. + * @param limit One more than the last code point in the enumeration range + * (the first one after the range). + * @param fn The function that is to be called for each character name. + * @param context An arbitrary pointer that is passed to the function. + * @param nameChoice Selector for which kind of names to enumerate. + * @param pErrorCode Pointer to a UErrorCode variable + * + * @see UCharNameChoice + * @see UEnumCharNamesFn + * @see u_charName + * @see u_charFromName + * @stable ICU 1.7 + */ +U_STABLE void U_EXPORT2 +u_enumCharNames(UChar32 start, UChar32 limit, + UEnumCharNamesFn *fn, + void *context, + UCharNameChoice nameChoice, + UErrorCode *pErrorCode); + +/** + * Return the Unicode name for a given property, as given in the + * Unicode database file PropertyAliases.txt. + * + * In addition, this function maps the property + * UCHAR_GENERAL_CATEGORY_MASK to the synthetic names "gcm" / + * "General_Category_Mask". These names are not in + * PropertyAliases.txt. + * + * @param property UProperty selector other than UCHAR_INVALID_CODE. + * If out of range, NULL is returned. + * + * @param nameChoice selector for which name to get. If out of range, + * NULL is returned. All properties have a long name. Most + * have a short name, but some do not. Unicode allows for + * additional names; if present these will be returned by + * U_LONG_PROPERTY_NAME + i, where i=1, 2,... + * + * @return a pointer to the name, or NULL if either the + * property or the nameChoice is out of range. If a given + * nameChoice returns NULL, then all larger values of + * nameChoice will return NULL, with one exception: if NULL is + * returned for U_SHORT_PROPERTY_NAME, then + * U_LONG_PROPERTY_NAME (and higher) may still return a + * non-NULL value. The returned pointer is valid until + * u_cleanup() is called. + * + * @see UProperty + * @see UPropertyNameChoice + * @stable ICU 2.4 + */ +U_STABLE const char* U_EXPORT2 +u_getPropertyName(UProperty property, + UPropertyNameChoice nameChoice); + +/** + * Return the UProperty enum for a given property name, as specified + * in the Unicode database file PropertyAliases.txt. Short, long, and + * any other variants are recognized. + * + * In addition, this function maps the synthetic names "gcm" / + * "General_Category_Mask" to the property + * UCHAR_GENERAL_CATEGORY_MASK. These names are not in + * PropertyAliases.txt. + * + * @param alias the property name to be matched. The name is compared + * using "loose matching" as described in PropertyAliases.txt. + * + * @return a UProperty enum, or UCHAR_INVALID_CODE if the given name + * does not match any property. + * + * @see UProperty + * @stable ICU 2.4 + */ +U_STABLE UProperty U_EXPORT2 +u_getPropertyEnum(const char* alias); + +/** + * Return the Unicode name for a given property value, as given in the + * Unicode database file PropertyValueAliases.txt. + * + * Note: Some of the names in PropertyValueAliases.txt can only be + * retrieved using UCHAR_GENERAL_CATEGORY_MASK, not + * UCHAR_GENERAL_CATEGORY. These include: "C" / "Other", "L" / + * "Letter", "LC" / "Cased_Letter", "M" / "Mark", "N" / "Number", "P" + * / "Punctuation", "S" / "Symbol", and "Z" / "Separator". + * + * @param property UProperty selector constant. + * Must be UCHAR_BINARY_START<=which2<=radix<=36 or if the + * value of c is not a valid digit in the specified + * radix, -1 is returned. A character is a valid digit + * if at least one of the following is true: + *

    + *
  • The character has a decimal digit value. + * Such characters have the general category "Nd" (decimal digit numbers) + * and a Numeric_Type of Decimal. + * In this case the value is the character's decimal digit value.
  • + *
  • The character is one of the uppercase Latin letters + * 'A' through 'Z'. + * In this case the value is c-'A'+10.
  • + *
  • The character is one of the lowercase Latin letters + * 'a' through 'z'. + * In this case the value is ch-'a'+10.
  • + *
  • Latin letters from both the ASCII range (0061..007A, 0041..005A) + * as well as from the Fullwidth ASCII range (FF41..FF5A, FF21..FF3A) + * are recognized.
  • + *
+ * + * Same as java.lang.Character.digit(). + * + * @param ch the code point to be tested. + * @param radix the radix. + * @return the numeric value represented by the character in the + * specified radix, + * or -1 if there is no value or if the value exceeds the radix. + * + * @see UCHAR_NUMERIC_TYPE + * @see u_forDigit + * @see u_charDigitValue + * @see u_isdigit + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +u_digit(UChar32 ch, int8_t radix); + +/** + * Determines the character representation for a specific digit in + * the specified radix. If the value of radix is not a + * valid radix, or the value of digit is not a valid + * digit in the specified radix, the null character + * (U+0000) is returned. + *

+ * The radix argument is valid if it is greater than or + * equal to 2 and less than or equal to 36. + * The digit argument is valid if + * 0 <= digit < radix. + *

+ * If the digit is less than 10, then + * '0' + digit is returned. Otherwise, the value + * 'a' + digit - 10 is returned. + * + * Same as java.lang.Character.forDigit(). + * + * @param digit the number to convert to a character. + * @param radix the radix. + * @return the char representation of the specified digit + * in the specified radix. + * + * @see u_digit + * @see u_charDigitValue + * @see u_isdigit + * @stable ICU 2.0 + */ +U_STABLE UChar32 U_EXPORT2 +u_forDigit(int32_t digit, int8_t radix); + +/** + * Get the "age" of the code point. + * The "age" is the Unicode version when the code point was first + * designated (as a non-character or for Private Use) + * or assigned a character. + * This can be useful to avoid emitting code points to receiving + * processes that do not accept newer characters. + * The data is from the UCD file DerivedAge.txt. + * + * @param c The code point. + * @param versionArray The Unicode version number array, to be filled in. + * + * @stable ICU 2.1 + */ +U_STABLE void U_EXPORT2 +u_charAge(UChar32 c, UVersionInfo versionArray); + +/** + * Gets the Unicode version information. + * The version array is filled in with the version information + * for the Unicode standard that is currently used by ICU. + * For example, Unicode version 3.1.1 is represented as an array with + * the values { 3, 1, 1, 0 }. + * + * @param versionArray an output array that will be filled in with + * the Unicode version number + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +u_getUnicodeVersion(UVersionInfo versionArray); + +/** + * Get the FC_NFKC_Closure property string for a character. + * See Unicode Standard Annex #15 for details, search for "FC_NFKC_Closure" + * or for "FNC": http://www.unicode.org/reports/tr15/ + * + * @param c The character (code point) for which to get the FC_NFKC_Closure string. + * It must be 0<=c<=0x10ffff. + * @param dest Destination address for copying the string. + * The string will be zero-terminated if possible. + * If there is no FC_NFKC_Closure string, + * then the buffer will be set to the empty string. + * @param destCapacity ==sizeof(dest) + * @param pErrorCode Pointer to a UErrorCode variable. + * @return The length of the string, or 0 if there is no FC_NFKC_Closure string for this character. + * If the destCapacity is less than or equal to the length, then the buffer + * contains the truncated name and the returned length indicates the full + * length of the name. + * The length does not include the zero-termination. + * + * @stable ICU 2.2 + */ +U_STABLE int32_t U_EXPORT2 +u_getFC_NFKC_Closure(UChar32 c, UChar *dest, int32_t destCapacity, UErrorCode *pErrorCode); + +U_CDECL_END + +#endif /*_UCHAR*/ +/*eof*/ diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/uchriter.h b/Build/source/libs/icu/icu-xetex/common/unicode/uchriter.h new file mode 100644 index 00000000000..6d5a990f7b7 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/uchriter.h @@ -0,0 +1,381 @@ +/* +********************************************************************** +* Copyright (C) 1998-2005, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +*/ + +#ifndef UCHRITER_H +#define UCHRITER_H + +#include "unicode/utypes.h" +#include "unicode/chariter.h" + +/** + * \file + * \brief C++ API: UChar Character Iterator + */ + +U_NAMESPACE_BEGIN + +/** + * A concrete subclass of CharacterIterator that iterates over the + * characters (code units or code points) in a UChar array. + * It's possible not only to create an + * iterator that iterates over an entire UChar array, but also to + * create one that iterates over only a subrange of a UChar array + * (iterators over different subranges of the same UChar array don't + * compare equal). + * @see CharacterIterator + * @see ForwardCharacterIterator + * @stable ICU 2.0 + */ +class U_COMMON_API UCharCharacterIterator : public CharacterIterator { +public: + /** + * Create an iterator over the UChar array referred to by "textPtr". + * The iteration range is 0 to length-1. + * text is only aliased, not adopted (the + * destructor will not delete it). + * @param textPtr The UChar array to be iterated over + * @param length The length of the UChar array + * @stable ICU 2.0 + */ + UCharCharacterIterator(const UChar* textPtr, int32_t length); + + /** + * Create an iterator over the UChar array referred to by "textPtr". + * The iteration range is 0 to length-1. + * text is only aliased, not adopted (the + * destructor will not delete it). + * The starting + * position is specified by "position". If "position" is outside the valid + * iteration range, the behavior of this object is undefined. + * @param textPtr The UChar array to be iteratd over + * @param length The length of the UChar array + * @param position The starting position of the iteration + * @stable ICU 2.0 + */ + UCharCharacterIterator(const UChar* textPtr, int32_t length, + int32_t position); + + /** + * Create an iterator over the UChar array referred to by "textPtr". + * The iteration range is 0 to end-1. + * text is only aliased, not adopted (the + * destructor will not delete it). + * The starting + * position is specified by "position". If begin and end do not + * form a valid iteration range or "position" is outside the valid + * iteration range, the behavior of this object is undefined. + * @param textPtr The UChar array to be iterated over + * @param length The length of the UChar array + * @param textBegin The begin position of the iteration range + * @param textEnd The end position of the iteration range + * @param position The starting position of the iteration + * @stable ICU 2.0 + */ + UCharCharacterIterator(const UChar* textPtr, int32_t length, + int32_t textBegin, + int32_t textEnd, + int32_t position); + + /** + * Copy constructor. The new iterator iterates over the same range + * of the same string as "that", and its initial position is the + * same as "that"'s current position. + * @param that The UCharCharacterIterator to be copied + * @stable ICU 2.0 + */ + UCharCharacterIterator(const UCharCharacterIterator& that); + + /** + * Destructor. + * @stable ICU 2.0 + */ + virtual ~UCharCharacterIterator(); + + /** + * Assignment operator. *this is altered to iterate over the sane + * range of the same string as "that", and refers to the same + * character within that string as "that" does. + * @param that The object to be copied + * @return the newly created object + * @stable ICU 2.0 + */ + UCharCharacterIterator& + operator=(const UCharCharacterIterator& that); + + /** + * Returns true if the iterators iterate over the same range of the + * same string and are pointing at the same character. + * @param that The ForwardCharacterIterator used to be compared for equality + * @return true if the iterators iterate over the same range of the + * same string and are pointing at the same character. + * @stable ICU 2.0 + */ + virtual UBool operator==(const ForwardCharacterIterator& that) const; + + /** + * Generates a hash code for this iterator. + * @return the hash code. + * @stable ICU 2.0 + */ + virtual int32_t hashCode(void) const; + + /** + * Returns a new UCharCharacterIterator referring to the same + * character in the same range of the same string as this one. The + * caller must delete the new iterator. + * @return the CharacterIterator newly created + * @stable ICU 2.0 + */ + virtual CharacterIterator* clone(void) const; + + /** + * Sets the iterator to refer to the first code unit in its + * iteration range, and returns that code unit. + * This can be used to begin an iteration with next(). + * @return the first code unit in its iteration range. + * @stable ICU 2.0 + */ + virtual UChar first(void); + + /** + * Sets the iterator to refer to the first code unit in its + * iteration range, returns that code unit, and moves the position + * to the second code unit. This is an alternative to setToStart() + * for forward iteration with nextPostInc(). + * @return the first code unit in its iteration range + * @stable ICU 2.0 + */ + virtual UChar firstPostInc(void); + + /** + * Sets the iterator to refer to the first code point in its + * iteration range, and returns that code unit, + * This can be used to begin an iteration with next32(). + * Note that an iteration with next32PostInc(), beginning with, + * e.g., setToStart() or firstPostInc(), is more efficient. + * @return the first code point in its iteration range + * @stable ICU 2.0 + */ + virtual UChar32 first32(void); + + /** + * Sets the iterator to refer to the first code point in its + * iteration range, returns that code point, and moves the position + * to the second code point. This is an alternative to setToStart() + * for forward iteration with next32PostInc(). + * @return the first code point in its iteration range. + * @stable ICU 2.0 + */ + virtual UChar32 first32PostInc(void); + + /** + * Sets the iterator to refer to the last code unit in its + * iteration range, and returns that code unit. + * This can be used to begin an iteration with previous(). + * @return the last code unit in its iteration range. + * @stable ICU 2.0 + */ + virtual UChar last(void); + + /** + * Sets the iterator to refer to the last code point in its + * iteration range, and returns that code unit. + * This can be used to begin an iteration with previous32(). + * @return the last code point in its iteration range. + * @stable ICU 2.0 + */ + virtual UChar32 last32(void); + + /** + * Sets the iterator to refer to the "position"-th code unit + * in the text-storage object the iterator refers to, and + * returns that code unit. + * @param position the position within the text-storage object + * @return the code unit + * @stable ICU 2.0 + */ + virtual UChar setIndex(int32_t position); + + /** + * Sets the iterator to refer to the beginning of the code point + * that contains the "position"-th code unit + * in the text-storage object the iterator refers to, and + * returns that code point. + * The current position is adjusted to the beginning of the code point + * (its first code unit). + * @param position the position within the text-storage object + * @return the code unit + * @stable ICU 2.0 + */ + virtual UChar32 setIndex32(int32_t position); + + /** + * Returns the code unit the iterator currently refers to. + * @return the code unit the iterator currently refers to. + * @stable ICU 2.0 + */ + virtual UChar current(void) const; + + /** + * Returns the code point the iterator currently refers to. + * @return the code point the iterator currently refers to. + * @stable ICU 2.0 + */ + virtual UChar32 current32(void) const; + + /** + * Advances to the next code unit in the iteration range (toward + * endIndex()), and returns that code unit. If there are no more + * code units to return, returns DONE. + * @return the next code unit in the iteration range. + * @stable ICU 2.0 + */ + virtual UChar next(void); + + /** + * Gets the current code unit for returning and advances to the next code unit + * in the iteration range + * (toward endIndex()). If there are + * no more code units to return, returns DONE. + * @return the current code unit. + * @stable ICU 2.0 + */ + virtual UChar nextPostInc(void); + + /** + * Advances to the next code point in the iteration range (toward + * endIndex()), and returns that code point. If there are no more + * code points to return, returns DONE. + * Note that iteration with "pre-increment" semantics is less + * efficient than iteration with "post-increment" semantics + * that is provided by next32PostInc(). + * @return the next code point in the iteration range. + * @stable ICU 2.0 + */ + virtual UChar32 next32(void); + + /** + * Gets the current code point for returning and advances to the next code point + * in the iteration range + * (toward endIndex()). If there are + * no more code points to return, returns DONE. + * @return the current point. + * @stable ICU 2.0 + */ + virtual UChar32 next32PostInc(void); + + /** + * Returns FALSE if there are no more code units or code points + * at or after the current position in the iteration range. + * This is used with nextPostInc() or next32PostInc() in forward + * iteration. + * @return FALSE if there are no more code units or code points + * at or after the current position in the iteration range. + * @stable ICU 2.0 + */ + virtual UBool hasNext(); + + /** + * Advances to the previous code unit in the iteration range (toward + * startIndex()), and returns that code unit. If there are no more + * code units to return, returns DONE. + * @return the previous code unit in the iteration range. + * @stable ICU 2.0 + */ + virtual UChar previous(void); + + /** + * Advances to the previous code point in the iteration range (toward + * startIndex()), and returns that code point. If there are no more + * code points to return, returns DONE. + * @return the previous code point in the iteration range. + * @stable ICU 2.0 + */ + virtual UChar32 previous32(void); + + /** + * Returns FALSE if there are no more code units or code points + * before the current position in the iteration range. + * This is used with previous() or previous32() in backward + * iteration. + * @return FALSE if there are no more code units or code points + * before the current position in the iteration range. + * @stable ICU 2.0 + */ + virtual UBool hasPrevious(); + + /** + * Moves the current position relative to the start or end of the + * iteration range, or relative to the current position itself. + * The movement is expressed in numbers of code units forward + * or backward by specifying a positive or negative delta. + * @param delta the position relative to origin. A positive delta means forward; + * a negative delta means backward. + * @param origin Origin enumeration {kStart, kCurrent, kEnd} + * @return the new position + * @stable ICU 2.0 + */ + virtual int32_t move(int32_t delta, EOrigin origin); + + /** + * Moves the current position relative to the start or end of the + * iteration range, or relative to the current position itself. + * The movement is expressed in numbers of code points forward + * or backward by specifying a positive or negative delta. + * @param delta the position relative to origin. A positive delta means forward; + * a negative delta means backward. + * @param origin Origin enumeration {kStart, kCurrent, kEnd} + * @return the new position + * @stable ICU 2.0 + */ + virtual int32_t move32(int32_t delta, EOrigin origin); + + /** + * Sets the iterator to iterate over a new range of text + * @stable ICU 2.0 + */ + void setText(const UChar* newText, int32_t newTextLength); + + /** + * Copies the UChar array under iteration into the UnicodeString + * referred to by "result". Even if this iterator iterates across + * only a part of this string, the whole string is copied. + * @param result Receives a copy of the text under iteration. + * @stable ICU 2.0 + */ + virtual void getText(UnicodeString& result); + + /** + * Return a class ID for this class (not really public) + * @return a class ID for this class + * @stable ICU 2.0 + */ + static UClassID U_EXPORT2 getStaticClassID(void); + + /** + * Return a class ID for this object (not really public) + * @return a class ID for this object. + * @stable ICU 2.0 + */ + virtual UClassID getDynamicClassID(void) const; + +protected: + /** + * Protected constructor + * @stable ICU 2.0 + */ + UCharCharacterIterator(); + /** + * Protected member text + * @stable ICU 2.0 + */ + const UChar* text; + +}; + +U_NAMESPACE_END +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/uclean.h b/Build/source/libs/icu/icu-xetex/common/unicode/uclean.h new file mode 100644 index 00000000000..a13924a0c3f --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/uclean.h @@ -0,0 +1,267 @@ +/* +****************************************************************************** +* * +* Copyright (C) 2001-2005, International Business Machines * +* Corporation and others. All Rights Reserved. * +* * +****************************************************************************** +* file name: uclean.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2001July05 +* created by: George Rhoten +*/ + +#ifndef __UCLEAN_H__ +#define __UCLEAN_H__ + +#include "unicode/utypes.h" +/** + * \file + * \brief C API: Initialize and clean up ICU + */ + +/** + * Initialize ICU. The description further below applies to ICU 2.6 to ICU 3.4. + * Starting with ICU 3.4, u_init() needs not be called any more for + * ensuring thread safety, but it can give an indication for whether ICU + * can load its data. In ICU 3.4, it will try to load the converter alias table + * (cnvalias.icu) and give an error code if that fails. + * This may change in the future. + *

+ * For ensuring the availability of necessary data, an application should + * open the service objects (converters, collators, etc.) that it will use + * and check for error codes there. + *

+ * Documentation for ICU 2.6 to ICU 3.4: + *

+ * This function loads and initializes data items + * that are required internally by various ICU functions. Use of this explicit + * initialization is required in multi-threaded applications; in + * single threaded apps, use is optional, but incurs little additional + * cost, and is thus recommended. + *

+ * In multi-threaded applications, u_init() should be called in the + * main thread before starting additional threads, or, alternatively + * it can be called in each individual thread once, before other ICU + * functions are called in that thread. In this second scenario, the + * application must guarantee that the first call to u_init() happen + * without contention, in a single thread only. + *

+ * If u_setMemoryFunctions() or + * u_setMutexFunctions are needed (uncommon), they must be + * called _before_ u_init(). + *

+ * Extra, repeated, or otherwise unneeded calls to u_init() do no harm, + * other than taking a small amount of time. + * + * @param status An ICU UErrorCode parameter. It must not be NULL. + * An Error will be returned if some required part of ICU data can not + * be loaded or initialized. + * The function returns immediately if the input error code indicates a + * failure, as usual. + * + * @stable ICU 2.6 + */ +U_STABLE void U_EXPORT2 +u_init(UErrorCode *status); + +/** + * Clean up the system resources, such as allocated memory or open files, + * used in all ICU libraries. This will free/delete all memory owned by the + * ICU libraries, and return them to their original load state. All open ICU + * items (collators, resource bundles, converters, etc.) must be closed before + * calling this function, otherwise ICU may not free its allocated memory + * (e.g. close your converters and resource bundles before calling this + * function). Generally, this function should be called once just before + * an application exits. For applications that dynamically load and unload + * the ICU libraries (relatively uncommon), u_cleanup() should be called + * just before the library unload. + *

+ * u_cleanup() also clears any ICU heap functions, mutex functions or + * trace functions that may have been set for the process. + * This has the effect of restoring ICU to its initial condition, before + * any of these override functions were installed. Refer to + * u_setMemoryFunctions(), u_setMutexFunctions and + * utrace_setFunctions(). If ICU is to be reinitialized after after + * calling u_cleanup(), these runtime override functions will need to + * be set up again if they are still required. + *

+ * u_cleanup() is not thread safe. All other threads should stop using ICU + * before calling this function. + *

+ * Any open ICU items will be left in an undefined state by u_cleanup(), + * and any subsequent attempt to use such an item will give unpredictable + * results. + *

+ * After calling u_cleanup(), an application may continue to use ICU by + * calling u_init(). An application must invoke u_init() first from one single + * thread before allowing other threads call u_init(). All threads existing + * at the time of the first thread's call to u_init() must also call + * u_init() themselves before continuing with other ICU operations. + *

+ * The use of u_cleanup() just before an application terminates is optional, + * but it should be called only once for performance reasons. The primary + * benefit is to eliminate reports of memory or resource leaks originating + * in ICU code from the results generated by heap analysis tools. + *

+ * Use this function with great care! + *

+ * + * @stable ICU 2.0 + * @system + */ +U_STABLE void U_EXPORT2 +u_cleanup(void); + + + + +/** + * An opaque pointer type that represents an ICU mutex. + * For user-implemented mutexes, the value will typically point to a + * struct or object that implements the mutex. + * @stable ICU 2.8 + * @system + */ +typedef void *UMTX; + +/** + * Function Pointer type for a user supplied mutex initialization function. + * The user-supplied function will be called by ICU whenever ICU needs to create a + * new mutex. The function implementation should create a mutex, and store a pointer + * to something that uniquely identifies the mutex into the UMTX that is supplied + * as a paramter. + * @param context user supplied value, obtained from from u_setMutexFunctions(). + * @param mutex Receives a pointer that identifies the new mutex. + * The mutex init function must set the UMTX to a non-null value. + * Subsequent calls by ICU to lock, unlock, or destroy a mutex will + * identify the mutex by the UMTX value. + * @param status Error status. Report errors back to ICU by setting this variable + * with an error code. + * @stable ICU 2.8 + * @system + */ +typedef void U_CALLCONV UMtxInitFn (const void *context, UMTX *mutex, UErrorCode* status); + + +/** + * Function Pointer type for a user supplied mutex functions. + * One of the user-supplied functions with this signature will be called by ICU + * whenever ICU needs to lock, unlock, or destroy a mutex. + * @param context user supplied value, obtained from from u_setMutexFunctions(). + * @param mutex specify the mutex on which to operate. + * @stable ICU 2.8 + * @system + */ +typedef void U_CALLCONV UMtxFn (const void *context, UMTX *mutex); + + +/** + * Set the functions that ICU will use for mutex operations + * Use of this function is optional; by default (without this function), ICU will + * directly access system functions for mutex operations + * This function can only be used when ICU is in an initial, unused state, before + * u_init() has been called. + * This function may be used even when ICU has been built without multi-threaded + * support (see ICU_USE_THREADS pre-processor variable, umutex.h) + * @param context This pointer value will be saved, and then (later) passed as + * a parameter to the user-supplied mutex functions each time they + * are called. + * @param init Pointer to a mutex initialization function. Must be non-null. + * @param destroy Pointer to the mutex destroy function. Must be non-null. + * @param lock pointer to the mutex lock function. Must be non-null. + * @param unlock Pointer to the mutex unlock function. Must be non-null. + * @param status Receives error values. + * @stable ICU 2.8 + * @system + */ +U_STABLE void U_EXPORT2 +u_setMutexFunctions(const void *context, UMtxInitFn *init, UMtxFn *destroy, UMtxFn *lock, UMtxFn *unlock, + UErrorCode *status); + + +/** + * Pointer type for a user supplied atomic increment or decrement function. + * @param context user supplied value, obtained from from u_setAtomicIncDecFunctions(). + * @param p Pointer to a 32 bit int to be incremented or decremented + * @return The value of the variable after the inc or dec operation. + * @stable ICU 2.8 + * @system + */ +typedef int32_t U_CALLCONV UMtxAtomicFn(const void *context, int32_t *p); + +/** + * Set the functions that ICU will use for atomic increment and decrement of int32_t values. + * Use of this function is optional; by default (without this function), ICU will + * use its own internal implementation of atomic increment/decrement. + * This function can only be used when ICU is in an initial, unused state, before + * u_init() has been called. + * @param context This pointer value will be saved, and then (later) passed as + * a parameter to the increment and decrement functions each time they + * are called. This function can only be called + * @param inc Pointer to a function to do an atomic increment operation. Must be non-null. + * @param dec Pointer to a function to do an atomic decrement operation. Must be non-null. + * @param status Receives error values. + * @stable ICU 2.8 + * @system + */ +U_STABLE void U_EXPORT2 +u_setAtomicIncDecFunctions(const void *context, UMtxAtomicFn *inc, UMtxAtomicFn *dec, + UErrorCode *status); + + + +/** + * Pointer type for a user supplied memory allocation function. + * @param context user supplied value, obtained from from u_setMemoryFunctions(). + * @param size The number of bytes to be allocated + * @return Pointer to the newly allocated memory, or NULL if the allocation failed. + * @stable ICU 2.8 + * @system + */ +typedef void *U_CALLCONV UMemAllocFn(const void *context, size_t size); +/** + * Pointer type for a user supplied memory re-allocation function. + * @param context user supplied value, obtained from from u_setMemoryFunctions(). + * @param size The number of bytes to be allocated + * @return Pointer to the newly allocated memory, or NULL if the allocation failed. + * @stable ICU 2.8 + * @system + */ +typedef void *U_CALLCONV UMemReallocFn(const void *context, void *mem, size_t size); +/** + * Pointer type for a user supplied memory free function. Behavior should be + * similar the standard C library free(). + * @param context user supplied value, obtained from from u_setMemoryFunctions(). + * @param mem Pointer to the memory block to be resized + * @param size The new size for the block + * @return Pointer to the resized memory block, or NULL if the resizing failed. + * @stable ICU 2.8 + * @system + */ +typedef void U_CALLCONV UMemFreeFn (const void *context, void *mem); + +/** + * Set the functions that ICU will use for memory allocation. + * Use of this function is optional; by default (without this function), ICU will + * use the standard C library malloc() and free() functions. + * This function can only be used when ICU is in an initial, unused state, before + * u_init() has been called. + * @param context This pointer value will be saved, and then (later) passed as + * a parameter to the memory functions each time they + * are called. + * @param a Pointer to a user-supplied malloc function. + * @param r Pointer to a user-supplied realloc function. + * @param f Pointer to a user-supplied free function. + * @param status Receives error values. + * @stable ICU 2.8 + * @system + */ +U_STABLE void U_EXPORT2 +u_setMemoryFunctions(const void *context, UMemAllocFn *a, UMemReallocFn *r, UMemFreeFn *f, + UErrorCode *status); + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/ucnv.h b/Build/source/libs/icu/icu-xetex/common/unicode/ucnv.h new file mode 100644 index 00000000000..5e49a83e323 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/ucnv.h @@ -0,0 +1,1954 @@ +/* +********************************************************************** +* Copyright (C) 1999-2007, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** + * ucnv.h: + * External APIs for the ICU's codeset conversion library + * Bertrand A. Damiba + * + * Modification History: + * + * Date Name Description + * 04/04/99 helena Fixed internal header inclusion. + * 05/11/00 helena Added setFallback and usesFallback APIs. + * 06/29/2000 helena Major rewrite of the callback APIs. + * 12/07/2000 srl Update of documentation + */ + +/** + * \file + * \brief C API: Character conversion + * + *

Character Conversion C API

+ * + *

This API is used to convert codepage or character encoded data to and + * from UTF-16. You can open a converter with {@link ucnv_open() }. With that + * converter, you can get its properties, set options, convert your data and + * close the converter.

+ * + *

Since many software programs recogize different converter names for + * different types of converters, there are other functions in this API to + * iterate over the converter aliases. The functions {@link ucnv_getAvailableName() }, + * {@link ucnv_getAlias() } and {@link ucnv_getStandardName() } are some of the + * more frequently used alias functions to get this information.

+ * + *

When a converter encounters an illegal, irregular, invalid or unmappable character + * its default behavior is to use a substitution character to replace the + * bad byte sequence. This behavior can be changed by using {@link ucnv_setFromUCallBack() } + * or {@link ucnv_setToUCallBack() } on the converter. The header ucnv_err.h defines + * many other callback actions that can be used instead of a character substitution.

+ * + *

More information about this API can be found in our + * User's + * Guide.

+ */ + +#ifndef UCNV_H +#define UCNV_H + +#include "unicode/ucnv_err.h" +#include "unicode/uenum.h" + +#ifndef __USET_H__ + +/** + * USet is the C API type for Unicode sets. + * It is forward-declared here to avoid including the header file if related + * conversion APIs are not used. + * See unicode/uset.h + * + * @see ucnv_getUnicodeSet + * @stable ICU 2.6 + */ +struct USet; +/** @stable ICU 2.6 */ +typedef struct USet USet; + +#endif + +#if !UCONFIG_NO_CONVERSION + +U_CDECL_BEGIN + +/** Maximum length of a converter name including the terminating NULL @stable ICU 2.0 */ +#define UCNV_MAX_CONVERTER_NAME_LENGTH 60 +/** Maximum length of a converter name including path and terminating NULL @stable ICU 2.0 */ +#define UCNV_MAX_FULL_FILE_NAME_LENGTH (600+UCNV_MAX_CONVERTER_NAME_LENGTH) + +/** Shift in for EBDCDIC_STATEFUL and iso2022 states @stable ICU 2.0 */ +#define UCNV_SI 0x0F +/** Shift out for EBDCDIC_STATEFUL and iso2022 states @stable ICU 2.0 */ +#define UCNV_SO 0x0E + +/** + * Enum for specifying basic types of converters + * @see ucnv_getType + * @stable ICU 2.0 + */ +typedef enum { + UCNV_UNSUPPORTED_CONVERTER = -1, + UCNV_SBCS = 0, + UCNV_DBCS = 1, + UCNV_MBCS = 2, + UCNV_LATIN_1 = 3, + UCNV_UTF8 = 4, + UCNV_UTF16_BigEndian = 5, + UCNV_UTF16_LittleEndian = 6, + UCNV_UTF32_BigEndian = 7, + UCNV_UTF32_LittleEndian = 8, + UCNV_EBCDIC_STATEFUL = 9, + UCNV_ISO_2022 = 10, + + UCNV_LMBCS_1 = 11, + UCNV_LMBCS_2, + UCNV_LMBCS_3, + UCNV_LMBCS_4, + UCNV_LMBCS_5, + UCNV_LMBCS_6, + UCNV_LMBCS_8, + UCNV_LMBCS_11, + UCNV_LMBCS_16, + UCNV_LMBCS_17, + UCNV_LMBCS_18, + UCNV_LMBCS_19, + UCNV_LMBCS_LAST = UCNV_LMBCS_19, + UCNV_HZ, + UCNV_SCSU, + UCNV_ISCII, + UCNV_US_ASCII, + UCNV_UTF7, + UCNV_BOCU1, + UCNV_UTF16, + UCNV_UTF32, + UCNV_CESU8, + UCNV_IMAP_MAILBOX, + + /* Number of converter types for which we have conversion routines. */ + UCNV_NUMBER_OF_SUPPORTED_CONVERTER_TYPES + +} UConverterType; + +/** + * Enum for specifying which platform a converter ID refers to. + * The use of platform/CCSID is not recommended. See ucnv_openCCSID(). + * + * @see ucnv_getPlatform + * @see ucnv_openCCSID + * @see ucnv_getCCSID + * @stable ICU 2.0 + */ +typedef enum { + UCNV_UNKNOWN = -1, + UCNV_IBM = 0 +} UConverterPlatform; + +/** + * Function pointer for error callback in the codepage to unicode direction. + * Called when an error has occured in conversion to unicode, or on open/close of the callback (see reason). + * @param context Pointer to the callback's private data + * @param args Information about the conversion in progress + * @param codeUnits Points to 'length' bytes of the concerned codepage sequence + * @param length Size (in bytes) of the concerned codepage sequence + * @param reason Defines the reason the callback was invoked + * @param pErrorCode ICU error code in/out parameter. + * For converter callback functions, set to a conversion error + * before the call, and the callback may reset it to U_ZERO_ERROR. + * @see ucnv_setToUCallBack + * @see UConverterToUnicodeArgs + * @stable ICU 2.0 + */ +typedef void (U_EXPORT2 *UConverterToUCallback) ( + const void* context, + UConverterToUnicodeArgs *args, + const char *codeUnits, + int32_t length, + UConverterCallbackReason reason, + UErrorCode *pErrorCode); + +/** + * Function pointer for error callback in the unicode to codepage direction. + * Called when an error has occured in conversion from unicode, or on open/close of the callback (see reason). + * @param context Pointer to the callback's private data + * @param args Information about the conversion in progress + * @param codeUnits Points to 'length' UChars of the concerned Unicode sequence + * @param length Size (in bytes) of the concerned codepage sequence + * @param codePoint Single UChar32 (UTF-32) containing the concerend Unicode codepoint. + * @param reason Defines the reason the callback was invoked + * @param pErrorCode ICU error code in/out parameter. + * For converter callback functions, set to a conversion error + * before the call, and the callback may reset it to U_ZERO_ERROR. + * @see ucnv_setFromUCallBack + * @stable ICU 2.0 + */ +typedef void (U_EXPORT2 *UConverterFromUCallback) ( + const void* context, + UConverterFromUnicodeArgs *args, + const UChar* codeUnits, + int32_t length, + UChar32 codePoint, + UConverterCallbackReason reason, + UErrorCode *pErrorCode); + +U_CDECL_END + +/** + * Character that separates converter names from options and options from each other. + * @see ucnv_open + * @stable ICU 2.0 + */ +#define UCNV_OPTION_SEP_CHAR ',' + +/** + * String version of UCNV_OPTION_SEP_CHAR. + * @see ucnv_open + * @stable ICU 2.0 + */ +#define UCNV_OPTION_SEP_STRING "," + +/** + * Character that separates a converter option from its value. + * @see ucnv_open + * @stable ICU 2.0 + */ +#define UCNV_VALUE_SEP_CHAR '=' + +/** + * String version of UCNV_VALUE_SEP_CHAR. + * @see ucnv_open + * @stable ICU 2.0 + */ +#define UCNV_VALUE_SEP_STRING "=" + +/** + * Converter option for specifying a locale. + * For example, ucnv_open("SCSU,locale=ja", &errorCode); + * See convrtrs.txt. + * + * @see ucnv_open + * @stable ICU 2.0 + */ +#define UCNV_LOCALE_OPTION_STRING ",locale=" + +/** + * Converter option for specifying a version selector (0..9) for some converters. + * For example, ucnv_open("UTF-7,version=1", &errorCode); + * See convrtrs.txt. + * + * @see ucnv_open + * @stable ICU 2.4 + */ +#define UCNV_VERSION_OPTION_STRING ",version=" + +/** + * Converter option for EBCDIC SBCS or mixed-SBCS/DBCS (stateful) codepages. + * Swaps Unicode mappings for EBCDIC LF and NL codes, as used on + * S/390 (z/OS) Unix System Services (Open Edition). + * For example, ucnv_open("ibm-1047,swaplfnl", &errorCode); + * See convrtrs.txt. + * + * @see ucnv_open + * @stable ICU 2.4 + */ +#define UCNV_SWAP_LFNL_OPTION_STRING ",swaplfnl" + +/** + * Do a fuzzy compare of two converter/alias names. + * The comparison is case-insensitive, ignores leading zeroes if they are not + * followed by further digits, and ignores all but letters and digits. + * Thus the strings "UTF-8", "utf_8", "u*T@f08" and "Utf 8" are exactly equivalent. + * See section 1.4, Charset Alias Matching in Unicode Technical Standard #22 + * at http://www.unicode.org/reports/tr22/ + * + * @param name1 a converter name or alias, zero-terminated + * @param name2 a converter name or alias, zero-terminated + * @return 0 if the names match, or a negative value if the name1 + * lexically precedes name2, or a positive value if the name1 + * lexically follows name2. + * @stable ICU 2.0 + */ +U_STABLE int U_EXPORT2 +ucnv_compareNames(const char *name1, const char *name2); + + +/** + * Creates a UConverter object with the name of a coded character set specified as a C string. + * The actual name will be resolved with the alias file + * using a case-insensitive string comparison that ignores + * leading zeroes and all non-alphanumeric characters. + * E.g., the names "UTF8", "utf-8", "u*T@f08" and "Utf 8" are all equivalent. + * (See also ucnv_compareNames().) + * If NULL is passed for the converter name, it will create one with the + * getDefaultName return value. + * + *

A converter name for ICU 1.5 and above may contain options + * like a locale specification to control the specific behavior of + * the newly instantiated converter. + * The meaning of the options depends on the particular converter. + * If an option is not defined for or recognized by a given converter, then it is ignored.

+ * + *

Options are appended to the converter name string, with a + * UCNV_OPTION_SEP_CHAR between the name and the first option and + * also between adjacent options.

+ * + *

If the alias is ambiguous, then the preferred converter is used + * and the status is set to U_AMBIGUOUS_ALIAS_WARNING.

+ * + *

The conversion behavior and names can vary between platforms. ICU may + * convert some characters differently from other platforms. Details on this topic + * are in the User's + * Guide. Aliases starting with a "cp" prefix have no specific meaning + * other than its an alias starting with the letters "cp". Please do not + * associate any meaning to these aliases.

+ * + * @param converterName Name of the coded character set table. + * This may have options appended to the string. + * IANA alias character set names, IBM CCSIDs starting with "ibm-", + * Windows codepage numbers starting with "windows-" are frequently + * used for this parameter. See ucnv_getAvailableName and + * ucnv_getAlias for a complete list that is available. + * If this parameter is NULL, the default converter will be used. + * @param err outgoing error status U_MEMORY_ALLOCATION_ERROR, U_FILE_ACCESS_ERROR + * @return the created Unicode converter object, or NULL if an error occured + * @see ucnv_openU + * @see ucnv_openCCSID + * @see ucnv_getAvailableName + * @see ucnv_getAlias + * @see ucnv_getDefaultName + * @see ucnv_close + * @ee ucnv_compareNames + * @stable ICU 2.0 + */ +U_STABLE UConverter* U_EXPORT2 +ucnv_open(const char *converterName, UErrorCode *err); + + +/** + * Creates a Unicode converter with the names specified as unicode string. + * The name should be limited to the ASCII-7 alphanumerics range. + * The actual name will be resolved with the alias file + * using a case-insensitive string comparison that ignores + * leading zeroes and all non-alphanumeric characters. + * E.g., the names "UTF8", "utf-8", "u*T@f08" and "Utf 8" are all equivalent. + * (See also ucnv_compareNames().) + * If NULL is passed for the converter name, it will create + * one with the ucnv_getDefaultName() return value. + * If the alias is ambiguous, then the preferred converter is used + * and the status is set to U_AMBIGUOUS_ALIAS_WARNING. + * + *

See ucnv_open for the complete details

+ * @param name Name of the UConverter table in a zero terminated + * Unicode string + * @param err outgoing error status U_MEMORY_ALLOCATION_ERROR, + * U_FILE_ACCESS_ERROR + * @return the created Unicode converter object, or NULL if an + * error occured + * @see ucnv_open + * @see ucnv_openCCSID + * @see ucnv_close + * @ee ucnv_compareNames + * @stable ICU 2.0 + */ +U_STABLE UConverter* U_EXPORT2 +ucnv_openU(const UChar *name, + UErrorCode *err); + +/** + * Creates a UConverter object from a CCSID number and platform pair. + * Note that the usefulness of this function is limited to platforms with numeric + * encoding IDs. Only IBM and Microsoft platforms use numeric (16-bit) identifiers for + * encodings. + * + * In addition, IBM CCSIDs and Unicode conversion tables are not 1:1 related. + * For many IBM CCSIDs there are multiple (up to six) Unicode conversion tables, and + * for some Unicode conversion tables there are multiple CCSIDs. + * Some "alternate" Unicode conversion tables are provided by the + * IBM CDRA conversion table registry. + * The most prominent example of a systematic modification of conversion tables that is + * not provided in the form of conversion table files in the repository is + * that S/390 Unix System Services swaps the codes for Line Feed and New Line in all + * EBCDIC codepages, which requires such a swap in the Unicode conversion tables as well. + * + * Only IBM default conversion tables are accessible with ucnv_openCCSID(). + * ucnv_getCCSID() will return the same CCSID for all conversion tables that are associated + * with that CCSID. + * + * Currently, the only "platform" supported in the ICU converter API is UCNV_IBM. + * + * In summary, the use of CCSIDs and the associated API functions is not recommended. + * + * In order to open a converter with the default IBM CDRA Unicode conversion table, + * you can use this function or use the prefix "ibm-": + * \code + * char name[20]; + * sprintf(name, "ibm-%hu", ccsid); + * cnv=ucnv_open(name, &errorCode); + * \endcode + * + * In order to open a converter with the IBM S/390 Unix System Services variant + * of a Unicode/EBCDIC conversion table, + * you can use the prefix "ibm-" together with the option string UCNV_SWAP_LFNL_OPTION_STRING: + * \code + * char name[20]; + * sprintf(name, "ibm-%hu" UCNV_SWAP_LFNL_OPTION_STRING, ccsid); + * cnv=ucnv_open(name, &errorCode); + * \endcode + * + * In order to open a converter from a Microsoft codepage number, use the prefix "cp": + * \code + * char name[20]; + * sprintf(name, "cp%hu", codepageID); + * cnv=ucnv_open(name, &errorCode); + * \endcode + * + * If the alias is ambiguous, then the preferred converter is used + * and the status is set to U_AMBIGUOUS_ALIAS_WARNING. + * + * @param codepage codepage number to create + * @param platform the platform in which the codepage number exists + * @param err error status U_MEMORY_ALLOCATION_ERROR, U_FILE_ACCESS_ERROR + * @return the created Unicode converter object, or NULL if an error + * occured. + * @see ucnv_open + * @see ucnv_openU + * @see ucnv_close + * @see ucnv_getCCSID + * @see ucnv_getPlatform + * @see UConverterPlatform + * @stable ICU 2.0 + */ +U_STABLE UConverter* U_EXPORT2 +ucnv_openCCSID(int32_t codepage, + UConverterPlatform platform, + UErrorCode * err); + +/** + *

Creates a UConverter object specified from a packageName and a converterName.

+ * + *

The packageName and converterName must point to an ICU udata object, as defined by + * udata_open( packageName, "cnv", converterName, err) or equivalent. + * Typically, packageName will refer to a (.dat) file, or to a package registered with + * udata_setAppData(). Using a full file or directory pathname for packageName is deprecated.

+ * + *

The name will NOT be looked up in the alias mechanism, nor will the converter be + * stored in the converter cache or the alias table. The only way to open further converters + * is call this function multiple times, or use the ucnv_safeClone() function to clone a + * 'master' converter.

+ * + *

A future version of ICU may add alias table lookups and/or caching + * to this function.

+ * + *

Example Use: + * cnv = ucnv_openPackage("myapp", "myconverter", &err); + *

+ * + * @param packageName name of the package (equivalent to 'path' in udata_open() call) + * @param converterName name of the data item to be used, without suffix. + * @param err outgoing error status U_MEMORY_ALLOCATION_ERROR, U_FILE_ACCESS_ERROR + * @return the created Unicode converter object, or NULL if an error occured + * @see udata_open + * @see ucnv_open + * @see ucnv_safeClone + * @see ucnv_close + * @stable ICU 2.2 + */ +U_STABLE UConverter* U_EXPORT2 +ucnv_openPackage(const char *packageName, const char *converterName, UErrorCode *err); + +/** + * Thread safe converter cloning operation. + * For most efficient operation, pass in a stackBuffer (and a *pBufferSize) + * with at least U_CNV_SAFECLONE_BUFFERSIZE bytes of space. + * If the buffer size is sufficient, then the clone will use the stack buffer; + * otherwise, it will be allocated, and *pBufferSize will indicate + * the actual size. (This should not occur with U_CNV_SAFECLONE_BUFFERSIZE.) + * + * You must ucnv_close() the clone in any case. + * + * If *pBufferSize==0, (regardless of whether stackBuffer==NULL or not) + * then *pBufferSize will be changed to a sufficient size + * for cloning this converter, + * without actually cloning the converter ("pure pre-flighting"). + * + * If *pBufferSize is greater than zero but not large enough for a stack-based + * clone, then the converter is cloned using newly allocated memory + * and *pBufferSize is changed to the necessary size. + * + * If the converter clone fits into the stack buffer but the stack buffer is not + * sufficiently aligned for the clone, then the clone will use an + * adjusted pointer and use an accordingly smaller buffer size. + * + * @param cnv converter to be cloned + * @param stackBuffer user allocated space for the new clone. If NULL new memory will be allocated. + * If buffer is not large enough, new memory will be allocated. + * Clients can use the U_CNV_SAFECLONE_BUFFERSIZE. This will probably be enough to avoid memory allocations. + * @param pBufferSize pointer to size of allocated space. pBufferSize must not be NULL. + * @param status to indicate whether the operation went on smoothly or there were errors + * An informational status value, U_SAFECLONE_ALLOCATED_WARNING, + * is used if any allocations were necessary. + * However, it is better to check if *pBufferSize grew for checking for + * allocations because warning codes can be overridden by subsequent + * function calls. + * @return pointer to the new clone + * @stable ICU 2.0 + */ +U_STABLE UConverter * U_EXPORT2 +ucnv_safeClone(const UConverter *cnv, + void *stackBuffer, + int32_t *pBufferSize, + UErrorCode *status); + +/** + * \def U_CNV_SAFECLONE_BUFFERSIZE + * Definition of a buffer size that is designed to be large enough for + * converters to be cloned with ucnv_safeClone(). + * @stable ICU 2.0 + */ +#define U_CNV_SAFECLONE_BUFFERSIZE 1024 + +/** + * Deletes the unicode converter and releases resources associated + * with just this instance. + * Does not free up shared converter tables. + * + * @param converter the converter object to be deleted + * @see ucnv_open + * @see ucnv_openU + * @see ucnv_openCCSID + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ucnv_close(UConverter * converter); + +/** + * Fills in the output parameter, subChars, with the substitution characters + * as multiple bytes. + * If ucnv_setSubstString() set a Unicode string because the converter is + * stateful, then subChars will be an empty string. + * + * @param converter the Unicode converter + * @param subChars the subsitution characters + * @param len on input the capacity of subChars, on output the number + * of bytes copied to it + * @param err the outgoing error status code. + * If the substitution character array is too small, an + * U_INDEX_OUTOFBOUNDS_ERROR will be returned. + * @see ucnv_setSubstString + * @see ucnv_setSubstChars + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ucnv_getSubstChars(const UConverter *converter, + char *subChars, + int8_t *len, + UErrorCode *err); + +/** + * Sets the substitution chars when converting from unicode to a codepage. The + * substitution is specified as a string of 1-4 bytes, and may contain + * NULL bytes. + * The subChars must represent a single character. The caller needs to know the + * byte sequence of a valid character in the converter's charset. + * For some converters, for example some ISO 2022 variants, only single-byte + * substitution characters may be supported. + * The newer ucnv_setSubstString() function relaxes these limitations. + * + * @param converter the Unicode converter + * @param subChars the substitution character byte sequence we want set + * @param len the number of bytes in subChars + * @param err the error status code. U_INDEX_OUTOFBOUNDS_ERROR if + * len is bigger than the maximum number of bytes allowed in subchars + * @see ucnv_setSubstString + * @see ucnv_getSubstChars + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ucnv_setSubstChars(UConverter *converter, + const char *subChars, + int8_t len, + UErrorCode *err); + +/** + * Set a substitution string for converting from Unicode to a charset. + * The caller need not know the charset byte sequence for each charset. + * + * Unlike ucnv_setSubstChars() which is designed to set a charset byte sequence + * for a single character, this function takes a Unicode string with + * zero, one or more characters, and immediately verifies that the string can be + * converted to the charset. + * If not, or if the result is too long (more than 32 bytes as of ICU 3.6), + * then the function returns with an error accordingly. + * + * Also unlike ucnv_setSubstChars(), this function works for stateful charsets + * by converting on the fly at the point of substitution rather than setting + * a fixed byte sequence. + * + * @param cnv The UConverter object. + * @param s The Unicode string. + * @param length The number of UChars in s, or -1 for a NUL-terminated string. + * @param err Pointer to a standard ICU error code. Its input value must + * pass the U_SUCCESS() test, or else the function returns + * immediately. Check for U_FAILURE() on output or use with + * function chaining. (See User Guide for details.) + * + * @see ucnv_setSubstChars + * @see ucnv_getSubstChars + * @stable ICU 3.6 + */ +U_STABLE void U_EXPORT2 +ucnv_setSubstString(UConverter *cnv, + const UChar *s, + int32_t length, + UErrorCode *err); + +/** + * Fills in the output parameter, errBytes, with the error characters from the + * last failing conversion. + * + * @param converter the Unicode converter + * @param errBytes the codepage bytes which were in error + * @param len on input the capacity of errBytes, on output the number of + * bytes which were copied to it + * @param err the error status code. + * If the substitution character array is too small, an + * U_INDEX_OUTOFBOUNDS_ERROR will be returned. + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ucnv_getInvalidChars(const UConverter *converter, + char *errBytes, + int8_t *len, + UErrorCode *err); + +/** + * Fills in the output parameter, errChars, with the error characters from the + * last failing conversion. + * + * @param converter the Unicode converter + * @param errUChars the UChars which were in error + * @param len on input the capacity of errUChars, on output the number of + * UChars which were copied to it + * @param err the error status code. + * If the substitution character array is too small, an + * U_INDEX_OUTOFBOUNDS_ERROR will be returned. + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ucnv_getInvalidUChars(const UConverter *converter, + UChar *errUChars, + int8_t *len, + UErrorCode *err); + +/** + * Resets the state of a converter to the default state. This is used + * in the case of an error, to restart a conversion from a known default state. + * It will also empty the internal output buffers. + * @param converter the Unicode converter + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ucnv_reset(UConverter *converter); + +/** + * Resets the to-Unicode part of a converter state to the default state. + * This is used in the case of an error to restart a conversion to + * Unicode to a known default state. It will also empty the internal + * output buffers used for the conversion to Unicode codepoints. + * @param converter the Unicode converter + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ucnv_resetToUnicode(UConverter *converter); + +/** + * Resets the from-Unicode part of a converter state to the default state. + * This is used in the case of an error to restart a conversion from + * Unicode to a known default state. It will also empty the internal output + * buffers used for the conversion from Unicode codepoints. + * @param converter the Unicode converter + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ucnv_resetFromUnicode(UConverter *converter); + +/** + * Returns the maximum number of bytes that are output per UChar in conversion + * from Unicode using this converter. + * The returned number can be used with UCNV_GET_MAX_BYTES_FOR_STRING + * to calculate the size of a target buffer for conversion from Unicode. + * + * Note: Before ICU 2.8, this function did not return reliable numbers for + * some stateful converters (EBCDIC_STATEFUL, ISO-2022) and LMBCS. + * + * This number may not be the same as the maximum number of bytes per + * "conversion unit". In other words, it may not be the intuitively expected + * number of bytes per character that would be published for a charset, + * and may not fulfill any other purpose than the allocation of an output + * buffer of guaranteed sufficient size for a given input length and converter. + * + * Examples for special cases that are taken into account: + * - Supplementary code points may convert to more bytes than BMP code points. + * This function returns bytes per UChar (UTF-16 code unit), not per + * Unicode code point, for efficient buffer allocation. + * - State-shifting output (SI/SO, escapes, etc.) from stateful converters. + * - When m input UChars are converted to n output bytes, then the maximum m/n + * is taken into account. + * + * The number returned here does not take into account + * (see UCNV_GET_MAX_BYTES_FOR_STRING): + * - callbacks which output more than one charset character sequence per call, + * like escape callbacks + * - initial and final non-character bytes that are output by some converters + * (automatic BOMs, initial escape sequence, final SI, etc.) + * + * Examples for returned values: + * - SBCS charsets: 1 + * - Shift-JIS: 2 + * - UTF-16: 2 (2 per BMP, 4 per surrogate _pair_, BOM not counted) + * - UTF-8: 3 (3 per BMP, 4 per surrogate _pair_) + * - EBCDIC_STATEFUL (EBCDIC mixed SBCS/DBCS): 3 (SO + DBCS) + * - ISO-2022: 3 (always outputs UTF-8) + * - ISO-2022-JP: 6 (4-byte escape sequences + DBCS) + * - ISO-2022-CN: 8 (4-byte designator sequences + 2-byte SS2/SS3 + DBCS) + * + * @param converter The Unicode converter. + * @return The maximum number of bytes per UChar that are output by ucnv_fromUnicode(), + * to be used together with UCNV_GET_MAX_BYTES_FOR_STRING for buffer allocation. + * + * @see UCNV_GET_MAX_BYTES_FOR_STRING + * @see ucnv_getMinCharSize + * @stable ICU 2.0 + */ +U_STABLE int8_t U_EXPORT2 +ucnv_getMaxCharSize(const UConverter *converter); + +/** + * Calculates the size of a buffer for conversion from Unicode to a charset. + * The calculated size is guaranteed to be sufficient for this conversion. + * + * It takes into account initial and final non-character bytes that are output + * by some converters. + * It does not take into account callbacks which output more than one charset + * character sequence per call, like escape callbacks. + * The default (substitution) callback only outputs one charset character sequence. + * + * @param length Number of UChars to be converted. + * @param maxCharSize Return value from ucnv_getMaxCharSize() for the converter + * that will be used. + * @return Size of a buffer that will be large enough to hold the output bytes of + * converting length UChars with the converter that returned the maxCharSize. + * + * @see ucnv_getMaxCharSize + * @stable ICU 2.8 + */ +#define UCNV_GET_MAX_BYTES_FOR_STRING(length, maxCharSize) \ + (((int32_t)(length)+10)*(int32_t)(maxCharSize)) + +/** + * Returns the minimum byte length for characters in this codepage. + * This is usually either 1 or 2. + * @param converter the Unicode converter + * @return the minimum number of bytes allowed by this particular converter + * @see ucnv_getMaxCharSize + * @stable ICU 2.0 + */ +U_STABLE int8_t U_EXPORT2 +ucnv_getMinCharSize(const UConverter *converter); + +/** + * Returns the display name of the converter passed in based on the Locale + * passed in. If the locale contains no display name, the internal ASCII + * name will be filled in. + * + * @param converter the Unicode converter. + * @param displayLocale is the specific Locale we want to localised for + * @param displayName user provided buffer to be filled in + * @param displayNameCapacity size of displayName Buffer + * @param err error status code + * @return displayNameLength number of UChar needed in displayName + * @see ucnv_getName + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +ucnv_getDisplayName(const UConverter *converter, + const char *displayLocale, + UChar *displayName, + int32_t displayNameCapacity, + UErrorCode *err); + +/** + * Gets the internal, canonical name of the converter (zero-terminated). + * The lifetime of the returned string will be that of the converter + * passed to this function. + * @param converter the Unicode converter + * @param err UErrorCode status + * @return the internal name of the converter + * @see ucnv_getDisplayName + * @stable ICU 2.0 + */ +U_STABLE const char * U_EXPORT2 +ucnv_getName(const UConverter *converter, UErrorCode *err); + +/** + * Gets a codepage number associated with the converter. This is not guaranteed + * to be the one used to create the converter. Some converters do not represent + * platform registered codepages and return zero for the codepage number. + * The error code fill-in parameter indicates if the codepage number + * is available. + * Does not check if the converter is NULL or if converter's data + * table is NULL. + * + * Important: The use of CCSIDs is not recommended because it is limited + * to only two platforms in principle and only one (UCNV_IBM) in the current + * ICU converter API. + * Also, CCSIDs are insufficient to identify IBM Unicode conversion tables precisely. + * For more details see ucnv_openCCSID(). + * + * @param converter the Unicode converter + * @param err the error status code. + * @return If any error occurrs, -1 will be returned otherwise, the codepage number + * will be returned + * @see ucnv_openCCSID + * @see ucnv_getPlatform + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +ucnv_getCCSID(const UConverter *converter, + UErrorCode *err); + +/** + * Gets a codepage platform associated with the converter. Currently, + * only UCNV_IBM will be returned. + * Does not test if the converter is NULL or if converter's data + * table is NULL. + * @param converter the Unicode converter + * @param err the error status code. + * @return The codepage platform + * @stable ICU 2.0 + */ +U_STABLE UConverterPlatform U_EXPORT2 +ucnv_getPlatform(const UConverter *converter, + UErrorCode *err); + +/** + * Gets the type of the converter + * e.g. SBCS, MBCS, DBCS, UTF8, UTF16_BE, UTF16_LE, ISO_2022, + * EBCDIC_STATEFUL, LATIN_1 + * @param converter a valid, opened converter + * @return the type of the converter + * @stable ICU 2.0 + */ +U_STABLE UConverterType U_EXPORT2 +ucnv_getType(const UConverter * converter); + +/** + * Gets the "starter" (lead) bytes for converters of type MBCS. + * Will fill in an U_ILLEGAL_ARGUMENT_ERROR if converter passed in + * is not MBCS. Fills in an array of type UBool, with the value of the byte + * as offset to the array. For example, if (starters[0x20] == TRUE) at return, + * it means that the byte 0x20 is a starter byte in this converter. + * Context pointers are always owned by the caller. + * + * @param converter a valid, opened converter of type MBCS + * @param starters an array of size 256 to be filled in + * @param err error status, U_ILLEGAL_ARGUMENT_ERROR if the + * converter is not a type which can return starters. + * @see ucnv_getType + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ucnv_getStarters(const UConverter* converter, + UBool starters[256], + UErrorCode* err); + + +/** + * Selectors for Unicode sets that can be returned by ucnv_getUnicodeSet(). + * @see ucnv_getUnicodeSet + * @stable ICU 2.6 + */ +typedef enum UConverterUnicodeSet { + /** Select the set of roundtrippable Unicode code points. @stable ICU 2.6 */ + UCNV_ROUNDTRIP_SET, + /** Number of UConverterUnicodeSet selectors. @stable ICU 2.6 */ + UCNV_SET_COUNT +} UConverterUnicodeSet; + + +/** + * Returns the set of Unicode code points that can be converted by an ICU converter. + * + * The current implementation returns only one kind of set (UCNV_ROUNDTRIP_SET): + * The set of all Unicode code points that can be roundtrip-converted + * (converted without any data loss) with the converter. + * This set will not include code points that have fallback mappings + * or are only the result of reverse fallback mappings. + * See UTR #22 "Character Mapping Markup Language" + * at http://www.unicode.org/reports/tr22/ + * + * This is useful for example for + * - checking that a string or document can be roundtrip-converted with a converter, + * without/before actually performing the conversion + * - testing if a converter can be used for text for typical text for a certain locale, + * by comparing its roundtrip set with the set of ExemplarCharacters from + * ICU's locale data or other sources + * + * In the future, there may be more UConverterUnicodeSet choices to select + * sets with different properties. + * + * @param cnv The converter for which a set is requested. + * @param setFillIn A valid USet *. It will be cleared by this function before + * the converter's specific set is filled into the USet. + * @param whichSet A UConverterUnicodeSet selector; + * currently UCNV_ROUNDTRIP_SET is the only supported value. + * @param pErrorCode ICU error code in/out parameter. + * Must fulfill U_SUCCESS before the function call. + * + * @see UConverterUnicodeSet + * @see uset_open + * @see uset_close + * @stable ICU 2.6 + */ +U_STABLE void U_EXPORT2 +ucnv_getUnicodeSet(const UConverter *cnv, + USet *setFillIn, + UConverterUnicodeSet whichSet, + UErrorCode *pErrorCode); + +/** + * Gets the current calback function used by the converter when an illegal + * or invalid codepage sequence is found. + * Context pointers are always owned by the caller. + * + * @param converter the unicode converter + * @param action fillin: returns the callback function pointer + * @param context fillin: returns the callback's private void* context + * @see ucnv_setToUCallBack + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ucnv_getToUCallBack (const UConverter * converter, + UConverterToUCallback *action, + const void **context); + +/** + * Gets the current callback function used by the converter when illegal + * or invalid Unicode sequence is found. + * Context pointers are always owned by the caller. + * + * @param converter the unicode converter + * @param action fillin: returns the callback function pointer + * @param context fillin: returns the callback's private void* context + * @see ucnv_setFromUCallBack + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ucnv_getFromUCallBack (const UConverter * converter, + UConverterFromUCallback *action, + const void **context); + +/** + * Changes the callback function used by the converter when + * an illegal or invalid sequence is found. + * Context pointers are always owned by the caller. + * Predefined actions and contexts can be found in the ucnv_err.h header. + * + * @param converter the unicode converter + * @param newAction the new callback function + * @param newContext the new toUnicode callback context pointer. This can be NULL. + * @param oldAction fillin: returns the old callback function pointer. This can be NULL. + * @param oldContext fillin: returns the old callback's private void* context. This can be NULL. + * @param err The error code status + * @see ucnv_getToUCallBack + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ucnv_setToUCallBack (UConverter * converter, + UConverterToUCallback newAction, + const void* newContext, + UConverterToUCallback *oldAction, + const void** oldContext, + UErrorCode * err); + +/** + * Changes the current callback function used by the converter when + * an illegal or invalid sequence is found. + * Context pointers are always owned by the caller. + * Predefined actions and contexts can be found in the ucnv_err.h header. + * + * @param converter the unicode converter + * @param newAction the new callback function + * @param newContext the new fromUnicode callback context pointer. This can be NULL. + * @param oldAction fillin: returns the old callback function pointer. This can be NULL. + * @param oldContext fillin: returns the old callback's private void* context. This can be NULL. + * @param err The error code status + * @see ucnv_getFromUCallBack + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ucnv_setFromUCallBack (UConverter * converter, + UConverterFromUCallback newAction, + const void *newContext, + UConverterFromUCallback *oldAction, + const void **oldContext, + UErrorCode * err); + +/** + * Converts an array of unicode characters to an array of codepage + * characters. This function is optimized for converting a continuous + * stream of data in buffer-sized chunks, where the entire source and + * target does not fit in available buffers. + * + * The source pointer is an in/out parameter. It starts out pointing where the + * conversion is to begin, and ends up pointing after the last UChar consumed. + * + * Target similarly starts out pointer at the first available byte in the output + * buffer, and ends up pointing after the last byte written to the output. + * + * The converter always attempts to consume the entire source buffer, unless + * (1.) the target buffer is full, or (2.) a failing error is returned from the + * current callback function. When a successful error status has been + * returned, it means that all of the source buffer has been + * consumed. At that point, the caller should reset the source and + * sourceLimit pointers to point to the next chunk. + * + * At the end of the stream (flush==TRUE), the input is completely consumed + * when *source==sourceLimit and no error code is set. + * The converter object is then automatically reset by this function. + * (This means that a converter need not be reset explicitly between data + * streams if it finishes the previous stream without errors.) + * + * This is a stateful conversion. Additionally, even when all source data has + * been consumed, some data may be in the converters' internal state. + * Call this function repeatedly, updating the target pointers with + * the next empty chunk of target in case of a + * U_BUFFER_OVERFLOW_ERROR, and updating the source pointers + * with the next chunk of source when a successful error status is + * returned, until there are no more chunks of source data. + * @param converter the Unicode converter + * @param target I/O parameter. Input : Points to the beginning of the buffer to copy + * codepage characters to. Output : points to after the last codepage character copied + * to target. + * @param targetLimit the pointer just after last of the target buffer + * @param source I/O parameter, pointer to pointer to the source Unicode character buffer. + * @param sourceLimit the pointer just after the last of the source buffer + * @param offsets if NULL is passed, nothing will happen to it, otherwise it needs to have the same number + * of allocated cells as target. Will fill in offsets from target to source pointer + * e.g: offsets[3] is equal to 6, it means that the target[3] was a result of transcoding source[6] + * For output data carried across calls, and other data without a specific source character + * (such as from escape sequences or callbacks) -1 will be placed for offsets. + * @param flush set to TRUE if the current source buffer is the last available + * chunk of the source, FALSE otherwise. Note that if a failing status is returned, + * this function may have to be called multiple times with flush set to TRUE until + * the source buffer is consumed. + * @param err the error status. U_ILLEGAL_ARGUMENT_ERROR will be set if the + * converter is NULL. + * U_BUFFER_OVERFLOW_ERROR will be set if the target is full and there is + * still data to be written to the target. + * @see ucnv_fromUChars + * @see ucnv_convert + * @see ucnv_getMinCharSize + * @see ucnv_setToUCallBack + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ucnv_fromUnicode (UConverter * converter, + char **target, + const char *targetLimit, + const UChar ** source, + const UChar * sourceLimit, + int32_t* offsets, + UBool flush, + UErrorCode * err); + +/** + * Converts a buffer of codepage bytes into an array of unicode UChars + * characters. This function is optimized for converting a continuous + * stream of data in buffer-sized chunks, where the entire source and + * target does not fit in available buffers. + * + * The source pointer is an in/out parameter. It starts out pointing where the + * conversion is to begin, and ends up pointing after the last byte of source consumed. + * + * Target similarly starts out pointer at the first available UChar in the output + * buffer, and ends up pointing after the last UChar written to the output. + * It does NOT necessarily keep UChar sequences together. + * + * The converter always attempts to consume the entire source buffer, unless + * (1.) the target buffer is full, or (2.) a failing error is returned from the + * current callback function. When a successful error status has been + * returned, it means that all of the source buffer has been + * consumed. At that point, the caller should reset the source and + * sourceLimit pointers to point to the next chunk. + * + * At the end of the stream (flush==TRUE), the input is completely consumed + * when *source==sourceLimit and no error code is set + * The converter object is then automatically reset by this function. + * (This means that a converter need not be reset explicitly between data + * streams if it finishes the previous stream without errors.) + * + * This is a stateful conversion. Additionally, even when all source data has + * been consumed, some data may be in the converters' internal state. + * Call this function repeatedly, updating the target pointers with + * the next empty chunk of target in case of a + * U_BUFFER_OVERFLOW_ERROR, and updating the source pointers + * with the next chunk of source when a successful error status is + * returned, until there are no more chunks of source data. + * @param converter the Unicode converter + * @param target I/O parameter. Input : Points to the beginning of the buffer to copy + * UChars into. Output : points to after the last UChar copied. + * @param targetLimit the pointer just after the end of the target buffer + * @param source I/O parameter, pointer to pointer to the source codepage buffer. + * @param sourceLimit the pointer to the byte after the end of the source buffer + * @param offsets if NULL is passed, nothing will happen to it, otherwise it needs to have the same number + * of allocated cells as target. Will fill in offsets from target to source pointer + * e.g: offsets[3] is equal to 6, it means that the target[3] was a result of transcoding source[6] + * For output data carried across calls, and other data without a specific source character + * (such as from escape sequences or callbacks) -1 will be placed for offsets. + * @param flush set to TRUE if the current source buffer is the last available + * chunk of the source, FALSE otherwise. Note that if a failing status is returned, + * this function may have to be called multiple times with flush set to TRUE until + * the source buffer is consumed. + * @param err the error status. U_ILLEGAL_ARGUMENT_ERROR will be set if the + * converter is NULL. + * U_BUFFER_OVERFLOW_ERROR will be set if the target is full and there is + * still data to be written to the target. + * @see ucnv_fromUChars + * @see ucnv_convert + * @see ucnv_getMinCharSize + * @see ucnv_setFromUCallBack + * @see ucnv_getNextUChar + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ucnv_toUnicode(UConverter *converter, + UChar **target, + const UChar *targetLimit, + const char **source, + const char *sourceLimit, + int32_t *offsets, + UBool flush, + UErrorCode *err); + +/** + * Convert the Unicode string into a codepage string using an existing UConverter. + * The output string is NUL-terminated if possible. + * + * This function is a more convenient but less powerful version of ucnv_fromUnicode(). + * It is only useful for whole strings, not for streaming conversion. + * + * The maximum output buffer capacity required (barring output from callbacks) will be + * UCNV_GET_MAX_BYTES_FOR_STRING(srcLength, ucnv_getMaxCharSize(cnv)). + * + * @param cnv the converter object to be used (ucnv_resetFromUnicode() will be called) + * @param src the input Unicode string + * @param srcLength the input string length, or -1 if NUL-terminated + * @param dest destination string buffer, can be NULL if destCapacity==0 + * @param destCapacity the number of chars available at dest + * @param pErrorCode normal ICU error code; + * common error codes that may be set by this function include + * U_BUFFER_OVERFLOW_ERROR, U_STRING_NOT_TERMINATED_WARNING, + * U_ILLEGAL_ARGUMENT_ERROR, and conversion errors + * @return the length of the output string, not counting the terminating NUL; + * if the length is greater than destCapacity, then the string will not fit + * and a buffer of the indicated length would need to be passed in + * @see ucnv_fromUnicode + * @see ucnv_convert + * @see UCNV_GET_MAX_BYTES_FOR_STRING + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +ucnv_fromUChars(UConverter *cnv, + char *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + UErrorCode *pErrorCode); + +/** + * Convert the codepage string into a Unicode string using an existing UConverter. + * The output string is NUL-terminated if possible. + * + * This function is a more convenient but less powerful version of ucnv_toUnicode(). + * It is only useful for whole strings, not for streaming conversion. + * + * The maximum output buffer capacity required (barring output from callbacks) will be + * 2*srcLength (each char may be converted into a surrogate pair). + * + * @param cnv the converter object to be used (ucnv_resetToUnicode() will be called) + * @param src the input codepage string + * @param srcLength the input string length, or -1 if NUL-terminated + * @param dest destination string buffer, can be NULL if destCapacity==0 + * @param destCapacity the number of UChars available at dest + * @param pErrorCode normal ICU error code; + * common error codes that may be set by this function include + * U_BUFFER_OVERFLOW_ERROR, U_STRING_NOT_TERMINATED_WARNING, + * U_ILLEGAL_ARGUMENT_ERROR, and conversion errors + * @return the length of the output string, not counting the terminating NUL; + * if the length is greater than destCapacity, then the string will not fit + * and a buffer of the indicated length would need to be passed in + * @see ucnv_toUnicode + * @see ucnv_convert + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +ucnv_toUChars(UConverter *cnv, + UChar *dest, int32_t destCapacity, + const char *src, int32_t srcLength, + UErrorCode *pErrorCode); + +/** + * Convert a codepage buffer into Unicode one character at a time. + * The input is completely consumed when the U_INDEX_OUTOFBOUNDS_ERROR is set. + * + * Advantage compared to ucnv_toUnicode() or ucnv_toUChars(): + * - Faster for small amounts of data, for most converters, e.g., + * US-ASCII, ISO-8859-1, UTF-8/16/32, and most "normal" charsets. + * (For complex converters, e.g., SCSU, UTF-7 and ISO 2022 variants, + * it uses ucnv_toUnicode() internally.) + * - Convenient. + * + * Limitations compared to ucnv_toUnicode(): + * - Always assumes flush=TRUE. + * This makes ucnv_getNextUChar() unsuitable for "streaming" conversion, + * that is, for where the input is supplied in multiple buffers, + * because ucnv_getNextUChar() will assume the end of the input at the end + * of the first buffer. + * - Does not provide offset output. + * + * It is possible to "mix" ucnv_getNextUChar() and ucnv_toUnicode() because + * ucnv_getNextUChar() uses the current state of the converter + * (unlike ucnv_toUChars() which always resets first). + * However, if ucnv_getNextUChar() is called after ucnv_toUnicode() + * stopped in the middle of a character sequence (with flush=FALSE), + * then ucnv_getNextUChar() will always use the slower ucnv_toUnicode() + * internally until the next character boundary. + * (This is new in ICU 2.6. In earlier releases, ucnv_getNextUChar() had to + * start at a character boundary.) + * + * Instead of using ucnv_getNextUChar(), it is recommended + * to convert using ucnv_toUnicode() or ucnv_toUChars() + * and then iterate over the text using U16_NEXT() or a UCharIterator (uiter.h) + * or a C++ CharacterIterator or similar. + * This allows streaming conversion and offset output, for example. + * + *

Handling of surrogate pairs and supplementary-plane code points:
+ * There are two different kinds of codepages that provide mappings for surrogate characters: + *

    + *
  • Codepages like UTF-8, UTF-32, and GB 18030 provide direct representations for Unicode + * code points U+10000-U+10ffff as well as for single surrogates U+d800-U+dfff. + * Each valid sequence will result in exactly one returned code point. + * If a sequence results in a single surrogate, then that will be returned + * by itself, even if a neighboring sequence encodes the matching surrogate.
  • + *
  • Codepages like SCSU and LMBCS (and UTF-16) provide direct representations only for BMP code points + * including surrogates. Code points in supplementary planes are represented with + * two sequences, each encoding a surrogate. + * For these codepages, matching pairs of surrogates will be combined into single + * code points for returning from this function. + * (Note that SCSU is actually a mix of these codepage types.)
  • + *

+ * + * @param converter an open UConverter + * @param source the address of a pointer to the codepage buffer, will be + * updated to point after the bytes consumed in the conversion call. + * @param sourceLimit points to the end of the input buffer + * @param err fills in error status (see ucnv_toUnicode) + * U_INDEX_OUTOFBOUNDS_ERROR will be set if the input + * is empty or does not convert to any output (e.g.: pure state-change + * codes SI/SO, escape sequences for ISO 2022, + * or if the callback did not output anything, ...). + * This function will not set a U_BUFFER_OVERFLOW_ERROR because + * the "buffer" is the return code. However, there might be subsequent output + * stored in the converter object + * that will be returned in following calls to this function. + * @return a UChar32 resulting from the partial conversion of source + * @see ucnv_toUnicode + * @see ucnv_toUChars + * @see ucnv_convert + * @stable ICU 2.0 + */ +U_STABLE UChar32 U_EXPORT2 +ucnv_getNextUChar(UConverter * converter, + const char **source, + const char * sourceLimit, + UErrorCode * err); + +/** + * Convert from one external charset to another using two existing UConverters. + * Internally, two conversions - ucnv_toUnicode() and ucnv_fromUnicode() - + * are used, "pivoting" through 16-bit Unicode. + * + * Important: For streaming conversion (multiple function calls for successive + * parts of a text stream), the caller must provide a pivot buffer explicitly, + * and must preserve the pivot buffer and associated pointers from one + * call to another. (The buffer may be moved if its contents and the relative + * pointer positions are preserved.) + * + * There is a similar function, ucnv_convert(), + * which has the following limitations: + * - it takes charset names, not converter objects, so that + * - two converters are opened for each call + * - only single-string conversion is possible, not streaming operation + * - it does not provide enough information to find out, + * in case of failure, whether the toUnicode or + * the fromUnicode conversion failed + * + * By contrast, ucnv_convertEx() + * - takes UConverter parameters instead of charset names + * - fully exposes the pivot buffer for streaming conversion and complete error handling + * + * ucnv_convertEx() also provides further convenience: + * - an option to reset the converters at the beginning + * (if reset==TRUE, see parameters; + * also sets *pivotTarget=*pivotSource=pivotStart) + * - allow NUL-terminated input + * (only a single NUL byte, will not work for charsets with multi-byte NULs) + * (if sourceLimit==NULL, see parameters) + * - terminate with a NUL on output + * (only a single NUL byte, not useful for charsets with multi-byte NULs), + * or set U_STRING_NOT_TERMINATED_WARNING if the output exactly fills + * the target buffer + * - the pivot buffer can be provided internally; + * possible only for whole-string conversion, not streaming conversion; + * in this case, the caller will not be able to get details about where an + * error occurred + * (if pivotStart==NULL, see below) + * + * The function returns when one of the following is true: + * - the entire source text has been converted successfully to the target buffer + * - a target buffer overflow occurred (U_BUFFER_OVERFLOW_ERROR) + * - a conversion error occurred + * (other U_FAILURE(), see description of pErrorCode) + * + * Limitation compared to the direct use of + * ucnv_fromUnicode() and ucnv_toUnicode(): + * ucnv_convertEx() does not provide offset information. + * + * Limitation compared to ucnv_fromUChars() and ucnv_toUChars(): + * ucnv_convertEx() does not support preflighting directly. + * + * Sample code for converting a single string from + * one external charset to UTF-8, ignoring the location of errors: + * + * \code + * int32_t + * myToUTF8(UConverter *cnv, + * const char *s, int32_t length, + * char *u8, int32_t capacity, + * UErrorCode *pErrorCode) { + * UConverter *utf8Cnv; + * char *target; + * + * if(U_FAILURE(*pErrorCode)) { + * return 0; + * } + * + * utf8Cnv=myGetCachedUTF8Converter(pErrorCode); + * if(U_FAILURE(*pErrorCode)) { + * return 0; + * } + * + * if(length<0) { + * length=strlen(s); + * } + * target=u8; + * ucnv_convertEx(cnv, utf8Cnv, + * &target, u8+capacity, + * &s, s+length, + * NULL, NULL, NULL, NULL, + * TRUE, TRUE, + * pErrorCode); + * + * myReleaseCachedUTF8Converter(utf8Cnv); + * + * // return the output string length, but without preflighting + * return (int32_t)(target-u8); + * } + * \endcode + * + * @param targetCnv Output converter, used to convert from the UTF-16 pivot + * to the target using ucnv_fromUnicode(). + * @param sourceCnv Input converter, used to convert from the source to + * the UTF-16 pivot using ucnv_toUnicode(). + * @param target I/O parameter, same as for ucnv_fromUChars(). + * Input: *target points to the beginning of the target buffer. + * Output: *target points to the first unit after the last char written. + * @param targetLimit Pointer to the first unit after the target buffer. + * @param source I/O parameter, same as for ucnv_toUChars(). + * Input: *source points to the beginning of the source buffer. + * Output: *source points to the first unit after the last char read. + * @param sourceLimit Pointer to the first unit after the source buffer. + * @param pivotStart Pointer to the UTF-16 pivot buffer. If pivotStart==NULL, + * then an internal buffer is used and the other pivot + * arguments are ignored and can be NULL as well. + * @param pivotSource I/O parameter, same as source in ucnv_fromUChars() for + * conversion from the pivot buffer to the target buffer. + * @param pivotTarget I/O parameter, same as target in ucnv_toUChars() for + * conversion from the source buffer to the pivot buffer. + * It must be pivotStart<=*pivotSource<=*pivotTarget<=pivotLimit + * and pivotStart[0..ucnv_countAvaiable()]) + * @return a pointer a string (library owned), or NULL if the index is out of bounds. + * @see ucnv_countAvailable + * @stable ICU 2.0 + */ +U_STABLE const char* U_EXPORT2 +ucnv_getAvailableName(int32_t n); + +/** + * Returns a UEnumeration to enumerate all of the canonical converter + * names, as per the alias file, regardless of the ability to open each + * converter. + * + * @return A UEnumeration object for getting all the recognized canonical + * converter names. + * @see ucnv_getAvailableName + * @see uenum_close + * @see uenum_next + * @stable ICU 2.4 + */ +U_STABLE UEnumeration * U_EXPORT2 +ucnv_openAllNames(UErrorCode *pErrorCode); + +/** + * Gives the number of aliases for a given converter or alias name. + * If the alias is ambiguous, then the preferred converter is used + * and the status is set to U_AMBIGUOUS_ALIAS_WARNING. + * This method only enumerates the listed entries in the alias file. + * @param alias alias name + * @param pErrorCode error status + * @return number of names on alias list for given alias + * @stable ICU 2.0 + */ +U_STABLE uint16_t U_EXPORT2 +ucnv_countAliases(const char *alias, UErrorCode *pErrorCode); + +/** + * Gives the name of the alias at given index of alias list. + * This method only enumerates the listed entries in the alias file. + * If the alias is ambiguous, then the preferred converter is used + * and the status is set to U_AMBIGUOUS_ALIAS_WARNING. + * @param alias alias name + * @param n index in alias list + * @param pErrorCode result of operation + * @return returns the name of the alias at given index + * @see ucnv_countAliases + * @stable ICU 2.0 + */ +U_STABLE const char * U_EXPORT2 +ucnv_getAlias(const char *alias, uint16_t n, UErrorCode *pErrorCode); + +/** + * Fill-up the list of alias names for the given alias. + * This method only enumerates the listed entries in the alias file. + * If the alias is ambiguous, then the preferred converter is used + * and the status is set to U_AMBIGUOUS_ALIAS_WARNING. + * @param alias alias name + * @param aliases fill-in list, aliases is a pointer to an array of + * ucnv_countAliases() string-pointers + * (const char *) that will be filled in. + * The strings themselves are owned by the library. + * @param pErrorCode result of operation + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ucnv_getAliases(const char *alias, const char **aliases, UErrorCode *pErrorCode); + +/** + * Return a new UEnumeration object for enumerating all the + * alias names for a given converter that are recognized by a standard. + * This method only enumerates the listed entries in the alias file. + * The convrtrs.txt file can be modified to change the results of + * this function. + * The first result in this list is the same result given by + * ucnv_getStandardName, which is the default alias for + * the specified standard name. The returned object must be closed with + * uenum_close when you are done with the object. + * + * @param convName original converter name + * @param standard name of the standard governing the names; MIME and IANA + * are such standards + * @param pErrorCode The error code + * @return A UEnumeration object for getting all aliases that are recognized + * by a standard. If any of the parameters are invalid, NULL + * is returned. + * @see ucnv_getStandardName + * @see uenum_close + * @see uenum_next + * @stable ICU 2.2 + */ +U_STABLE UEnumeration * U_EXPORT2 +ucnv_openStandardNames(const char *convName, + const char *standard, + UErrorCode *pErrorCode); + +/** + * Gives the number of standards associated to converter names. + * @return number of standards + * @stable ICU 2.0 + */ +U_STABLE uint16_t U_EXPORT2 +ucnv_countStandards(void); + +/** + * Gives the name of the standard at given index of standard list. + * @param n index in standard list + * @param pErrorCode result of operation + * @return returns the name of the standard at given index. Owned by the library. + * @stable ICU 2.0 + */ +U_STABLE const char * U_EXPORT2 +ucnv_getStandard(uint16_t n, UErrorCode *pErrorCode); + +/** + * Returns a standard name for a given converter name. + *

+ * Example alias table:
+ * conv alias1 { STANDARD1 } alias2 { STANDARD1* } + *

+ * Result of ucnv_getStandardName("conv", "STANDARD1") from example + * alias table:
+ * "alias2" + * + * @param name original converter name + * @param standard name of the standard governing the names; MIME and IANA + * are such standards + * @param pErrorCode result of operation + * @return returns the standard converter name; + * if a standard converter name cannot be determined, + * then NULL is returned. Owned by the library. + * @stable ICU 2.0 + */ +U_STABLE const char * U_EXPORT2 +ucnv_getStandardName(const char *name, const char *standard, UErrorCode *pErrorCode); + +/** + * This function will return the internal canonical converter name of the + * tagged alias. This is the opposite of ucnv_openStandardNames, which + * returns the tagged alias given the canonical name. + *

+ * Example alias table:
+ * conv alias1 { STANDARD1 } alias2 { STANDARD1* } + *

+ * Result of ucnv_getStandardName("alias1", "STANDARD1") from example + * alias table:
+ * "conv" + * + * @return returns the canonical converter name; + * if a standard or alias name cannot be determined, + * then NULL is returned. The returned string is + * owned by the library. + * @see ucnv_getStandardName + * @stable ICU 2.4 + */ +U_STABLE const char * U_EXPORT2 +ucnv_getCanonicalName(const char *alias, const char *standard, UErrorCode *pErrorCode); + +/** + * Returns the current default converter name. If you want to open + * a default converter, you do not need to use this function. + * It is faster if you pass a NULL argument to ucnv_open the + * default converter. + * + * @return returns the current default converter name. + * Storage owned by the library + * @see ucnv_setDefaultName + * @stable ICU 2.0 + */ +U_STABLE const char * U_EXPORT2 +ucnv_getDefaultName(void); + +/** + * This function is not thread safe. DO NOT call this function when ANY ICU + * function is being used from more than one thread! This function sets the + * current default converter name. If this function needs to be called, it + * should be called during application initialization. Most of the time, the + * results from ucnv_getDefaultName() or ucnv_open with a NULL string argument + * is sufficient for your application. + * @param name the converter name to be the default (must be known by ICU). + * @see ucnv_getDefaultName + * @system + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ucnv_setDefaultName(const char *name); + +/** + * Fixes the backslash character mismapping. For example, in SJIS, the backslash + * character in the ASCII portion is also used to represent the yen currency sign. + * When mapping from Unicode character 0x005C, it's unclear whether to map the + * character back to yen or backslash in SJIS. This function will take the input + * buffer and replace all the yen sign characters with backslash. This is necessary + * when the user tries to open a file with the input buffer on Windows. + * This function will test the converter to see whether such mapping is + * required. You can sometimes avoid using this function by using the correct version + * of Shift-JIS. + * + * @param cnv The converter representing the target codepage. + * @param source the input buffer to be fixed + * @param sourceLen the length of the input buffer + * @see ucnv_isAmbiguous + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ucnv_fixFileSeparator(const UConverter *cnv, UChar *source, int32_t sourceLen); + +/** + * Determines if the converter contains ambiguous mappings of the same + * character or not. + * @param cnv the converter to be tested + * @return TRUE if the converter contains ambiguous mapping of the same + * character, FALSE otherwise. + * @stable ICU 2.0 + */ +U_STABLE UBool U_EXPORT2 +ucnv_isAmbiguous(const UConverter *cnv); + +/** + * Sets the converter to use fallback mappings or not. + * Regardless of this flag, the converter will always use + * fallbacks from Unicode Private Use code points, as well as + * reverse fallbacks (to Unicode). + * For details see ".ucm File Format" + * in the Conversion Data chapter of the ICU User Guide: + * http://www.icu-project.org/userguide/conversion-data.html#ucmformat + * + * @param cnv The converter to set the fallback mapping usage on. + * @param usesFallback TRUE if the user wants the converter to take advantage of the fallback + * mapping, FALSE otherwise. + * @stable ICU 2.0 + * @see ucnv_usesFallback + */ +U_STABLE void U_EXPORT2 +ucnv_setFallback(UConverter *cnv, UBool usesFallback); + +/** + * Determines if the converter uses fallback mappings or not. + * This flag has restrictions, see ucnv_setFallback(). + * + * @param cnv The converter to be tested + * @return TRUE if the converter uses fallback, FALSE otherwise. + * @stable ICU 2.0 + * @see ucnv_setFallback + */ +U_STABLE UBool U_EXPORT2 +ucnv_usesFallback(const UConverter *cnv); + +/** + * Detects Unicode signature byte sequences at the start of the byte stream + * and returns the charset name of the indicated Unicode charset. + * NULL is returned when no Unicode signature is recognized. + * The number of bytes in the signature is output as well. + * + * The caller can ucnv_open() a converter using the charset name. + * The first code unit (UChar) from the start of the stream will be U+FEFF + * (the Unicode BOM/signature character) and can usually be ignored. + * + * For most Unicode charsets it is also possible to ignore the indicated + * number of initial stream bytes and start converting after them. + * However, there are stateful Unicode charsets (UTF-7 and BOCU-1) for which + * this will not work. Therefore, it is best to ignore the first output UChar + * instead of the input signature bytes. + *

+ * Usage: + * @code + * UErrorCode err = U_ZERO_ERROR; + * char input[] = { '\xEF','\xBB', '\xBF','\x41','\x42','\x43' }; + * int32_t signatureLength = 0; + * char *encoding = ucnv_detectUnicodeSignature(input,sizeof(input),&signatureLength,&err); + * UConverter *conv = NULL; + * UChar output[100]; + * UChar *target = output, *out; + * char *source = input; + * if(encoding!=NULL && U_SUCCESS(err)){ + * // should signature be discarded ? + * conv = ucnv_open(encoding, &err); + * // do the conversion + * ucnv_toUnicode(conv, + * target, output + sizeof(output)/U_SIZEOF_UCHAR, + * source, input + sizeof(input), + * NULL, TRUE, &err); + * out = output; + * if (discardSignature){ + * ++out; // ignore initial U+FEFF + * } + * while(out != target) { + * printf("%04x ", *out++); + * } + * puts(""); + * } + * + * @endcode + * + * @param source The source string in which the signature should be detected. + * @param sourceLength Length of the input string, or -1 if terminated with a NUL byte. + * @param signatureLength A pointer to int32_t to receive the number of bytes that make up the signature + * of the detected UTF. 0 if not detected. + * Can be a NULL pointer. + * @param pErrorCode ICU error code in/out parameter. + * Must fulfill U_SUCCESS before the function call. + * @return The name of the encoding detected. NULL if encoding is not detected. + * @stable ICU 2.4 + */ +U_STABLE const char* U_EXPORT2 +ucnv_detectUnicodeSignature(const char* source, + int32_t sourceLength, + int32_t *signatureLength, + UErrorCode *pErrorCode); + +/** + * Returns the number of UChars held in the converter's internal state + * because more input is needed for completing the conversion. This function is + * useful for mapping semantics of ICU's converter interface to those of iconv, + * and this information is not needed for normal conversion. + * @param cnv The converter in which the input is held + * @param status ICU error code in/out parameter. + * Must fulfill U_SUCCESS before the function call. + * @return The number of UChars in the state. -1 if an error is encountered. + * @stable ICU 3.4 + */ +U_STABLE int32_t U_EXPORT2 +ucnv_fromUCountPending(const UConverter* cnv, UErrorCode* status); + +/** + * Returns the number of chars held in the converter's internal state + * because more input is needed for completing the conversion. This function is + * useful for mapping semantics of ICU's converter interface to those of iconv, + * and this information is not needed for normal conversion. + * @param cnv The converter in which the input is held as internal state + * @param status ICU error code in/out parameter. + * Must fulfill U_SUCCESS before the function call. + * @return The number of chars in the state. -1 if an error is encountered. + * @stable ICU 3.4 + */ +U_STABLE int32_t U_EXPORT2 +ucnv_toUCountPending(const UConverter* cnv, UErrorCode* status); + +#endif + +#endif +/*_UCNV*/ diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/ucnv_cb.h b/Build/source/libs/icu/icu-xetex/common/unicode/ucnv_cb.h new file mode 100644 index 00000000000..f0e67ba11e0 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/ucnv_cb.h @@ -0,0 +1,162 @@ +/* +********************************************************************** +* Copyright (C) 2000-2004, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** + * ucnv_cb.h: + * External APIs for the ICU's codeset conversion library + * Helena Shih + * + * Modification History: + * + * Date Name Description + */ + +/** + * \file + * \brief C UConverter functions to aid the writers of callbacks + * + *

Callback API for UConverter

+ * + * These functions are provided here for the convenience of the callback + * writer. If you are just looking for callback functions to use, please + * see ucnv_err.h. DO NOT call these functions directly when you are + * working with converters, unless your code has been called as a callback + * via ucnv_setFromUCallback or ucnv_setToUCallback !! + * + * A note about error codes and overflow. Unlike other ICU functions, + * these functions do not expect the error status to be U_ZERO_ERROR. + * Callbacks must be much more careful about their error codes. + * The error codes used here are in/out parameters, which should be passed + * back in the callback's error parameter. + * + * For example, if you call ucnv_cbfromUWriteBytes to write data out + * to the output codepage, it may return U_BUFFER_OVERFLOW_ERROR if + * the data did not fit in the target. But this isn't a failing error, + * in fact, ucnv_cbfromUWriteBytes may be called AGAIN with the error + * status still U_BUFFER_OVERFLOW_ERROR to attempt to write further bytes, + * which will also go into the internal overflow buffers. + * + * Concerning offsets, the 'offset' parameters here are relative to the start + * of SOURCE. For example, Suppose the string "ABCD" was being converted + * from Unicode into a codepage which doesn't have a mapping for 'B'. + * 'A' will be written out correctly, but + * The FromU Callback will be called on an unassigned character for 'B'. + * At this point, this is the state of the world: + * Target: A [..] [points after A] + * Source: A B [C] D [points to C - B has been consumed] + * 0 1 2 3 + * codePoint = "B" [the unassigned codepoint] + * + * Now, suppose a callback wants to write the substitution character '?' to + * the target. It calls ucnv_cbFromUWriteBytes() to write the ?. + * It should pass ZERO as the offset, because the offset as far as the + * callback is concerned is relative to the SOURCE pointer [which points + * before 'C'.] If the callback goes into the args and consumes 'C' also, + * it would call FromUWriteBytes with an offset of 1 (and advance the source + * pointer). + * + */ + +#ifndef UCNV_CB_H +#define UCNV_CB_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "unicode/ucnv.h" +#include "unicode/ucnv_err.h" + +/** + * ONLY used by FromU callback functions. + * Writes out the specified byte output bytes to the target byte buffer or to converter internal buffers. + * + * @param args callback fromUnicode arguments + * @param source source bytes to write + * @param length length of bytes to write + * @param offsetIndex the relative offset index from callback. + * @param err error status. If U_BUFFER_OVERFLOW is returned, then U_BUFFER_OVERFLOW must + * be returned to the user, because it means that not all data could be written into the target buffer, and some is + * in the converter error buffer. + * @see ucnv_cbFromUWriteSub + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ucnv_cbFromUWriteBytes (UConverterFromUnicodeArgs *args, + const char* source, + int32_t length, + int32_t offsetIndex, + UErrorCode * err); + +/** + * ONLY used by FromU callback functions. + * This function will write out the correct substitution character sequence + * to the target. + * + * @param args callback fromUnicode arguments + * @param offsetIndex the relative offset index from the current source pointer to be used + * @param err error status. If U_BUFFER_OVERFLOW is returned, then U_BUFFER_OVERFLOW must + * be returned to the user, because it means that not all data could be written into the target buffer, and some is + * in the converter error buffer. + * @see ucnv_cbFromUWriteBytes + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ucnv_cbFromUWriteSub (UConverterFromUnicodeArgs *args, + int32_t offsetIndex, + UErrorCode * err); + +/** + * ONLY used by fromU callback functions. + * This function will write out the error character(s) to the target UChar buffer. + * + * @param args callback fromUnicode arguments + * @param source pointer to pointer to first UChar to write [on exit: 1 after last UChar processed] + * @param sourceLimit pointer after last UChar to write + * @param offsetIndex the relative offset index from callback which will be set + * @param err error status U_BUFFER_OVERFLOW + * @see ucnv_cbToUWriteSub + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 ucnv_cbFromUWriteUChars(UConverterFromUnicodeArgs *args, + const UChar** source, + const UChar* sourceLimit, + int32_t offsetIndex, + UErrorCode * err); + +/** + * ONLY used by ToU callback functions. + * This function will write out the specified characters to the target + * UChar buffer. + * + * @param args callback toUnicode arguments + * @param source source string to write + * @param length the length of source string + * @param offsetIndex the relative offset index which will be written. + * @param err error status U_BUFFER_OVERFLOW + * @see ucnv_cbToUWriteSub + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 ucnv_cbToUWriteUChars (UConverterToUnicodeArgs *args, + const UChar* source, + int32_t length, + int32_t offsetIndex, + UErrorCode * err); + +/** + * ONLY used by ToU callback functions. + * This function will write out the Unicode substitution character (U+FFFD). + * + * @param args callback fromUnicode arguments + * @param offsetIndex the relative offset index from callback. + * @param err error status U_BUFFER_OVERFLOW + * @see ucnv_cbToUWriteUChars + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 ucnv_cbToUWriteSub (UConverterToUnicodeArgs *args, + int32_t offsetIndex, + UErrorCode * err); +#endif + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/ucnv_err.h b/Build/source/libs/icu/icu-xetex/common/unicode/ucnv_err.h new file mode 100644 index 00000000000..b0db7bc17c8 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/ucnv_err.h @@ -0,0 +1,456 @@ +/* +********************************************************************** +* Copyright (C) 1999-2005, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** + * + * + * ucnv_err.h: + */ + +/** + * \file + * \brief C UConverter predefined error callbacks + * + *

Error Behaviour Functions

+ * Defines some error behaviour functions called by ucnv_{from,to}Unicode + * These are provided as part of ICU and many are stable, but they + * can also be considered only as an example of what can be done with + * callbacks. You may of course write your own. + * + * If you want to write your own, you may also find the functions from + * ucnv_cb.h useful when writing your own callbacks. + * + * These functions, although public, should NEVER be called directly. + * They should be used as parameters to the ucnv_setFromUCallback + * and ucnv_setToUCallback functions, to set the behaviour of a converter + * when it encounters ILLEGAL/UNMAPPED/INVALID sequences. + * + * usage example: 'STOP' doesn't need any context, but newContext + * could be set to something other than 'NULL' if needed. The available + * contexts in this header can modify the default behavior of the callback. + * + * \code + * UErrorCode err = U_ZERO_ERROR; + * UConverter *myConverter = ucnv_open("ibm-949", &err); + * const void *oldContext; + * UConverterFromUCallback oldAction; + * + * + * if (U_SUCCESS(err)) + * { + * ucnv_setFromUCallBack(myConverter, + * UCNV_FROM_U_CALLBACK_STOP, + * NULL, + * &oldAction, + * &oldContext, + * &status); + * } + * \endcode + * + * The code above tells "myConverter" to stop when it encounters an + * ILLEGAL/TRUNCATED/INVALID sequences when it is used to convert from + * Unicode -> Codepage. The behavior from Codepage to Unicode is not changed, + * and ucnv_setToUCallBack would need to be called in order to change + * that behavior too. + * + * Here is an example with a context: + * + * \code + * UErrorCode err = U_ZERO_ERROR; + * UConverter *myConverter = ucnv_open("ibm-949", &err); + * const void *oldContext; + * UConverterFromUCallback oldAction; + * + * + * if (U_SUCCESS(err)) + * { + * ucnv_setToUCallBack(myConverter, + * UCNV_TO_U_CALLBACK_SUBSTITUTE, + * UCNV_SUB_STOP_ON_ILLEGAL, + * &oldAction, + * &oldContext, + * &status); + * } + * \endcode + * + * The code above tells "myConverter" to stop when it encounters an + * ILLEGAL/TRUNCATED/INVALID sequences when it is used to convert from + * Codepage -> Unicode. Any unmapped and legal characters will be + * substituted to be the default substitution character. + */ + +#ifndef UCNV_ERR_H +#define UCNV_ERR_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +/** Forward declaring the UConverter structure. @stable ICU 2.0 */ +struct UConverter; + +/** @stable ICU 2.0 */ +typedef struct UConverter UConverter; + +/** + * FROM_U, TO_U context options for sub callback + * @stable ICU 2.0 + */ +#define UCNV_SUB_STOP_ON_ILLEGAL "i" + +/** + * FROM_U, TO_U context options for skip callback + * @stable ICU 2.0 + */ +#define UCNV_SKIP_STOP_ON_ILLEGAL "i" + +/** + * FROM_U_CALLBACK_ESCAPE context option to escape the code unit according to ICU (%UXXXX) + * @stable ICU 2.0 + */ +#define UCNV_ESCAPE_ICU NULL +/** + * FROM_U_CALLBACK_ESCAPE context option to escape the code unit according to JAVA (\\uXXXX) + * @stable ICU 2.0 + */ +#define UCNV_ESCAPE_JAVA "J" +/** + * FROM_U_CALLBACK_ESCAPE context option to escape the code unit according to C (\\uXXXX \\UXXXXXXXX) + * TO_U_CALLBACK_ESCAPE option to escape the character value accoding to C (\\xXXXX) + * @stable ICU 2.0 + */ +#define UCNV_ESCAPE_C "C" +/** + * FROM_U_CALLBACK_ESCAPE context option to escape the code unit according to XML Decimal escape \htmlonly(&#DDDD;)\endhtmlonly + * TO_U_CALLBACK_ESCAPE context option to escape the character value accoding to XML Decimal escape \htmlonly(&#DDDD;)\endhtmlonly + * @stable ICU 2.0 + */ +#define UCNV_ESCAPE_XML_DEC "D" +/** + * FROM_U_CALLBACK_ESCAPE context option to escape the code unit according to XML Hex escape \htmlonly(&#xXXXX;)\endhtmlonly + * TO_U_CALLBACK_ESCAPE context option to escape the character value accoding to XML Hex escape \htmlonly(&#xXXXX;)\endhtmlonly + * @stable ICU 2.0 + */ +#define UCNV_ESCAPE_XML_HEX "X" +/** + * FROM_U_CALLBACK_ESCAPE context option to escape teh code unit according to Unicode (U+XXXXX) + * @stable ICU 2.0 + */ +#define UCNV_ESCAPE_UNICODE "U" + +/** + * The process condition code to be used with the callbacks. + * Codes which are greater than UCNV_IRREGULAR should be + * passed on to any chained callbacks. + * @stable ICU 2.0 + */ +typedef enum { + UCNV_UNASSIGNED = 0, /**< The code point is unassigned. + The error code U_INVALID_CHAR_FOUND will be set. */ + UCNV_ILLEGAL = 1, /**< The code point is illegal. For example, + \\x81\\x2E is illegal in SJIS because \\x2E + is not a valid trail byte for the \\x81 + lead byte. + Also, starting with Unicode 3.0.1, non-shortest byte sequences + in UTF-8 (like \\xC1\\xA1 instead of \\x61 for U+0061) + are also illegal, not just irregular. + The error code U_ILLEGAL_CHAR_FOUND will be set. */ + UCNV_IRREGULAR = 2, /**< The codepoint is not a regular sequence in + the encoding. For example, \\xED\\xA0\\x80..\\xED\\xBF\\xBF + are irregular UTF-8 byte sequences for single surrogate + code points. + The error code U_INVALID_CHAR_FOUND will be set. */ + UCNV_RESET = 3, /**< The callback is called with this reason when a + 'reset' has occured. Callback should reset all + state. */ + UCNV_CLOSE = 4, /**< Called when the converter is closed. The + callback should release any allocated memory.*/ + UCNV_CLONE = 5 /**< Called when ucnv_safeClone() is called on the + converter. the pointer available as the + 'context' is an alias to the original converters' + context pointer. If the context must be owned + by the new converter, the callback must clone + the data and call ucnv_setFromUCallback + (or setToUCallback) with the correct pointer. + @stable ICU 2.2 + */ +} UConverterCallbackReason; + + +/** + * The structure for the fromUnicode callback function parameter. + * @stable ICU 2.0 + */ +typedef struct { + uint16_t size; /**< The size of this struct. @stable ICU 2.0 */ + UBool flush; /**< The internal state of converter will be reset and data flushed if set to TRUE. @stable ICU 2.0 */ + UConverter *converter; /**< Pointer to the converter that is opened and to which this struct is passed as an argument. @stable ICU 2.0 */ + const UChar *source; /**< Pointer to the source source buffer. @stable ICU 2.0 */ + const UChar *sourceLimit; /**< Pointer to the limit (end + 1) of source buffer. @stable ICU 2.0 */ + char *target; /**< Pointer to the target buffer. @stable ICU 2.0 */ + const char *targetLimit; /**< Pointer to the limit (end + 1) of target buffer. @stable ICU 2.0 */ + int32_t *offsets; /**< Pointer to the buffer that recieves the offsets. *offset = blah ; offset++;. @stable ICU 2.0 */ +} UConverterFromUnicodeArgs; + + +/** + * The structure for the toUnicode callback function parameter. + * @stable ICU 2.0 + */ +typedef struct { + uint16_t size; /**< The size of this struct @stable ICU 2.0 */ + UBool flush; /**< The internal state of converter will be reset and data flushed if set to TRUE. @stable ICU 2.0 */ + UConverter *converter; /**< Pointer to the converter that is opened and to which this struct is passed as an argument. @stable ICU 2.0 */ + const char *source; /**< Pointer to the source source buffer. @stable ICU 2.0 */ + const char *sourceLimit; /**< Pointer to the limit (end + 1) of source buffer. @stable ICU 2.0 */ + UChar *target; /**< Pointer to the target buffer. @stable ICU 2.0 */ + const UChar *targetLimit; /**< Pointer to the limit (end + 1) of target buffer. @stable ICU 2.0 */ + int32_t *offsets; /**< Pointer to the buffer that recieves the offsets. *offset = blah ; offset++;. @stable ICU 2.0 */ +} UConverterToUnicodeArgs; + + +/** + * DO NOT CALL THIS FUNCTION DIRECTLY! + * This From Unicode callback STOPS at the ILLEGAL_SEQUENCE, + * returning the error code back to the caller immediately. + * + * @param context Pointer to the callback's private data + * @param fromUArgs Information about the conversion in progress + * @param codeUnits Points to 'length' UChars of the concerned Unicode sequence + * @param length Size (in bytes) of the concerned codepage sequence + * @param codePoint Single UChar32 (UTF-32) containing the concerend Unicode codepoint. + * @param reason Defines the reason the callback was invoked + * @param err This should always be set to a failure status prior to calling. + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 UCNV_FROM_U_CALLBACK_STOP ( + const void *context, + UConverterFromUnicodeArgs *fromUArgs, + const UChar* codeUnits, + int32_t length, + UChar32 codePoint, + UConverterCallbackReason reason, + UErrorCode * err); + + + +/** + * DO NOT CALL THIS FUNCTION DIRECTLY! + * This To Unicode callback STOPS at the ILLEGAL_SEQUENCE, + * returning the error code back to the caller immediately. + * + * @param context Pointer to the callback's private data + * @param toUArgs Information about the conversion in progress + * @param codeUnits Points to 'length' bytes of the concerned codepage sequence + * @param length Size (in bytes) of the concerned codepage sequence + * @param reason Defines the reason the callback was invoked + * @param err This should always be set to a failure status prior to calling. + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 UCNV_TO_U_CALLBACK_STOP ( + const void *context, + UConverterToUnicodeArgs *toUArgs, + const char* codeUnits, + int32_t length, + UConverterCallbackReason reason, + UErrorCode * err); + +/** + * DO NOT CALL THIS FUNCTION DIRECTLY! + * This From Unicode callback skips any ILLEGAL_SEQUENCE, or + * skips only UNASSINGED_SEQUENCE depending on the context parameter + * simply ignoring those characters. + * + * @param context The function currently recognizes the callback options: + * UCNV_SKIP_STOP_ON_ILLEGAL: STOPS at the ILLEGAL_SEQUENCE, + * returning the error code back to the caller immediately. + * NULL: Skips any ILLEGAL_SEQUENCE + * @param fromUArgs Information about the conversion in progress + * @param codeUnits Points to 'length' UChars of the concerned Unicode sequence + * @param length Size (in bytes) of the concerned codepage sequence + * @param codePoint Single UChar32 (UTF-32) containing the concerend Unicode codepoint. + * @param reason Defines the reason the callback was invoked + * @param err Return value will be set to success if the callback was handled, + * otherwise this value will be set to a failure status. + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 UCNV_FROM_U_CALLBACK_SKIP ( + const void *context, + UConverterFromUnicodeArgs *fromUArgs, + const UChar* codeUnits, + int32_t length, + UChar32 codePoint, + UConverterCallbackReason reason, + UErrorCode * err); + +/** + * DO NOT CALL THIS FUNCTION DIRECTLY! + * This From Unicode callback will Substitute the ILLEGAL SEQUENCE, or + * UNASSIGNED_SEQUENCE depending on context parameter, with the + * current substitution string for the converter. This is the default + * callback. + * + * @param context The function currently recognizes the callback options: + * UCNV_SUB_STOP_ON_ILLEGAL: STOPS at the ILLEGAL_SEQUENCE, + * returning the error code back to the caller immediately. + * NULL: Substitutes any ILLEGAL_SEQUENCE + * @param fromUArgs Information about the conversion in progress + * @param codeUnits Points to 'length' UChars of the concerned Unicode sequence + * @param length Size (in bytes) of the concerned codepage sequence + * @param codePoint Single UChar32 (UTF-32) containing the concerend Unicode codepoint. + * @param reason Defines the reason the callback was invoked + * @param err Return value will be set to success if the callback was handled, + * otherwise this value will be set to a failure status. + * @see ucnv_setSubstChars + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 UCNV_FROM_U_CALLBACK_SUBSTITUTE ( + const void *context, + UConverterFromUnicodeArgs *fromUArgs, + const UChar* codeUnits, + int32_t length, + UChar32 codePoint, + UConverterCallbackReason reason, + UErrorCode * err); + +/** + * DO NOT CALL THIS FUNCTION DIRECTLY! + * This From Unicode callback will Substitute the ILLEGAL SEQUENCE with the + * hexadecimal representation of the illegal codepoints + * + * @param context The function currently recognizes the callback options: + *
    + *
  • UCNV_ESCAPE_ICU: Substitues the ILLEGAL SEQUENCE with the hexadecimal + * representation in the format %UXXXX, e.g. "%uFFFE%u00AC%uC8FE"). + * In the Event the converter doesn't support the characters {%,U}[A-F][0-9], + * it will substitute the illegal sequence with the substitution characters. + * Note that codeUnit(32bit int eg: unit of a surrogate pair) is represented as + * %UD84D%UDC56
  • + *
  • UCNV_ESCAPE_JAVA: Substitues the ILLEGAL SEQUENCE with the hexadecimal + * representation in the format \\uXXXX, e.g. "\\uFFFE\\u00AC\\uC8FE"). + * In the Event the converter doesn't support the characters {\,u}[A-F][0-9], + * it will substitute the illegal sequence with the substitution characters. + * Note that codeUnit(32bit int eg: unit of a surrogate pair) is represented as + * \\uD84D\\uDC56
  • + *
  • UCNV_ESCAPE_C: Substitues the ILLEGAL SEQUENCE with the hexadecimal + * representation in the format \\uXXXX, e.g. "\\uFFFE\\u00AC\\uC8FE"). + * In the Event the converter doesn't support the characters {\,u,U}[A-F][0-9], + * it will substitute the illegal sequence with the substitution characters. + * Note that codeUnit(32bit int eg: unit of a surrogate pair) is represented as + * \\U00023456
  • + *
  • UCNV_ESCAPE_XML_DEC: Substitues the ILLEGAL SEQUENCE with the decimal + * representation in the format \htmlonly&#DDDDDDDD;, e.g. "&#65534;&#172;&#51454;")\endhtmlonly. + * In the Event the converter doesn't support the characters {&,#}[0-9], + * it will substitute the illegal sequence with the substitution characters. + * Note that codeUnit(32bit int eg: unit of a surrogate pair) is represented as + * &#144470; and Zero padding is ignored.
  • + *
  • UCNV_ESCAPE_XML_HEX:Substitues the ILLEGAL SEQUENCE with the decimal + * representation in the format \htmlonly&#xXXXX; e.g. "&#xFFFE;&#x00AC;&#xC8FE;")\endhtmlonly. + * In the Event the converter doesn't support the characters {&,#,x}[0-9], + * it will substitute the illegal sequence with the substitution characters. + * Note that codeUnit(32bit int eg: unit of a surrogate pair) is represented as + * \htmlonly&#x23456;\endhtmlonly
  • + *
+ * @param fromUArgs Information about the conversion in progress + * @param codeUnits Points to 'length' UChars of the concerned Unicode sequence + * @param length Size (in bytes) of the concerned codepage sequence + * @param codePoint Single UChar32 (UTF-32) containing the concerend Unicode codepoint. + * @param reason Defines the reason the callback was invoked + * @param err Return value will be set to success if the callback was handled, + * otherwise this value will be set to a failure status. + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 UCNV_FROM_U_CALLBACK_ESCAPE ( + const void *context, + UConverterFromUnicodeArgs *fromUArgs, + const UChar* codeUnits, + int32_t length, + UChar32 codePoint, + UConverterCallbackReason reason, + UErrorCode * err); + + +/** + * DO NOT CALL THIS FUNCTION DIRECTLY! + * This To Unicode callback skips any ILLEGAL_SEQUENCE, or + * skips only UNASSINGED_SEQUENCE depending on the context parameter + * simply ignoring those characters. + * + * @param context The function currently recognizes the callback options: + * UCNV_SKIP_STOP_ON_ILLEGAL: STOPS at the ILLEGAL_SEQUENCE, + * returning the error code back to the caller immediately. + * NULL: Skips any ILLEGAL_SEQUENCE + * @param toUArgs Information about the conversion in progress + * @param codeUnits Points to 'length' bytes of the concerned codepage sequence + * @param length Size (in bytes) of the concerned codepage sequence + * @param reason Defines the reason the callback was invoked + * @param err Return value will be set to success if the callback was handled, + * otherwise this value will be set to a failure status. + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 UCNV_TO_U_CALLBACK_SKIP ( + const void *context, + UConverterToUnicodeArgs *toUArgs, + const char* codeUnits, + int32_t length, + UConverterCallbackReason reason, + UErrorCode * err); + +/** + * DO NOT CALL THIS FUNCTION DIRECTLY! + * This To Unicode callback will Substitute the ILLEGAL SEQUENCE,or + * UNASSIGNED_SEQUENCE depending on context parameter, with the + * Unicode substitution character, U+FFFD. + * + * @param context The function currently recognizes the callback options: + * UCNV_SUB_STOP_ON_ILLEGAL: STOPS at the ILLEGAL_SEQUENCE, + * returning the error code back to the caller immediately. + * NULL: Substitutes any ILLEGAL_SEQUENCE + * @param toUArgs Information about the conversion in progress + * @param codeUnits Points to 'length' bytes of the concerned codepage sequence + * @param length Size (in bytes) of the concerned codepage sequence + * @param reason Defines the reason the callback was invoked + * @param err Return value will be set to success if the callback was handled, + * otherwise this value will be set to a failure status. + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 UCNV_TO_U_CALLBACK_SUBSTITUTE ( + const void *context, + UConverterToUnicodeArgs *toUArgs, + const char* codeUnits, + int32_t length, + UConverterCallbackReason reason, + UErrorCode * err); + +/** + * DO NOT CALL THIS FUNCTION DIRECTLY! + * This To Unicode callback will Substitute the ILLEGAL SEQUENCE with the + * hexadecimal representation of the illegal bytes + * (in the format %XNN, e.g. "%XFF%X0A%XC8%X03"). + * + * @param context This function currently recognizes the callback options: + * UCNV_ESCAPE_ICU, UCNV_ESCAPE_JAVA, UCNV_ESCAPE_C, UCNV_ESCAPE_XML_DEC, + * UCNV_ESCAPE_XML_HEX and UCNV_ESCAPE_UNICODE. + * @param toUArgs Information about the conversion in progress + * @param codeUnits Points to 'length' bytes of the concerned codepage sequence + * @param length Size (in bytes) of the concerned codepage sequence + * @param reason Defines the reason the callback was invoked + * @param err Return value will be set to success if the callback was handled, + * otherwise this value will be set to a failure status. + * @stable ICU 2.0 + */ + +U_STABLE void U_EXPORT2 UCNV_TO_U_CALLBACK_ESCAPE ( + const void *context, + UConverterToUnicodeArgs *toUArgs, + const char* codeUnits, + int32_t length, + UConverterCallbackReason reason, + UErrorCode * err); + +#endif + +#endif + +/*UCNV_ERR_H*/ diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/uconfig.h b/Build/source/libs/icu/icu-xetex/common/unicode/uconfig.h new file mode 100644 index 00000000000..24cfe4518e7 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/uconfig.h @@ -0,0 +1,215 @@ +/* +********************************************************************** +* Copyright (C) 2002-2007, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* file name: uconfig.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2002sep19 +* created by: Markus W. Scherer +*/ + +#ifndef __UCONFIG_H__ +#define __UCONFIG_H__ + +/*! + * \file + * \brief Switches for excluding parts of ICU library code modules. + * + * Allows to build partial, smaller libraries for special purposes. + * By default, all modules are built. + * The switches are fairly coarse, controlling large modules. + * Basic services cannot be turned off. + * + * Building with any of these options does not guarantee that the + * ICU build process will completely work. It is recommended that + * the ICU libraries and data be built using the normal build. + * At that time you should remove the data used by those services. + * After building the ICU data library, you should rebuild the ICU + * libraries with these switches customized to your needs. + * + * @stable ICU 2.4 + */ + +/** + * \def UCONFIG_ONLY_COLLATION + * This switch turns off modules that are not needed for collation. + * + * It does not turn off legacy conversion because that is necessary + * for ICU to work on EBCDIC platforms (for the default converter). + * If you want "only collation" and do not build for EBCDIC, + * then you can define UCONFIG_NO_LEGACY_CONVERSION 1 as well. + * + * @stable ICU 2.4 + */ +#ifndef UCONFIG_ONLY_COLLATION +# define UCONFIG_ONLY_COLLATION 0 +#endif + +#if UCONFIG_ONLY_COLLATION + /* common library */ +# define UCONFIG_NO_BREAK_ITERATION 1 +# define UCONFIG_NO_IDNA 1 + + /* i18n library */ +# if UCONFIG_NO_COLLATION +# error Contradictory collation switches in uconfig.h. +# endif +# define UCONFIG_NO_FORMATTING 1 +# define UCONFIG_NO_TRANSLITERATION 1 +# define UCONFIG_NO_REGULAR_EXPRESSIONS 1 +#endif + +/* common library switches -------------------------------------------------- */ + +/** + * \def UCONFIG_NO_FILE_IO + * This switch turns off all file access in the common library + * where file access is only used for data loading. + * ICU data must then be provided in the form of a data DLL (or with an + * equivalent way to link to the data residing in an executable, + * as in building a combined library with both the common library's code and + * the data), or via udata_setCommonData(). + * Application data must be provided via udata_setAppData() or by using + * "open" functions that take pointers to data, for example ucol_openBinary(). + * + * File access is not used at all in the i18n library. + * + * File access cannot be turned off for the icuio library or for the ICU + * test suites and ICU tools. + * + * @stable ICU 3.6 + */ +#ifndef UCONFIG_NO_FILE_IO +# define UCONFIG_NO_FILE_IO 0 +#endif + +/** + * \def UCONFIG_NO_CONVERSION + * ICU will not completely build with this switch turned on. + * This switch turns off all converters. + * + * @stable ICU 3.2 + */ +#ifndef UCONFIG_NO_CONVERSION +# define UCONFIG_NO_CONVERSION 0 +#endif + +#if UCONFIG_NO_CONVERSION +# define UCONFIG_NO_LEGACY_CONVERSION 1 +#endif + +/** + * \def UCONFIG_NO_LEGACY_CONVERSION + * This switch turns off all converters except for + * - Unicode charsets (UTF-7/8/16/32, CESU-8, SCSU, BOCU-1) + * - US-ASCII + * - ISO-8859-1 + * + * Turning off legacy conversion is not possible on EBCDIC platforms + * because they need ibm-37 or ibm-1047 default converters. + * + * @stable ICU 2.4 + */ +#ifndef UCONFIG_NO_LEGACY_CONVERSION +# define UCONFIG_NO_LEGACY_CONVERSION 0 +#endif + +/** + * \def UCONFIG_NO_NORMALIZATION + * This switch turns off normalization. + * It implies turning off several other services as well, for example + * collation and IDNA. + * + * @stable ICU 2.6 + */ +#ifndef UCONFIG_NO_NORMALIZATION +# define UCONFIG_NO_NORMALIZATION 0 +#elif UCONFIG_NO_NORMALIZATION + /* common library */ +# define UCONFIG_NO_IDNA 1 + + /* i18n library */ +# if UCONFIG_ONLY_COLLATION +# error Contradictory collation switches in uconfig.h. +# endif +# define UCONFIG_NO_COLLATION 1 +# define UCONFIG_NO_TRANSLITERATION 1 +#endif + +/** + * \def UCONFIG_NO_BREAK_ITERATION + * This switch turns off break iteration. + * + * @stable ICU 2.4 + */ +#ifndef UCONFIG_NO_BREAK_ITERATION +# define UCONFIG_NO_BREAK_ITERATION 0 +#endif + +/** + * \def UCONFIG_NO_IDNA + * This switch turns off IDNA. + * + * @stable ICU 2.6 + */ +#ifndef UCONFIG_NO_IDNA +# define UCONFIG_NO_IDNA 0 +#endif + +/* i18n library switches ---------------------------------------------------- */ + +/** + * \def UCONFIG_NO_COLLATION + * This switch turns off collation and collation-based string search. + * + * @stable ICU 2.4 + */ +#ifndef UCONFIG_NO_COLLATION +# define UCONFIG_NO_COLLATION 0 +#endif + +/** + * \def UCONFIG_NO_FORMATTING + * This switch turns off formatting and calendar/timezone services. + * + * @stable ICU 2.4 + */ +#ifndef UCONFIG_NO_FORMATTING +# define UCONFIG_NO_FORMATTING 0 +#endif + +/** + * \def UCONFIG_NO_TRANSLITERATION + * This switch turns off transliteration. + * + * @stable ICU 2.4 + */ +#ifndef UCONFIG_NO_TRANSLITERATION +# define UCONFIG_NO_TRANSLITERATION 0 +#endif + +/** + * \def UCONFIG_NO_REGULAR_EXPRESSIONS + * This switch turns off regular expressions. + * + * @stable ICU 2.4 + */ +#ifndef UCONFIG_NO_REGULAR_EXPRESSIONS +# define UCONFIG_NO_REGULAR_EXPRESSIONS 0 +#endif + +/** + * \def UCONFIG_NO_SERVICE + * This switch turns off service registration. + * + * @stable ICU 3.2 + */ +#ifndef UCONFIG_NO_SERVICE +# define UCONFIG_NO_SERVICE 0 +#endif + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/udata.h b/Build/source/libs/icu/icu-xetex/common/unicode/udata.h new file mode 100644 index 00000000000..95348fb954d --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/udata.h @@ -0,0 +1,389 @@ +/* +****************************************************************************** +* +* Copyright (C) 1999-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: udata.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 1999oct25 +* created by: Markus W. Scherer +*/ + +#ifndef __UDATA_H__ +#define __UDATA_H__ + +#include "unicode/utypes.h" + +U_CDECL_BEGIN + +/** + * \file + * \brief C API: Data loading interface + * + *

Information about data loading interface

+ * + * This API is used to find and efficiently load data for ICU and applications + * using ICU. It provides an abstract interface that specifies a data type and + * name to find and load the data. Normally this API is used by other ICU APIs + * to load required data out of the ICU data library, but it can be used to + * load data out of other places. + * + * See the User Guide Data Management chapter. + */ + +#ifndef U_HIDE_INTERNAL_API +/** + * Character used to separate package names from tree names + * @internal ICU 3.0 + */ +#define U_TREE_SEPARATOR '-' + +/** + * String used to separate package names from tree names + * @internal ICU 3.0 + */ +#define U_TREE_SEPARATOR_STRING "-" + +/** + * Character used to separate parts of entry names + * @internal ICU 3.0 + */ +#define U_TREE_ENTRY_SEP_CHAR '/' + +/** + * String used to separate parts of entry names + * @internal ICU 3.0 + */ +#define U_TREE_ENTRY_SEP_STRING "/" + +/** + * Alias for standard ICU data + * @internal ICU 3.0 + */ +#define U_ICUDATA_ALIAS "ICUDATA" + +#endif /* U_HIDE_INTERNAL_API */ + +/** + * UDataInfo contains the properties about the requested data. + * This is meta data. + * + *

This structure may grow in the future, indicated by the + * size field.

+ * + *

The platform data property fields help determine if a data + * file can be efficiently used on a given machine. + * The particular fields are of importance only if the data + * is affected by the properties - if there is integer data + * with word sizes > 1 byte, char* text, or UChar* text.

+ * + *

The implementation for the udata_open[Choice]() + * functions may reject data based on the value in isBigEndian. + * No other field is used by the udata API implementation.

+ * + *

The dataFormat may be used to identify + * the kind of data, e.g. a converter table.

+ * + *

The formatVersion field should be used to + * make sure that the format can be interpreted. + * I may be a good idea to check only for the one or two highest + * of the version elements to allow the data memory to + * get more or somewhat rearranged contents, for as long + * as the using code can still interpret the older contents.

+ * + *

The dataVersion field is intended to be a + * common place to store the source version of the data; + * for data from the Unicode character database, this could + * reflect the Unicode version.

+ * @stable ICU 2.0 + */ +typedef struct { + /** sizeof(UDataInfo) + * @stable ICU 2.0 */ + uint16_t size; + + /** unused, set to 0 + * @stable ICU 2.0*/ + uint16_t reservedWord; + + /* platform data properties */ + /** 0 for little-endian machine, 1 for big-endian + * @stable ICU 2.0 */ + uint8_t isBigEndian; + + /** see U_CHARSET_FAMILY values in utypes.h + * @stable ICU 2.0*/ + uint8_t charsetFamily; + + /** sizeof(UChar), one of { 1, 2, 4 } + * @stable ICU 2.0*/ + uint8_t sizeofUChar; + + /** unused, set to 0 + * @stable ICU 2.0*/ + uint8_t reservedByte; + + /** data format identifier + * @stable ICU 2.0*/ + uint8_t dataFormat[4]; + + /** versions: [0] major [1] minor [2] milli [3] micro + * @stable ICU 2.0*/ + uint8_t formatVersion[4]; + + /** versions: [0] major [1] minor [2] milli [3] micro + * @stable ICU 2.0*/ + uint8_t dataVersion[4]; +} UDataInfo; + +/* API for reading data -----------------------------------------------------*/ + +/** + * Forward declaration of the data memory type. + * @stable ICU 2.0 + */ +typedef struct UDataMemory UDataMemory; + +/** + * Callback function for udata_openChoice(). + * @param context parameter passed into udata_openChoice(). + * @param type The type of the data as passed into udata_openChoice(). + * It may be NULL. + * @param name The name of the data as passed into udata_openChoice(). + * @param pInfo A pointer to the UDataInfo structure + * of data that has been loaded and will be returned + * by udata_openChoice() if this function + * returns TRUE. + * @return TRUE if the current data memory is acceptable + * @stable ICU 2.0 + */ +typedef UBool U_CALLCONV +UDataMemoryIsAcceptable(void *context, + const char *type, const char *name, + const UDataInfo *pInfo); + + +/** + * Convenience function. + * This function works the same as udata_openChoice + * except that any data that matches the type and name + * is assumed to be acceptable. + * @param path Specifies an absolute path and/or a basename for the + * finding of the data in the file system. + * NULL for ICU data. + * @param type A string that specifies the type of data to be loaded. + * For example, resource bundles are loaded with type "res", + * conversion tables with type "cnv". + * This may be NULL or empty. + * @param name A string that specifies the name of the data. + * @param pErrorCode An ICU UErrorCode parameter. It must not be NULL. + * @return A pointer (handle) to a data memory object, or NULL + * if an error occurs. Call udata_getMemory() + * to get a pointer to the actual data. + * + * @see udata_openChoice + * @stable ICU 2.0 + */ +U_STABLE UDataMemory * U_EXPORT2 +udata_open(const char *path, const char *type, const char *name, + UErrorCode *pErrorCode); + +/** + * Data loading function. + * This function is used to find and load efficiently data for + * ICU and applications using ICU. + * It provides an abstract interface that allows to specify a data + * type and name to find and load the data. + * + *

The implementation depends on platform properties and user preferences + * and may involve loading shared libraries (DLLs), mapping + * files into memory, or fopen()/fread() files. + * It may also involve using static memory or database queries etc. + * Several or all data items may be combined into one entity + * (DLL, memory-mappable file).

+ * + *

The data is always preceded by a header that includes + * a UDataInfo structure. + * The caller's isAcceptable() function is called to make + * sure that the data is useful. It may be called several times if it + * rejects the data and there is more than one location with data + * matching the type and name.

+ * + *

If path==NULL, then ICU data is loaded. + * Otherwise, it is separated into a basename and a basename-less directory string. + * The basename is used as the data package name, and the directory is + * logically prepended to the ICU data directory string.

+ * + *

For details about ICU data loading see the User Guide + * Data Management chapter. (http://icu-project.org/userguide/icudata.html)

+ * + * @param path Specifies an absolute path and/or a basename for the + * finding of the data in the file system. + * NULL for ICU data. + * @param type A string that specifies the type of data to be loaded. + * For example, resource bundles are loaded with type "res", + * conversion tables with type "cnv". + * This may be NULL or empty. + * @param name A string that specifies the name of the data. + * @param isAcceptable This function is called to verify that loaded data + * is useful for the client code. If it returns FALSE + * for all data items, then udata_openChoice() + * will return with an error. + * @param context Arbitrary parameter to be passed into isAcceptable. + * @param pErrorCode An ICU UErrorCode parameter. It must not be NULL. + * @return A pointer (handle) to a data memory object, or NULL + * if an error occurs. Call udata_getMemory() + * to get a pointer to the actual data. + * @stable ICU 2.0 + */ +U_STABLE UDataMemory * U_EXPORT2 +udata_openChoice(const char *path, const char *type, const char *name, + UDataMemoryIsAcceptable *isAcceptable, void *context, + UErrorCode *pErrorCode); + +/** + * Close the data memory. + * This function must be called to allow the system to + * release resources associated with this data memory. + * @param pData The pointer to data memory object + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +udata_close(UDataMemory *pData); + +/** + * Get the pointer to the actual data inside the data memory. + * The data is read-only. + * @param pData The pointer to data memory object + * @stable ICU 2.0 + */ +U_STABLE const void * U_EXPORT2 +udata_getMemory(UDataMemory *pData); + +/** + * Get the information from the data memory header. + * This allows to get access to the header containing + * platform data properties etc. which is not part of + * the data itself and can therefore not be accessed + * via the pointer that udata_getMemory() returns. + * + * @param pData pointer to the data memory object + * @param pInfo pointer to a UDataInfo object; + * its size field must be set correctly, + * typically to sizeof(UDataInfo). + * + * *pInfo will be filled with the UDataInfo structure + * in the data memory object. If this structure is smaller than + * pInfo->size, then the size will be + * adjusted and only part of the structure will be filled. + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +udata_getInfo(UDataMemory *pData, UDataInfo *pInfo); + +/** + * This function bypasses the normal ICU data loading process and + * allows you to force ICU's system data to come out of a user-specified + * area in memory. + * + * The format of this data is that of the icu common data file, as is + * generated by the pkgdata tool with mode=common or mode=dll. + * You can read in a whole common mode file and pass the address to the start of the + * data, or (with the appropriate link options) pass in the pointer to + * the data that has been loaded from a dll by the operating system, + * as shown in this code: + * + * extern const char U_IMPORT U_ICUDATA_ENTRY_POINT []; + * // U_ICUDATA_ENTRY_POINT is same as entry point specified to pkgdata tool + * UErrorCode status = U_ZERO_ERROR; + * + * udata_setCommonData(&U_ICUDATA_ENTRY_POINT, &status); + * + * Warning: ICU must NOT have even attempted to access its data yet + * when this call is made, or U_USING_DEFAULT_WARNING code will + * be returned. Be careful of UnicodeStrings in static initialization which + * may attempt to load a converter (use the UNICODE_STRING(x) macro instead). + * + * Also note that it is important that the declaration be as above. The entry point + * must not be declared as an extern void*. + * + * This function has no effect on application (non ICU) data. See udata_setAppData() + * for similar functionality for application data. + * + * @param data pointer to ICU common data + * @param err outgoing error status U_USING_DEFAULT_WARNING, U_UNSUPPORTED_ERROR + * @stable ICU 2.0 + */ + +U_STABLE void U_EXPORT2 +udata_setCommonData(const void *data, UErrorCode *err); + + +/** + * This function bypasses the normal ICU data loading process for application-specific + * data and allows you to force the it to come out of a user-specified + * pointer. + * + * The format of this data is that of the icu common data file, like 'icudt26l.dat' + * or the corresponding shared library (DLL) file. + * The application must read in or otherwise construct an image of the data and then + * pass the address of it to this function. + * + * + * Warning: setAppData will set a U_USING_DEFAULT_WARNING code if + * data with the specifed path that has already been opened, or + * if setAppData with the same path has already been called. + * Any such calls to setAppData will have no effect. + * + * + * @param packageName the package name by which the application will refer + * to (open) this data + * @param data pointer to the data + * @param err outgoing error status U_USING_DEFAULT_WARNING, U_UNSUPPORTED_ERROR + * @see udata_setCommonData + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +udata_setAppData(const char *packageName, const void *data, UErrorCode *err); + +/** + * Possible settings for udata_setFileAccess() + * @see udata_setFileAccess + * @stable ICU 3.4 + */ +typedef enum UDataFileAccess { + /** ICU does not access the file system for data loading. */ + UDATA_NO_FILES, + /** ICU only loads data from packages, not from single files. */ + UDATA_ONLY_PACKAGES, + /** ICU loads data from packages first, and only from single files + if the data cannot be found in a package. */ + UDATA_PACKAGES_FIRST, + /** ICU looks for data in single files first, then in packages. (default) */ + UDATA_FILES_FIRST, + /** An alias for the default access mode. */ + UDATA_DEFAULT_ACCESS = UDATA_FILES_FIRST, + UDATA_FILE_ACCESS_COUNT +} UDataFileAccess; + +/** + * This function may be called to control how ICU loads data. It must be called + * before any ICU data is loaded, including application data loaded with ures/ResourceBundle or + * udata APIs. It should be called before u_init. This function is not multithread safe. + * The results of calling it while other threads are loading data are undefined. + * @param access The type of file access to be used + * @param status Error code. + * @see UDataFileAccess + * @stable ICU 3.4 + */ +U_STABLE void U_EXPORT2 +udata_setFileAccess(UDataFileAccess access, UErrorCode *status); + +U_CDECL_END + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/udeprctd.h b/Build/source/libs/icu/icu-xetex/common/unicode/udeprctd.h new file mode 100644 index 00000000000..158967efe25 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/udeprctd.h @@ -0,0 +1,50 @@ +/* +******************************************************************************* +* Copyright (C) 2004-2007, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* +* file name: +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* Created by: genheaders.pl, a perl script written by Ram Viswanadha +* +* Contains data for commenting out APIs. +* Gets included by umachine.h +* +* THIS FILE IS MACHINE-GENERATED, DON'T PLAY WITH IT IF YOU DON'T KNOW WHAT +* YOU ARE DOING, OTHERWISE VERY BAD THINGS WILL HAPPEN! +*/ + +#ifndef UDEPRCTD_H +#define UDEPRCTD_H + +#ifdef U_HIDE_DEPRECATED_API + +# if U_DISABLE_RENAMING +# define ucol_getContractions ucol_getContractions_DEPRECATED_API_DO_NOT_USE +# define ucol_getLocale ucol_getLocale_DEPRECATED_API_DO_NOT_USE +# define ures_countArrayItems ures_countArrayItems_DEPRECATED_API_DO_NOT_USE +# define ures_getLocale ures_getLocale_DEPRECATED_API_DO_NOT_USE +# define ures_getVersionNumber ures_getVersionNumber_DEPRECATED_API_DO_NOT_USE +# define utrans_getAvailableID utrans_getAvailableID_DEPRECATED_API_DO_NOT_USE +# define utrans_getID utrans_getID_DEPRECATED_API_DO_NOT_USE +# define utrans_open utrans_open_DEPRECATED_API_DO_NOT_USE +# define utrans_unregister utrans_unregister_DEPRECATED_API_DO_NOT_USE +# else +# define ucol_getContractions_3_8 ucol_getContractions_DEPRECATED_API_DO_NOT_USE +# define ucol_getLocale_3_8 ucol_getLocale_DEPRECATED_API_DO_NOT_USE +# define ures_countArrayItems_3_8 ures_countArrayItems_DEPRECATED_API_DO_NOT_USE +# define ures_getLocale_3_8 ures_getLocale_DEPRECATED_API_DO_NOT_USE +# define ures_getVersionNumber_3_8 ures_getVersionNumber_DEPRECATED_API_DO_NOT_USE +# define utrans_getAvailableID_3_8 utrans_getAvailableID_DEPRECATED_API_DO_NOT_USE +# define utrans_getID_3_8 utrans_getID_DEPRECATED_API_DO_NOT_USE +# define utrans_open_3_8 utrans_open_DEPRECATED_API_DO_NOT_USE +# define utrans_unregister_3_8 utrans_unregister_DEPRECATED_API_DO_NOT_USE +# endif /* U_DISABLE_RENAMING */ + +#endif /* U_HIDE_DEPRECATED_API */ +#endif /* UDEPRCTD_H */ + diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/udraft.h b/Build/source/libs/icu/icu-xetex/common/unicode/udraft.h new file mode 100644 index 00000000000..f869b240eec --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/udraft.h @@ -0,0 +1,186 @@ +/* +******************************************************************************* +* Copyright (C) 2004-2007, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* +* file name: +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* Created by: genheaders.pl, a perl script written by Ram Viswanadha +* +* Contains data for commenting out APIs. +* Gets included by umachine.h +* +* THIS FILE IS MACHINE-GENERATED, DON'T PLAY WITH IT IF YOU DON'T KNOW WHAT +* YOU ARE DOING, OTHERWISE VERY BAD THINGS WILL HAPPEN! +*/ + +#ifndef UDRAFT_H +#define UDRAFT_H + +#ifdef U_HIDE_DRAFT_API + +# if U_DISABLE_RENAMING +# define u_fclose u_fclose_DRAFT_API_DO_NOT_USE +# define u_feof u_feof_DRAFT_API_DO_NOT_USE +# define u_fflush u_fflush_DRAFT_API_DO_NOT_USE +# define u_fgetConverter u_fgetConverter_DRAFT_API_DO_NOT_USE +# define u_fgetc u_fgetc_DRAFT_API_DO_NOT_USE +# define u_fgetcodepage u_fgetcodepage_DRAFT_API_DO_NOT_USE +# define u_fgetcx u_fgetcx_DRAFT_API_DO_NOT_USE +# define u_fgetfile u_fgetfile_DRAFT_API_DO_NOT_USE +# define u_fgetlocale u_fgetlocale_DRAFT_API_DO_NOT_USE +# define u_fgets u_fgets_DRAFT_API_DO_NOT_USE +# define u_file_read u_file_read_DRAFT_API_DO_NOT_USE +# define u_file_write u_file_write_DRAFT_API_DO_NOT_USE +# define u_finit u_finit_DRAFT_API_DO_NOT_USE +# define u_fopen u_fopen_DRAFT_API_DO_NOT_USE +# define u_fprintf u_fprintf_DRAFT_API_DO_NOT_USE +# define u_fprintf_u u_fprintf_u_DRAFT_API_DO_NOT_USE +# define u_fputc u_fputc_DRAFT_API_DO_NOT_USE +# define u_fputs u_fputs_DRAFT_API_DO_NOT_USE +# define u_frewind u_frewind_DRAFT_API_DO_NOT_USE +# define u_fscanf u_fscanf_DRAFT_API_DO_NOT_USE +# define u_fscanf_u u_fscanf_u_DRAFT_API_DO_NOT_USE +# define u_fsetcodepage u_fsetcodepage_DRAFT_API_DO_NOT_USE +# define u_fsetlocale u_fsetlocale_DRAFT_API_DO_NOT_USE +# define u_fsettransliterator u_fsettransliterator_DRAFT_API_DO_NOT_USE +# define u_fstropen u_fstropen_DRAFT_API_DO_NOT_USE +# define u_fungetc u_fungetc_DRAFT_API_DO_NOT_USE +# define u_snprintf u_snprintf_DRAFT_API_DO_NOT_USE +# define u_snprintf_u u_snprintf_u_DRAFT_API_DO_NOT_USE +# define u_sprintf u_sprintf_DRAFT_API_DO_NOT_USE +# define u_sprintf_u u_sprintf_u_DRAFT_API_DO_NOT_USE +# define u_sscanf u_sscanf_DRAFT_API_DO_NOT_USE +# define u_sscanf_u u_sscanf_u_DRAFT_API_DO_NOT_USE +# define u_vfprintf u_vfprintf_DRAFT_API_DO_NOT_USE +# define u_vfprintf_u u_vfprintf_u_DRAFT_API_DO_NOT_USE +# define u_vfscanf u_vfscanf_DRAFT_API_DO_NOT_USE +# define u_vfscanf_u u_vfscanf_u_DRAFT_API_DO_NOT_USE +# define u_vsnprintf u_vsnprintf_DRAFT_API_DO_NOT_USE +# define u_vsnprintf_u u_vsnprintf_u_DRAFT_API_DO_NOT_USE +# define u_vsprintf u_vsprintf_DRAFT_API_DO_NOT_USE +# define u_vsprintf_u u_vsprintf_u_DRAFT_API_DO_NOT_USE +# define u_vsscanf u_vsscanf_DRAFT_API_DO_NOT_USE +# define u_vsscanf_u u_vsscanf_u_DRAFT_API_DO_NOT_USE +# define ucal_getTZDataVersion ucal_getTZDataVersion_DRAFT_API_DO_NOT_USE +# define ucasemap_getBreakIterator ucasemap_getBreakIterator_DRAFT_API_DO_NOT_USE +# define ucasemap_setBreakIterator ucasemap_setBreakIterator_DRAFT_API_DO_NOT_USE +# define ucasemap_toTitle ucasemap_toTitle_DRAFT_API_DO_NOT_USE +# define ucasemap_utf8FoldCase ucasemap_utf8FoldCase_DRAFT_API_DO_NOT_USE +# define ucasemap_utf8ToTitle ucasemap_utf8ToTitle_DRAFT_API_DO_NOT_USE +# define udatpg_addPattern udatpg_addPattern_DRAFT_API_DO_NOT_USE +# define udatpg_clone udatpg_clone_DRAFT_API_DO_NOT_USE +# define udatpg_close udatpg_close_DRAFT_API_DO_NOT_USE +# define udatpg_getAppendItemFormat udatpg_getAppendItemFormat_DRAFT_API_DO_NOT_USE +# define udatpg_getAppendItemName udatpg_getAppendItemName_DRAFT_API_DO_NOT_USE +# define udatpg_getBaseSkeleton udatpg_getBaseSkeleton_DRAFT_API_DO_NOT_USE +# define udatpg_getBestPattern udatpg_getBestPattern_DRAFT_API_DO_NOT_USE +# define udatpg_getDateTimeFormat udatpg_getDateTimeFormat_DRAFT_API_DO_NOT_USE +# define udatpg_getDecimal udatpg_getDecimal_DRAFT_API_DO_NOT_USE +# define udatpg_getPatternForSkeleton udatpg_getPatternForSkeleton_DRAFT_API_DO_NOT_USE +# define udatpg_getSkeleton udatpg_getSkeleton_DRAFT_API_DO_NOT_USE +# define udatpg_open udatpg_open_DRAFT_API_DO_NOT_USE +# define udatpg_openBaseSkeletons udatpg_openBaseSkeletons_DRAFT_API_DO_NOT_USE +# define udatpg_openEmpty udatpg_openEmpty_DRAFT_API_DO_NOT_USE +# define udatpg_openSkeletons udatpg_openSkeletons_DRAFT_API_DO_NOT_USE +# define udatpg_replaceFieldTypes udatpg_replaceFieldTypes_DRAFT_API_DO_NOT_USE +# define udatpg_setAppendItemFormat udatpg_setAppendItemFormat_DRAFT_API_DO_NOT_USE +# define udatpg_setAppendItemName udatpg_setAppendItemName_DRAFT_API_DO_NOT_USE +# define udatpg_setDateTimeFormat udatpg_setDateTimeFormat_DRAFT_API_DO_NOT_USE +# define udatpg_setDecimal udatpg_setDecimal_DRAFT_API_DO_NOT_USE +# define uloc_getLocaleForLCID uloc_getLocaleForLCID_DRAFT_API_DO_NOT_USE +# define uset_clone uset_clone_DRAFT_API_DO_NOT_USE +# define uset_cloneAsThawed uset_cloneAsThawed_DRAFT_API_DO_NOT_USE +# define uset_freeze uset_freeze_DRAFT_API_DO_NOT_USE +# define uset_isFrozen uset_isFrozen_DRAFT_API_DO_NOT_USE +# define uset_span uset_span_DRAFT_API_DO_NOT_USE +# define uset_spanBack uset_spanBack_DRAFT_API_DO_NOT_USE +# define uset_spanBackUTF8 uset_spanBackUTF8_DRAFT_API_DO_NOT_USE +# define uset_spanUTF8 uset_spanUTF8_DRAFT_API_DO_NOT_USE +# else +# define u_fclose_3_8 u_fclose_DRAFT_API_DO_NOT_USE +# define u_feof_3_8 u_feof_DRAFT_API_DO_NOT_USE +# define u_fflush_3_8 u_fflush_DRAFT_API_DO_NOT_USE +# define u_fgetConverter_3_8 u_fgetConverter_DRAFT_API_DO_NOT_USE +# define u_fgetc_3_8 u_fgetc_DRAFT_API_DO_NOT_USE +# define u_fgetcodepage_3_8 u_fgetcodepage_DRAFT_API_DO_NOT_USE +# define u_fgetcx_3_8 u_fgetcx_DRAFT_API_DO_NOT_USE +# define u_fgetfile_3_8 u_fgetfile_DRAFT_API_DO_NOT_USE +# define u_fgetlocale_3_8 u_fgetlocale_DRAFT_API_DO_NOT_USE +# define u_fgets_3_8 u_fgets_DRAFT_API_DO_NOT_USE +# define u_file_read_3_8 u_file_read_DRAFT_API_DO_NOT_USE +# define u_file_write_3_8 u_file_write_DRAFT_API_DO_NOT_USE +# define u_finit_3_8 u_finit_DRAFT_API_DO_NOT_USE +# define u_fopen_3_8 u_fopen_DRAFT_API_DO_NOT_USE +# define u_fprintf_3_8 u_fprintf_DRAFT_API_DO_NOT_USE +# define u_fprintf_u_3_8 u_fprintf_u_DRAFT_API_DO_NOT_USE +# define u_fputc_3_8 u_fputc_DRAFT_API_DO_NOT_USE +# define u_fputs_3_8 u_fputs_DRAFT_API_DO_NOT_USE +# define u_frewind_3_8 u_frewind_DRAFT_API_DO_NOT_USE +# define u_fscanf_3_8 u_fscanf_DRAFT_API_DO_NOT_USE +# define u_fscanf_u_3_8 u_fscanf_u_DRAFT_API_DO_NOT_USE +# define u_fsetcodepage_3_8 u_fsetcodepage_DRAFT_API_DO_NOT_USE +# define u_fsetlocale_3_8 u_fsetlocale_DRAFT_API_DO_NOT_USE +# define u_fsettransliterator_3_8 u_fsettransliterator_DRAFT_API_DO_NOT_USE +# define u_fstropen_3_8 u_fstropen_DRAFT_API_DO_NOT_USE +# define u_fungetc_3_8 u_fungetc_DRAFT_API_DO_NOT_USE +# define u_snprintf_3_8 u_snprintf_DRAFT_API_DO_NOT_USE +# define u_snprintf_u_3_8 u_snprintf_u_DRAFT_API_DO_NOT_USE +# define u_sprintf_3_8 u_sprintf_DRAFT_API_DO_NOT_USE +# define u_sprintf_u_3_8 u_sprintf_u_DRAFT_API_DO_NOT_USE +# define u_sscanf_3_8 u_sscanf_DRAFT_API_DO_NOT_USE +# define u_sscanf_u_3_8 u_sscanf_u_DRAFT_API_DO_NOT_USE +# define u_vfprintf_3_8 u_vfprintf_DRAFT_API_DO_NOT_USE +# define u_vfprintf_u_3_8 u_vfprintf_u_DRAFT_API_DO_NOT_USE +# define u_vfscanf_3_8 u_vfscanf_DRAFT_API_DO_NOT_USE +# define u_vfscanf_u_3_8 u_vfscanf_u_DRAFT_API_DO_NOT_USE +# define u_vsnprintf_3_8 u_vsnprintf_DRAFT_API_DO_NOT_USE +# define u_vsnprintf_u_3_8 u_vsnprintf_u_DRAFT_API_DO_NOT_USE +# define u_vsprintf_3_8 u_vsprintf_DRAFT_API_DO_NOT_USE +# define u_vsprintf_u_3_8 u_vsprintf_u_DRAFT_API_DO_NOT_USE +# define u_vsscanf_3_8 u_vsscanf_DRAFT_API_DO_NOT_USE +# define u_vsscanf_u_3_8 u_vsscanf_u_DRAFT_API_DO_NOT_USE +# define ucal_getTZDataVersion_3_8 ucal_getTZDataVersion_DRAFT_API_DO_NOT_USE +# define ucasemap_getBreakIterator_3_8 ucasemap_getBreakIterator_DRAFT_API_DO_NOT_USE +# define ucasemap_setBreakIterator_3_8 ucasemap_setBreakIterator_DRAFT_API_DO_NOT_USE +# define ucasemap_toTitle_3_8 ucasemap_toTitle_DRAFT_API_DO_NOT_USE +# define ucasemap_utf8FoldCase_3_8 ucasemap_utf8FoldCase_DRAFT_API_DO_NOT_USE +# define ucasemap_utf8ToTitle_3_8 ucasemap_utf8ToTitle_DRAFT_API_DO_NOT_USE +# define udatpg_addPattern_3_8 udatpg_addPattern_DRAFT_API_DO_NOT_USE +# define udatpg_clone_3_8 udatpg_clone_DRAFT_API_DO_NOT_USE +# define udatpg_close_3_8 udatpg_close_DRAFT_API_DO_NOT_USE +# define udatpg_getAppendItemFormat_3_8 udatpg_getAppendItemFormat_DRAFT_API_DO_NOT_USE +# define udatpg_getAppendItemName_3_8 udatpg_getAppendItemName_DRAFT_API_DO_NOT_USE +# define udatpg_getBaseSkeleton_3_8 udatpg_getBaseSkeleton_DRAFT_API_DO_NOT_USE +# define udatpg_getBestPattern_3_8 udatpg_getBestPattern_DRAFT_API_DO_NOT_USE +# define udatpg_getDateTimeFormat_3_8 udatpg_getDateTimeFormat_DRAFT_API_DO_NOT_USE +# define udatpg_getDecimal_3_8 udatpg_getDecimal_DRAFT_API_DO_NOT_USE +# define udatpg_getPatternForSkeleton_3_8 udatpg_getPatternForSkeleton_DRAFT_API_DO_NOT_USE +# define udatpg_getSkeleton_3_8 udatpg_getSkeleton_DRAFT_API_DO_NOT_USE +# define udatpg_openBaseSkeletons_3_8 udatpg_openBaseSkeletons_DRAFT_API_DO_NOT_USE +# define udatpg_openEmpty_3_8 udatpg_openEmpty_DRAFT_API_DO_NOT_USE +# define udatpg_openSkeletons_3_8 udatpg_openSkeletons_DRAFT_API_DO_NOT_USE +# define udatpg_open_3_8 udatpg_open_DRAFT_API_DO_NOT_USE +# define udatpg_replaceFieldTypes_3_8 udatpg_replaceFieldTypes_DRAFT_API_DO_NOT_USE +# define udatpg_setAppendItemFormat_3_8 udatpg_setAppendItemFormat_DRAFT_API_DO_NOT_USE +# define udatpg_setAppendItemName_3_8 udatpg_setAppendItemName_DRAFT_API_DO_NOT_USE +# define udatpg_setDateTimeFormat_3_8 udatpg_setDateTimeFormat_DRAFT_API_DO_NOT_USE +# define udatpg_setDecimal_3_8 udatpg_setDecimal_DRAFT_API_DO_NOT_USE +# define uloc_getLocaleForLCID_3_8 uloc_getLocaleForLCID_DRAFT_API_DO_NOT_USE +# define uset_cloneAsThawed_3_8 uset_cloneAsThawed_DRAFT_API_DO_NOT_USE +# define uset_clone_3_8 uset_clone_DRAFT_API_DO_NOT_USE +# define uset_freeze_3_8 uset_freeze_DRAFT_API_DO_NOT_USE +# define uset_isFrozen_3_8 uset_isFrozen_DRAFT_API_DO_NOT_USE +# define uset_spanBackUTF8_3_8 uset_spanBackUTF8_DRAFT_API_DO_NOT_USE +# define uset_spanBack_3_8 uset_spanBack_DRAFT_API_DO_NOT_USE +# define uset_spanUTF8_3_8 uset_spanUTF8_DRAFT_API_DO_NOT_USE +# define uset_span_3_8 uset_span_DRAFT_API_DO_NOT_USE +# endif /* U_DISABLE_RENAMING */ + +#endif /* U_HIDE_DRAFT_API */ +#endif /* UDRAFT_H */ + diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/uenum.h b/Build/source/libs/icu/icu-xetex/common/unicode/uenum.h new file mode 100644 index 00000000000..ff9b29997e9 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/uenum.h @@ -0,0 +1,134 @@ +/* +******************************************************************************* +* +* Copyright (C) 2002-2005, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: uenum.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:2 +* +* created on: 2002jul08 +* created by: Vladimir Weinstein +*/ + +#ifndef __UENUM_H +#define __UENUM_H + +#include "unicode/utypes.h" + +/** + * \file + * \brief C API: String Enumeration + */ + +/** + * An enumeration object. + * For usage in C programs. + * @stable ICU 2.2 + */ +struct UEnumeration; +/** structure representing an enumeration object instance @stable ICU 2.2 */ +typedef struct UEnumeration UEnumeration; + +/** + * Disposes of resources in use by the iterator. If en is NULL, + * does nothing. After this call, any char* or UChar* pointer + * returned by uenum_unext() or uenum_next() is invalid. + * @param en UEnumeration structure pointer + * @stable ICU 2.2 + */ +U_STABLE void U_EXPORT2 +uenum_close(UEnumeration* en); + +/** + * Returns the number of elements that the iterator traverses. If + * the iterator is out-of-sync with its service, status is set to + * U_ENUM_OUT_OF_SYNC_ERROR. + * This is a convenience function. It can end up being very + * expensive as all the items might have to be pre-fetched (depending + * on the type of data being traversed). Use with caution and only + * when necessary. + * @param en UEnumeration structure pointer + * @param status error code, can be U_ENUM_OUT_OF_SYNC_ERROR if the + * iterator is out of sync. + * @return number of elements in the iterator + * @stable ICU 2.2 + */ +U_STABLE int32_t U_EXPORT2 +uenum_count(UEnumeration* en, UErrorCode* status); + +/** + * Returns the next element in the iterator's list. If there are + * no more elements, returns NULL. If the iterator is out-of-sync + * with its service, status is set to U_ENUM_OUT_OF_SYNC_ERROR and + * NULL is returned. If the native service string is a char* string, + * it is converted to UChar* with the invariant converter. + * The result is terminated by (UChar)0. + * @param en the iterator object + * @param resultLength pointer to receive the length of the result + * (not including the terminating \\0). + * If the pointer is NULL it is ignored. + * @param status the error code, set to U_ENUM_OUT_OF_SYNC_ERROR if + * the iterator is out of sync with its service. + * @return a pointer to the string. The string will be + * zero-terminated. The return pointer is owned by this iterator + * and must not be deleted by the caller. The pointer is valid + * until the next call to any uenum_... method, including + * uenum_next() or uenum_unext(). When all strings have been + * traversed, returns NULL. + * @stable ICU 2.2 + */ +U_STABLE const UChar* U_EXPORT2 +uenum_unext(UEnumeration* en, + int32_t* resultLength, + UErrorCode* status); + +/** + * Returns the next element in the iterator's list. If there are + * no more elements, returns NULL. If the iterator is out-of-sync + * with its service, status is set to U_ENUM_OUT_OF_SYNC_ERROR and + * NULL is returned. If the native service string is a UChar* + * string, it is converted to char* with the invariant converter. + * The result is terminated by (char)0. If the conversion fails + * (because a character cannot be converted) then status is set to + * U_INVARIANT_CONVERSION_ERROR and the return value is undefined + * (but non-NULL). + * @param en the iterator object + * @param resultLength pointer to receive the length of the result + * (not including the terminating \\0). + * If the pointer is NULL it is ignored. + * @param status the error code, set to U_ENUM_OUT_OF_SYNC_ERROR if + * the iterator is out of sync with its service. Set to + * U_INVARIANT_CONVERSION_ERROR if the underlying native string is + * UChar* and conversion to char* with the invariant converter + * fails. This error pertains only to current string, so iteration + * might be able to continue successfully. + * @return a pointer to the string. The string will be + * zero-terminated. The return pointer is owned by this iterator + * and must not be deleted by the caller. The pointer is valid + * until the next call to any uenum_... method, including + * uenum_next() or uenum_unext(). When all strings have been + * traversed, returns NULL. + * @stable ICU 2.2 + */ +U_STABLE const char* U_EXPORT2 +uenum_next(UEnumeration* en, + int32_t* resultLength, + UErrorCode* status); + +/** + * Resets the iterator to the current list of service IDs. This + * re-establishes sync with the service and rewinds the iterator + * to start at the first element. + * @param en the iterator object + * @param status the error code, set to U_ENUM_OUT_OF_SYNC_ERROR if + * the iterator is out of sync with its service. + * @stable ICU 2.2 + */ +U_STABLE void U_EXPORT2 +uenum_reset(UEnumeration* en, UErrorCode* status); + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/uidna.h b/Build/source/libs/icu/icu-xetex/common/unicode/uidna.h new file mode 100644 index 00000000000..52aa6e9104e --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/uidna.h @@ -0,0 +1,312 @@ +/* + ******************************************************************************* + * + * Copyright (C) 2003-2007, International Business Machines + * Corporation and others. All Rights Reserved. + * + ******************************************************************************* + * file name: uidna.h + * encoding: US-ASCII + * tab size: 8 (not used) + * indentation:4 + * + * created on: 2003feb1 + * created by: Ram Viswanadha + */ + +#ifndef __UIDNA_H__ +#define __UIDNA_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_IDNA + +#include "unicode/parseerr.h" + +/** + * \file + * \brief C API: Internationalized Domain Names in Applications Tranformation + * + * UIDNA API implements the IDNA protocol as defined in the IDNA RFC + * (http://www.ietf.org/rfc/rfc3490.txt). + * The RFC defines 2 operations: ToASCII and ToUnicode. Domain labels + * containing non-ASCII code points are required to be processed by + * ToASCII operation before passing it to resolver libraries. Domain names + * that are obtained from resolver libraries are required to be processed by + * ToUnicode operation before displaying the domain name to the user. + * IDNA requires that implementations process input strings with Nameprep + * (http://www.ietf.org/rfc/rfc3491.txt), + * which is a profile of Stringprep (http://www.ietf.org/rfc/rfc3454.txt), + * and then with Punycode (http://www.ietf.org/rfc/rfc3492.txt). + * Implementations of IDNA MUST fully implement Nameprep and Punycode; + * neither Nameprep nor Punycode are optional. + * The input and output of ToASCII and ToUnicode operations are Unicode + * and are designed to be chainable, i.e., applying ToASCII or ToUnicode operations + * multiple times to an input string will yield the same result as applying the operation + * once. + * ToUnicode(ToUnicode(ToUnicode...(ToUnicode(string)))) == ToUnicode(string) + * ToASCII(ToASCII(ToASCII...(ToASCII(string))) == ToASCII(string). + * + */ + +/** + * Option to prohibit processing of unassigned codepoints in the input and + * do not check if the input conforms to STD-3 ASCII rules. + * + * @see uidna_toASCII uidna_toUnicode + * @stable ICU 2.6 + */ +#define UIDNA_DEFAULT 0x0000 +/** + * Option to allow processing of unassigned codepoints in the input + * + * @see uidna_toASCII uidna_toUnicode + * @stable ICU 2.6 + */ +#define UIDNA_ALLOW_UNASSIGNED 0x0001 +/** + * Option to check if input conforms to STD-3 ASCII rules + * + * @see uidna_toASCII uidna_toUnicode + * @stable ICU 2.6 + */ +#define UIDNA_USE_STD3_RULES 0x0002 + +/** + * This function implements the ToASCII operation as defined in the IDNA RFC. + * This operation is done on single labels before sending it to something that expects + * ASCII names. A label is an individual part of a domain name. Labels are usually + * separated by dots; e.g." "www.example.com" is composed of 3 labels + * "www","example", and "com". + * + * + * @param src Input UChar array containing label in Unicode. + * @param srcLength Number of UChars in src, or -1 if NUL-terminated. + * @param dest Output UChar array with ASCII (ACE encoded) label. + * @param destCapacity Size of dest. + * @param options A bit set of options: + * + * - UIDNA_DEFAULT Use default options, i.e., do not process unassigned code points + * and do not use STD3 ASCII rules + * If unassigned code points are found the operation fails with + * U_UNASSIGNED_ERROR error code. + * + * - UIDNA_ALLOW_UNASSIGNED Unassigned values can be converted to ASCII for query operations + * If this option is set, the unassigned code points are in the input + * are treated as normal Unicode code points. + * + * - UIDNA_USE_STD3_RULES Use STD3 ASCII rules for host name syntax restrictions + * If this option is set and the input does not satisfy STD3 rules, + * the operation will fail with U_IDNA_STD3_ASCII_RULES_ERROR + * + * @param parseError Pointer to UParseError struct to receive information on position + * of error if an error is encountered. Can be NULL. + * @param status ICU in/out error code parameter. + * U_INVALID_CHAR_FOUND if src contains + * unmatched single surrogates. + * U_INDEX_OUTOFBOUNDS_ERROR if src contains + * too many code points. + * U_BUFFER_OVERFLOW_ERROR if destCapacity is not enough + * @return The length of the result string, if successful - or in case of a buffer overflow, + * in which case it will be greater than destCapacity. + * @stable ICU 2.6 + */ +U_STABLE int32_t U_EXPORT2 +uidna_toASCII(const UChar* src, int32_t srcLength, + UChar* dest, int32_t destCapacity, + int32_t options, + UParseError* parseError, + UErrorCode* status); + + +/** + * This function implements the ToUnicode operation as defined in the IDNA RFC. + * This operation is done on single labels before sending it to something that expects + * Unicode names. A label is an individual part of a domain name. Labels are usually + * separated by dots; for e.g." "www.example.com" is composed of 3 labels + * "www","example", and "com". + * + * @param src Input UChar array containing ASCII (ACE encoded) label. + * @param srcLength Number of UChars in src, or -1 if NUL-terminated. + * @param dest Output Converted UChar array containing Unicode equivalent of label. + * @param destCapacity Size of dest. + * @param options A bit set of options: + * + * - UIDNA_DEFAULT Use default options, i.e., do not process unassigned code points + * and do not use STD3 ASCII rules + * If unassigned code points are found the operation fails with + * U_UNASSIGNED_ERROR error code. + * + * - UIDNA_ALLOW_UNASSIGNED Unassigned values can be converted to ASCII for query operations + * If this option is set, the unassigned code points are in the input + * are treated as normal Unicode code points. Note: This option is + * required on toUnicode operation because the RFC mandates + * verification of decoded ACE input by applying toASCII and comparing + * its output with source + * + * + * + * - UIDNA_USE_STD3_RULES Use STD3 ASCII rules for host name syntax restrictions + * If this option is set and the input does not satisfy STD3 rules, + * the operation will fail with U_IDNA_STD3_ASCII_RULES_ERROR + * + * @param parseError Pointer to UParseError struct to receive information on position + * of error if an error is encountered. Can be NULL. + * @param status ICU in/out error code parameter. + * U_INVALID_CHAR_FOUND if src contains + * unmatched single surrogates. + * U_INDEX_OUTOFBOUNDS_ERROR if src contains + * too many code points. + * U_BUFFER_OVERFLOW_ERROR if destCapacity is not enough + * @return The length of the result string, if successful - or in case of a buffer overflow, + * in which case it will be greater than destCapacity. + * @stable ICU 2.6 + */ +U_STABLE int32_t U_EXPORT2 +uidna_toUnicode(const UChar* src, int32_t srcLength, + UChar* dest, int32_t destCapacity, + int32_t options, + UParseError* parseError, + UErrorCode* status); + + +/** + * Convenience function that implements the IDNToASCII operation as defined in the IDNA RFC. + * This operation is done on complete domain names, e.g: "www.example.com". + * It is important to note that this operation can fail. If it fails, then the input + * domain name cannot be used as an Internationalized Domain Name and the application + * should have methods defined to deal with the failure. + * + * Note: IDNA RFC specifies that a conformant application should divide a domain name + * into separate labels, decide whether to apply allowUnassigned and useSTD3ASCIIRules on each, + * and then convert. This function does not offer that level of granularity. The options once + * set will apply to all labels in the domain name + * + * @param src Input UChar array containing IDN in Unicode. + * @param srcLength Number of UChars in src, or -1 if NUL-terminated. + * @param dest Output UChar array with ASCII (ACE encoded) IDN. + * @param destCapacity Size of dest. + * @param options A bit set of options: + * + * - UIDNA_DEFAULT Use default options, i.e., do not process unassigned code points + * and do not use STD3 ASCII rules + * If unassigned code points are found the operation fails with + * U_UNASSIGNED_CODE_POINT_FOUND error code. + * + * - UIDNA_ALLOW_UNASSIGNED Unassigned values can be converted to ASCII for query operations + * If this option is set, the unassigned code points are in the input + * are treated as normal Unicode code points. + * + * - UIDNA_USE_STD3_RULES Use STD3 ASCII rules for host name syntax restrictions + * If this option is set and the input does not satisfy STD3 rules, + * the operation will fail with U_IDNA_STD3_ASCII_RULES_ERROR + * + * @param parseError Pointer to UParseError struct to receive information on position + * of error if an error is encountered. Can be NULL. + * @param status ICU in/out error code parameter. + * U_INVALID_CHAR_FOUND if src contains + * unmatched single surrogates. + * U_INDEX_OUTOFBOUNDS_ERROR if src contains + * too many code points. + * U_BUFFER_OVERFLOW_ERROR if destCapacity is not enough + * @return The length of the result string, if successful - or in case of a buffer overflow, + * in which case it will be greater than destCapacity. + * @stable ICU 2.6 + */ +U_STABLE int32_t U_EXPORT2 +uidna_IDNToASCII( const UChar* src, int32_t srcLength, + UChar* dest, int32_t destCapacity, + int32_t options, + UParseError* parseError, + UErrorCode* status); + +/** + * Convenience function that implements the IDNToUnicode operation as defined in the IDNA RFC. + * This operation is done on complete domain names, e.g: "www.example.com". + * + * Note: IDNA RFC specifies that a conformant application should divide a domain name + * into separate labels, decide whether to apply allowUnassigned and useSTD3ASCIIRules on each, + * and then convert. This function does not offer that level of granularity. The options once + * set will apply to all labels in the domain name + * + * @param src Input UChar array containing IDN in ASCII (ACE encoded) form. + * @param srcLength Number of UChars in src, or -1 if NUL-terminated. + * @param dest Output UChar array containing Unicode equivalent of source IDN. + * @param destCapacity Size of dest. + * @param options A bit set of options: + * + * - UIDNA_DEFAULT Use default options, i.e., do not process unassigned code points + * and do not use STD3 ASCII rules + * If unassigned code points are found the operation fails with + * U_UNASSIGNED_CODE_POINT_FOUND error code. + * + * - UIDNA_ALLOW_UNASSIGNED Unassigned values can be converted to ASCII for query operations + * If this option is set, the unassigned code points are in the input + * are treated as normal Unicode code points. + * + * - UIDNA_USE_STD3_RULES Use STD3 ASCII rules for host name syntax restrictions + * If this option is set and the input does not satisfy STD3 rules, + * the operation will fail with U_IDNA_STD3_ASCII_RULES_ERROR + * + * @param parseError Pointer to UParseError struct to receive information on position + * of error if an error is encountered. Can be NULL. + * @param status ICU in/out error code parameter. + * U_INVALID_CHAR_FOUND if src contains + * unmatched single surrogates. + * U_INDEX_OUTOFBOUNDS_ERROR if src contains + * too many code points. + * U_BUFFER_OVERFLOW_ERROR if destCapacity is not enough + * @return The length of the result string, if successful - or in case of a buffer overflow, + * in which case it will be greater than destCapacity. + * @stable ICU 2.6 + */ +U_STABLE int32_t U_EXPORT2 +uidna_IDNToUnicode( const UChar* src, int32_t srcLength, + UChar* dest, int32_t destCapacity, + int32_t options, + UParseError* parseError, + UErrorCode* status); + +/** + * Compare two IDN strings for equivalence. + * This function splits the domain names into labels and compares them. + * According to IDN RFC, whenever two labels are compared, they are + * considered equal if and only if their ASCII forms (obtained by + * applying toASCII) match using an case-insensitive ASCII comparison. + * Two domain names are considered a match if and only if all labels + * match regardless of whether label separators match. + * + * @param s1 First source string. + * @param length1 Length of first source string, or -1 if NUL-terminated. + * + * @param s2 Second source string. + * @param length2 Length of second source string, or -1 if NUL-terminated. + * @param options A bit set of options: + * + * - UIDNA_DEFAULT Use default options, i.e., do not process unassigned code points + * and do not use STD3 ASCII rules + * If unassigned code points are found the operation fails with + * U_UNASSIGNED_CODE_POINT_FOUND error code. + * + * - UIDNA_ALLOW_UNASSIGNED Unassigned values can be converted to ASCII for query operations + * If this option is set, the unassigned code points are in the input + * are treated as normal Unicode code points. + * + * - UIDNA_USE_STD3_RULES Use STD3 ASCII rules for host name syntax restrictions + * If this option is set and the input does not satisfy STD3 rules, + * the operation will fail with U_IDNA_STD3_ASCII_RULES_ERROR + * + * @param status ICU error code in/out parameter. + * Must fulfill U_SUCCESS before the function call. + * @return <0 or 0 or >0 as usual for string comparisons + * @stable ICU 2.6 + */ +U_STABLE int32_t U_EXPORT2 +uidna_compare( const UChar *s1, int32_t length1, + const UChar *s2, int32_t length2, + int32_t options, + UErrorCode* status); + +#endif /* #if !UCONFIG_NO_IDNA */ + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/uintrnal.h b/Build/source/libs/icu/icu-xetex/common/unicode/uintrnal.h new file mode 100644 index 00000000000..dedae51050c --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/uintrnal.h @@ -0,0 +1,180 @@ +/* +******************************************************************************* +* Copyright (C) 2004-2007, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* +* file name: +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* Created by: genheaders.pl, a perl script written by Ram Viswanadha +* +* Contains data for commenting out APIs. +* Gets included by umachine.h +* +* THIS FILE IS MACHINE-GENERATED, DON'T PLAY WITH IT IF YOU DON'T KNOW WHAT +* YOU ARE DOING, OTHERWISE VERY BAD THINGS WILL HAPPEN! +*/ + +#ifndef UINTRNAL_H +#define UINTRNAL_H + +#ifdef U_HIDE_INTERNAL_API + +# if U_DISABLE_RENAMING +# define RegexPatternDump RegexPatternDump_INTERNAL_API_DO_NOT_USE +# define pl_addFontRun pl_addFontRun_INTERNAL_API_DO_NOT_USE +# define pl_addLocaleRun pl_addLocaleRun_INTERNAL_API_DO_NOT_USE +# define pl_addValueRun pl_addValueRun_INTERNAL_API_DO_NOT_USE +# define pl_close pl_close_INTERNAL_API_DO_NOT_USE +# define pl_closeFontRuns pl_closeFontRuns_INTERNAL_API_DO_NOT_USE +# define pl_closeLine pl_closeLine_INTERNAL_API_DO_NOT_USE +# define pl_closeLocaleRuns pl_closeLocaleRuns_INTERNAL_API_DO_NOT_USE +# define pl_closeValueRuns pl_closeValueRuns_INTERNAL_API_DO_NOT_USE +# define pl_countLineRuns pl_countLineRuns_INTERNAL_API_DO_NOT_USE +# define pl_create pl_create_INTERNAL_API_DO_NOT_USE +# define pl_getAscent pl_getAscent_INTERNAL_API_DO_NOT_USE +# define pl_getDescent pl_getDescent_INTERNAL_API_DO_NOT_USE +# define pl_getFontRunCount pl_getFontRunCount_INTERNAL_API_DO_NOT_USE +# define pl_getFontRunFont pl_getFontRunFont_INTERNAL_API_DO_NOT_USE +# define pl_getFontRunLastLimit pl_getFontRunLastLimit_INTERNAL_API_DO_NOT_USE +# define pl_getFontRunLimit pl_getFontRunLimit_INTERNAL_API_DO_NOT_USE +# define pl_getLeading pl_getLeading_INTERNAL_API_DO_NOT_USE +# define pl_getLineAscent pl_getLineAscent_INTERNAL_API_DO_NOT_USE +# define pl_getLineDescent pl_getLineDescent_INTERNAL_API_DO_NOT_USE +# define pl_getLineLeading pl_getLineLeading_INTERNAL_API_DO_NOT_USE +# define pl_getLineVisualRun pl_getLineVisualRun_INTERNAL_API_DO_NOT_USE +# define pl_getLineWidth pl_getLineWidth_INTERNAL_API_DO_NOT_USE +# define pl_getLocaleRunCount pl_getLocaleRunCount_INTERNAL_API_DO_NOT_USE +# define pl_getLocaleRunLastLimit pl_getLocaleRunLastLimit_INTERNAL_API_DO_NOT_USE +# define pl_getLocaleRunLimit pl_getLocaleRunLimit_INTERNAL_API_DO_NOT_USE +# define pl_getLocaleRunLocale pl_getLocaleRunLocale_INTERNAL_API_DO_NOT_USE +# define pl_getParagraphLevel pl_getParagraphLevel_INTERNAL_API_DO_NOT_USE +# define pl_getTextDirection pl_getTextDirection_INTERNAL_API_DO_NOT_USE +# define pl_getValueRunCount pl_getValueRunCount_INTERNAL_API_DO_NOT_USE +# define pl_getValueRunLastLimit pl_getValueRunLastLimit_INTERNAL_API_DO_NOT_USE +# define pl_getValueRunLimit pl_getValueRunLimit_INTERNAL_API_DO_NOT_USE +# define pl_getValueRunValue pl_getValueRunValue_INTERNAL_API_DO_NOT_USE +# define pl_getVisualRunAscent pl_getVisualRunAscent_INTERNAL_API_DO_NOT_USE +# define pl_getVisualRunDescent pl_getVisualRunDescent_INTERNAL_API_DO_NOT_USE +# define pl_getVisualRunDirection pl_getVisualRunDirection_INTERNAL_API_DO_NOT_USE +# define pl_getVisualRunFont pl_getVisualRunFont_INTERNAL_API_DO_NOT_USE +# define pl_getVisualRunGlyphCount pl_getVisualRunGlyphCount_INTERNAL_API_DO_NOT_USE +# define pl_getVisualRunGlyphToCharMap pl_getVisualRunGlyphToCharMap_INTERNAL_API_DO_NOT_USE +# define pl_getVisualRunGlyphs pl_getVisualRunGlyphs_INTERNAL_API_DO_NOT_USE +# define pl_getVisualRunLeading pl_getVisualRunLeading_INTERNAL_API_DO_NOT_USE +# define pl_getVisualRunPositions pl_getVisualRunPositions_INTERNAL_API_DO_NOT_USE +# define pl_isComplex pl_isComplex_INTERNAL_API_DO_NOT_USE +# define pl_line pl_line_INTERNAL_API_DO_NOT_USE +# define pl_nextLine pl_nextLine_INTERNAL_API_DO_NOT_USE +# define pl_openEmptyFontRuns pl_openEmptyFontRuns_INTERNAL_API_DO_NOT_USE +# define pl_openEmptyLocaleRuns pl_openEmptyLocaleRuns_INTERNAL_API_DO_NOT_USE +# define pl_openEmptyValueRuns pl_openEmptyValueRuns_INTERNAL_API_DO_NOT_USE +# define pl_openFontRuns pl_openFontRuns_INTERNAL_API_DO_NOT_USE +# define pl_openLocaleRuns pl_openLocaleRuns_INTERNAL_API_DO_NOT_USE +# define pl_openValueRuns pl_openValueRuns_INTERNAL_API_DO_NOT_USE +# define pl_paragraph pl_paragraph_INTERNAL_API_DO_NOT_USE +# define pl_reflow pl_reflow_INTERNAL_API_DO_NOT_USE +# define pl_resetFontRuns pl_resetFontRuns_INTERNAL_API_DO_NOT_USE +# define pl_resetLocaleRuns pl_resetLocaleRuns_INTERNAL_API_DO_NOT_USE +# define pl_resetValueRuns pl_resetValueRuns_INTERNAL_API_DO_NOT_USE +# define pl_visualRun pl_visualRun_INTERNAL_API_DO_NOT_USE +# define ucol_collatorToIdentifier ucol_collatorToIdentifier_INTERNAL_API_DO_NOT_USE +# define ucol_equals ucol_equals_INTERNAL_API_DO_NOT_USE +# define ucol_forgetUCA ucol_forgetUCA_INTERNAL_API_DO_NOT_USE +# define ucol_getAttributeOrDefault ucol_getAttributeOrDefault_INTERNAL_API_DO_NOT_USE +# define ucol_getUnsafeSet ucol_getUnsafeSet_INTERNAL_API_DO_NOT_USE +# define ucol_identifierToShortString ucol_identifierToShortString_INTERNAL_API_DO_NOT_USE +# define ucol_openFromIdentifier ucol_openFromIdentifier_INTERNAL_API_DO_NOT_USE +# define ucol_prepareShortStringOpen ucol_prepareShortStringOpen_INTERNAL_API_DO_NOT_USE +# define ucol_shortStringToIdentifier ucol_shortStringToIdentifier_INTERNAL_API_DO_NOT_USE +# define uprv_getDefaultCodepage uprv_getDefaultCodepage_INTERNAL_API_DO_NOT_USE +# define uprv_getDefaultLocaleID uprv_getDefaultLocaleID_INTERNAL_API_DO_NOT_USE +# define ures_openFillIn ures_openFillIn_INTERNAL_API_DO_NOT_USE +# define utf8_appendCharSafeBody utf8_appendCharSafeBody_INTERNAL_API_DO_NOT_USE +# define utf8_back1SafeBody utf8_back1SafeBody_INTERNAL_API_DO_NOT_USE +# define utf8_countTrailBytes utf8_countTrailBytes_INTERNAL_API_DO_NOT_USE +# define utf8_nextCharSafeBody utf8_nextCharSafeBody_INTERNAL_API_DO_NOT_USE +# define utf8_prevCharSafeBody utf8_prevCharSafeBody_INTERNAL_API_DO_NOT_USE +# else +# define RegexPatternDump_3_8 RegexPatternDump_INTERNAL_API_DO_NOT_USE +# define pl_addFontRun_3_8 pl_addFontRun_INTERNAL_API_DO_NOT_USE +# define pl_addLocaleRun_3_8 pl_addLocaleRun_INTERNAL_API_DO_NOT_USE +# define pl_addValueRun_3_8 pl_addValueRun_INTERNAL_API_DO_NOT_USE +# define pl_closeFontRuns_3_8 pl_closeFontRuns_INTERNAL_API_DO_NOT_USE +# define pl_closeLine_3_8 pl_closeLine_INTERNAL_API_DO_NOT_USE +# define pl_closeLocaleRuns_3_8 pl_closeLocaleRuns_INTERNAL_API_DO_NOT_USE +# define pl_closeValueRuns_3_8 pl_closeValueRuns_INTERNAL_API_DO_NOT_USE +# define pl_close_3_8 pl_close_INTERNAL_API_DO_NOT_USE +# define pl_countLineRuns_3_8 pl_countLineRuns_INTERNAL_API_DO_NOT_USE +# define pl_create_3_8 pl_create_INTERNAL_API_DO_NOT_USE +# define pl_getAscent_3_8 pl_getAscent_INTERNAL_API_DO_NOT_USE +# define pl_getDescent_3_8 pl_getDescent_INTERNAL_API_DO_NOT_USE +# define pl_getFontRunCount_3_8 pl_getFontRunCount_INTERNAL_API_DO_NOT_USE +# define pl_getFontRunFont_3_8 pl_getFontRunFont_INTERNAL_API_DO_NOT_USE +# define pl_getFontRunLastLimit_3_8 pl_getFontRunLastLimit_INTERNAL_API_DO_NOT_USE +# define pl_getFontRunLimit_3_8 pl_getFontRunLimit_INTERNAL_API_DO_NOT_USE +# define pl_getLeading_3_8 pl_getLeading_INTERNAL_API_DO_NOT_USE +# define pl_getLineAscent_3_8 pl_getLineAscent_INTERNAL_API_DO_NOT_USE +# define pl_getLineDescent_3_8 pl_getLineDescent_INTERNAL_API_DO_NOT_USE +# define pl_getLineLeading_3_8 pl_getLineLeading_INTERNAL_API_DO_NOT_USE +# define pl_getLineVisualRun_3_8 pl_getLineVisualRun_INTERNAL_API_DO_NOT_USE +# define pl_getLineWidth_3_8 pl_getLineWidth_INTERNAL_API_DO_NOT_USE +# define pl_getLocaleRunCount_3_8 pl_getLocaleRunCount_INTERNAL_API_DO_NOT_USE +# define pl_getLocaleRunLastLimit_3_8 pl_getLocaleRunLastLimit_INTERNAL_API_DO_NOT_USE +# define pl_getLocaleRunLimit_3_8 pl_getLocaleRunLimit_INTERNAL_API_DO_NOT_USE +# define pl_getLocaleRunLocale_3_8 pl_getLocaleRunLocale_INTERNAL_API_DO_NOT_USE +# define pl_getParagraphLevel_3_8 pl_getParagraphLevel_INTERNAL_API_DO_NOT_USE +# define pl_getTextDirection_3_8 pl_getTextDirection_INTERNAL_API_DO_NOT_USE +# define pl_getValueRunCount_3_8 pl_getValueRunCount_INTERNAL_API_DO_NOT_USE +# define pl_getValueRunLastLimit_3_8 pl_getValueRunLastLimit_INTERNAL_API_DO_NOT_USE +# define pl_getValueRunLimit_3_8 pl_getValueRunLimit_INTERNAL_API_DO_NOT_USE +# define pl_getValueRunValue_3_8 pl_getValueRunValue_INTERNAL_API_DO_NOT_USE +# define pl_getVisualRunAscent_3_8 pl_getVisualRunAscent_INTERNAL_API_DO_NOT_USE +# define pl_getVisualRunDescent_3_8 pl_getVisualRunDescent_INTERNAL_API_DO_NOT_USE +# define pl_getVisualRunDirection_3_8 pl_getVisualRunDirection_INTERNAL_API_DO_NOT_USE +# define pl_getVisualRunFont_3_8 pl_getVisualRunFont_INTERNAL_API_DO_NOT_USE +# define pl_getVisualRunGlyphCount_3_8 pl_getVisualRunGlyphCount_INTERNAL_API_DO_NOT_USE +# define pl_getVisualRunGlyphToCharMap_3_8 pl_getVisualRunGlyphToCharMap_INTERNAL_API_DO_NOT_USE +# define pl_getVisualRunGlyphs_3_8 pl_getVisualRunGlyphs_INTERNAL_API_DO_NOT_USE +# define pl_getVisualRunLeading_3_8 pl_getVisualRunLeading_INTERNAL_API_DO_NOT_USE +# define pl_getVisualRunPositions_3_8 pl_getVisualRunPositions_INTERNAL_API_DO_NOT_USE +# define pl_isComplex_3_8 pl_isComplex_INTERNAL_API_DO_NOT_USE +# define pl_line_3_8 pl_line_INTERNAL_API_DO_NOT_USE +# define pl_nextLine_3_8 pl_nextLine_INTERNAL_API_DO_NOT_USE +# define pl_openEmptyFontRuns_3_8 pl_openEmptyFontRuns_INTERNAL_API_DO_NOT_USE +# define pl_openEmptyLocaleRuns_3_8 pl_openEmptyLocaleRuns_INTERNAL_API_DO_NOT_USE +# define pl_openEmptyValueRuns_3_8 pl_openEmptyValueRuns_INTERNAL_API_DO_NOT_USE +# define pl_openFontRuns_3_8 pl_openFontRuns_INTERNAL_API_DO_NOT_USE +# define pl_openLocaleRuns_3_8 pl_openLocaleRuns_INTERNAL_API_DO_NOT_USE +# define pl_openValueRuns_3_8 pl_openValueRuns_INTERNAL_API_DO_NOT_USE +# define pl_paragraph_3_8 pl_paragraph_INTERNAL_API_DO_NOT_USE +# define pl_reflow_3_8 pl_reflow_INTERNAL_API_DO_NOT_USE +# define pl_resetFontRuns_3_8 pl_resetFontRuns_INTERNAL_API_DO_NOT_USE +# define pl_resetLocaleRuns_3_8 pl_resetLocaleRuns_INTERNAL_API_DO_NOT_USE +# define pl_resetValueRuns_3_8 pl_resetValueRuns_INTERNAL_API_DO_NOT_USE +# define pl_visualRun_3_8 pl_visualRun_INTERNAL_API_DO_NOT_USE +# define ucol_collatorToIdentifier_3_8 ucol_collatorToIdentifier_INTERNAL_API_DO_NOT_USE +# define ucol_equals_3_8 ucol_equals_INTERNAL_API_DO_NOT_USE +# define ucol_forgetUCA_3_8 ucol_forgetUCA_INTERNAL_API_DO_NOT_USE +# define ucol_getAttributeOrDefault_3_8 ucol_getAttributeOrDefault_INTERNAL_API_DO_NOT_USE +# define ucol_getUnsafeSet_3_8 ucol_getUnsafeSet_INTERNAL_API_DO_NOT_USE +# define ucol_identifierToShortString_3_8 ucol_identifierToShortString_INTERNAL_API_DO_NOT_USE +# define ucol_openFromIdentifier_3_8 ucol_openFromIdentifier_INTERNAL_API_DO_NOT_USE +# define ucol_prepareShortStringOpen_3_8 ucol_prepareShortStringOpen_INTERNAL_API_DO_NOT_USE +# define ucol_shortStringToIdentifier_3_8 ucol_shortStringToIdentifier_INTERNAL_API_DO_NOT_USE +# define uprv_getDefaultCodepage_3_8 uprv_getDefaultCodepage_INTERNAL_API_DO_NOT_USE +# define uprv_getDefaultLocaleID_3_8 uprv_getDefaultLocaleID_INTERNAL_API_DO_NOT_USE +# define ures_openFillIn_3_8 ures_openFillIn_INTERNAL_API_DO_NOT_USE +# define utf8_appendCharSafeBody_3_8 utf8_appendCharSafeBody_INTERNAL_API_DO_NOT_USE +# define utf8_back1SafeBody_3_8 utf8_back1SafeBody_INTERNAL_API_DO_NOT_USE +# define utf8_countTrailBytes_3_8 utf8_countTrailBytes_INTERNAL_API_DO_NOT_USE +# define utf8_nextCharSafeBody_3_8 utf8_nextCharSafeBody_INTERNAL_API_DO_NOT_USE +# define utf8_prevCharSafeBody_3_8 utf8_prevCharSafeBody_INTERNAL_API_DO_NOT_USE +# endif /* U_DISABLE_RENAMING */ + +#endif /* U_HIDE_INTERNAL_API */ +#endif /* UINTRNAL_H */ + diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/uiter.h b/Build/source/libs/icu/icu-xetex/common/unicode/uiter.h new file mode 100644 index 00000000000..9409f01ee9a --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/uiter.h @@ -0,0 +1,707 @@ +/* +******************************************************************************* +* +* Copyright (C) 2002-2006, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: uiter.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2002jan18 +* created by: Markus W. Scherer +*/ + +#ifndef __UITER_H__ +#define __UITER_H__ + +/** + * \file + * \brief C API: Unicode Character Iteration + * + * @see UCharIterator + */ + +#include "unicode/utypes.h" + +#ifdef XP_CPLUSPLUS + U_NAMESPACE_BEGIN + + class CharacterIterator; + class Replaceable; + + U_NAMESPACE_END +#endif + +U_CDECL_BEGIN + +struct UCharIterator; +typedef struct UCharIterator UCharIterator; /**< C typedef for struct UCharIterator. @stable ICU 2.1 */ + +/** + * Origin constants for UCharIterator.getIndex() and UCharIterator.move(). + * @see UCharIteratorMove + * @see UCharIterator + * @stable ICU 2.1 + */ +typedef enum UCharIteratorOrigin { + UITER_START, UITER_CURRENT, UITER_LIMIT, UITER_ZERO, UITER_LENGTH +} UCharIteratorOrigin; + +/** Constants for UCharIterator. @stable ICU 2.6 */ +enum { + /** + * Constant value that may be returned by UCharIteratorMove + * indicating that the final UTF-16 index is not known, but that the move succeeded. + * This can occur when moving relative to limit or length, or + * when moving relative to the current index after a setState() + * when the current UTF-16 index is not known. + * + * It would be very inefficient to have to count from the beginning of the text + * just to get the current/limit/length index after moving relative to it. + * The actual index can be determined with getIndex(UITER_CURRENT) + * which will count the UChars if necessary. + * + * @stable ICU 2.6 + */ + UITER_UNKNOWN_INDEX=-2 +}; + + +/** + * Constant for UCharIterator getState() indicating an error or + * an unknown state. + * Returned by uiter_getState()/UCharIteratorGetState + * when an error occurs. + * Also, some UCharIterator implementations may not be able to return + * a valid state for each position. This will be clearly documented + * for each such iterator (none of the public ones here). + * + * @stable ICU 2.6 + */ +#define UITER_NO_STATE ((uint32_t)0xffffffff) + +/** + * Function type declaration for UCharIterator.getIndex(). + * + * Gets the current position, or the start or limit of the + * iteration range. + * + * This function may perform slowly for UITER_CURRENT after setState() was called, + * or for UITER_LENGTH, because an iterator implementation may have to count + * UChars if the underlying storage is not UTF-16. + * + * @param iter the UCharIterator structure ("this pointer") + * @param origin get the 0, start, limit, length, or current index + * @return the requested index, or U_SENTINEL in an error condition + * + * @see UCharIteratorOrigin + * @see UCharIterator + * @stable ICU 2.1 + */ +typedef int32_t U_CALLCONV +UCharIteratorGetIndex(UCharIterator *iter, UCharIteratorOrigin origin); + +/** + * Function type declaration for UCharIterator.move(). + * + * Use iter->move(iter, index, UITER_ZERO) like CharacterIterator::setIndex(index). + * + * Moves the current position relative to the start or limit of the + * iteration range, or relative to the current position itself. + * The movement is expressed in numbers of code units forward + * or backward by specifying a positive or negative delta. + * Out of bounds movement will be pinned to the start or limit. + * + * This function may perform slowly for moving relative to UITER_LENGTH + * because an iterator implementation may have to count the rest of the + * UChars if the native storage is not UTF-16. + * + * When moving relative to the limit or length, or + * relative to the current position after setState() was called, + * move() may return UITER_UNKNOWN_INDEX (-2) to avoid an inefficient + * determination of the actual UTF-16 index. + * The actual index can be determined with getIndex(UITER_CURRENT) + * which will count the UChars if necessary. + * See UITER_UNKNOWN_INDEX for details. + * + * @param iter the UCharIterator structure ("this pointer") + * @param delta can be positive, zero, or negative + * @param origin move relative to the 0, start, limit, length, or current index + * @return the new index, or U_SENTINEL on an error condition, + * or UITER_UNKNOWN_INDEX when the index is not known. + * + * @see UCharIteratorOrigin + * @see UCharIterator + * @see UITER_UNKNOWN_INDEX + * @stable ICU 2.1 + */ +typedef int32_t U_CALLCONV +UCharIteratorMove(UCharIterator *iter, int32_t delta, UCharIteratorOrigin origin); + +/** + * Function type declaration for UCharIterator.hasNext(). + * + * Check if current() and next() can still + * return another code unit. + * + * @param iter the UCharIterator structure ("this pointer") + * @return boolean value for whether current() and next() can still return another code unit + * + * @see UCharIterator + * @stable ICU 2.1 + */ +typedef UBool U_CALLCONV +UCharIteratorHasNext(UCharIterator *iter); + +/** + * Function type declaration for UCharIterator.hasPrevious(). + * + * Check if previous() can still return another code unit. + * + * @param iter the UCharIterator structure ("this pointer") + * @return boolean value for whether previous() can still return another code unit + * + * @see UCharIterator + * @stable ICU 2.1 + */ +typedef UBool U_CALLCONV +UCharIteratorHasPrevious(UCharIterator *iter); + +/** + * Function type declaration for UCharIterator.current(). + * + * Return the code unit at the current position, + * or U_SENTINEL if there is none (index is at the limit). + * + * @param iter the UCharIterator structure ("this pointer") + * @return the current code unit + * + * @see UCharIterator + * @stable ICU 2.1 + */ +typedef UChar32 U_CALLCONV +UCharIteratorCurrent(UCharIterator *iter); + +/** + * Function type declaration for UCharIterator.next(). + * + * Return the code unit at the current index and increment + * the index (post-increment, like s[i++]), + * or return U_SENTINEL if there is none (index is at the limit). + * + * @param iter the UCharIterator structure ("this pointer") + * @return the current code unit (and post-increment the current index) + * + * @see UCharIterator + * @stable ICU 2.1 + */ +typedef UChar32 U_CALLCONV +UCharIteratorNext(UCharIterator *iter); + +/** + * Function type declaration for UCharIterator.previous(). + * + * Decrement the index and return the code unit from there + * (pre-decrement, like s[--i]), + * or return U_SENTINEL if there is none (index is at the start). + * + * @param iter the UCharIterator structure ("this pointer") + * @return the previous code unit (after pre-decrementing the current index) + * + * @see UCharIterator + * @stable ICU 2.1 + */ +typedef UChar32 U_CALLCONV +UCharIteratorPrevious(UCharIterator *iter); + +/** + * Function type declaration for UCharIterator.reservedFn(). + * Reserved for future use. + * + * @param iter the UCharIterator structure ("this pointer") + * @param something some integer argument + * @return some integer + * + * @see UCharIterator + * @stable ICU 2.1 + */ +typedef int32_t U_CALLCONV +UCharIteratorReserved(UCharIterator *iter, int32_t something); + +/** + * Function type declaration for UCharIterator.getState(). + * + * Get the "state" of the iterator in the form of a single 32-bit word. + * It is recommended that the state value be calculated to be as small as + * is feasible. For strings with limited lengths, fewer than 32 bits may + * be sufficient. + * + * This is used together with setState()/UCharIteratorSetState + * to save and restore the iterator position more efficiently than with + * getIndex()/move(). + * + * The iterator state is defined as a uint32_t value because it is designed + * for use in ucol_nextSortKeyPart() which provides 32 bits to store the state + * of the character iterator. + * + * With some UCharIterator implementations (e.g., UTF-8), + * getting and setting the UTF-16 index with existing functions + * (getIndex(UITER_CURRENT) followed by move(pos, UITER_ZERO)) is possible but + * relatively slow because the iterator has to "walk" from a known index + * to the requested one. + * This takes more time the farther it needs to go. + * + * An opaque state value allows an iterator implementation to provide + * an internal index (UTF-8: the source byte array index) for + * fast, constant-time restoration. + * + * After calling setState(), a getIndex(UITER_CURRENT) may be slow because + * the UTF-16 index may not be restored as well, but the iterator can deliver + * the correct text contents and move relative to the current position + * without performance degradation. + * + * Some UCharIterator implementations may not be able to return + * a valid state for each position, in which case they return UITER_NO_STATE instead. + * This will be clearly documented for each such iterator (none of the public ones here). + * + * @param iter the UCharIterator structure ("this pointer") + * @return the state word + * + * @see UCharIterator + * @see UCharIteratorSetState + * @see UITER_NO_STATE + * @stable ICU 2.6 + */ +typedef uint32_t U_CALLCONV +UCharIteratorGetState(const UCharIterator *iter); + +/** + * Function type declaration for UCharIterator.setState(). + * + * Restore the "state" of the iterator using a state word from a getState() call. + * The iterator object need not be the same one as for which getState() was called, + * but it must be of the same type (set up using the same uiter_setXYZ function) + * and it must iterate over the same string + * (binary identical regardless of memory address). + * For more about the state word see UCharIteratorGetState. + * + * After calling setState(), a getIndex(UITER_CURRENT) may be slow because + * the UTF-16 index may not be restored as well, but the iterator can deliver + * the correct text contents and move relative to the current position + * without performance degradation. + * + * @param iter the UCharIterator structure ("this pointer") + * @param state the state word from a getState() call + * on a same-type, same-string iterator + * @param pErrorCode Must be a valid pointer to an error code value, + * which must not indicate a failure before the function call. + * + * @see UCharIterator + * @see UCharIteratorGetState + * @stable ICU 2.6 + */ +typedef void U_CALLCONV +UCharIteratorSetState(UCharIterator *iter, uint32_t state, UErrorCode *pErrorCode); + + +/** + * C API for code unit iteration. + * This can be used as a C wrapper around + * CharacterIterator, Replaceable, or implemented using simple strings, etc. + * + * There are two roles for using UCharIterator: + * + * A "provider" sets the necessary function pointers and controls the "protected" + * fields of the UCharIterator structure. A "provider" passes a UCharIterator + * into C APIs that need a UCharIterator as an abstract, flexible string interface. + * + * Implementations of such C APIs are "callers" of UCharIterator functions; + * they only use the "public" function pointers and never access the "protected" + * fields directly. + * + * The current() and next() functions only check the current index against the + * limit, and previous() only checks the current index against the start, + * to see if the iterator already reached the end of the iteration range. + * + * The assumption - in all iterators - is that the index is moved via the API, + * which means it won't go out of bounds, or the index is modified by + * user code that knows enough about the iterator implementation to set valid + * index values. + * + * UCharIterator functions return code unit values 0..0xffff, + * or U_SENTINEL if the iteration bounds are reached. + * + * @stable ICU 2.1 + */ +struct UCharIterator { + /** + * (protected) Pointer to string or wrapped object or similar. + * Not used by caller. + * @stable ICU 2.1 + */ + const void *context; + + /** + * (protected) Length of string or similar. + * Not used by caller. + * @stable ICU 2.1 + */ + int32_t length; + + /** + * (protected) Start index or similar. + * Not used by caller. + * @stable ICU 2.1 + */ + int32_t start; + + /** + * (protected) Current index or similar. + * Not used by caller. + * @stable ICU 2.1 + */ + int32_t index; + + /** + * (protected) Limit index or similar. + * Not used by caller. + * @stable ICU 2.1 + */ + int32_t limit; + + /** + * (protected) Used by UTF-8 iterators and possibly others. + * @stable ICU 2.1 + */ + int32_t reservedField; + + /** + * (public) Returns the current position or the + * start or limit index of the iteration range. + * + * @see UCharIteratorGetIndex + * @stable ICU 2.1 + */ + UCharIteratorGetIndex *getIndex; + + /** + * (public) Moves the current position relative to the start or limit of the + * iteration range, or relative to the current position itself. + * The movement is expressed in numbers of code units forward + * or backward by specifying a positive or negative delta. + * + * @see UCharIteratorMove + * @stable ICU 2.1 + */ + UCharIteratorMove *move; + + /** + * (public) Check if current() and next() can still + * return another code unit. + * + * @see UCharIteratorHasNext + * @stable ICU 2.1 + */ + UCharIteratorHasNext *hasNext; + + /** + * (public) Check if previous() can still return another code unit. + * + * @see UCharIteratorHasPrevious + * @stable ICU 2.1 + */ + UCharIteratorHasPrevious *hasPrevious; + + /** + * (public) Return the code unit at the current position, + * or U_SENTINEL if there is none (index is at the limit). + * + * @see UCharIteratorCurrent + * @stable ICU 2.1 + */ + UCharIteratorCurrent *current; + + /** + * (public) Return the code unit at the current index and increment + * the index (post-increment, like s[i++]), + * or return U_SENTINEL if there is none (index is at the limit). + * + * @see UCharIteratorNext + * @stable ICU 2.1 + */ + UCharIteratorNext *next; + + /** + * (public) Decrement the index and return the code unit from there + * (pre-decrement, like s[--i]), + * or return U_SENTINEL if there is none (index is at the start). + * + * @see UCharIteratorPrevious + * @stable ICU 2.1 + */ + UCharIteratorPrevious *previous; + + /** + * (public) Reserved for future use. Currently NULL. + * + * @see UCharIteratorReserved + * @stable ICU 2.1 + */ + UCharIteratorReserved *reservedFn; + + /** + * (public) Return the state of the iterator, to be restored later with setState(). + * This function pointer is NULL if the iterator does not implement it. + * + * @see UCharIteratorGet + * @stable ICU 2.6 + */ + UCharIteratorGetState *getState; + + /** + * (public) Restore the iterator state from the state word from a call + * to getState(). + * This function pointer is NULL if the iterator does not implement it. + * + * @see UCharIteratorSet + * @stable ICU 2.6 + */ + UCharIteratorSetState *setState; +}; + +/** + * Helper function for UCharIterator to get the code point + * at the current index. + * + * Return the code point that includes the code unit at the current position, + * or U_SENTINEL if there is none (index is at the limit). + * If the current code unit is a lead or trail surrogate, + * then the following or preceding surrogate is used to form + * the code point value. + * + * @param iter the UCharIterator structure ("this pointer") + * @return the current code point + * + * @see UCharIterator + * @see U16_GET + * @see UnicodeString::char32At() + * @stable ICU 2.1 + */ +U_STABLE UChar32 U_EXPORT2 +uiter_current32(UCharIterator *iter); + +/** + * Helper function for UCharIterator to get the next code point. + * + * Return the code point at the current index and increment + * the index (post-increment, like s[i++]), + * or return U_SENTINEL if there is none (index is at the limit). + * + * @param iter the UCharIterator structure ("this pointer") + * @return the current code point (and post-increment the current index) + * + * @see UCharIterator + * @see U16_NEXT + * @stable ICU 2.1 + */ +U_STABLE UChar32 U_EXPORT2 +uiter_next32(UCharIterator *iter); + +/** + * Helper function for UCharIterator to get the previous code point. + * + * Decrement the index and return the code point from there + * (pre-decrement, like s[--i]), + * or return U_SENTINEL if there is none (index is at the start). + * + * @param iter the UCharIterator structure ("this pointer") + * @return the previous code point (after pre-decrementing the current index) + * + * @see UCharIterator + * @see U16_PREV + * @stable ICU 2.1 + */ +U_STABLE UChar32 U_EXPORT2 +uiter_previous32(UCharIterator *iter); + +/** + * Get the "state" of the iterator in the form of a single 32-bit word. + * This is a convenience function that calls iter->getState(iter) + * if iter->getState is not NULL; + * if it is NULL or any other error occurs, then UITER_NO_STATE is returned. + * + * Some UCharIterator implementations may not be able to return + * a valid state for each position, in which case they return UITER_NO_STATE instead. + * This will be clearly documented for each such iterator (none of the public ones here). + * + * @param iter the UCharIterator structure ("this pointer") + * @return the state word + * + * @see UCharIterator + * @see UCharIteratorGetState + * @see UITER_NO_STATE + * @stable ICU 2.6 + */ +U_STABLE uint32_t U_EXPORT2 +uiter_getState(const UCharIterator *iter); + +/** + * Restore the "state" of the iterator using a state word from a getState() call. + * This is a convenience function that calls iter->setState(iter, state, pErrorCode) + * if iter->setState is not NULL; if it is NULL, then U_UNSUPPORTED_ERROR is set. + * + * @param iter the UCharIterator structure ("this pointer") + * @param state the state word from a getState() call + * on a same-type, same-string iterator + * @param pErrorCode Must be a valid pointer to an error code value, + * which must not indicate a failure before the function call. + * + * @see UCharIterator + * @see UCharIteratorSetState + * @stable ICU 2.6 + */ +U_STABLE void U_EXPORT2 +uiter_setState(UCharIterator *iter, uint32_t state, UErrorCode *pErrorCode); + +/** + * Set up a UCharIterator to iterate over a string. + * + * Sets the UCharIterator function pointers for iteration over the string s + * with iteration boundaries start=index=0 and length=limit=string length. + * The "provider" may set the start, index, and limit values at any time + * within the range 0..length. + * The length field will be ignored. + * + * The string pointer s is set into UCharIterator.context without copying + * or reallocating the string contents. + * + * getState() simply returns the current index. + * move() will always return the final index. + * + * @param iter UCharIterator structure to be set for iteration + * @param s String to iterate over + * @param length Length of s, or -1 if NUL-terminated + * + * @see UCharIterator + * @stable ICU 2.1 + */ +U_STABLE void U_EXPORT2 +uiter_setString(UCharIterator *iter, const UChar *s, int32_t length); + +/** + * Set up a UCharIterator to iterate over a UTF-16BE string + * (byte vector with a big-endian pair of bytes per UChar). + * + * Everything works just like with a normal UChar iterator (uiter_setString), + * except that UChars are assembled from byte pairs, + * and that the length argument here indicates an even number of bytes. + * + * getState() simply returns the current index. + * move() will always return the final index. + * + * @param iter UCharIterator structure to be set for iteration + * @param s UTF-16BE string to iterate over + * @param length Length of s as an even number of bytes, or -1 if NUL-terminated + * (NUL means pair of 0 bytes at even index from s) + * + * @see UCharIterator + * @see uiter_setString + * @stable ICU 2.6 + */ +U_STABLE void U_EXPORT2 +uiter_setUTF16BE(UCharIterator *iter, const char *s, int32_t length); + +/** + * Set up a UCharIterator to iterate over a UTF-8 string. + * + * Sets the UCharIterator function pointers for iteration over the UTF-8 string s + * with UTF-8 iteration boundaries 0 and length. + * The implementation counts the UTF-16 index on the fly and + * lazily evaluates the UTF-16 length of the text. + * + * The start field is used as the UTF-8 offset, the limit field as the UTF-8 length. + * When the reservedField is not 0, then it contains a supplementary code point + * and the UTF-16 index is between the two corresponding surrogates. + * At that point, the UTF-8 index is behind that code point. + * + * The UTF-8 string pointer s is set into UCharIterator.context without copying + * or reallocating the string contents. + * + * getState() returns a state value consisting of + * - the current UTF-8 source byte index (bits 31..1) + * - a flag (bit 0) that indicates whether the UChar position is in the middle + * of a surrogate pair + * (from a 4-byte UTF-8 sequence for the corresponding supplementary code point) + * + * getState() cannot also encode the UTF-16 index in the state value. + * move(relative to limit or length), or + * move(relative to current) after setState(), may return UITER_UNKNOWN_INDEX. + * + * @param iter UCharIterator structure to be set for iteration + * @param s UTF-8 string to iterate over + * @param length Length of s in bytes, or -1 if NUL-terminated + * + * @see UCharIterator + * @stable ICU 2.6 + */ +U_STABLE void U_EXPORT2 +uiter_setUTF8(UCharIterator *iter, const char *s, int32_t length); + +#ifdef XP_CPLUSPLUS + +/** + * Set up a UCharIterator to wrap around a C++ CharacterIterator. + * + * Sets the UCharIterator function pointers for iteration using the + * CharacterIterator charIter. + * + * The CharacterIterator pointer charIter is set into UCharIterator.context + * without copying or cloning the CharacterIterator object. + * The other "protected" UCharIterator fields are set to 0 and will be ignored. + * The iteration index and boundaries are controlled by the CharacterIterator. + * + * getState() simply returns the current index. + * move() will always return the final index. + * + * @param iter UCharIterator structure to be set for iteration + * @param charIter CharacterIterator to wrap + * + * @see UCharIterator + * @stable ICU 2.1 + */ +U_STABLE void U_EXPORT2 +uiter_setCharacterIterator(UCharIterator *iter, U_NAMESPACE_QUALIFIER CharacterIterator *charIter); + +/** + * Set up a UCharIterator to iterate over a C++ Replaceable. + * + * Sets the UCharIterator function pointers for iteration over the + * Replaceable rep with iteration boundaries start=index=0 and + * length=limit=rep->length(). + * The "provider" may set the start, index, and limit values at any time + * within the range 0..length=rep->length(). + * The length field will be ignored. + * + * The Replaceable pointer rep is set into UCharIterator.context without copying + * or cloning/reallocating the Replaceable object. + * + * getState() simply returns the current index. + * move() will always return the final index. + * + * @param iter UCharIterator structure to be set for iteration + * @param rep Replaceable to iterate over + * + * @see UCharIterator + * @stable ICU 2.1 + */ +U_STABLE void U_EXPORT2 +uiter_setReplaceable(UCharIterator *iter, const U_NAMESPACE_QUALIFIER Replaceable *rep); + +#endif + +U_CDECL_END + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/uloc.h b/Build/source/libs/icu/icu-xetex/common/unicode/uloc.h new file mode 100644 index 00000000000..0d5d3b3b390 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/uloc.h @@ -0,0 +1,931 @@ +/* +********************************************************************** +* Copyright (C) 1997-2007, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* +* File ULOC.H +* +* Modification History: +* +* Date Name Description +* 04/01/97 aliu Creation. +* 08/22/98 stephen JDK 1.2 sync. +* 12/08/98 rtg New C API for Locale +* 03/30/99 damiba overhaul +* 03/31/99 helena Javadoc for uloc functions. +* 04/15/99 Madhu Updated Javadoc +******************************************************************************** +*/ + +#ifndef ULOC_H +#define ULOC_H + +#include "unicode/utypes.h" +#include "unicode/uenum.h" + +/** + * \file + * \brief C API: Locale + * + *

ULoc C API for Locale

+ * A Locale represents a specific geographical, political, + * or cultural region. An operation that requires a Locale to perform + * its task is called locale-sensitive and uses the Locale + * to tailor information for the user. For example, displaying a number + * is a locale-sensitive operation--the number should be formatted + * according to the customs/conventions of the user's native country, + * region, or culture. In the C APIs, a locales is simply a const char string. + * + *

+ * You create a Locale with one of the three options listed below. + * Each of the component is separated by '_' in the locale string. + * \htmlonly

\endhtmlonly + *
+ * \code
+ *       newLanguage
+ * 
+ *       newLanguage + newCountry
+ * 
+ *       newLanguage + newCountry + newVariant
+ * \endcode
+ * 
+ * \htmlonly
\endhtmlonly + * The first option is a valid ISO + * Language Code. These codes are the lower-case two-letter + * codes as defined by ISO-639. + * You can find a full list of these codes at a number of sites, such as: + *
+ * http://www.ics.uci.edu/pub/ietf/http/related/iso639.txt + * + *

+ * The second option includes an additonal ISO Country + * Code. These codes are the upper-case two-letter codes + * as defined by ISO-3166. + * You can find a full list of these codes at a number of sites, such as: + *
+ * http://www.chemie.fu-berlin.de/diverse/doc/ISO_3166.html + * + *

+ * The third option requires another additonal information--the + * Variant. + * The Variant codes are vendor and browser-specific. + * For example, use WIN for Windows, MAC for Macintosh, and POSIX for POSIX. + * Where there are two variants, separate them with an underscore, and + * put the most important one first. For + * example, a Traditional Spanish collation might be referenced, with + * "ES", "ES", "Traditional_WIN". + * + *

+ * Because a Locale is just an identifier for a region, + * no validity check is performed when you specify a Locale. + * If you want to see whether particular resources are available for the + * Locale you asked for, you must query those resources. For + * example, ask the UNumberFormat for the locales it supports + * using its getAvailable method. + *
Note: When you ask for a resource for a particular + * locale, you get back the best available match, not necessarily + * precisely what you asked for. For more information, look at + * UResourceBundle. + * + *

+ * The Locale provides a number of convenient constants + * that you can use to specify the commonly used + * locales. For example, the following refers to a locale + * for the United States: + * \htmlonly

\endhtmlonly + *
+ * \code
+ *       ULOC_US
+ * \endcode
+ * 
+ * \htmlonly
\endhtmlonly + * + *

+ * Once you've specified a locale you can query it for information about + * itself. Use uloc_getCountry to get the ISO Country Code and + * uloc_getLanguage to get the ISO Language Code. You can + * use uloc_getDisplayCountry to get the + * name of the country suitable for displaying to the user. Similarly, + * you can use uloc_getDisplayLanguage to get the name of + * the language suitable for displaying to the user. Interestingly, + * the uloc_getDisplayXXX methods are themselves locale-sensitive + * and have two versions: one that uses the default locale and one + * that takes a locale as an argument and displays the name or country in + * a language appropriate to that locale. + * + *

+ * The ICU provides a number of services that perform locale-sensitive + * operations. For example, the unum_xxx functions format + * numbers, currency, or percentages in a locale-sensitive manner. + *

+ * \htmlonly
\endhtmlonly + *
+ * \code
+ *     UErrorCode success = U_ZERO_ERROR;
+ *     UNumberFormat *nf;
+ *     const char* myLocale = "fr_FR";
+ * 
+ *     nf = unum_open( UNUM_DEFAULT, NULL, success );          
+ *     unum_close(nf);
+ *     nf = unum_open( UNUM_CURRENCY, NULL, success );
+ *     unum_close(nf);
+ *     nf = unum_open( UNUM_PERCENT, NULL, success );   
+ *     unum_close(nf);
+ * \endcode
+ * 
+ * \htmlonly
\endhtmlonly + * Each of these methods has two variants; one with an explicit locale + * and one without; the latter using the default locale. + * \htmlonly
\endhtmlonly + *
+ * \code 
+ * 
+ *     nf = unum_open( UNUM_DEFAULT, myLocale, success );          
+ *     unum_close(nf);
+ *     nf = unum_open( UNUM_CURRENCY, myLocale, success );
+ *     unum_close(nf);
+ *     nf = unum_open( UNUM_PERCENT, myLocale, success );   
+ *     unum_close(nf);
+ * \endcode
+ * 
+ * \htmlonly
\endhtmlonly + * A Locale is the mechanism for identifying the kind of services + * (UNumberFormat) that you would like to get. The locale is + * just a mechanism for identifying these services. + * + *

+ * Each international serivce that performs locale-sensitive operations + * allows you + * to get all the available objects of that type. You can sift + * through these objects by language, country, or variant, + * and use the display names to present a menu to the user. + * For example, you can create a menu of all the collation objects + * suitable for a given language. Such classes implement these + * three class methods: + * \htmlonly

\endhtmlonly + *
+ * \code
+ *       const char* uloc_getAvailable(int32_t index);
+ *       int32_t uloc_countAvailable();
+ *       int32_t
+ *       uloc_getDisplayName(const char* localeID,
+ *                 const char* inLocaleID, 
+ *                 UChar* result,
+ *                 int32_t maxResultSize,
+ *                  UErrorCode* err);
+ * 
+ * \endcode
+ * 
+ * \htmlonly
\endhtmlonly + *

+ * Concerning POSIX/RFC1766 Locale IDs, + * the getLanguage/getCountry/getVariant/getName functions do understand + * the POSIX type form of language_COUNTRY.ENCODING\@VARIANT + * and if there is not an ICU-stype variant, uloc_getVariant() for example + * will return the one listed after the \@at sign. As well, the hyphen + * "-" is recognized as a country/variant separator similarly to RFC1766. + * So for example, "en-us" will be interpreted as en_US. + * As a result, uloc_getName() is far from a no-op, and will have the + * effect of converting POSIX/RFC1766 IDs into ICU form, although it does + * NOT map any of the actual codes (i.e. russian->ru) in any way. + * Applications should call uloc_getName() at the point where a locale ID + * is coming from an external source (user entry, OS, web browser) + * and pass the resulting string to other ICU functions. For example, + * don't use de-de\@EURO as an argument to resourcebundle. + * + * @see UResourceBundle + */ + +/** Useful constant for this language. @stable ICU 2.0 */ +#define ULOC_CHINESE "zh" +/** Useful constant for this language. @stable ICU 2.0 */ +#define ULOC_ENGLISH "en" +/** Useful constant for this language. @stable ICU 2.0 */ +#define ULOC_FRENCH "fr" +/** Useful constant for this language. @stable ICU 2.0 */ +#define ULOC_GERMAN "de" +/** Useful constant for this language. @stable ICU 2.0 */ +#define ULOC_ITALIAN "it" +/** Useful constant for this language. @stable ICU 2.0 */ +#define ULOC_JAPANESE "ja" +/** Useful constant for this language. @stable ICU 2.0 */ +#define ULOC_KOREAN "ko" +/** Useful constant for this language. @stable ICU 2.0 */ +#define ULOC_SIMPLIFIED_CHINESE "zh_CN" +/** Useful constant for this language. @stable ICU 2.0 */ +#define ULOC_TRADITIONAL_CHINESE "zh_TW" + +/** Useful constant for this country/region. @stable ICU 2.0 */ +#define ULOC_CANADA "en_CA" +/** Useful constant for this country/region. @stable ICU 2.0 */ +#define ULOC_CANADA_FRENCH "fr_CA" +/** Useful constant for this country/region. @stable ICU 2.0 */ +#define ULOC_CHINA "zh_CN" +/** Useful constant for this country/region. @stable ICU 2.0 */ +#define ULOC_PRC "zh_CN" +/** Useful constant for this country/region. @stable ICU 2.0 */ +#define ULOC_FRANCE "fr_FR" +/** Useful constant for this country/region. @stable ICU 2.0 */ +#define ULOC_GERMANY "de_DE" +/** Useful constant for this country/region. @stable ICU 2.0 */ +#define ULOC_ITALY "it_IT" +/** Useful constant for this country/region. @stable ICU 2.0 */ +#define ULOC_JAPAN "ja_JP" +/** Useful constant for this country/region. @stable ICU 2.0 */ +#define ULOC_KOREA "ko_KR" +/** Useful constant for this country/region. @stable ICU 2.0 */ +#define ULOC_TAIWAN "zh_TW" +/** Useful constant for this country/region. @stable ICU 2.0 */ +#define ULOC_UK "en_GB" +/** Useful constant for this country/region. @stable ICU 2.0 */ +#define ULOC_US "en_US" + +/** + * Useful constant for the maximum size of the language part of a locale ID. + * (including the terminating NULL). + * @stable ICU 2.0 + */ +#define ULOC_LANG_CAPACITY 12 + +/** + * Useful constant for the maximum size of the country part of a locale ID + * (including the terminating NULL). + * @stable ICU 2.0 + */ +#define ULOC_COUNTRY_CAPACITY 4 +/** + * Useful constant for the maximum size of the whole locale ID + * (including the terminating NULL). + * @stable ICU 2.0 + */ +#define ULOC_FULLNAME_CAPACITY 56 + +/** + * Useful constant for the maximum size of the script part of a locale ID + * (including the terminating NULL). + * @stable ICU 2.8 + */ +#define ULOC_SCRIPT_CAPACITY 6 + +/** + * Useful constant for the maximum size of keywords in a locale + * @stable ICU 2.8 + */ +#define ULOC_KEYWORDS_CAPACITY 50 + +/** + * Useful constant for the maximum SIZE of keywords in a locale + * @stable ICU 2.8 + */ +#define ULOC_KEYWORD_AND_VALUES_CAPACITY 100 + +/** + * Character separating keywords from the locale string + * different for EBCDIC - TODO + * @stable ICU 2.8 + */ +#define ULOC_KEYWORD_SEPARATOR '@' +/** + * Character for assigning value to a keyword + * @stable ICU 2.8 + */ +#define ULOC_KEYWORD_ASSIGN '=' +/** + * Character separating keywords + * @stable ICU 2.8 + */ +#define ULOC_KEYWORD_ITEM_SEPARATOR ';' + +/** + * Constants for *_getLocale() + * Allow user to select whether she wants information on + * requested, valid or actual locale. + * For example, a collator for "en_US_CALIFORNIA" was + * requested. In the current state of ICU (2.0), + * the requested locale is "en_US_CALIFORNIA", + * the valid locale is "en_US" (most specific locale supported by ICU) + * and the actual locale is "root" (the collation data comes unmodified + * from the UCA) + * The locale is considered supported by ICU if there is a core ICU bundle + * for that locale (although it may be empty). + * @stable ICU 2.1 + */ +typedef enum { + /** This is locale the data actually comes from + * @stable ICU 2.1 + */ + ULOC_ACTUAL_LOCALE = 0, + /** This is the most specific locale supported by ICU + * @stable ICU 2.1 + */ + ULOC_VALID_LOCALE = 1, + +#ifndef U_HIDE_DEPRECATED_API + /** This is the requested locale + * @deprecated ICU 2.8 + */ + ULOC_REQUESTED_LOCALE = 2, +#endif /* U_HIDE_DEPRECATED_API */ + + ULOC_DATA_LOCALE_TYPE_LIMIT = 3 +} ULocDataLocaleType ; + + +/** + * Gets ICU's default locale. + * The returned string is a snapshot in time, and will remain valid + * and unchanged even when uloc_setDefault() is called. + * The returned storage is owned by ICU, and must not be altered or deleted + * by the caller. + * + * @return the ICU default locale + * @system + * @stable ICU 2.0 + */ +U_STABLE const char* U_EXPORT2 +uloc_getDefault(void); + +/** + * Sets ICU's default locale. + * By default (without calling this function), ICU's default locale will be based + * on information obtained from the underlying system environment. + *

+ * Changes to ICU's default locale do not propagate back to the + * system environment. + *

+ * Changes to ICU's default locale to not affect any ICU services that + * may already be open based on the previous default locale value. + * + * @param localeID the new ICU default locale. A value of NULL will try to get + * the system's default locale. + * @param status the error information if the setting of default locale fails + * @system + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +uloc_setDefault(const char* localeID, + UErrorCode* status); + +/** + * Gets the language code for the specified locale. + * + * @param localeID the locale to get the ISO language code with + * @param language the language code for localeID + * @param languageCapacity the size of the language buffer to store the + * language code with + * @param err error information if retrieving the language code failed + * @return the actual buffer size needed for the language code. If it's greater + * than languageCapacity, the returned language code will be truncated. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +uloc_getLanguage(const char* localeID, + char* language, + int32_t languageCapacity, + UErrorCode* err); + +/** + * Gets the script code for the specified locale. + * + * @param localeID the locale to get the ISO language code with + * @param script the language code for localeID + * @param scriptCapacity the size of the language buffer to store the + * language code with + * @param err error information if retrieving the language code failed + * @return the actual buffer size needed for the language code. If it's greater + * than scriptCapacity, the returned language code will be truncated. + * @stable ICU 2.8 + */ +U_STABLE int32_t U_EXPORT2 +uloc_getScript(const char* localeID, + char* script, + int32_t scriptCapacity, + UErrorCode* err); + +/** + * Gets the country code for the specified locale. + * + * @param localeID the locale to get the country code with + * @param country the country code for localeID + * @param countryCapacity the size of the country buffer to store the + * country code with + * @param err error information if retrieving the country code failed + * @return the actual buffer size needed for the country code. If it's greater + * than countryCapacity, the returned country code will be truncated. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +uloc_getCountry(const char* localeID, + char* country, + int32_t countryCapacity, + UErrorCode* err); + +/** + * Gets the variant code for the specified locale. + * + * @param localeID the locale to get the variant code with + * @param variant the variant code for localeID + * @param variantCapacity the size of the variant buffer to store the + * variant code with + * @param err error information if retrieving the variant code failed + * @return the actual buffer size needed for the variant code. If it's greater + * than variantCapacity, the returned variant code will be truncated. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +uloc_getVariant(const char* localeID, + char* variant, + int32_t variantCapacity, + UErrorCode* err); + + +/** + * Gets the full name for the specified locale. + * Note: This has the effect of 'canonicalizing' the ICU locale ID to + * a certain extent. Upper and lower case are set as needed. + * It does NOT map aliased names in any way. + * See the top of this header file. + * This API supports preflighting. + * + * @param localeID the locale to get the full name with + * @param name fill in buffer for the name without keywords. + * @param nameCapacity capacity of the fill in buffer. + * @param err error information if retrieving the full name failed + * @return the actual buffer size needed for the full name. If it's greater + * than nameCapacity, the returned full name will be truncated. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +uloc_getName(const char* localeID, + char* name, + int32_t nameCapacity, + UErrorCode* err); + +/** + * Gets the full name for the specified locale. + * Note: This has the effect of 'canonicalizing' the string to + * a certain extent. Upper and lower case are set as needed, + * and if the components were in 'POSIX' format they are changed to + * ICU format. It does NOT map aliased names in any way. + * See the top of this header file. + * + * @param localeID the locale to get the full name with + * @param name the full name for localeID + * @param nameCapacity the size of the name buffer to store the + * full name with + * @param err error information if retrieving the full name failed + * @return the actual buffer size needed for the full name. If it's greater + * than nameCapacity, the returned full name will be truncated. + * @stable ICU 2.8 + */ +U_STABLE int32_t U_EXPORT2 +uloc_canonicalize(const char* localeID, + char* name, + int32_t nameCapacity, + UErrorCode* err); + +/** + * Gets the ISO language code for the specified locale. + * + * @param localeID the locale to get the ISO language code with + * @return language the ISO language code for localeID + * @stable ICU 2.0 + */ +U_STABLE const char* U_EXPORT2 +uloc_getISO3Language(const char* localeID); + + +/** + * Gets the ISO country code for the specified locale. + * + * @param localeID the locale to get the ISO country code with + * @return country the ISO country code for localeID + * @stable ICU 2.0 + */ +U_STABLE const char* U_EXPORT2 +uloc_getISO3Country(const char* localeID); + +/** + * Gets the Win32 LCID value for the specified locale. + * If the ICU locale is not recognized by Windows, 0 will be returned. + * + * @param localeID the locale to get the Win32 LCID value with + * @return country the Win32 LCID for localeID + * @stable ICU 2.0 + */ +U_STABLE uint32_t U_EXPORT2 +uloc_getLCID(const char* localeID); + +/** + * Gets the language name suitable for display for the specified locale. + * + * @param locale the locale to get the ISO language code with + * @param displayLocale Specifies the locale to be used to display the name. In other words, + * if the locale's language code is "en", passing Locale::getFrench() for + * inLocale would result in "Anglais", while passing Locale::getGerman() + * for inLocale would result in "Englisch". + * @param language the displayable language code for localeID + * @param languageCapacity the size of the language buffer to store the + * displayable language code with + * @param status error information if retrieving the displayable language code failed + * @return the actual buffer size needed for the displayable language code. If it's greater + * than languageCapacity, the returned language code will be truncated. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +uloc_getDisplayLanguage(const char* locale, + const char* displayLocale, + UChar* language, + int32_t languageCapacity, + UErrorCode* status); + +/** + * Gets the script name suitable for display for the specified locale. + * + * @param locale the locale to get the displayable script code with. NULL may be used to specify the default. + * @param displayLocale Specifies the locale to be used to display the name. In other words, + * if the locale's language code is "en", passing Locale::getFrench() for + * inLocale would result in "", while passing Locale::getGerman() + * for inLocale would result in "". NULL may be used to specify the default. + * @param script the displayable country code for localeID + * @param scriptCapacity the size of the script buffer to store the + * displayable script code with + * @param status error information if retrieving the displayable script code failed + * @return the actual buffer size needed for the displayable script code. If it's greater + * than scriptCapacity, the returned displayable script code will be truncated. + * @stable ICU 2.8 + */ +U_STABLE int32_t U_EXPORT2 +uloc_getDisplayScript(const char* locale, + const char* displayLocale, + UChar* script, + int32_t scriptCapacity, + UErrorCode* status); + +/** + * Gets the country name suitable for display for the specified locale. + * + * @param locale the locale to get the displayable country code with. NULL may be used to specify the default. + * @param displayLocale Specifies the locale to be used to display the name. In other words, + * if the locale's language code is "en", passing Locale::getFrench() for + * inLocale would result in "Anglais", while passing Locale::getGerman() + * for inLocale would result in "Englisch". NULL may be used to specify the default. + * @param country the displayable country code for localeID + * @param countryCapacity the size of the country buffer to store the + * displayable country code with + * @param status error information if retrieving the displayable country code failed + * @return the actual buffer size needed for the displayable country code. If it's greater + * than countryCapacity, the returned displayable country code will be truncated. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +uloc_getDisplayCountry(const char* locale, + const char* displayLocale, + UChar* country, + int32_t countryCapacity, + UErrorCode* status); + + +/** + * Gets the variant name suitable for display for the specified locale. + * + * @param locale the locale to get the displayable variant code with. NULL may be used to specify the default. + * @param displayLocale Specifies the locale to be used to display the name. In other words, + * if the locale's language code is "en", passing Locale::getFrench() for + * inLocale would result in "Anglais", while passing Locale::getGerman() + * for inLocale would result in "Englisch". NULL may be used to specify the default. + * @param variant the displayable variant code for localeID + * @param variantCapacity the size of the variant buffer to store the + * displayable variant code with + * @param status error information if retrieving the displayable variant code failed + * @return the actual buffer size needed for the displayable variant code. If it's greater + * than variantCapacity, the returned displayable variant code will be truncated. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +uloc_getDisplayVariant(const char* locale, + const char* displayLocale, + UChar* variant, + int32_t variantCapacity, + UErrorCode* status); + +/** + * Gets the keyword name suitable for display for the specified locale. + * E.g: for the locale string de_DE\@collation=PHONEBOOK, this API gets the display + * string for the keyword collation. + * Usage: + * + * UErrorCode status = U_ZERO_ERROR; + * const char* keyword =NULL; + * int32_t keywordLen = 0; + * int32_t keywordCount = 0; + * UChar displayKeyword[256]; + * int32_t displayKeywordLen = 0; + * UEnumeration* keywordEnum = uloc_openKeywords("de_DE@collation=PHONEBOOK;calendar=TRADITIONAL", &status); + * for(keywordCount = uenum_count(keywordEnum, &status); keywordCount > 0 ; keywordCount--){ + * if(U_FAILURE(status)){ + * ...something went wrong so handle the error... + * break; + * } + * // the uenum_next returns NUL terminated string + * keyword = uenum_next(keywordEnum, &keywordLen, &status); + * displayKeywordLen = uloc_getDisplayKeyword(keyword, "en_US", displayKeyword, 256); + * ... do something interesting ..... + * } + * uenum_close(keywordEnum); + * + * @param keyword The keyword whose display string needs to be returned. + * @param displayLocale Specifies the locale to be used to display the name. In other words, + * if the locale's language code is "en", passing Locale::getFrench() for + * inLocale would result in "Anglais", while passing Locale::getGerman() + * for inLocale would result in "Englisch". NULL may be used to specify the default. + * @param dest the buffer to which the displayable keyword should be written. + * @param destCapacity The size of the buffer (number of UChars). If it is 0, then + * dest may be NULL and the function will only return the length of the + * result without writing any of the result string (pre-flighting). + * @param status error information if retrieving the displayable string failed. + * Should not be NULL and should not indicate failure on entry. + * @return the actual buffer size needed for the displayable variant code. + * @see #uloc_openKeywords + * @stable ICU 2.8 + */ +U_STABLE int32_t U_EXPORT2 +uloc_getDisplayKeyword(const char* keyword, + const char* displayLocale, + UChar* dest, + int32_t destCapacity, + UErrorCode* status); +/** + * Gets the value of the keyword suitable for display for the specified locale. + * E.g: for the locale string de_DE\@collation=PHONEBOOK, this API gets the display + * string for PHONEBOOK, in the display locale, when "collation" is specified as the keyword. + * + * @param locale The locale to get the displayable variant code with. NULL may be used to specify the default. + * @param keyword The keyword for whose value should be used. + * @param displayLocale Specifies the locale to be used to display the name. In other words, + * if the locale's language code is "en", passing Locale::getFrench() for + * inLocale would result in "Anglais", while passing Locale::getGerman() + * for inLocale would result in "Englisch". NULL may be used to specify the default. + * @param dest the buffer to which the displayable keyword should be written. + * @param destCapacity The size of the buffer (number of UChars). If it is 0, then + * dest may be NULL and the function will only return the length of the + * result without writing any of the result string (pre-flighting). + * @param status error information if retrieving the displayable string failed. + * Should not be NULL and must not indicate failure on entry. + * @return the actual buffer size needed for the displayable variant code. + * @stable ICU 2.8 + */ +U_STABLE int32_t U_EXPORT2 +uloc_getDisplayKeywordValue( const char* locale, + const char* keyword, + const char* displayLocale, + UChar* dest, + int32_t destCapacity, + UErrorCode* status); +/** + * Gets the full name suitable for display for the specified locale. + * + * @param localeID the locale to get the displayable name with. NULL may be used to specify the default. + * @param inLocaleID Specifies the locale to be used to display the name. In other words, + * if the locale's language code is "en", passing Locale::getFrench() for + * inLocale would result in "Anglais", while passing Locale::getGerman() + * for inLocale would result in "Englisch". NULL may be used to specify the default. + * @param result the displayable name for localeID + * @param maxResultSize the size of the name buffer to store the + * displayable full name with + * @param err error information if retrieving the displayable name failed + * @return the actual buffer size needed for the displayable name. If it's greater + * than maxResultSize, the returned displayable name will be truncated. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +uloc_getDisplayName(const char* localeID, + const char* inLocaleID, + UChar* result, + int32_t maxResultSize, + UErrorCode* err); + + +/** + * Gets the specified locale from a list of all available locales. + * The return value is a pointer to an item of + * a locale name array. Both this array and the pointers + * it contains are owned by ICU and should not be deleted or written through + * by the caller. The locale name is terminated by a null pointer. + * @param n the specific locale name index of the available locale list + * @return a specified locale name of all available locales + * @stable ICU 2.0 + */ +U_STABLE const char* U_EXPORT2 +uloc_getAvailable(int32_t n); + +/** + * Gets the size of the all available locale list. + * + * @return the size of the locale list + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 uloc_countAvailable(void); + +/** + * + * Gets a list of all available language codes defined in ISO 639. This is a pointer + * to an array of pointers to arrays of char. All of these pointers are owned + * by ICU-- do not delete them, and do not write through them. The array is + * terminated with a null pointer. + * @return a list of all available language codes + * @stable ICU 2.0 + */ +U_STABLE const char* const* U_EXPORT2 +uloc_getISOLanguages(void); + +/** + * + * Gets a list of all available 2-letter country codes defined in ISO 639. This is a + * pointer to an array of pointers to arrays of char. All of these pointers are + * owned by ICU-- do not delete them, and do not write through them. The array is + * terminated with a null pointer. + * @return a list of all available country codes + * @stable ICU 2.0 + */ +U_STABLE const char* const* U_EXPORT2 +uloc_getISOCountries(void); + +/** + * Truncate the locale ID string to get the parent locale ID. + * Copies the part of the string before the last underscore. + * The parent locale ID will be an empty string if there is no + * underscore, or if there is only one underscore at localeID[0]. + * + * @param localeID Input locale ID string. + * @param parent Output string buffer for the parent locale ID. + * @param parentCapacity Size of the output buffer. + * @param err A UErrorCode value. + * @return The length of the parent locale ID. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +uloc_getParent(const char* localeID, + char* parent, + int32_t parentCapacity, + UErrorCode* err); + + + + +/** + * Gets the full name for the specified locale. + * Note: This has the effect of 'canonicalizing' the string to + * a certain extent. Upper and lower case are set as needed, + * and if the components were in 'POSIX' format they are changed to + * ICU format. It does NOT map aliased names in any way. + * See the top of this header file. + * This API strips off the keyword part, so "de_DE\@collation=phonebook" + * will become "de_DE". + * This API supports preflighting. + * + * @param localeID the locale to get the full name with + * @param name fill in buffer for the name without keywords. + * @param nameCapacity capacity of the fill in buffer. + * @param err error information if retrieving the full name failed + * @return the actual buffer size needed for the full name. If it's greater + * than nameCapacity, the returned full name will be truncated. + * @stable ICU 2.8 + */ +U_STABLE int32_t U_EXPORT2 +uloc_getBaseName(const char* localeID, + char* name, + int32_t nameCapacity, + UErrorCode* err); + +/** + * Gets an enumeration of keywords for the specified locale. Enumeration + * must get disposed of by the client using uenum_close function. + * + * @param localeID the locale to get the variant code with + * @param status error information if retrieving the keywords failed + * @return enumeration of keywords or NULL if there are no keywords. + * @stable ICU 2.8 + */ +U_STABLE UEnumeration* U_EXPORT2 +uloc_openKeywords(const char* localeID, + UErrorCode* status); + +/** + * Get the value for a keyword. Locale name does not need to be normalized. + * + * @param localeID locale name containing the keyword ("de_DE@currency=EURO;collation=PHONEBOOK") + * @param keywordName name of the keyword for which we want the value. Case insensitive. + * @param buffer receiving buffer + * @param bufferCapacity capacity of receiving buffer + * @param status containing error code - buffer not big enough. + * @return the length of keyword value + * @stable ICU 2.8 + */ +U_STABLE int32_t U_EXPORT2 +uloc_getKeywordValue(const char* localeID, + const char* keywordName, + char* buffer, int32_t bufferCapacity, + UErrorCode* status); + + +/** + * Set the value of the specified keyword. + * NOTE: Unlike almost every other ICU function which takes a + * buffer, this function will NOT truncate the output text. If a + * BUFFER_OVERFLOW_ERROR is received, it means that the original + * buffer is untouched. This is done to prevent incorrect or possibly + * even malformed locales from being generated and used. + * + * @param keywordName name of the keyword to be set. Case insensitive. + * @param keywordValue value of the keyword to be set. If 0-length or + * NULL, will result in the keyword being removed. No error is given if + * that keyword does not exist. + * @param buffer input buffer containing locale to be modified. + * @param bufferCapacity capacity of receiving buffer + * @param status containing error code - buffer not big enough. + * @return the length needed for the buffer + * @see uloc_getKeywordValue + * @stable ICU 3.2 + */ +U_STABLE int32_t U_EXPORT2 +uloc_setKeywordValue(const char* keywordName, + const char* keywordValue, + char* buffer, int32_t bufferCapacity, + UErrorCode* status); + +/** + * enums for the 'outResult' parameter return value + * @see uloc_acceptLanguageFromHTTP + * @see uloc_acceptLanguage + * @stable ICU 3.2 + */ +typedef enum { + ULOC_ACCEPT_FAILED = 0, /* No exact match was found. */ + ULOC_ACCEPT_VALID = 1, /* An exact match was found. */ + ULOC_ACCEPT_FALLBACK = 2 /* A fallback was found, for example, + Accept list contained 'ja_JP' + which matched available locale 'ja'. */ +} UAcceptResult; + + +/** + * Based on a HTTP header from a web browser and a list of available locales, + * determine an acceptable locale for the user. + * @param result - buffer to accept the result locale + * @param resultAvailable the size of the result buffer. + * @param outResult - An out parameter that contains the fallback status + * @param httpAcceptLanguage - "Accept-Language:" header as per HTTP. + * @param availableLocales - list of available locales to match + * @param status Error status, may be BUFFER_OVERFLOW_ERROR + * @return length needed for the locale. + * @stable ICU 3.2 + */ +U_STABLE int32_t U_EXPORT2 +uloc_acceptLanguageFromHTTP(char *result, int32_t resultAvailable, + UAcceptResult *outResult, + const char *httpAcceptLanguage, + UEnumeration* availableLocales, + UErrorCode *status); + +/** + * Based on a list of available locales, + * determine an acceptable locale for the user. + * @param result - buffer to accept the result locale + * @param resultAvailable the size of the result buffer. + * @param outResult - An out parameter that contains the fallback status + * @param acceptList - list of acceptable languages + * @param acceptListCount - count of acceptList items + * @param availableLocales - list of available locales to match + * @param status Error status, may be BUFFER_OVERFLOW_ERROR + * @return length needed for the locale. + * @stable ICU 3.2 + */ +U_STABLE int32_t U_EXPORT2 +uloc_acceptLanguage(char *result, int32_t resultAvailable, + UAcceptResult *outResult, const char **acceptList, + int32_t acceptListCount, + UEnumeration* availableLocales, + UErrorCode *status); + + +/** + * Gets the ICU locale ID for the specified Win32 LCID value. + * + * @param hostID the Win32 LCID to translate + * @param locale the output buffer for the ICU locale ID, which will be NUL-terminated + * if there is room. + * @param localeCapacity the size of the output buffer + * @param status an error is returned if the LCID is unrecognized or the output buffer + * is too small + * @return actual the actual size of the locale ID, not including NUL-termination + * @draft ICU 3.8 + */ +U_DRAFT int32_t U_EXPORT2 +uloc_getLocaleForLCID(uint32_t hostID, char *locale, int32_t localeCapacity, + UErrorCode *status); + +#endif /*_ULOC*/ + + diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/umachine.h b/Build/source/libs/icu/icu-xetex/common/unicode/umachine.h new file mode 100644 index 00000000000..083f9cf014c --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/umachine.h @@ -0,0 +1,338 @@ +/* +****************************************************************************** +* +* Copyright (C) 1999-2006, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: umachine.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 1999sep13 +* created by: Markus W. Scherer +* +* This file defines basic types and constants for utf.h to be +* platform-independent. umachine.h and utf.h are included into +* utypes.h to provide all the general definitions for ICU. +* All of these definitions used to be in utypes.h before +* the UTF-handling macros made this unmaintainable. +*/ + +#ifndef __UMACHINE_H__ +#define __UMACHINE_H__ + + +/** + * \file + * \brief Basic types and constants for UTF + * + *

Basic types and constants for UTF

+ * This file defines basic types and constants for utf.h to be + * platform-independent. umachine.h and utf.h are included into + * utypes.h to provide all the general definitions for ICU. + * All of these definitions used to be in utypes.h before + * the UTF-handling macros made this unmaintainable. + * + */ +/*==========================================================================*/ +/* Include platform-dependent definitions */ +/* which are contained in the platform-specific file platform.h */ +/*==========================================================================*/ + +#if defined(U_PALMOS) +# include "unicode/ppalmos.h" +#elif defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64) +# include "unicode/pwin32.h" +#else +# include "unicode/platform.h" +#endif + +/* + * ANSI C headers: + * stddef.h defines wchar_t + */ +#include + +/*==========================================================================*/ +/* XP_CPLUSPLUS is a cross-platform symbol which should be defined when */ +/* using C++. It should not be defined when compiling under C. */ +/*==========================================================================*/ + +#ifdef __cplusplus +# ifndef XP_CPLUSPLUS +# define XP_CPLUSPLUS +# endif +#else +# undef XP_CPLUSPLUS +#endif + +/*==========================================================================*/ +/* For C wrappers, we use the symbol U_STABLE. */ +/* This works properly if the includer is C or C++. */ +/* Functions are declared U_STABLE return-type U_EXPORT2 function-name()... */ +/*==========================================================================*/ + +/** + * \def U_CFUNC + * This is used in a declaration of a library private ICU C function. + * @stable ICU 2.4 + */ + +/** + * \def U_CDECL_BEGIN + * This is used to begin a declaration of a library private ICU C API. + * @stable ICU 2.4 + */ + +/** + * \def U_CDECL_END + * This is used to end a declaration of a library private ICU C API + * @stable ICU 2.4 + */ + +#ifdef XP_CPLUSPLUS +# define U_CFUNC extern "C" +# define U_CDECL_BEGIN extern "C" { +# define U_CDECL_END } +#else +# define U_CFUNC extern +# define U_CDECL_BEGIN +# define U_CDECL_END +#endif + +/** This is used to declare a function as a public ICU C API @stable ICU 2.0*/ +#define U_CAPI U_CFUNC U_EXPORT +#define U_STABLE U_CAPI +#define U_DRAFT U_CAPI +#define U_DEPRECATED U_CAPI +#define U_OBSOLETE U_CAPI +#define U_INTERNAL U_CAPI + +/*==========================================================================*/ +/* limits for int32_t etc., like in POSIX inttypes.h */ +/*==========================================================================*/ + +#ifndef INT8_MIN +/** The smallest value an 8 bit signed integer can hold @stable ICU 2.0 */ +# define INT8_MIN ((int8_t)(-128)) +#endif +#ifndef INT16_MIN +/** The smallest value a 16 bit signed integer can hold @stable ICU 2.0 */ +# define INT16_MIN ((int16_t)(-32767-1)) +#endif +#ifndef INT32_MIN +/** The smallest value a 32 bit signed integer can hold @stable ICU 2.0 */ +# define INT32_MIN ((int32_t)(-2147483647-1)) +#endif + +#ifndef INT8_MAX +/** The largest value an 8 bit signed integer can hold @stable ICU 2.0 */ +# define INT8_MAX ((int8_t)(127)) +#endif +#ifndef INT16_MAX +/** The largest value a 16 bit signed integer can hold @stable ICU 2.0 */ +# define INT16_MAX ((int16_t)(32767)) +#endif +#ifndef INT32_MAX +/** The largest value a 32 bit signed integer can hold @stable ICU 2.0 */ +# define INT32_MAX ((int32_t)(2147483647)) +#endif + +#ifndef UINT8_MAX +/** The largest value an 8 bit unsigned integer can hold @stable ICU 2.0 */ +# define UINT8_MAX ((uint8_t)(255U)) +#endif +#ifndef UINT16_MAX +/** The largest value a 16 bit unsigned integer can hold @stable ICU 2.0 */ +# define UINT16_MAX ((uint16_t)(65535U)) +#endif +#ifndef UINT32_MAX +/** The largest value a 32 bit unsigned integer can hold @stable ICU 2.0 */ +# define UINT32_MAX ((uint32_t)(4294967295U)) +#endif + +#if defined(U_INT64_T_UNAVAILABLE) +# error int64_t is required for decimal format and rule-based number format. +#else +# ifndef INT64_C +/** + * Provides a platform independent way to specify a signed 64-bit integer constant. + * note: may be wrong for some 64 bit platforms - ensure your compiler provides INT64_C + * @stable ICU 2.8 + */ +# define INT64_C(c) c ## LL +# endif +# ifndef UINT64_C +/** + * Provides a platform independent way to specify an unsigned 64-bit integer constant. + * note: may be wrong for some 64 bit platforms - ensure your compiler provides UINT64_C + * @stable ICU 2.8 + */ +# define UINT64_C(c) c ## ULL +# endif +# ifndef U_INT64_MIN +/** The smallest value a 64 bit signed integer can hold @stable ICU 2.8 */ +# define U_INT64_MIN ((int64_t)(INT64_C(-9223372036854775807)-1)) +# endif +# ifndef U_INT64_MAX +/** The largest value a 64 bit signed integer can hold @stable ICU 2.8 */ +# define U_INT64_MAX ((int64_t)(INT64_C(9223372036854775807))) +# endif +# ifndef U_UINT64_MAX +/** The largest value a 64 bit unsigned integer can hold @stable ICU 2.8 */ +# define U_UINT64_MAX ((uint64_t)(UINT64_C(18446744073709551615))) +# endif +#endif + +/*==========================================================================*/ +/* Boolean data type */ +/*==========================================================================*/ + +/** The ICU boolean type @stable ICU 2.0 */ +typedef int8_t UBool; + +#ifndef TRUE +/** The TRUE value of a UBool @stable ICU 2.0 */ +# define TRUE 1 +#endif +#ifndef FALSE +/** The FALSE value of a UBool @stable ICU 2.0 */ +# define FALSE 0 +#endif + + +/*==========================================================================*/ +/* Unicode data types */ +/*==========================================================================*/ + +/* wchar_t-related definitions -------------------------------------------- */ + +/** + * \def U_HAVE_WCHAR_H + * Indicates whether is available (1) or not (0). Set to 1 by default. + * + * @stable ICU 2.0 + */ +#ifndef U_HAVE_WCHAR_H +# define U_HAVE_WCHAR_H 1 +#endif + +/** + * \def U_SIZEOF_WCHAR_T + * U_SIZEOF_WCHAR_T==sizeof(wchar_t) (0 means it is not defined or autoconf could not set it) + * + * @stable ICU 2.0 + */ +#if U_SIZEOF_WCHAR_T==0 +# undef U_SIZEOF_WCHAR_T +# define U_SIZEOF_WCHAR_T 4 +#endif + +/* + * \def U_WCHAR_IS_UTF16 + * Defined if wchar_t uses UTF-16. + * + * @stable ICU 2.0 + */ +/* + * \def U_WCHAR_IS_UTF32 + * Defined if wchar_t uses UTF-32. + * + * @stable ICU 2.0 + */ +#if !defined(U_WCHAR_IS_UTF16) && !defined(U_WCHAR_IS_UTF32) +# ifdef __STDC_ISO_10646__ +# if (U_SIZEOF_WCHAR_T==2) +# define U_WCHAR_IS_UTF16 +# elif (U_SIZEOF_WCHAR_T==4) +# define U_WCHAR_IS_UTF32 +# endif +# elif defined __UCS2__ +# if (__OS390__ || __OS400__) && (U_SIZEOF_WCHAR_T==2) +# define U_WCHAR_IS_UTF16 +# endif +# elif defined __UCS4__ +# if (U_SIZEOF_WCHAR_T==4) +# define U_WCHAR_IS_UTF32 +# endif +# elif defined(U_WINDOWS) +# define U_WCHAR_IS_UTF16 +# endif +#endif + +/* UChar and UChar32 definitions -------------------------------------------- */ + +/** Number of bytes in a UChar. @stable ICU 2.0 */ +#define U_SIZEOF_UCHAR 2 + +/** + * \var UChar + * Define UChar to be wchar_t if that is 16 bits wide; always assumed to be unsigned. + * If wchar_t is not 16 bits wide, then define UChar to be uint16_t. + * This makes the definition of UChar platform-dependent + * but allows direct string type compatibility with platforms with + * 16-bit wchar_t types. + * + * @stable ICU 2.0 + */ + +/* Define UChar to be compatible with wchar_t if possible. */ +#if U_SIZEOF_WCHAR_T==2 + typedef wchar_t UChar; +#else + typedef uint16_t UChar; +#endif + +/** + * Define UChar32 as a type for single Unicode code points. + * UChar32 is a signed 32-bit integer (same as int32_t). + * + * The Unicode code point range is 0..0x10ffff. + * All other values (negative or >=0x110000) are illegal as Unicode code points. + * They may be used as sentinel values to indicate "done", "error" + * or similar non-code point conditions. + * + * Before ICU 2.4 (Jitterbug 2146), UChar32 was defined + * to be wchar_t if that is 32 bits wide (wchar_t may be signed or unsigned) + * or else to be uint32_t. + * That is, the definition of UChar32 was platform-dependent. + * + * @see U_SENTINEL + * @stable ICU 2.4 + */ +typedef int32_t UChar32; + +/*==========================================================================*/ +/* U_INLINE and U_ALIGN_CODE Set default values if these are not already */ +/* defined. Definitions normally are in */ +/* platform.h or the corresponding file for */ +/* the OS in use. */ +/*==========================================================================*/ + +#ifndef U_HIDE_INTERNAL_API + +/** + * \def U_ALIGN_CODE + * This is used to align code fragments to a specific byte boundary. + * This is useful for getting consistent performance test results. + * @internal + */ +#ifndef U_ALIGN_CODE +# define U_ALIGN_CODE(n) +#endif + +#endif /* U_HIDE_INTERNAL_API */ + +#ifndef U_INLINE +# ifdef XP_CPLUSPLUS +# define U_INLINE inline +# else +# define U_INLINE +# endif +#endif + +#include "unicode/urename.h" + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/umisc.h b/Build/source/libs/icu/icu-xetex/common/unicode/umisc.h new file mode 100644 index 00000000000..d85451fc767 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/umisc.h @@ -0,0 +1,60 @@ +/* +********************************************************************** +* Copyright (C) 1999-2006, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* file name: umisc.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 1999oct15 +* created by: Markus W. Scherer +*/ + +#ifndef UMISC_H +#define UMISC_H + +#include "unicode/utypes.h" + +/** + * \file + * \brief C API:misc definitions + * + * This file contains miscellaneous definitions for the C APIs. + */ + +U_CDECL_BEGIN + +/** A struct representing a range of text containing a specific field + * @stable ICU 2.0 + */ +typedef struct UFieldPosition { + /** + * The field + * @stable ICU 2.0 + */ + int32_t field; + /** + * The start of the text range containing field + * @stable ICU 2.0 + */ + int32_t beginIndex; + /** + * The limit of the text range containing field + * @stable ICU 2.0 + */ + int32_t endIndex; +} UFieldPosition; + +#if !UCONFIG_NO_SERVICE +/** + * Opaque type returned by registerInstance, registerFactory and unregister for service registration. + * @stable ICU 2.6 + */ +typedef const void* URegistryKey; +#endif + +U_CDECL_END + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/unifilt.h b/Build/source/libs/icu/icu-xetex/common/unicode/unifilt.h new file mode 100644 index 00000000000..5bf1ba4e4fd --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/unifilt.h @@ -0,0 +1,127 @@ +/* +********************************************************************** +* Copyright (C) 1999-2006, International Business Machines Corporation and others. +* All Rights Reserved. +********************************************************************** +* Date Name Description +* 11/17/99 aliu Creation. +********************************************************************** +*/ +#ifndef UNIFILT_H +#define UNIFILT_H + +#include "unicode/unifunct.h" +#include "unicode/unimatch.h" + +/** + * \file + * \brief C++ API: Unicode Filter + */ + +U_NAMESPACE_BEGIN + +/** + * U_ETHER is used to represent character values for positions outside + * a range. For example, transliterator uses this to represent + * characters outside the range contextStart..contextLimit-1. This + * allows explicit matching by rules and UnicodeSets of text outside a + * defined range. + * @stable ICU 3.0 + */ +#define U_ETHER ((UChar)0xFFFF) + +/** + * + * UnicodeFilter defines a protocol for selecting a + * subset of the full range (U+0000 to U+10FFFF) of Unicode characters. + * Currently, filters are used in conjunction with classes like {@link + * Transliterator} to only process selected characters through a + * transformation. + * + *

Note: UnicodeFilter currently stubs out two pure virtual methods + * of its base class, UnicodeMatcher. These methods are toPattern() + * and matchesIndexValue(). This is done so that filter classes that + * are not actually used as matchers -- specifically, those in the + * UnicodeFilterLogic component, and those in tests -- can continue to + * work without defining these methods. As long as a filter is not + * used in an RBT during real transliteration, these methods will not + * be called. However, this breaks the UnicodeMatcher base class + * protocol, and it is not a correct solution. + * + *

In the future we may revisit the UnicodeMatcher / UnicodeFilter + * hierarchy and either redesign it, or simply remove the stubs in + * UnicodeFilter and force subclasses to implement the full + * UnicodeMatcher protocol. + * + * @see UnicodeFilterLogic + * @stable ICU 2.0 + */ +class U_COMMON_API UnicodeFilter : public UnicodeFunctor, public UnicodeMatcher { + +public: + /** + * Destructor + * @stable ICU 2.0 + */ + virtual ~UnicodeFilter(); + + /** + * Returns true for characters that are in the selected + * subset. In other words, if a character is to be + * filtered, then contains() returns + * false. + * @stable ICU 2.0 + */ + virtual UBool contains(UChar32 c) const = 0; + + /** + * UnicodeFunctor API. Cast 'this' to a UnicodeMatcher* pointer + * and return the pointer. + * @stable ICU 2.4 + */ + virtual UnicodeMatcher* toMatcher() const; + + /** + * Implement UnicodeMatcher API. + * @stable ICU 2.4 + */ + virtual UMatchDegree matches(const Replaceable& text, + int32_t& offset, + int32_t limit, + UBool incremental); + + /** + * UnicodeFunctor API. Nothing to do. + * @stable ICU 2.4 + */ + virtual void setData(const TransliterationRuleData*); + + /** + * ICU "poor man's RTTI", returns a UClassID for the actual class. + * + * @stable ICU 2.2 + */ + virtual UClassID getDynamicClassID() const = 0; + + /** + * ICU "poor man's RTTI", returns a UClassID for this class. + * + * @stable ICU 2.2 + */ + static UClassID U_EXPORT2 getStaticClassID(); + +protected: + + /* + * Since this class has pure virtual functions, + * a constructor can't be used. + * @stable ICU 2.0 + */ +/* UnicodeFilter();*/ +}; + +/*inline UnicodeFilter::UnicodeFilter() {}*/ + +U_NAMESPACE_END + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/unifunct.h b/Build/source/libs/icu/icu-xetex/common/unicode/unifunct.h new file mode 100644 index 00000000000..3aa7b0358c0 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/unifunct.h @@ -0,0 +1,125 @@ +/* +********************************************************************** +* Copyright (c) 2002-2005, International Business Machines Corporation +* and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 01/14/2002 aliu Creation. +********************************************************************** +*/ +#ifndef UNIFUNCT_H +#define UNIFUNCT_H + +#include "unicode/utypes.h" +#include "unicode/uobject.h" + +/** + * \file + * \brief C++ API: Unicode Functor + */ + +U_NAMESPACE_BEGIN + +class UnicodeMatcher; +class UnicodeReplacer; +class TransliterationRuleData; + +/** + * UnicodeFunctor is an abstract base class for objects + * that perform match and/or replace operations on Unicode strings. + * @author Alan Liu + * @stable ICU 2.4 + */ +class U_COMMON_API UnicodeFunctor : public UObject { + +public: + + /** + * Destructor + * @stable ICU 2.4 + */ + virtual ~UnicodeFunctor(); + + /** + * Return a copy of this object. All UnicodeFunctor objects + * have to support cloning in order to allow classes using + * UnicodeFunctor to implement cloning. + * @stable ICU 2.4 + */ + virtual UnicodeFunctor* clone() const = 0; + + /** + * Cast 'this' to a UnicodeMatcher* pointer and return the + * pointer, or null if this is not a UnicodeMatcher*. Subclasses + * that mix in UnicodeMatcher as a base class must override this. + * This protocol is required because a pointer to a UnicodeFunctor + * cannot be cast to a pointer to a UnicodeMatcher, since + * UnicodeMatcher is a mixin that does not derive from + * UnicodeFunctor. + * @stable ICU 2.4 + */ + virtual UnicodeMatcher* toMatcher() const; + + /** + * Cast 'this' to a UnicodeReplacer* pointer and return the + * pointer, or null if this is not a UnicodeReplacer*. Subclasses + * that mix in UnicodeReplacer as a base class must override this. + * This protocol is required because a pointer to a UnicodeFunctor + * cannot be cast to a pointer to a UnicodeReplacer, since + * UnicodeReplacer is a mixin that does not derive from + * UnicodeFunctor. + * @stable ICU 2.4 + */ + virtual UnicodeReplacer* toReplacer() const; + + /** + * Return the class ID for this class. This is useful only for + * comparing to a return value from getDynamicClassID(). + * @return The class ID for all objects of this class. + * @stable ICU 2.0 + */ + static UClassID U_EXPORT2 getStaticClassID(void); + + /** + * Returns a unique class ID polymorphically. This method + * is to implement a simple version of RTTI, since not all C++ + * compilers support genuine RTTI. Polymorphic operator==() and + * clone() methods call this method. + * + *

Concrete subclasses of UnicodeFunctor should use the macro + * UOBJECT_DEFINE_RTTI_IMPLEMENTATION from uobject.h to + * provide definitios getStaticClassID and getDynamicClassID. + * + * @return The class ID for this object. All objects of a given + * class have the same class ID. Objects of other classes have + * different class IDs. + * @stable ICU 2.4 + */ + virtual UClassID getDynamicClassID(void) const = 0; + + /** + * Set the data object associated with this functor. The data + * object provides context for functor-to-standin mapping. This + * method is required when assigning a functor to a different data + * object. This function MAY GO AWAY later if the architecture is + * changed to pass data object pointers through the API. + * @internal ICU 2.1 + */ + virtual void setData(const TransliterationRuleData*) = 0; + +protected: + + /** + * Since this class has pure virtual functions, + * a constructor can't be used. + * @stable ICU 2.0 + */ + /*UnicodeFunctor();*/ + +}; + +/*inline UnicodeFunctor::UnicodeFunctor() {}*/ + +U_NAMESPACE_END + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/unimatch.h b/Build/source/libs/icu/icu-xetex/common/unicode/unimatch.h new file mode 100644 index 00000000000..0dbb14efc20 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/unimatch.h @@ -0,0 +1,163 @@ +/* +* Copyright (C) 2001-2005, International Business Machines Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 07/18/01 aliu Creation. +********************************************************************** +*/ +#ifndef UNIMATCH_H +#define UNIMATCH_H + +#include "unicode/utypes.h" + +/** + * \file + * \brief C++ API: Unicode Matcher + */ + + +U_NAMESPACE_BEGIN + +class Replaceable; +class UnicodeString; +class UnicodeSet; + +/** + * Constants returned by UnicodeMatcher::matches() + * indicating the degree of match. + * @stable ICU 2.4 + */ +enum UMatchDegree { + /** + * Constant returned by matches() indicating a + * mismatch between the text and this matcher. The text contains + * a character which does not match, or the text does not contain + * all desired characters for a non-incremental match. + * @stable ICU 2.4 + */ + U_MISMATCH, + + /** + * Constant returned by matches() indicating a + * partial match between the text and this matcher. This value is + * only returned for incremental match operations. All characters + * of the text match, but more characters are required for a + * complete match. Alternatively, for variable-length matchers, + * all characters of the text match, and if more characters were + * supplied at limit, they might also match. + * @stable ICU 2.4 + */ + U_PARTIAL_MATCH, + + /** + * Constant returned by matches() indicating a + * complete match between the text and this matcher. For an + * incremental variable-length match, this value is returned if + * the given text matches, and it is known that additional + * characters would not alter the extent of the match. + * @stable ICU 2.4 + */ + U_MATCH +}; + +/** + * UnicodeMatcher defines a protocol for objects that can + * match a range of characters in a Replaceable string. + * @stable ICU 2.4 + */ +class U_COMMON_API UnicodeMatcher /* not : public UObject because this is an interface/mixin class */ { + +public: + /** + * Destructor. + * @stable ICU 2.4 + */ + virtual ~UnicodeMatcher(); + + /** + * Return a UMatchDegree value indicating the degree of match for + * the given text at the given offset. Zero, one, or more + * characters may be matched. + * + * Matching in the forward direction is indicated by limit > + * offset. Characters from offset forwards to limit-1 will be + * considered for matching. + * + * Matching in the reverse direction is indicated by limit < + * offset. Characters from offset backwards to limit+1 will be + * considered for matching. + * + * If limit == offset then the only match possible is a zero + * character match (which subclasses may implement if desired). + * + * As a side effect, advance the offset parameter to the limit of + * the matched substring. In the forward direction, this will be + * the index of the last matched character plus one. In the + * reverse direction, this will be the index of the last matched + * character minus one. + * + *

Note: This method is not const because some classes may + * modify their state as the result of a match. + * + * @param text the text to be matched + * @param offset on input, the index into text at which to begin + * matching. On output, the limit of the matched text. The + * number of matched characters is the output value of offset + * minus the input value. Offset should always point to the + * HIGH SURROGATE (leading code unit) of a pair of surrogates, + * both on entry and upon return. + * @param limit the limit index of text to be matched. Greater + * than offset for a forward direction match, less than offset for + * a backward direction match. The last character to be + * considered for matching will be text.charAt(limit-1) in the + * forward direction or text.charAt(limit+1) in the backward + * direction. + * @param incremental if TRUE, then assume further characters may + * be inserted at limit and check for partial matching. Otherwise + * assume the text as given is complete. + * @return a match degree value indicating a full match, a partial + * match, or a mismatch. If incremental is FALSE then + * U_PARTIAL_MATCH should never be returned. + * @stable ICU 2.4 + */ + virtual UMatchDegree matches(const Replaceable& text, + int32_t& offset, + int32_t limit, + UBool incremental) = 0; + + /** + * Returns a string representation of this matcher. If the result of + * calling this function is passed to the appropriate parser, it + * will produce another matcher that is equal to this one. + * @param result the string to receive the pattern. Previous + * contents will be deleted. + * @param escapeUnprintable if TRUE then convert unprintable + * character to their hex escape representations, \\uxxxx or + * \\Uxxxxxxxx. Unprintable characters are those other than + * U+000A, U+0020..U+007E. + * @stable ICU 2.4 + */ + virtual UnicodeString& toPattern(UnicodeString& result, + UBool escapeUnprintable = FALSE) const = 0; + + /** + * Returns TRUE if this matcher will match a character c, where c + * & 0xFF == v, at offset, in the forward direction (with limit > + * offset). This is used by RuleBasedTransliterator for + * indexing. + * @stable ICU 2.4 + */ + virtual UBool matchesIndexValue(uint8_t v) const = 0; + + /** + * Union the set of all characters that may be matched by this object + * into the given set. + * @param toUnionTo the set into which to union the source characters + * @stable ICU 2.4 + */ + virtual void addMatchSetTo(UnicodeSet& toUnionTo) const = 0; +}; + +U_NAMESPACE_END + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/uniset.h b/Build/source/libs/icu/icu-xetex/common/unicode/uniset.h new file mode 100644 index 00000000000..2d49410c56b --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/uniset.h @@ -0,0 +1,1508 @@ +/* +*************************************************************************** +* Copyright (C) 1999-2007, International Business Machines Corporation +* and others. All Rights Reserved. +*************************************************************************** +* Date Name Description +* 10/20/99 alan Creation. +*************************************************************************** +*/ + +#ifndef UNICODESET_H +#define UNICODESET_H + +#include "unicode/unifilt.h" +#include "unicode/unistr.h" +#include "unicode/uset.h" + +/** + * \file + * \brief C++ API: Unicode Set + */ + +U_NAMESPACE_BEGIN + +class BMPSet; +class ParsePosition; +class SymbolTable; +class UnicodeSetStringSpan; +class UVector; +class RuleCharacterIterator; + +/** + * A mutable set of Unicode characters and multicharacter strings. Objects of this class + * represent character classes used in regular expressions. + * A character specifies a subset of Unicode code points. Legal + * code points are U+0000 to U+10FFFF, inclusive. + * + *

The UnicodeSet class is not designed to be subclassed. + * + *

UnicodeSet supports two APIs. The first is the + * operand API that allows the caller to modify the value of + * a UnicodeSet object. It conforms to Java 2's + * java.util.Set interface, although + * UnicodeSet does not actually implement that + * interface. All methods of Set are supported, with the + * modification that they take a character range or single character + * instead of an Object, and they take a + * UnicodeSet instead of a Collection. The + * operand API may be thought of in terms of boolean logic: a boolean + * OR is implemented by add, a boolean AND is implemented + * by retain, a boolean XOR is implemented by + * complement taking an argument, and a boolean NOT is + * implemented by complement with no argument. In terms + * of traditional set theory function names, add is a + * union, retain is an intersection, remove + * is an asymmetric difference, and complement with no + * argument is a set complement with respect to the superset range + * MIN_VALUE-MAX_VALUE + * + *

The second API is the + * applyPattern()/toPattern() API from the + * java.text.Format-derived classes. Unlike the + * methods that add characters, add categories, and control the logic + * of the set, the method applyPattern() sets all + * attributes of a UnicodeSet at once, based on a + * string pattern. + * + *

Pattern syntax

+ * + * Patterns are accepted by the constructors and the + * applyPattern() methods and returned by the + * toPattern() method. These patterns follow a syntax + * similar to that employed by version 8 regular expression character + * classes. Here are some simple examples: + * + * \htmlonly
\endhtmlonly + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
[]No characters
[a]The character 'a'
[ae]The characters 'a' and 'e'
[a-e]The characters 'a' through 'e' inclusive, in Unicode code + * point order
[\\u4E01]The character U+4E01
[a{ab}{ac}]The character 'a' and the multicharacter strings "ab" and + * "ac"
[\\p{Lu}]All characters in the general category Uppercase Letter
+ * \htmlonly
\endhtmlonly + * + * Any character may be preceded by a backslash in order to remove any special + * meaning. White space characters, as defined by UCharacter.isWhitespace(), are + * ignored, unless they are escaped. + * + *

Property patterns specify a set of characters having a certain + * property as defined by the Unicode standard. Both the POSIX-like + * "[:Lu:]" and the Perl-like syntax "\\p{Lu}" are recognized. For a + * complete list of supported property patterns, see the User's Guide + * for UnicodeSet at + * + * http://icu-project.org/userguide/unicodeSet.html. + * Actual determination of property data is defined by the underlying + * Unicode database as implemented by UCharacter. + * + *

Patterns specify individual characters, ranges of characters, and + * Unicode property sets. When elements are concatenated, they + * specify their union. To complement a set, place a '^' immediately + * after the opening '['. Property patterns are inverted by modifying + * their delimiters; "[:^foo]" and "\\P{foo}". In any other location, + * '^' has no special meaning. + * + *

Ranges are indicated by placing two a '-' between two + * characters, as in "a-z". This specifies the range of all + * characters from the left to the right, in Unicode order. If the + * left character is greater than or equal to the + * right character it is a syntax error. If a '-' occurs as the first + * character after the opening '[' or '[^', or if it occurs as the + * last character before the closing ']', then it is taken as a + * literal. Thus "[a\-b]", "[-ab]", and "[ab-]" all indicate the same + * set of three characters, 'a', 'b', and '-'. + * + *

Sets may be intersected using the '&' operator or the asymmetric + * set difference may be taken using the '-' operator, for example, + * "[[:L:]&[\\u0000-\\u0FFF]]" indicates the set of all Unicode letters + * with values less than 4096. Operators ('&' and '|') have equal + * precedence and bind left-to-right. Thus + * "[[:L:]-[a-z]-[\\u0100-\\u01FF]]" is equivalent to + * "[[[:L:]-[a-z]]-[\\u0100-\\u01FF]]". This only really matters for + * difference; intersection is commutative. + * + * + *
[a]The set containing 'a' + *
[a-z]The set containing 'a' + * through 'z' and all letters in between, in Unicode order + *
[^a-z]The set containing + * all characters but 'a' through 'z', + * that is, U+0000 through 'a'-1 and 'z'+1 through U+10FFFF + *
[[pat1][pat2]] + * The union of sets specified by pat1 and pat2 + *
[[pat1]&[pat2]] + * The intersection of sets specified by pat1 and pat2 + *
[[pat1]-[pat2]] + * The asymmetric difference of sets specified by pat1 and + * pat2 + *
[:Lu:] or \\p{Lu} + * The set of characters having the specified + * Unicode property; in + * this case, Unicode uppercase letters + *
[:^Lu:] or \\P{Lu} + * The set of characters not having the given + * Unicode property + *
+ * + *

Warning: you cannot add an empty string ("") to a UnicodeSet.

+ * + *

Formal syntax

+ * + * \htmlonly
\endhtmlonly + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
pattern :=  ('[' '^'? item* ']') | + * property
item :=  char | (char '-' char) | pattern-expr
+ *
pattern-expr :=  pattern | pattern-expr pattern | + * pattern-expr op pattern
+ *
op :=  '&' | '-'
+ *
special :=  '[' | ']' | '-'
+ *
char :=  any character that is not special
+ * | ('\'
any character)
+ * | ('\\u' hex hex hex hex)
+ *
hex :=  any character for which + * Character.digit(c, 16) + * returns a non-negative result
property :=  a Unicode property set pattern
+ *
+ * + * + * + * + *
Legend: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
a := b  a may be replaced by b
a?zero or one instance of a
+ *
a*one or more instances of a
+ *
a | beither a or b
+ *
'a'the literal string between the quotes
+ *
+ * \htmlonly
\endhtmlonly + * + * @author Alan Liu + * @stable ICU 2.0 + */ +class U_COMMON_API UnicodeSet : public UnicodeFilter { + + int32_t len; // length of list used; 0 <= len <= capacity + int32_t capacity; // capacity of list + UChar32* list; // MUST be terminated with HIGH + BMPSet *bmpSet; // The set is frozen iff either bmpSet or stringSpan is not NULL. + UChar32* buffer; // internal buffer, may be NULL + int32_t bufferCapacity; // capacity of buffer + int32_t patLen; + + /** + * The pattern representation of this set. This may not be the + * most economical pattern. It is the pattern supplied to + * applyPattern(), with variables substituted and whitespace + * removed. For sets constructed without applyPattern(), or + * modified using the non-pattern API, this string will be empty, + * indicating that toPattern() must generate a pattern + * representation from the inversion list. + */ + UChar *pat; + UVector* strings; // maintained in sorted order + UnicodeSetStringSpan *stringSpan; + +public: + + enum { + /** + * Minimum value that can be stored in a UnicodeSet. + * @stable ICU 2.4 + */ + MIN_VALUE = 0, + + /** + * Maximum value that can be stored in a UnicodeSet. + * @stable ICU 2.4 + */ + MAX_VALUE = 0x10ffff + }; + + //---------------------------------------------------------------- + // Constructors &c + //---------------------------------------------------------------- + +public: + + /** + * Constructs an empty set. + * @stable ICU 2.0 + */ + UnicodeSet(); + + /** + * Constructs a set containing the given range. If end > + * start then an empty set is created. + * + * @param start first character, inclusive, of range + * @param end last character, inclusive, of range + * @stable ICU 2.4 + */ + UnicodeSet(UChar32 start, UChar32 end); + + /** + * Constructs a set from the given pattern. See the class + * description for the syntax of the pattern language. + * @param pattern a string specifying what characters are in the set + * @param status returns U_ILLEGAL_ARGUMENT_ERROR if the pattern + * contains a syntax error. + * @stable ICU 2.0 + */ + UnicodeSet(const UnicodeString& pattern, + UErrorCode& status); + + /** + * Constructs a set from the given pattern. See the class + * description for the syntax of the pattern language. + * @param pattern a string specifying what characters are in the set + * @param options bitmask for options to apply to the pattern. + * Valid options are USET_IGNORE_SPACE and USET_CASE_INSENSITIVE. + * @param symbols a symbol table mapping variable names to values + * and stand-in characters to UnicodeSets; may be NULL + * @param status returns U_ILLEGAL_ARGUMENT_ERROR if the pattern + * contains a syntax error. + * @internal + */ + UnicodeSet(const UnicodeString& pattern, + uint32_t options, + const SymbolTable* symbols, + UErrorCode& status); + + /** + * Constructs a set from the given pattern. See the class description + * for the syntax of the pattern language. + * @param pattern a string specifying what characters are in the set + * @param pos on input, the position in pattern at which to start parsing. + * On output, the position after the last character parsed. + * @param options bitmask for options to apply to the pattern. + * Valid options are USET_IGNORE_SPACE and USET_CASE_INSENSITIVE. + * @param symbols a symbol table mapping variable names to values + * and stand-in characters to UnicodeSets; may be NULL + * @param status input-output error code + * @stable ICU 2.8 + */ + UnicodeSet(const UnicodeString& pattern, ParsePosition& pos, + uint32_t options, + const SymbolTable* symbols, + UErrorCode& status); + + /** + * Constructs a set that is identical to the given UnicodeSet. + * @stable ICU 2.0 + */ + UnicodeSet(const UnicodeSet& o); + + /** + * Destructs the set. + * @stable ICU 2.0 + */ + virtual ~UnicodeSet(); + + /** + * Assigns this object to be a copy of another. + * A frozen set will not be modified. + * @stable ICU 2.0 + */ + UnicodeSet& operator=(const UnicodeSet& o); + + /** + * Compares the specified object with this set for equality. Returns + * true if the two sets + * have the same size, and every member of the specified set is + * contained in this set (or equivalently, every member of this set is + * contained in the specified set). + * + * @param o set to be compared for equality with this set. + * @return true if the specified set is equal to this set. + * @stable ICU 2.0 + */ + virtual UBool operator==(const UnicodeSet& o) const; + + /** + * Compares the specified object with this set for equality. Returns + * true if the specified set is not equal to this set. + * @stable ICU 2.0 + */ + UBool operator!=(const UnicodeSet& o) const; + + /** + * Returns a copy of this object. All UnicodeFunctor objects have + * to support cloning in order to allow classes using + * UnicodeFunctors, such as Transliterator, to implement cloning. + * If this set is frozen, then the clone will be frozen as well. + * Use cloneAsThawed() for a mutable clone of a frozen set. + * @see cloneAsThawed + * @stable ICU 2.0 + */ + virtual UnicodeFunctor* clone() const; + + /** + * Returns the hash code value for this set. + * + * @return the hash code value for this set. + * @see Object#hashCode() + * @stable ICU 2.0 + */ + virtual int32_t hashCode(void) const; + + //---------------------------------------------------------------- + // Freezable API + //---------------------------------------------------------------- + + /** + * Determines whether the set has been frozen (made immutable) or not. + * See the ICU4J Freezable interface for details. + * @return TRUE/FALSE for whether the set has been frozen + * @see freeze + * @see cloneAsThawed + * @draft ICU 3.8 + */ + inline UBool isFrozen() const; + + /** + * Freeze the set (make it immutable). + * Once frozen, it cannot be unfrozen and is therefore thread-safe + * until it is deleted. + * See the ICU4J Freezable interface for details. + * Freezing the set may also make some operations faster, for example + * contains() and span(). + * A frozen set will not be modified. (It remains frozen.) + * @return this set. + * @see isFrozen + * @see cloneAsThawed + * @draft ICU 3.8 + */ + UnicodeFunctor *freeze(); + + /** + * Clone the set and make the clone mutable. + * See the ICU4J Freezable interface for details. + * @return the mutable clone + * @see freeze + * @see isFrozen + * @draft ICU 3.8 + */ + UnicodeFunctor *cloneAsThawed() const; + + //---------------------------------------------------------------- + // Public API + //---------------------------------------------------------------- + + /** + * Make this object represent the range start - end. + * If end > start then this object is set to an + * an empty range. + * A frozen set will not be modified. + * + * @param start first character in the set, inclusive + * @param end last character in the set, inclusive + * @stable ICU 2.4 + */ + UnicodeSet& set(UChar32 start, UChar32 end); + + /** + * Return true if the given position, in the given pattern, appears + * to be the start of a UnicodeSet pattern. + * @stable ICU 2.4 + */ + static UBool resemblesPattern(const UnicodeString& pattern, + int32_t pos); + + /** + * Modifies this set to represent the set specified by the given + * pattern, optionally ignoring white space. See the class + * description for the syntax of the pattern language. + * A frozen set will not be modified. + * @param pattern a string specifying what characters are in the set + * @param status returns U_ILLEGAL_ARGUMENT_ERROR if the pattern + * contains a syntax error. + * Empties the set passed before applying the pattern. + * @return a reference to this + * @stable ICU 2.0 + */ + UnicodeSet& applyPattern(const UnicodeString& pattern, + UErrorCode& status); + + /** + * Modifies this set to represent the set specified by the given + * pattern, optionally ignoring white space. See the class + * description for the syntax of the pattern language. + * A frozen set will not be modified. + * @param pattern a string specifying what characters are in the set + * @param options bitmask for options to apply to the pattern. + * Valid options are USET_IGNORE_SPACE and USET_CASE_INSENSITIVE. + * @param symbols a symbol table mapping variable names to + * values and stand-ins to UnicodeSets; may be NULL + * @param status returns U_ILLEGAL_ARGUMENT_ERROR if the pattern + * contains a syntax error. + * Empties the set passed before applying the pattern. + * @return a reference to this + * @internal + */ + UnicodeSet& applyPattern(const UnicodeString& pattern, + uint32_t options, + const SymbolTable* symbols, + UErrorCode& status); + + /** + * Parses the given pattern, starting at the given position. The + * character at pattern.charAt(pos.getIndex()) must be '[', or the + * parse fails. Parsing continues until the corresponding closing + * ']'. If a syntax error is encountered between the opening and + * closing brace, the parse fails. Upon return from a successful + * parse, the ParsePosition is updated to point to the character + * following the closing ']', and a StringBuffer containing a + * pairs list for the parsed pattern is returned. This method calls + * itself recursively to parse embedded subpatterns. + * Empties the set passed before applying the pattern. + * A frozen set will not be modified. + * + * @param pattern the string containing the pattern to be parsed. + * The portion of the string from pos.getIndex(), which must be a + * '[', to the corresponding closing ']', is parsed. + * @param pos upon entry, the position at which to being parsing. + * The character at pattern.charAt(pos.getIndex()) must be a '['. + * Upon return from a successful parse, pos.getIndex() is either + * the character after the closing ']' of the parsed pattern, or + * pattern.length() if the closing ']' is the last character of + * the pattern string. + * @param options bitmask for options to apply to the pattern. + * Valid options are USET_IGNORE_SPACE and USET_CASE_INSENSITIVE. + * @param symbols a symbol table mapping variable names to + * values and stand-ins to UnicodeSets; may be NULL + * @param status returns U_ILLEGAL_ARGUMENT_ERROR if the pattern + * contains a syntax error. + * @return a reference to this + * @stable ICU 2.8 + */ + UnicodeSet& applyPattern(const UnicodeString& pattern, + ParsePosition& pos, + uint32_t options, + const SymbolTable* symbols, + UErrorCode& status); + + /** + * Returns a string representation of this set. If the result of + * calling this function is passed to a UnicodeSet constructor, it + * will produce another set that is equal to this one. + * A frozen set will not be modified. + * @param result the string to receive the rules. Previous + * contents will be deleted. + * @param escapeUnprintable if TRUE then convert unprintable + * character to their hex escape representations, \\uxxxx or + * \\Uxxxxxxxx. Unprintable characters are those other than + * U+000A, U+0020..U+007E. + * @stable ICU 2.0 + */ + virtual UnicodeString& toPattern(UnicodeString& result, + UBool escapeUnprintable = FALSE) const; + + /** + * Modifies this set to contain those code points which have the given value + * for the given binary or enumerated property, as returned by + * u_getIntPropertyValue. Prior contents of this set are lost. + * A frozen set will not be modified. + * + * @param prop a property in the range UCHAR_BIN_START..UCHAR_BIN_LIMIT-1 + * or UCHAR_INT_START..UCHAR_INT_LIMIT-1 + * or UCHAR_MASK_START..UCHAR_MASK_LIMIT-1. + * + * @param value a value in the range u_getIntPropertyMinValue(prop).. + * u_getIntPropertyMaxValue(prop), with one exception. If prop is + * UCHAR_GENERAL_CATEGORY_MASK, then value should not be a UCharCategory, but + * rather a mask value produced by U_GET_GC_MASK(). This allows grouped + * categories such as [:L:] to be represented. + * + * @param ec error code input/output parameter + * + * @return a reference to this set + * + * @stable ICU 2.4 + */ + UnicodeSet& applyIntPropertyValue(UProperty prop, + int32_t value, + UErrorCode& ec); + + /** + * Modifies this set to contain those code points which have the + * given value for the given property. Prior contents of this + * set are lost. + * A frozen set will not be modified. + * + * @param prop a property alias, either short or long. The name is matched + * loosely. See PropertyAliases.txt for names and a description of loose + * matching. If the value string is empty, then this string is interpreted + * as either a General_Category value alias, a Script value alias, a binary + * property alias, or a special ID. Special IDs are matched loosely and + * correspond to the following sets: + * + * "ANY" = [\\u0000-\\U0010FFFF], + * "ASCII" = [\\u0000-\\u007F], + * "Assigned" = [:^Cn:]. + * + * @param value a value alias, either short or long. The name is matched + * loosely. See PropertyValueAliases.txt for names and a description of + * loose matching. In addition to aliases listed, numeric values and + * canonical combining classes may be expressed numerically, e.g., ("nv", + * "0.5") or ("ccc", "220"). The value string may also be empty. + * + * @param ec error code input/output parameter + * + * @return a reference to this set + * + * @stable ICU 2.4 + */ + UnicodeSet& applyPropertyAlias(const UnicodeString& prop, + const UnicodeString& value, + UErrorCode& ec); + + /** + * Returns the number of elements in this set (its cardinality). + * Note than the elements of a set may include both individual + * codepoints and strings. + * + * @return the number of elements in this set (its cardinality). + * @stable ICU 2.0 + */ + virtual int32_t size(void) const; + + /** + * Returns true if this set contains no elements. + * + * @return true if this set contains no elements. + * @stable ICU 2.0 + */ + virtual UBool isEmpty(void) const; + + /** + * Returns true if this set contains the given character. + * This function works faster with a frozen set. + * @param c character to be checked for containment + * @return true if the test condition is met + * @stable ICU 2.0 + */ + virtual UBool contains(UChar32 c) const; + + /** + * Returns true if this set contains every character + * of the given range. + * @param start first character, inclusive, of the range + * @param end last character, inclusive, of the range + * @return true if the test condition is met + * @stable ICU 2.0 + */ + virtual UBool contains(UChar32 start, UChar32 end) const; + + /** + * Returns true if this set contains the given + * multicharacter string. + * @param s string to be checked for containment + * @return true if this set contains the specified string + * @stable ICU 2.4 + */ + UBool contains(const UnicodeString& s) const; + + /** + * Returns true if this set contains all the characters and strings + * of the given set. + * @param c set to be checked for containment + * @return true if the test condition is met + * @stable ICU 2.4 + */ + virtual UBool containsAll(const UnicodeSet& c) const; + + /** + * Returns true if this set contains all the characters + * of the given string. + * @param s string containing characters to be checked for containment + * @return true if the test condition is met + * @stable ICU 2.4 + */ + UBool containsAll(const UnicodeString& s) const; + + /** + * Returns true if this set contains none of the characters + * of the given range. + * @param start first character, inclusive, of the range + * @param end last character, inclusive, of the range + * @return true if the test condition is met + * @stable ICU 2.4 + */ + UBool containsNone(UChar32 start, UChar32 end) const; + + /** + * Returns true if this set contains none of the characters and strings + * of the given set. + * @param c set to be checked for containment + * @return true if the test condition is met + * @stable ICU 2.4 + */ + UBool containsNone(const UnicodeSet& c) const; + + /** + * Returns true if this set contains none of the characters + * of the given string. + * @param s string containing characters to be checked for containment + * @return true if the test condition is met + * @stable ICU 2.4 + */ + UBool containsNone(const UnicodeString& s) const; + + /** + * Returns true if this set contains one or more of the characters + * in the given range. + * @param start first character, inclusive, of the range + * @param end last character, inclusive, of the range + * @return true if the condition is met + * @stable ICU 2.4 + */ + inline UBool containsSome(UChar32 start, UChar32 end) const; + + /** + * Returns true if this set contains one or more of the characters + * and strings of the given set. + * @param s The set to be checked for containment + * @return true if the condition is met + * @stable ICU 2.4 + */ + inline UBool containsSome(const UnicodeSet& s) const; + + /** + * Returns true if this set contains one or more of the characters + * of the given string. + * @param s string containing characters to be checked for containment + * @return true if the condition is met + * @stable ICU 2.4 + */ + inline UBool containsSome(const UnicodeString& s) const; + + /** + * Returns the length of the initial substring of the input string which + * consists only of characters and strings that are contained in this set + * (USET_SPAN_CONTAINED, USET_SPAN_SIMPLE), + * or only of characters and strings that are not contained + * in this set (USET_SPAN_NOT_CONTAINED). + * See USetSpanCondition for details. + * Similar to the strspn() C library function. + * Unpaired surrogates are treated according to contains() of their surrogate code points. + * This function works faster with a frozen set and with a non-negative string length argument. + * @param s start of the string + * @param length of the string; can be -1 for NUL-terminated + * @param spanCondition specifies the containment condition + * @return the length of the initial substring according to the spanCondition; + * 0 if the start of the string does not fit the spanCondition + * @draft ICU 3.8 + * @see USetSpanCondition + */ + int32_t span(const UChar *s, int32_t length, USetSpanCondition spanCondition) const; + + /** + * Returns the start of the trailing substring of the input string which + * consists only of characters and strings that are contained in this set + * (USET_SPAN_CONTAINED, USET_SPAN_SIMPLE), + * or only of characters and strings that are not contained + * in this set (USET_SPAN_NOT_CONTAINED). + * See USetSpanCondition for details. + * Unpaired surrogates are treated according to contains() of their surrogate code points. + * This function works faster with a frozen set and with a non-negative string length argument. + * @param s start of the string + * @param length of the string; can be -1 for NUL-terminated + * @param spanCondition specifies the containment condition + * @return the start of the trailing substring according to the spanCondition; + * the string length if the end of the string does not fit the spanCondition + * @draft ICU 3.8 + * @see USetSpanCondition + */ + int32_t spanBack(const UChar *s, int32_t length, USetSpanCondition spanCondition) const; + + /** + * Returns the length of the initial substring of the input string which + * consists only of characters and strings that are contained in this set + * (USET_SPAN_CONTAINED, USET_SPAN_SIMPLE), + * or only of characters and strings that are not contained + * in this set (USET_SPAN_NOT_CONTAINED). + * See USetSpanCondition for details. + * Similar to the strspn() C library function. + * Malformed byte sequences are treated according to contains(0xfffd). + * This function works faster with a frozen set and with a non-negative string length argument. + * @param s start of the string (UTF-8) + * @param length of the string; can be -1 for NUL-terminated + * @param spanCondition specifies the containment condition + * @return the length of the initial substring according to the spanCondition; + * 0 if the start of the string does not fit the spanCondition + * @draft ICU 3.8 + * @see USetSpanCondition + */ + int32_t spanUTF8(const char *s, int32_t length, USetSpanCondition spanCondition) const; + + /** + * Returns the start of the trailing substring of the input string which + * consists only of characters and strings that are contained in this set + * (USET_SPAN_CONTAINED, USET_SPAN_SIMPLE), + * or only of characters and strings that are not contained + * in this set (USET_SPAN_NOT_CONTAINED). + * See USetSpanCondition for details. + * Malformed byte sequences are treated according to contains(0xfffd). + * This function works faster with a frozen set and with a non-negative string length argument. + * @param s start of the string (UTF-8) + * @param length of the string; can be -1 for NUL-terminated + * @param spanCondition specifies the containment condition + * @return the start of the trailing substring according to the spanCondition; + * the string length if the end of the string does not fit the spanCondition + * @draft ICU 3.8 + * @see USetSpanCondition + */ + int32_t spanBackUTF8(const char *s, int32_t length, USetSpanCondition spanCondition) const; + + /** + * Implement UnicodeMatcher::matches() + * @stable ICU 2.4 + */ + virtual UMatchDegree matches(const Replaceable& text, + int32_t& offset, + int32_t limit, + UBool incremental); + +private: + /** + * Returns the longest match for s in text at the given position. + * If limit > start then match forward from start+1 to limit + * matching all characters except s.charAt(0). If limit < start, + * go backward starting from start-1 matching all characters + * except s.charAt(s.length()-1). This method assumes that the + * first character, text.charAt(start), matches s, so it does not + * check it. + * @param text the text to match + * @param start the first character to match. In the forward + * direction, text.charAt(start) is matched against s.charAt(0). + * In the reverse direction, it is matched against + * s.charAt(s.length()-1). + * @param limit the limit offset for matching, either last+1 in + * the forward direction, or last-1 in the reverse direction, + * where last is the index of the last character to match. + * @return If part of s matches up to the limit, return |limit - + * start|. If all of s matches before reaching the limit, return + * s.length(). If there is a mismatch between s and text, return + * 0 + */ + static int32_t matchRest(const Replaceable& text, + int32_t start, int32_t limit, + const UnicodeString& s); + + /** + * Returns the smallest value i such that c < list[i]. Caller + * must ensure that c is a legal value or this method will enter + * an infinite loop. This method performs a binary search. + * @param c a character in the range MIN_VALUE..MAX_VALUE + * inclusive + * @return the smallest integer i in the range 0..len-1, + * inclusive, such that c < list[i] + */ + int32_t findCodePoint(UChar32 c) const; + +public: + + /** + * Implementation of UnicodeMatcher API. Union the set of all + * characters that may be matched by this object into the given + * set. + * @param toUnionTo the set into which to union the source characters + * @stable ICU 2.4 + */ + virtual void addMatchSetTo(UnicodeSet& toUnionTo) const; + + /** + * Returns the index of the given character within this set, where + * the set is ordered by ascending code point. If the character + * is not in this set, return -1. The inverse of this method is + * charAt(). + * @return an index from 0..size()-1, or -1 + * @stable ICU 2.4 + */ + int32_t indexOf(UChar32 c) const; + + /** + * Returns the character at the given index within this set, where + * the set is ordered by ascending code point. If the index is + * out of range, return (UChar32)-1. The inverse of this method is + * indexOf(). + * @param index an index from 0..size()-1 + * @return the character at the given index, or (UChar32)-1. + * @stable ICU 2.4 + */ + UChar32 charAt(int32_t index) const; + + /** + * Adds the specified range to this set if it is not already + * present. If this set already contains the specified range, + * the call leaves this set unchanged. If end > start + * then an empty range is added, leaving the set unchanged. + * This is equivalent to a boolean logic OR, or a set UNION. + * A frozen set will not be modified. + * + * @param start first character, inclusive, of range to be added + * to this set. + * @param end last character, inclusive, of range to be added + * to this set. + * @stable ICU 2.0 + */ + virtual UnicodeSet& add(UChar32 start, UChar32 end); + + /** + * Adds the specified character to this set if it is not already + * present. If this set already contains the specified character, + * the call leaves this set unchanged. + * A frozen set will not be modified. + * @stable ICU 2.0 + */ + UnicodeSet& add(UChar32 c); + + /** + * Adds the specified multicharacter to this set if it is not already + * present. If this set already contains the multicharacter, + * the call leaves this set unchanged. + * Thus "ch" => {"ch"} + *
Warning: you cannot add an empty string ("") to a UnicodeSet. + * A frozen set will not be modified. + * @param s the source string + * @return this object, for chaining + * @stable ICU 2.4 + */ + UnicodeSet& add(const UnicodeString& s); + + private: + /** + * @return a code point IF the string consists of a single one. + * otherwise returns -1. + * @param string to test + */ + static int32_t getSingleCP(const UnicodeString& s); + + void _add(const UnicodeString& s); + + public: + /** + * Adds each of the characters in this string to the set. Thus "ch" => {"c", "h"} + * If this set already any particular character, it has no effect on that character. + * A frozen set will not be modified. + * @param s the source string + * @return this object, for chaining + * @stable ICU 2.4 + */ + UnicodeSet& addAll(const UnicodeString& s); + + /** + * Retains EACH of the characters in this string. Note: "ch" == {"c", "h"} + * If this set already any particular character, it has no effect on that character. + * A frozen set will not be modified. + * @param s the source string + * @return this object, for chaining + * @stable ICU 2.4 + */ + UnicodeSet& retainAll(const UnicodeString& s); + + /** + * Complement EACH of the characters in this string. Note: "ch" == {"c", "h"} + * If this set already any particular character, it has no effect on that character. + * A frozen set will not be modified. + * @param s the source string + * @return this object, for chaining + * @stable ICU 2.4 + */ + UnicodeSet& complementAll(const UnicodeString& s); + + /** + * Remove EACH of the characters in this string. Note: "ch" == {"c", "h"} + * If this set already any particular character, it has no effect on that character. + * A frozen set will not be modified. + * @param s the source string + * @return this object, for chaining + * @stable ICU 2.4 + */ + UnicodeSet& removeAll(const UnicodeString& s); + + /** + * Makes a set from a multicharacter string. Thus "ch" => {"ch"} + *
Warning: you cannot add an empty string ("") to a UnicodeSet. + * @param s the source string + * @return a newly created set containing the given string. + * The caller owns the return object and is responsible for deleting it. + * @stable ICU 2.4 + */ + static UnicodeSet* U_EXPORT2 createFrom(const UnicodeString& s); + + + /** + * Makes a set from each of the characters in the string. Thus "ch" => {"c", "h"} + * @param s the source string + * @return a newly created set containing the given characters + * The caller owns the return object and is responsible for deleting it. + * @stable ICU 2.4 + */ + static UnicodeSet* U_EXPORT2 createFromAll(const UnicodeString& s); + + /** + * Retain only the elements in this set that are contained in the + * specified range. If end > start then an empty range is + * retained, leaving the set empty. This is equivalent to + * a boolean logic AND, or a set INTERSECTION. + * A frozen set will not be modified. + * + * @param start first character, inclusive, of range to be retained + * to this set. + * @param end last character, inclusive, of range to be retained + * to this set. + * @stable ICU 2.0 + */ + virtual UnicodeSet& retain(UChar32 start, UChar32 end); + + + /** + * Retain the specified character from this set if it is present. + * A frozen set will not be modified. + * @stable ICU 2.0 + */ + UnicodeSet& retain(UChar32 c); + + /** + * Removes the specified range from this set if it is present. + * The set will not contain the specified range once the call + * returns. If end > start then an empty range is + * removed, leaving the set unchanged. + * A frozen set will not be modified. + * + * @param start first character, inclusive, of range to be removed + * from this set. + * @param end last character, inclusive, of range to be removed + * from this set. + * @stable ICU 2.0 + */ + virtual UnicodeSet& remove(UChar32 start, UChar32 end); + + /** + * Removes the specified character from this set if it is present. + * The set will not contain the specified range once the call + * returns. + * A frozen set will not be modified. + * @stable ICU 2.0 + */ + UnicodeSet& remove(UChar32 c); + + /** + * Removes the specified string from this set if it is present. + * The set will not contain the specified character once the call + * returns. + * A frozen set will not be modified. + * @param s the source string + * @return this object, for chaining + * @stable ICU 2.4 + */ + UnicodeSet& remove(const UnicodeString& s); + + /** + * Inverts this set. This operation modifies this set so that + * its value is its complement. This is equivalent to + * complement(MIN_VALUE, MAX_VALUE). + * A frozen set will not be modified. + * @stable ICU 2.0 + */ + virtual UnicodeSet& complement(void); + + /** + * Complements the specified range in this set. Any character in + * the range will be removed if it is in this set, or will be + * added if it is not in this set. If end > start + * then an empty range is complemented, leaving the set unchanged. + * This is equivalent to a boolean logic XOR. + * A frozen set will not be modified. + * + * @param start first character, inclusive, of range to be removed + * from this set. + * @param end last character, inclusive, of range to be removed + * from this set. + * @stable ICU 2.0 + */ + virtual UnicodeSet& complement(UChar32 start, UChar32 end); + + /** + * Complements the specified character in this set. The character + * will be removed if it is in this set, or will be added if it is + * not in this set. + * A frozen set will not be modified. + * @stable ICU 2.0 + */ + UnicodeSet& complement(UChar32 c); + + /** + * Complement the specified string in this set. + * The set will not contain the specified string once the call + * returns. + *
Warning: you cannot add an empty string ("") to a UnicodeSet. + * A frozen set will not be modified. + * @param s the string to complement + * @return this object, for chaining + * @stable ICU 2.4 + */ + UnicodeSet& complement(const UnicodeString& s); + + /** + * Adds all of the elements in the specified set to this set if + * they're not already present. This operation effectively + * modifies this set so that its value is the union of the two + * sets. The behavior of this operation is unspecified if the specified + * collection is modified while the operation is in progress. + * A frozen set will not be modified. + * + * @param c set whose elements are to be added to this set. + * @see #add(UChar32, UChar32) + * @stable ICU 2.0 + */ + virtual UnicodeSet& addAll(const UnicodeSet& c); + + /** + * Retains only the elements in this set that are contained in the + * specified set. In other words, removes from this set all of + * its elements that are not contained in the specified set. This + * operation effectively modifies this set so that its value is + * the intersection of the two sets. + * A frozen set will not be modified. + * + * @param c set that defines which elements this set will retain. + * @stable ICU 2.0 + */ + virtual UnicodeSet& retainAll(const UnicodeSet& c); + + /** + * Removes from this set all of its elements that are contained in the + * specified set. This operation effectively modifies this + * set so that its value is the asymmetric set difference of + * the two sets. + * A frozen set will not be modified. + * + * @param c set that defines which elements will be removed from + * this set. + * @stable ICU 2.0 + */ + virtual UnicodeSet& removeAll(const UnicodeSet& c); + + /** + * Complements in this set all elements contained in the specified + * set. Any character in the other set will be removed if it is + * in this set, or will be added if it is not in this set. + * A frozen set will not be modified. + * + * @param c set that defines which elements will be xor'ed from + * this set. + * @stable ICU 2.4 + */ + virtual UnicodeSet& complementAll(const UnicodeSet& c); + + /** + * Removes all of the elements from this set. This set will be + * empty after this call returns. + * A frozen set will not be modified. + * @stable ICU 2.0 + */ + virtual UnicodeSet& clear(void); + + /** + * Close this set over the given attribute. For the attribute + * USET_CASE, the result is to modify this set so that: + * + * 1. For each character or string 'a' in this set, all strings or + * characters 'b' such that foldCase(a) == foldCase(b) are added + * to this set. + * + * 2. For each string 'e' in the resulting set, if e != + * foldCase(e), 'e' will be removed. + * + * Example: [aq\\u00DF{Bc}{bC}{Fi}] => [aAqQ\\u00DF\\uFB01{ss}{bc}{fi}] + * + * (Here foldCase(x) refers to the operation u_strFoldCase, and a + * == b denotes that the contents are the same, not pointer + * comparison.) + * + * A frozen set will not be modified. + * + * @param attribute bitmask for attributes to close over. + * Currently only the USET_CASE bit is supported. Any undefined bits + * are ignored. + * @return a reference to this set. + * @internal + */ + UnicodeSet& closeOver(int32_t attribute); + + /** + * Iteration method that returns the number of ranges contained in + * this set. + * @see #getRangeStart + * @see #getRangeEnd + * @stable ICU 2.4 + */ + virtual int32_t getRangeCount(void) const; + + /** + * Iteration method that returns the first character in the + * specified range of this set. + * @see #getRangeCount + * @see #getRangeEnd + * @stable ICU 2.4 + */ + virtual UChar32 getRangeStart(int32_t index) const; + + /** + * Iteration method that returns the last character in the + * specified range of this set. + * @see #getRangeStart + * @see #getRangeEnd + * @stable ICU 2.4 + */ + virtual UChar32 getRangeEnd(int32_t index) const; + + /** + * Serializes this set into an array of 16-bit integers. Serialization + * (currently) only records the characters in the set; multicharacter + * strings are ignored. + * + * The array has following format (each line is one 16-bit + * integer): + * + * length = (n+2*m) | (m!=0?0x8000:0) + * bmpLength = n; present if m!=0 + * bmp[0] + * bmp[1] + * ... + * bmp[n-1] + * supp-high[0] + * supp-low[0] + * supp-high[1] + * supp-low[1] + * ... + * supp-high[m-1] + * supp-low[m-1] + * + * The array starts with a header. After the header are n bmp + * code points, then m supplementary code points. Either n or m + * or both may be zero. n+2*m is always <= 0x7FFF. + * + * If there are no supplementary characters (if m==0) then the + * header is one 16-bit integer, 'length', with value n. + * + * If there are supplementary characters (if m!=0) then the header + * is two 16-bit integers. The first, 'length', has value + * (n+2*m)|0x8000. The second, 'bmpLength', has value n. + * + * After the header the code points are stored in ascending order. + * Supplementary code points are stored as most significant 16 + * bits followed by least significant 16 bits. + * + * @param dest pointer to buffer of destCapacity 16-bit integers. + * May be NULL only if destCapacity is zero. + * @param destCapacity size of dest, or zero. Must not be negative. + * @param ec error code. Will be set to U_INDEX_OUTOFBOUNDS_ERROR + * if n+2*m > 0x7FFF. Will be set to U_BUFFER_OVERFLOW_ERROR if + * n+2*m+(m!=0?2:1) > destCapacity. + * @return the total length of the serialized format, including + * the header, that is, n+2*m+(m!=0?2:1), or 0 on error other + * than U_BUFFER_OVERFLOW_ERROR. + * @stable ICU 2.4 + */ + int32_t serialize(uint16_t *dest, int32_t destCapacity, UErrorCode& ec) const; + + /** + * Reallocate this objects internal structures to take up the least + * possible space, without changing this object's value. + * A frozen set will not be modified. + * @stable ICU 2.4 + */ + virtual UnicodeSet& compact(); + + /** + * Return the class ID for this class. This is useful only for + * comparing to a return value from getDynamicClassID(). For example: + *
+     * .      Base* polymorphic_pointer = createPolymorphicObject();
+     * .      if (polymorphic_pointer->getDynamicClassID() ==
+     * .          Derived::getStaticClassID()) ...
+     * 
+ * @return The class ID for all objects of this class. + * @stable ICU 2.0 + */ + static UClassID U_EXPORT2 getStaticClassID(void); + + /** + * Implement UnicodeFunctor API. + * + * @return The class ID for this object. All objects of a given + * class have the same class ID. Objects of other classes have + * different class IDs. + * @stable ICU 2.4 + */ + virtual UClassID getDynamicClassID(void) const; + +private: + + // Private API for the USet API + + friend class USetAccess; + + int32_t getStringCount() const; + + const UnicodeString* getString(int32_t index) const; + + //---------------------------------------------------------------- + // RuleBasedTransliterator support + //---------------------------------------------------------------- + +private: + + /** + * Returns true if this set contains any character whose low byte + * is the given value. This is used by RuleBasedTransliterator for + * indexing. + */ + virtual UBool matchesIndexValue(uint8_t v) const; + +private: + + //---------------------------------------------------------------- + // Implementation: Clone as thawed (see ICU4J Freezable) + //---------------------------------------------------------------- + + UnicodeSet(const UnicodeSet& o, UBool /* asThawed */); + + //---------------------------------------------------------------- + // Implementation: Pattern parsing + //---------------------------------------------------------------- + + void applyPattern(RuleCharacterIterator& chars, + const SymbolTable* symbols, + UnicodeString& rebuiltPat, + uint32_t options, + UErrorCode& ec); + + //---------------------------------------------------------------- + // Implementation: Utility methods + //---------------------------------------------------------------- + + void ensureCapacity(int32_t newLen); + + void ensureBufferCapacity(int32_t newLen); + + void swapBuffers(void); + + UBool allocateStrings(UErrorCode &status); + + UnicodeString& _toPattern(UnicodeString& result, + UBool escapeUnprintable) const; + + UnicodeString& _generatePattern(UnicodeString& result, + UBool escapeUnprintable) const; + + static void _appendToPat(UnicodeString& buf, const UnicodeString& s, UBool escapeUnprintable); + + static void _appendToPat(UnicodeString& buf, UChar32 c, UBool escapeUnprintable); + + //---------------------------------------------------------------- + // Implementation: Fundamental operators + //---------------------------------------------------------------- + + void exclusiveOr(const UChar32* other, int32_t otherLen, int8_t polarity); + + void add(const UChar32* other, int32_t otherLen, int8_t polarity); + + void retain(const UChar32* other, int32_t otherLen, int8_t polarity); + + /** + * Return true if the given position, in the given pattern, appears + * to be the start of a property set pattern [:foo:], \\p{foo}, or + * \\P{foo}, or \\N{name}. + */ + static UBool resemblesPropertyPattern(const UnicodeString& pattern, + int32_t pos); + + static UBool resemblesPropertyPattern(RuleCharacterIterator& chars, + int32_t iterOpts); + + /** + * Parse the given property pattern at the given parse position + * and set this UnicodeSet to the result. + * + * The original design document is out of date, but still useful. + * Ignore the property and value names: + * http://source.icu-project.org/repos/icu/icuhtml/trunk/design/unicodeset_properties.html + * + * Recognized syntax: + * + * [:foo:] [:^foo:] - white space not allowed within "[:" or ":]" + * \\p{foo} \\P{foo} - white space not allowed within "\\p" or "\\P" + * \\N{name} - white space not allowed within "\\N" + * + * Other than the above restrictions, white space is ignored. Case + * is ignored except in "\\p" and "\\P" and "\\N". In 'name' leading + * and trailing space is deleted, and internal runs of whitespace + * are collapsed to a single space. + * + * We support binary properties, enumerated properties, and the + * following non-enumerated properties: + * + * Numeric_Value + * Name + * Unicode_1_Name + * + * @param pattern the pattern string + * @param ppos on entry, the position at which to begin parsing. + * This should be one of the locations marked '^': + * + * [:blah:] \\p{blah} \\P{blah} \\N{name} + * ^ % ^ % ^ % ^ % + * + * On return, the position after the last character parsed, that is, + * the locations marked '%'. If the parse fails, ppos is returned + * unchanged. + * @return a reference to this. + */ + UnicodeSet& applyPropertyPattern(const UnicodeString& pattern, + ParsePosition& ppos, + UErrorCode &ec); + + void applyPropertyPattern(RuleCharacterIterator& chars, + UnicodeString& rebuiltPat, + UErrorCode& ec); + + /** + * A filter that returns TRUE if the given code point should be + * included in the UnicodeSet being constructed. + */ + typedef UBool (*Filter)(UChar32 codePoint, void* context); + + /** + * Given a filter, set this UnicodeSet to the code points + * contained by that filter. The filter MUST be + * property-conformant. That is, if it returns value v for one + * code point, then it must return v for all affiliated code + * points, as defined by the inclusions list. See + * getInclusions(). + * src is a UPropertySource value. + */ + void applyFilter(Filter filter, + void* context, + int32_t src, + UErrorCode &status); + + /** + * Set the new pattern to cache. + */ + void setPattern(const UnicodeString& newPat); + /** + * Release existing cached pattern. + */ + void releasePattern(); + + friend class UnicodeSetIterator; +}; + +inline UBool UnicodeSet::operator!=(const UnicodeSet& o) const { + return !operator==(o); +} + +inline UBool UnicodeSet::isFrozen() const { + return (UBool)(bmpSet!=NULL || stringSpan!=NULL); +} + +inline UBool UnicodeSet::containsSome(UChar32 start, UChar32 end) const { + return !containsNone(start, end); +} + +inline UBool UnicodeSet::containsSome(const UnicodeSet& s) const { + return !containsNone(s); +} + +inline UBool UnicodeSet::containsSome(const UnicodeString& s) const { + return !containsNone(s); +} + +U_NAMESPACE_END + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/unistr.h b/Build/source/libs/icu/icu-xetex/common/unicode/unistr.h new file mode 100644 index 00000000000..575998e4ed4 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/unistr.h @@ -0,0 +1,4149 @@ +/* +********************************************************************** +* Copyright (C) 1998-2007, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* +* File unistr.h +* +* Modification History: +* +* Date Name Description +* 09/25/98 stephen Creation. +* 11/11/98 stephen Changed per 11/9 code review. +* 04/20/99 stephen Overhauled per 4/16 code review. +* 11/18/99 aliu Made to inherit from Replaceable. Added method +* handleReplaceBetween(); other methods unchanged. +* 06/25/01 grhoten Remove dependency on iostream. +****************************************************************************** +*/ + +#ifndef UNISTR_H +#define UNISTR_H + +/** + * \file + * \brief C++ API: Unicode String + */ + +#include "unicode/rep.h" + +struct UConverter; // unicode/ucnv.h +class StringThreadTest; + +#ifndef U_COMPARE_CODE_POINT_ORDER +/* see also ustring.h and unorm.h */ +/** + * Option bit for u_strCaseCompare, u_strcasecmp, unorm_compare, etc: + * Compare strings in code point order instead of code unit order. + * @stable ICU 2.2 + */ +#define U_COMPARE_CODE_POINT_ORDER 0x8000 +#endif + +#ifndef USTRING_H +/** + * \ingroup ustring_ustrlen + */ +U_STABLE int32_t U_EXPORT2 +u_strlen(const UChar *s); +#endif + +U_NAMESPACE_BEGIN + +class Locale; // unicode/locid.h +class StringCharacterIterator; +class BreakIterator; // unicode/brkiter.h + +/* The include has been moved to unicode/ustream.h */ + +/** + * Constant to be used in the UnicodeString(char *, int32_t, EInvariant) constructor + * which constructs a Unicode string from an invariant-character char * string. + * About invariant characters see utypes.h. + * This constructor has no runtime dependency on conversion code and is + * therefore recommended over ones taking a charset name string + * (where the empty string "" indicates invariant-character conversion). + * + * @stable ICU 3.2 + */ +#define US_INV U_NAMESPACE_QUALIFIER UnicodeString::kInvariant + +/** + * Unicode String literals in C++. + * Dependent on the platform properties, different UnicodeString + * constructors should be used to create a UnicodeString object from + * a string literal. + * The macros are defined for maximum performance. + * They work only for strings that contain "invariant characters", i.e., + * only latin letters, digits, and some punctuation. + * See utypes.h for details. + * + * The string parameter must be a C string literal. + * The length of the string, not including the terminating + * NUL, must be specified as a constant. + * The U_STRING_DECL macro should be invoked exactly once for one + * such string variable before it is used. + * @stable ICU 2.0 + */ +#if defined(U_DECLARE_UTF16) +# define UNICODE_STRING(cs, _length) U_NAMESPACE_QUALIFIER UnicodeString(TRUE, (const UChar *)U_DECLARE_UTF16(cs), _length) +#elif U_SIZEOF_WCHAR_T==U_SIZEOF_UCHAR && (U_CHARSET_FAMILY==U_ASCII_FAMILY || (U_SIZEOF_UCHAR == 2 && defined(U_WCHAR_IS_UTF16))) +# define UNICODE_STRING(cs, _length) U_NAMESPACE_QUALIFIER UnicodeString(TRUE, (const UChar *)L ## cs, _length) +#elif U_SIZEOF_UCHAR==1 && U_CHARSET_FAMILY==U_ASCII_FAMILY +# define UNICODE_STRING(cs, _length) U_NAMESPACE_QUALIFIER UnicodeString(TRUE, (const UChar *)cs, _length) +#else +# define UNICODE_STRING(cs, _length) U_NAMESPACE_QUALIFIER UnicodeString(cs, _length, US_INV) +#endif + +/** + * Unicode String literals in C++. + * Dependent on the platform properties, different UnicodeString + * constructors should be used to create a UnicodeString object from + * a string literal. + * The macros are defined for improved performance. + * They work only for strings that contain "invariant characters", i.e., + * only latin letters, digits, and some punctuation. + * See utypes.h for details. + * + * The string parameter must be a C string literal. + * @stable ICU 2.0 + */ +#define UNICODE_STRING_SIMPLE(cs) UNICODE_STRING(cs, -1) + +/** + * UnicodeString is a string class that stores Unicode characters directly and provides + * similar functionality as the Java String and StringBuffer classes. + * It is a concrete implementation of the abstract class Replaceable (for transliteration). + * + * The UnicodeString class is not suitable for subclassing. + * + *

For an overview of Unicode strings in C and C++ see the + * User Guide Strings chapter.

+ * + *

In ICU, a Unicode string consists of 16-bit Unicode code units. + * A Unicode character may be stored with either one code unit + * (the most common case) or with a matched pair of special code units + * ("surrogates"). The data type for code units is UChar. + * For single-character handling, a Unicode character code point is a value + * in the range 0..0x10ffff. ICU uses the UChar32 type for code points.

+ * + *

Indexes and offsets into and lengths of strings always count code units, not code points. + * This is the same as with multi-byte char* strings in traditional string handling. + * Operations on partial strings typically do not test for code point boundaries. + * If necessary, the user needs to take care of such boundaries by testing for the code unit + * values or by using functions like + * UnicodeString::getChar32Start() and UnicodeString::getChar32Limit() + * (or, in C, the equivalent macros U16_SET_CP_START() and U16_SET_CP_LIMIT(), see utf.h).

+ * + * UnicodeString methods are more lenient with regard to input parameter values + * than other ICU APIs. In particular: + * - If indexes are out of bounds for a UnicodeString object + * (<0 or >length()) then they are "pinned" to the nearest boundary. + * - If primitive string pointer values (e.g., const UChar * or char *) + * for input strings are NULL, then those input string parameters are treated + * as if they pointed to an empty string. + * However, this is not the case for char * parameters for charset names + * or other IDs. + * - Most UnicodeString methods do not take a UErrorCode parameter because + * there are usually very few opportunities for failure other than a shortage + * of memory, error codes in low-level C++ string methods would be inconvenient, + * and the error code as the last parameter (ICU convention) would prevent + * the use of default parameter values. + * Instead, such methods set the UnicodeString into a "bogus" state + * (see isBogus()) if an error occurs. + * + * In string comparisons, two UnicodeString objects that are both "bogus" + * compare equal (to be transitive and prevent endless loops in sorting), + * and a "bogus" string compares less than any non-"bogus" one. + * + * Const UnicodeString methods are thread-safe. Multiple threads can use + * const methods on the same UnicodeString object simultaneously, + * but non-const methods must not be called concurrently (in multiple threads) + * with any other (const or non-const) methods. + * + * Similarly, const UnicodeString & parameters are thread-safe. + * One object may be passed in as such a parameter concurrently in multiple threads. + * This includes the const UnicodeString & parameters for + * copy construction, assignment, and cloning. + * + *

UnicodeString uses several storage methods. + * String contents can be stored inside the UnicodeString object itself, + * in an allocated and shared buffer, or in an outside buffer that is "aliased". + * Most of this is done transparently, but careful aliasing in particular provides + * significant performance improvements. + * Also, the internal buffer is accessible via special functions. + * For details see the + * User Guide Strings chapter.

+ * + * @see utf.h + * @see CharacterIterator + * @stable ICU 2.0 + */ +class U_COMMON_API UnicodeString : public Replaceable +{ +public: + + /** + * Constant to be used in the UnicodeString(char *, int32_t, EInvariant) constructor + * which constructs a Unicode string from an invariant-character char * string. + * Use the macro US_INV instead of the full qualification for this value. + * + * @see US_INV + * @stable ICU 3.2 + */ + enum EInvariant { + /** + * @see EInvariant + * @stable ICU 3.2 + */ + kInvariant + }; + + //======================================== + // Read-only operations + //======================================== + + /* Comparison - bitwise only - for international comparison use collation */ + + /** + * Equality operator. Performs only bitwise comparison. + * @param text The UnicodeString to compare to this one. + * @return TRUE if text contains the same characters as this one, + * FALSE otherwise. + * @stable ICU 2.0 + */ + inline UBool operator== (const UnicodeString& text) const; + + /** + * Inequality operator. Performs only bitwise comparison. + * @param text The UnicodeString to compare to this one. + * @return FALSE if text contains the same characters as this one, + * TRUE otherwise. + * @stable ICU 2.0 + */ + inline UBool operator!= (const UnicodeString& text) const; + + /** + * Greater than operator. Performs only bitwise comparison. + * @param text The UnicodeString to compare to this one. + * @return TRUE if the characters in this are bitwise + * greater than the characters in text, FALSE otherwise + * @stable ICU 2.0 + */ + inline UBool operator> (const UnicodeString& text) const; + + /** + * Less than operator. Performs only bitwise comparison. + * @param text The UnicodeString to compare to this one. + * @return TRUE if the characters in this are bitwise + * less than the characters in text, FALSE otherwise + * @stable ICU 2.0 + */ + inline UBool operator< (const UnicodeString& text) const; + + /** + * Greater than or equal operator. Performs only bitwise comparison. + * @param text The UnicodeString to compare to this one. + * @return TRUE if the characters in this are bitwise + * greater than or equal to the characters in text, FALSE otherwise + * @stable ICU 2.0 + */ + inline UBool operator>= (const UnicodeString& text) const; + + /** + * Less than or equal operator. Performs only bitwise comparison. + * @param text The UnicodeString to compare to this one. + * @return TRUE if the characters in this are bitwise + * less than or equal to the characters in text, FALSE otherwise + * @stable ICU 2.0 + */ + inline UBool operator<= (const UnicodeString& text) const; + + /** + * Compare the characters bitwise in this UnicodeString to + * the characters in text. + * @param text The UnicodeString to compare to this one. + * @return The result of bitwise character comparison: 0 if this + * contains the same characters as text, -1 if the characters in + * this are bitwise less than the characters in text, +1 if the + * characters in this are bitwise greater than the characters + * in text. + * @stable ICU 2.0 + */ + inline int8_t compare(const UnicodeString& text) const; + + /** + * Compare the characters bitwise in the range + * [start, start + length) with the characters + * in text + * @param start the offset at which the compare operation begins + * @param length the number of characters of text to compare. + * @param text the other text to be compared against this string. + * @return The result of bitwise character comparison: 0 if this + * contains the same characters as text, -1 if the characters in + * this are bitwise less than the characters in text, +1 if the + * characters in this are bitwise greater than the characters + * in text. + * @stable ICU 2.0 + */ + inline int8_t compare(int32_t start, + int32_t length, + const UnicodeString& text) const; + + /** + * Compare the characters bitwise in the range + * [start, start + length) with the characters + * in srcText in the range + * [srcStart, srcStart + srcLength). + * @param start the offset at which the compare operation begins + * @param length the number of characters in this to compare. + * @param srcText the text to be compared + * @param srcStart the offset into srcText to start comparison + * @param srcLength the number of characters in src to compare + * @return The result of bitwise character comparison: 0 if this + * contains the same characters as srcText, -1 if the characters in + * this are bitwise less than the characters in srcText, +1 if the + * characters in this are bitwise greater than the characters + * in srcText. + * @stable ICU 2.0 + */ + inline int8_t compare(int32_t start, + int32_t length, + const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLength) const; + + /** + * Compare the characters bitwise in this UnicodeString with the first + * srcLength characters in srcChars. + * @param srcChars The characters to compare to this UnicodeString. + * @param srcLength the number of characters in srcChars to compare + * @return The result of bitwise character comparison: 0 if this + * contains the same characters as srcChars, -1 if the characters in + * this are bitwise less than the characters in srcChars, +1 if the + * characters in this are bitwise greater than the characters + * in srcChars. + * @stable ICU 2.0 + */ + inline int8_t compare(const UChar *srcChars, + int32_t srcLength) const; + + /** + * Compare the characters bitwise in the range + * [start, start + length) with the first + * length characters in srcChars + * @param start the offset at which the compare operation begins + * @param length the number of characters to compare. + * @param srcChars the characters to be compared + * @return The result of bitwise character comparison: 0 if this + * contains the same characters as srcChars, -1 if the characters in + * this are bitwise less than the characters in srcChars, +1 if the + * characters in this are bitwise greater than the characters + * in srcChars. + * @stable ICU 2.0 + */ + inline int8_t compare(int32_t start, + int32_t length, + const UChar *srcChars) const; + + /** + * Compare the characters bitwise in the range + * [start, start + length) with the characters + * in srcChars in the range + * [srcStart, srcStart + srcLength). + * @param start the offset at which the compare operation begins + * @param length the number of characters in this to compare + * @param srcChars the characters to be compared + * @param srcStart the offset into srcChars to start comparison + * @param srcLength the number of characters in srcChars to compare + * @return The result of bitwise character comparison: 0 if this + * contains the same characters as srcChars, -1 if the characters in + * this are bitwise less than the characters in srcChars, +1 if the + * characters in this are bitwise greater than the characters + * in srcChars. + * @stable ICU 2.0 + */ + inline int8_t compare(int32_t start, + int32_t length, + const UChar *srcChars, + int32_t srcStart, + int32_t srcLength) const; + + /** + * Compare the characters bitwise in the range + * [start, limit) with the characters + * in srcText in the range + * [srcStart, srcLimit). + * @param start the offset at which the compare operation begins + * @param limit the offset immediately following the compare operation + * @param srcText the text to be compared + * @param srcStart the offset into srcText to start comparison + * @param srcLimit the offset into srcText to limit comparison + * @return The result of bitwise character comparison: 0 if this + * contains the same characters as srcText, -1 if the characters in + * this are bitwise less than the characters in srcText, +1 if the + * characters in this are bitwise greater than the characters + * in srcText. + * @stable ICU 2.0 + */ + inline int8_t compareBetween(int32_t start, + int32_t limit, + const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLimit) const; + + /** + * Compare two Unicode strings in code point order. + * The result may be different from the results of compare(), operator<, etc. + * if supplementary characters are present: + * + * In UTF-16, supplementary characters (with code points U+10000 and above) are + * stored with pairs of surrogate code units. These have values from 0xd800 to 0xdfff, + * which means that they compare as less than some other BMP characters like U+feff. + * This function compares Unicode strings in code point order. + * If either of the UTF-16 strings is malformed (i.e., it contains unpaired surrogates), then the result is not defined. + * + * @param text Another string to compare this one to. + * @return a negative/zero/positive integer corresponding to whether + * this string is less than/equal to/greater than the second one + * in code point order + * @stable ICU 2.0 + */ + inline int8_t compareCodePointOrder(const UnicodeString& text) const; + + /** + * Compare two Unicode strings in code point order. + * The result may be different from the results of compare(), operator<, etc. + * if supplementary characters are present: + * + * In UTF-16, supplementary characters (with code points U+10000 and above) are + * stored with pairs of surrogate code units. These have values from 0xd800 to 0xdfff, + * which means that they compare as less than some other BMP characters like U+feff. + * This function compares Unicode strings in code point order. + * If either of the UTF-16 strings is malformed (i.e., it contains unpaired surrogates), then the result is not defined. + * + * @param start The start offset in this string at which the compare operation begins. + * @param length The number of code units from this string to compare. + * @param srcText Another string to compare this one to. + * @return a negative/zero/positive integer corresponding to whether + * this string is less than/equal to/greater than the second one + * in code point order + * @stable ICU 2.0 + */ + inline int8_t compareCodePointOrder(int32_t start, + int32_t length, + const UnicodeString& srcText) const; + + /** + * Compare two Unicode strings in code point order. + * The result may be different from the results of compare(), operator<, etc. + * if supplementary characters are present: + * + * In UTF-16, supplementary characters (with code points U+10000 and above) are + * stored with pairs of surrogate code units. These have values from 0xd800 to 0xdfff, + * which means that they compare as less than some other BMP characters like U+feff. + * This function compares Unicode strings in code point order. + * If either of the UTF-16 strings is malformed (i.e., it contains unpaired surrogates), then the result is not defined. + * + * @param start The start offset in this string at which the compare operation begins. + * @param length The number of code units from this string to compare. + * @param srcText Another string to compare this one to. + * @param srcStart The start offset in that string at which the compare operation begins. + * @param srcLength The number of code units from that string to compare. + * @return a negative/zero/positive integer corresponding to whether + * this string is less than/equal to/greater than the second one + * in code point order + * @stable ICU 2.0 + */ + inline int8_t compareCodePointOrder(int32_t start, + int32_t length, + const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLength) const; + + /** + * Compare two Unicode strings in code point order. + * The result may be different from the results of compare(), operator<, etc. + * if supplementary characters are present: + * + * In UTF-16, supplementary characters (with code points U+10000 and above) are + * stored with pairs of surrogate code units. These have values from 0xd800 to 0xdfff, + * which means that they compare as less than some other BMP characters like U+feff. + * This function compares Unicode strings in code point order. + * If either of the UTF-16 strings is malformed (i.e., it contains unpaired surrogates), then the result is not defined. + * + * @param srcChars A pointer to another string to compare this one to. + * @param srcLength The number of code units from that string to compare. + * @return a negative/zero/positive integer corresponding to whether + * this string is less than/equal to/greater than the second one + * in code point order + * @stable ICU 2.0 + */ + inline int8_t compareCodePointOrder(const UChar *srcChars, + int32_t srcLength) const; + + /** + * Compare two Unicode strings in code point order. + * The result may be different from the results of compare(), operator<, etc. + * if supplementary characters are present: + * + * In UTF-16, supplementary characters (with code points U+10000 and above) are + * stored with pairs of surrogate code units. These have values from 0xd800 to 0xdfff, + * which means that they compare as less than some other BMP characters like U+feff. + * This function compares Unicode strings in code point order. + * If either of the UTF-16 strings is malformed (i.e., it contains unpaired surrogates), then the result is not defined. + * + * @param start The start offset in this string at which the compare operation begins. + * @param length The number of code units from this string to compare. + * @param srcChars A pointer to another string to compare this one to. + * @return a negative/zero/positive integer corresponding to whether + * this string is less than/equal to/greater than the second one + * in code point order + * @stable ICU 2.0 + */ + inline int8_t compareCodePointOrder(int32_t start, + int32_t length, + const UChar *srcChars) const; + + /** + * Compare two Unicode strings in code point order. + * The result may be different from the results of compare(), operator<, etc. + * if supplementary characters are present: + * + * In UTF-16, supplementary characters (with code points U+10000 and above) are + * stored with pairs of surrogate code units. These have values from 0xd800 to 0xdfff, + * which means that they compare as less than some other BMP characters like U+feff. + * This function compares Unicode strings in code point order. + * If either of the UTF-16 strings is malformed (i.e., it contains unpaired surrogates), then the result is not defined. + * + * @param start The start offset in this string at which the compare operation begins. + * @param length The number of code units from this string to compare. + * @param srcChars A pointer to another string to compare this one to. + * @param srcStart The start offset in that string at which the compare operation begins. + * @param srcLength The number of code units from that string to compare. + * @return a negative/zero/positive integer corresponding to whether + * this string is less than/equal to/greater than the second one + * in code point order + * @stable ICU 2.0 + */ + inline int8_t compareCodePointOrder(int32_t start, + int32_t length, + const UChar *srcChars, + int32_t srcStart, + int32_t srcLength) const; + + /** + * Compare two Unicode strings in code point order. + * The result may be different from the results of compare(), operator<, etc. + * if supplementary characters are present: + * + * In UTF-16, supplementary characters (with code points U+10000 and above) are + * stored with pairs of surrogate code units. These have values from 0xd800 to 0xdfff, + * which means that they compare as less than some other BMP characters like U+feff. + * This function compares Unicode strings in code point order. + * If either of the UTF-16 strings is malformed (i.e., it contains unpaired surrogates), then the result is not defined. + * + * @param start The start offset in this string at which the compare operation begins. + * @param limit The offset after the last code unit from this string to compare. + * @param srcText Another string to compare this one to. + * @param srcStart The start offset in that string at which the compare operation begins. + * @param srcLimit The offset after the last code unit from that string to compare. + * @return a negative/zero/positive integer corresponding to whether + * this string is less than/equal to/greater than the second one + * in code point order + * @stable ICU 2.0 + */ + inline int8_t compareCodePointOrderBetween(int32_t start, + int32_t limit, + const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLimit) const; + + /** + * Compare two strings case-insensitively using full case folding. + * This is equivalent to this->foldCase(options).compare(text.foldCase(options)). + * + * @param text Another string to compare this one to. + * @param options A bit set of options: + * - U_FOLD_CASE_DEFAULT or 0 is used for default options: + * Comparison in code unit order with default case folding. + * + * - U_COMPARE_CODE_POINT_ORDER + * Set to choose code point order instead of code unit order + * (see u_strCompare for details). + * + * - U_FOLD_CASE_EXCLUDE_SPECIAL_I + * + * @return A negative, zero, or positive integer indicating the comparison result. + * @stable ICU 2.0 + */ + inline int8_t caseCompare(const UnicodeString& text, uint32_t options) const; + + /** + * Compare two strings case-insensitively using full case folding. + * This is equivalent to this->foldCase(options).compare(srcText.foldCase(options)). + * + * @param start The start offset in this string at which the compare operation begins. + * @param length The number of code units from this string to compare. + * @param srcText Another string to compare this one to. + * @param options A bit set of options: + * - U_FOLD_CASE_DEFAULT or 0 is used for default options: + * Comparison in code unit order with default case folding. + * + * - U_COMPARE_CODE_POINT_ORDER + * Set to choose code point order instead of code unit order + * (see u_strCompare for details). + * + * - U_FOLD_CASE_EXCLUDE_SPECIAL_I + * + * @return A negative, zero, or positive integer indicating the comparison result. + * @stable ICU 2.0 + */ + inline int8_t caseCompare(int32_t start, + int32_t length, + const UnicodeString& srcText, + uint32_t options) const; + + /** + * Compare two strings case-insensitively using full case folding. + * This is equivalent to this->foldCase(options).compare(srcText.foldCase(options)). + * + * @param start The start offset in this string at which the compare operation begins. + * @param length The number of code units from this string to compare. + * @param srcText Another string to compare this one to. + * @param srcStart The start offset in that string at which the compare operation begins. + * @param srcLength The number of code units from that string to compare. + * @param options A bit set of options: + * - U_FOLD_CASE_DEFAULT or 0 is used for default options: + * Comparison in code unit order with default case folding. + * + * - U_COMPARE_CODE_POINT_ORDER + * Set to choose code point order instead of code unit order + * (see u_strCompare for details). + * + * - U_FOLD_CASE_EXCLUDE_SPECIAL_I + * + * @return A negative, zero, or positive integer indicating the comparison result. + * @stable ICU 2.0 + */ + inline int8_t caseCompare(int32_t start, + int32_t length, + const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLength, + uint32_t options) const; + + /** + * Compare two strings case-insensitively using full case folding. + * This is equivalent to this->foldCase(options).compare(srcChars.foldCase(options)). + * + * @param srcChars A pointer to another string to compare this one to. + * @param srcLength The number of code units from that string to compare. + * @param options A bit set of options: + * - U_FOLD_CASE_DEFAULT or 0 is used for default options: + * Comparison in code unit order with default case folding. + * + * - U_COMPARE_CODE_POINT_ORDER + * Set to choose code point order instead of code unit order + * (see u_strCompare for details). + * + * - U_FOLD_CASE_EXCLUDE_SPECIAL_I + * + * @return A negative, zero, or positive integer indicating the comparison result. + * @stable ICU 2.0 + */ + inline int8_t caseCompare(const UChar *srcChars, + int32_t srcLength, + uint32_t options) const; + + /** + * Compare two strings case-insensitively using full case folding. + * This is equivalent to this->foldCase(options).compare(srcChars.foldCase(options)). + * + * @param start The start offset in this string at which the compare operation begins. + * @param length The number of code units from this string to compare. + * @param srcChars A pointer to another string to compare this one to. + * @param options A bit set of options: + * - U_FOLD_CASE_DEFAULT or 0 is used for default options: + * Comparison in code unit order with default case folding. + * + * - U_COMPARE_CODE_POINT_ORDER + * Set to choose code point order instead of code unit order + * (see u_strCompare for details). + * + * - U_FOLD_CASE_EXCLUDE_SPECIAL_I + * + * @return A negative, zero, or positive integer indicating the comparison result. + * @stable ICU 2.0 + */ + inline int8_t caseCompare(int32_t start, + int32_t length, + const UChar *srcChars, + uint32_t options) const; + + /** + * Compare two strings case-insensitively using full case folding. + * This is equivalent to this->foldCase(options).compare(srcChars.foldCase(options)). + * + * @param start The start offset in this string at which the compare operation begins. + * @param length The number of code units from this string to compare. + * @param srcChars A pointer to another string to compare this one to. + * @param srcStart The start offset in that string at which the compare operation begins. + * @param srcLength The number of code units from that string to compare. + * @param options A bit set of options: + * - U_FOLD_CASE_DEFAULT or 0 is used for default options: + * Comparison in code unit order with default case folding. + * + * - U_COMPARE_CODE_POINT_ORDER + * Set to choose code point order instead of code unit order + * (see u_strCompare for details). + * + * - U_FOLD_CASE_EXCLUDE_SPECIAL_I + * + * @return A negative, zero, or positive integer indicating the comparison result. + * @stable ICU 2.0 + */ + inline int8_t caseCompare(int32_t start, + int32_t length, + const UChar *srcChars, + int32_t srcStart, + int32_t srcLength, + uint32_t options) const; + + /** + * Compare two strings case-insensitively using full case folding. + * This is equivalent to this->foldCase(options).compareBetween(text.foldCase(options)). + * + * @param start The start offset in this string at which the compare operation begins. + * @param limit The offset after the last code unit from this string to compare. + * @param srcText Another string to compare this one to. + * @param srcStart The start offset in that string at which the compare operation begins. + * @param srcLimit The offset after the last code unit from that string to compare. + * @param options A bit set of options: + * - U_FOLD_CASE_DEFAULT or 0 is used for default options: + * Comparison in code unit order with default case folding. + * + * - U_COMPARE_CODE_POINT_ORDER + * Set to choose code point order instead of code unit order + * (see u_strCompare for details). + * + * - U_FOLD_CASE_EXCLUDE_SPECIAL_I + * + * @return A negative, zero, or positive integer indicating the comparison result. + * @stable ICU 2.0 + */ + inline int8_t caseCompareBetween(int32_t start, + int32_t limit, + const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLimit, + uint32_t options) const; + + /** + * Determine if this starts with the characters in text + * @param text The text to match. + * @return TRUE if this starts with the characters in text, + * FALSE otherwise + * @stable ICU 2.0 + */ + inline UBool startsWith(const UnicodeString& text) const; + + /** + * Determine if this starts with the characters in srcText + * in the range [srcStart, srcStart + srcLength). + * @param srcText The text to match. + * @param srcStart the offset into srcText to start matching + * @param srcLength the number of characters in srcText to match + * @return TRUE if this starts with the characters in text, + * FALSE otherwise + * @stable ICU 2.0 + */ + inline UBool startsWith(const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLength) const; + + /** + * Determine if this starts with the characters in srcChars + * @param srcChars The characters to match. + * @param srcLength the number of characters in srcChars + * @return TRUE if this starts with the characters in srcChars, + * FALSE otherwise + * @stable ICU 2.0 + */ + inline UBool startsWith(const UChar *srcChars, + int32_t srcLength) const; + + /** + * Determine if this ends with the characters in srcChars + * in the range [srcStart, srcStart + srcLength). + * @param srcChars The characters to match. + * @param srcStart the offset into srcText to start matching + * @param srcLength the number of characters in srcChars to match + * @return TRUE if this ends with the characters in srcChars, FALSE otherwise + * @stable ICU 2.0 + */ + inline UBool startsWith(const UChar *srcChars, + int32_t srcStart, + int32_t srcLength) const; + + /** + * Determine if this ends with the characters in text + * @param text The text to match. + * @return TRUE if this ends with the characters in text, + * FALSE otherwise + * @stable ICU 2.0 + */ + inline UBool endsWith(const UnicodeString& text) const; + + /** + * Determine if this ends with the characters in srcText + * in the range [srcStart, srcStart + srcLength). + * @param srcText The text to match. + * @param srcStart the offset into srcText to start matching + * @param srcLength the number of characters in srcText to match + * @return TRUE if this ends with the characters in text, + * FALSE otherwise + * @stable ICU 2.0 + */ + inline UBool endsWith(const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLength) const; + + /** + * Determine if this ends with the characters in srcChars + * @param srcChars The characters to match. + * @param srcLength the number of characters in srcChars + * @return TRUE if this ends with the characters in srcChars, + * FALSE otherwise + * @stable ICU 2.0 + */ + inline UBool endsWith(const UChar *srcChars, + int32_t srcLength) const; + + /** + * Determine if this ends with the characters in srcChars + * in the range [srcStart, srcStart + srcLength). + * @param srcChars The characters to match. + * @param srcStart the offset into srcText to start matching + * @param srcLength the number of characters in srcChars to match + * @return TRUE if this ends with the characters in srcChars, + * FALSE otherwise + * @stable ICU 2.0 + */ + inline UBool endsWith(const UChar *srcChars, + int32_t srcStart, + int32_t srcLength) const; + + + /* Searching - bitwise only */ + + /** + * Locate in this the first occurrence of the characters in text, + * using bitwise comparison. + * @param text The text to search for. + * @return The offset into this of the start of text, + * or -1 if not found. + * @stable ICU 2.0 + */ + inline int32_t indexOf(const UnicodeString& text) const; + + /** + * Locate in this the first occurrence of the characters in text + * starting at offset start, using bitwise comparison. + * @param text The text to search for. + * @param start The offset at which searching will start. + * @return The offset into this of the start of text, + * or -1 if not found. + * @stable ICU 2.0 + */ + inline int32_t indexOf(const UnicodeString& text, + int32_t start) const; + + /** + * Locate in this the first occurrence in the range + * [start, start + length) of the characters + * in text, using bitwise comparison. + * @param text The text to search for. + * @param start The offset at which searching will start. + * @param length The number of characters to search + * @return The offset into this of the start of text, + * or -1 if not found. + * @stable ICU 2.0 + */ + inline int32_t indexOf(const UnicodeString& text, + int32_t start, + int32_t length) const; + + /** + * Locate in this the first occurrence in the range + * [start, start + length) of the characters + * in srcText in the range + * [srcStart, srcStart + srcLength), + * using bitwise comparison. + * @param srcText The text to search for. + * @param srcStart the offset into srcText at which + * to start matching + * @param srcLength the number of characters in srcText to match + * @param start the offset into this at which to start matching + * @param length the number of characters in this to search + * @return The offset into this of the start of text, + * or -1 if not found. + * @stable ICU 2.0 + */ + inline int32_t indexOf(const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLength, + int32_t start, + int32_t length) const; + + /** + * Locate in this the first occurrence of the characters in + * srcChars + * starting at offset start, using bitwise comparison. + * @param srcChars The text to search for. + * @param srcLength the number of characters in srcChars to match + * @param start the offset into this at which to start matching + * @return The offset into this of the start of text, + * or -1 if not found. + * @stable ICU 2.0 + */ + inline int32_t indexOf(const UChar *srcChars, + int32_t srcLength, + int32_t start) const; + + /** + * Locate in this the first occurrence in the range + * [start, start + length) of the characters + * in srcChars, using bitwise comparison. + * @param srcChars The text to search for. + * @param srcLength the number of characters in srcChars + * @param start The offset at which searching will start. + * @param length The number of characters to search + * @return The offset into this of the start of srcChars, + * or -1 if not found. + * @stable ICU 2.0 + */ + inline int32_t indexOf(const UChar *srcChars, + int32_t srcLength, + int32_t start, + int32_t length) const; + + /** + * Locate in this the first occurrence in the range + * [start, start + length) of the characters + * in srcChars in the range + * [srcStart, srcStart + srcLength), + * using bitwise comparison. + * @param srcChars The text to search for. + * @param srcStart the offset into srcChars at which + * to start matching + * @param srcLength the number of characters in srcChars to match + * @param start the offset into this at which to start matching + * @param length the number of characters in this to search + * @return The offset into this of the start of text, + * or -1 if not found. + * @stable ICU 2.0 + */ + int32_t indexOf(const UChar *srcChars, + int32_t srcStart, + int32_t srcLength, + int32_t start, + int32_t length) const; + + /** + * Locate in this the first occurrence of the BMP code point c, + * using bitwise comparison. + * @param c The code unit to search for. + * @return The offset into this of c, or -1 if not found. + * @stable ICU 2.0 + */ + inline int32_t indexOf(UChar c) const; + + /** + * Locate in this the first occurrence of the code point c, + * using bitwise comparison. + * + * @param c The code point to search for. + * @return The offset into this of c, or -1 if not found. + * @stable ICU 2.0 + */ + inline int32_t indexOf(UChar32 c) const; + + /** + * Locate in this the first occurrence of the BMP code point c, + * starting at offset start, using bitwise comparison. + * @param c The code unit to search for. + * @param start The offset at which searching will start. + * @return The offset into this of c, or -1 if not found. + * @stable ICU 2.0 + */ + inline int32_t indexOf(UChar c, + int32_t start) const; + + /** + * Locate in this the first occurrence of the code point c + * starting at offset start, using bitwise comparison. + * + * @param c The code point to search for. + * @param start The offset at which searching will start. + * @return The offset into this of c, or -1 if not found. + * @stable ICU 2.0 + */ + inline int32_t indexOf(UChar32 c, + int32_t start) const; + + /** + * Locate in this the first occurrence of the BMP code point c + * in the range [start, start + length), + * using bitwise comparison. + * @param c The code unit to search for. + * @param start the offset into this at which to start matching + * @param length the number of characters in this to search + * @return The offset into this of c, or -1 if not found. + * @stable ICU 2.0 + */ + inline int32_t indexOf(UChar c, + int32_t start, + int32_t length) const; + + /** + * Locate in this the first occurrence of the code point c + * in the range [start, start + length), + * using bitwise comparison. + * + * @param c The code point to search for. + * @param start the offset into this at which to start matching + * @param length the number of characters in this to search + * @return The offset into this of c, or -1 if not found. + * @stable ICU 2.0 + */ + inline int32_t indexOf(UChar32 c, + int32_t start, + int32_t length) const; + + /** + * Locate in this the last occurrence of the characters in text, + * using bitwise comparison. + * @param text The text to search for. + * @return The offset into this of the start of text, + * or -1 if not found. + * @stable ICU 2.0 + */ + inline int32_t lastIndexOf(const UnicodeString& text) const; + + /** + * Locate in this the last occurrence of the characters in text + * starting at offset start, using bitwise comparison. + * @param text The text to search for. + * @param start The offset at which searching will start. + * @return The offset into this of the start of text, + * or -1 if not found. + * @stable ICU 2.0 + */ + inline int32_t lastIndexOf(const UnicodeString& text, + int32_t start) const; + + /** + * Locate in this the last occurrence in the range + * [start, start + length) of the characters + * in text, using bitwise comparison. + * @param text The text to search for. + * @param start The offset at which searching will start. + * @param length The number of characters to search + * @return The offset into this of the start of text, + * or -1 if not found. + * @stable ICU 2.0 + */ + inline int32_t lastIndexOf(const UnicodeString& text, + int32_t start, + int32_t length) const; + + /** + * Locate in this the last occurrence in the range + * [start, start + length) of the characters + * in srcText in the range + * [srcStart, srcStart + srcLength), + * using bitwise comparison. + * @param srcText The text to search for. + * @param srcStart the offset into srcText at which + * to start matching + * @param srcLength the number of characters in srcText to match + * @param start the offset into this at which to start matching + * @param length the number of characters in this to search + * @return The offset into this of the start of text, + * or -1 if not found. + * @stable ICU 2.0 + */ + inline int32_t lastIndexOf(const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLength, + int32_t start, + int32_t length) const; + + /** + * Locate in this the last occurrence of the characters in srcChars + * starting at offset start, using bitwise comparison. + * @param srcChars The text to search for. + * @param srcLength the number of characters in srcChars to match + * @param start the offset into this at which to start matching + * @return The offset into this of the start of text, + * or -1 if not found. + * @stable ICU 2.0 + */ + inline int32_t lastIndexOf(const UChar *srcChars, + int32_t srcLength, + int32_t start) const; + + /** + * Locate in this the last occurrence in the range + * [start, start + length) of the characters + * in srcChars, using bitwise comparison. + * @param srcChars The text to search for. + * @param srcLength the number of characters in srcChars + * @param start The offset at which searching will start. + * @param length The number of characters to search + * @return The offset into this of the start of srcChars, + * or -1 if not found. + * @stable ICU 2.0 + */ + inline int32_t lastIndexOf(const UChar *srcChars, + int32_t srcLength, + int32_t start, + int32_t length) const; + + /** + * Locate in this the last occurrence in the range + * [start, start + length) of the characters + * in srcChars in the range + * [srcStart, srcStart + srcLength), + * using bitwise comparison. + * @param srcChars The text to search for. + * @param srcStart the offset into srcChars at which + * to start matching + * @param srcLength the number of characters in srcChars to match + * @param start the offset into this at which to start matching + * @param length the number of characters in this to search + * @return The offset into this of the start of text, + * or -1 if not found. + * @stable ICU 2.0 + */ + int32_t lastIndexOf(const UChar *srcChars, + int32_t srcStart, + int32_t srcLength, + int32_t start, + int32_t length) const; + + /** + * Locate in this the last occurrence of the BMP code point c, + * using bitwise comparison. + * @param c The code unit to search for. + * @return The offset into this of c, or -1 if not found. + * @stable ICU 2.0 + */ + inline int32_t lastIndexOf(UChar c) const; + + /** + * Locate in this the last occurrence of the code point c, + * using bitwise comparison. + * + * @param c The code point to search for. + * @return The offset into this of c, or -1 if not found. + * @stable ICU 2.0 + */ + inline int32_t lastIndexOf(UChar32 c) const; + + /** + * Locate in this the last occurrence of the BMP code point c + * starting at offset start, using bitwise comparison. + * @param c The code unit to search for. + * @param start The offset at which searching will start. + * @return The offset into this of c, or -1 if not found. + * @stable ICU 2.0 + */ + inline int32_t lastIndexOf(UChar c, + int32_t start) const; + + /** + * Locate in this the last occurrence of the code point c + * starting at offset start, using bitwise comparison. + * + * @param c The code point to search for. + * @param start The offset at which searching will start. + * @return The offset into this of c, or -1 if not found. + * @stable ICU 2.0 + */ + inline int32_t lastIndexOf(UChar32 c, + int32_t start) const; + + /** + * Locate in this the last occurrence of the BMP code point c + * in the range [start, start + length), + * using bitwise comparison. + * @param c The code unit to search for. + * @param start the offset into this at which to start matching + * @param length the number of characters in this to search + * @return The offset into this of c, or -1 if not found. + * @stable ICU 2.0 + */ + inline int32_t lastIndexOf(UChar c, + int32_t start, + int32_t length) const; + + /** + * Locate in this the last occurrence of the code point c + * in the range [start, start + length), + * using bitwise comparison. + * + * @param c The code point to search for. + * @param start the offset into this at which to start matching + * @param length the number of characters in this to search + * @return The offset into this of c, or -1 if not found. + * @stable ICU 2.0 + */ + inline int32_t lastIndexOf(UChar32 c, + int32_t start, + int32_t length) const; + + + /* Character access */ + + /** + * Return the code unit at offset offset. + * If the offset is not valid (0..length()-1) then U+ffff is returned. + * @param offset a valid offset into the text + * @return the code unit at offset offset + * or 0xffff if the offset is not valid for this string + * @stable ICU 2.0 + */ + inline UChar charAt(int32_t offset) const; + + /** + * Return the code unit at offset offset. + * If the offset is not valid (0..length()-1) then U+ffff is returned. + * @param offset a valid offset into the text + * @return the code unit at offset offset + * @stable ICU 2.0 + */ + inline UChar operator[] (int32_t offset) const; + + /** + * Return the code point that contains the code unit + * at offset offset. + * If the offset is not valid (0..length()-1) then U+ffff is returned. + * @param offset a valid offset into the text + * that indicates the text offset of any of the code units + * that will be assembled into a code point (21-bit value) and returned + * @return the code point of text at offset + * or 0xffff if the offset is not valid for this string + * @stable ICU 2.0 + */ + inline UChar32 char32At(int32_t offset) const; + + /** + * Adjust a random-access offset so that + * it points to the beginning of a Unicode character. + * The offset that is passed in points to + * any code unit of a code point, + * while the returned offset will point to the first code unit + * of the same code point. + * In UTF-16, if the input offset points to a second surrogate + * of a surrogate pair, then the returned offset will point + * to the first surrogate. + * @param offset a valid offset into one code point of the text + * @return offset of the first code unit of the same code point + * @see U16_SET_CP_START + * @stable ICU 2.0 + */ + inline int32_t getChar32Start(int32_t offset) const; + + /** + * Adjust a random-access offset so that + * it points behind a Unicode character. + * The offset that is passed in points behind + * any code unit of a code point, + * while the returned offset will point behind the last code unit + * of the same code point. + * In UTF-16, if the input offset points behind the first surrogate + * (i.e., to the second surrogate) + * of a surrogate pair, then the returned offset will point + * behind the second surrogate (i.e., to the first surrogate). + * @param offset a valid offset after any code unit of a code point of the text + * @return offset of the first code unit after the same code point + * @see U16_SET_CP_LIMIT + * @stable ICU 2.0 + */ + inline int32_t getChar32Limit(int32_t offset) const; + + /** + * Move the code unit index along the string by delta code points. + * Interpret the input index as a code unit-based offset into the string, + * move the index forward or backward by delta code points, and + * return the resulting index. + * The input index should point to the first code unit of a code point, + * if there is more than one. + * + * Both input and output indexes are code unit-based as for all + * string indexes/offsets in ICU (and other libraries, like MBCS char*). + * If delta<0 then the index is moved backward (toward the start of the string). + * If delta>0 then the index is moved forward (toward the end of the string). + * + * This behaves like CharacterIterator::move32(delta, kCurrent). + * + * Behavior for out-of-bounds indexes: + * moveIndex32 pins the input index to 0..length(), i.e., + * if the input index<0 then it is pinned to 0; + * if it is index>length() then it is pinned to length(). + * Afterwards, the index is moved by delta code points + * forward or backward, + * but no further backward than to 0 and no further forward than to length(). + * The resulting index return value will be in between 0 and length(), inclusively. + * + * Examples: + *
+   * // s has code points 'a' U+10000 'b' U+10ffff U+2029
+   * UnicodeString s=UNICODE_STRING("a\\U00010000b\\U0010ffff\\u2029", 31).unescape();
+   *
+   * // initial index: position of U+10000
+   * int32_t index=1;
+   *
+   * // the following examples will all result in index==4, position of U+10ffff
+   *
+   * // skip 2 code points from some position in the string
+   * index=s.moveIndex32(index, 2); // skips U+10000 and 'b'
+   *
+   * // go to the 3rd code point from the start of s (0-based)
+   * index=s.moveIndex32(0, 3); // skips 'a', U+10000, and 'b'
+   *
+   * // go to the next-to-last code point of s
+   * index=s.moveIndex32(s.length(), -2); // backward-skips U+2029 and U+10ffff
+   * 
+ * + * @param index input code unit index + * @param delta (signed) code point count to move the index forward or backward + * in the string + * @return the resulting code unit index + * @stable ICU 2.0 + */ + int32_t moveIndex32(int32_t index, int32_t delta) const; + + /* Substring extraction */ + + /** + * Copy the characters in the range + * [start, start + length) into the array dst, + * beginning at dstStart. + * If the string aliases to dst itself as an external buffer, + * then extract() will not copy the contents. + * + * @param start offset of first character which will be copied into the array + * @param length the number of characters to extract + * @param dst array in which to copy characters. The length of dst + * must be at least (dstStart + length). + * @param dstStart the offset in dst where the first character + * will be extracted + * @stable ICU 2.0 + */ + inline void extract(int32_t start, + int32_t length, + UChar *dst, + int32_t dstStart = 0) const; + + /** + * Copy the contents of the string into dest. + * This is a convenience function that + * checks if there is enough space in dest, + * extracts the entire string if possible, + * and NUL-terminates dest if possible. + * + * If the string fits into dest but cannot be NUL-terminated + * (length()==destCapacity) then the error code is set to U_STRING_NOT_TERMINATED_WARNING. + * If the string itself does not fit into dest + * (length()>destCapacity) then the error code is set to U_BUFFER_OVERFLOW_ERROR. + * + * If the string aliases to dest itself as an external buffer, + * then extract() will not copy the contents. + * + * @param dest Destination string buffer. + * @param destCapacity Number of UChars available at dest. + * @param errorCode ICU error code. + * @return length() + * @stable ICU 2.0 + */ + int32_t + extract(UChar *dest, int32_t destCapacity, + UErrorCode &errorCode) const; + + /** + * Copy the characters in the range + * [start, start + length) into the UnicodeString + * target. + * @param start offset of first character which will be copied + * @param length the number of characters to extract + * @param target UnicodeString into which to copy characters. + * @return A reference to target + * @stable ICU 2.0 + */ + inline void extract(int32_t start, + int32_t length, + UnicodeString& target) const; + + /** + * Copy the characters in the range [start, limit) + * into the array dst, beginning at dstStart. + * @param start offset of first character which will be copied into the array + * @param limit offset immediately following the last character to be copied + * @param dst array in which to copy characters. The length of dst + * must be at least (dstStart + (limit - start)). + * @param dstStart the offset in dst where the first character + * will be extracted + * @stable ICU 2.0 + */ + inline void extractBetween(int32_t start, + int32_t limit, + UChar *dst, + int32_t dstStart = 0) const; + + /** + * Copy the characters in the range [start, limit) + * into the UnicodeString target. Replaceable API. + * @param start offset of first character which will be copied + * @param limit offset immediately following the last character to be copied + * @param target UnicodeString into which to copy characters. + * @return A reference to target + * @stable ICU 2.0 + */ + virtual void extractBetween(int32_t start, + int32_t limit, + UnicodeString& target) const; + + /** + * Copy the characters in the range + * [start, start + length) into an array of characters. + * All characters must be invariant (see utypes.h). + * Use US_INV as the last, signature-distinguishing parameter. + * + * This function does not write any more than targetLength + * characters but returns the length of the entire output string + * so that one can allocate a larger buffer and call the function again + * if necessary. + * The output string is NUL-terminated if possible. + * + * @param start offset of first character which will be copied + * @param startLength the number of characters to extract + * @param target the target buffer for extraction, can be NULL + * if targetLength is 0 + * @param targetCapacity the length of the target buffer + * @param inv Signature-distinguishing paramater, use US_INV. + * @return the output string length, not including the terminating NUL + * @stable ICU 3.2 + */ + int32_t extract(int32_t start, + int32_t startLength, + char *target, + int32_t targetCapacity, + enum EInvariant inv) const; + +#if !UCONFIG_NO_CONVERSION + + /** + * Copy the characters in the range + * [start, start + length) into an array of characters + * in a specified codepage. + * The output string is NUL-terminated. + * + * Recommendation: For invariant-character strings use + * extract(int32_t start, int32_t length, char *target, int32_t targetCapacity, enum EInvariant inv) const + * because it avoids object code dependencies of UnicodeString on + * the conversion code. + * + * @param start offset of first character which will be copied + * @param startLength the number of characters to extract + * @param target the target buffer for extraction + * @param codepage the desired codepage for the characters. 0 has + * the special meaning of the default codepage + * If codepage is an empty string (""), + * then a simple conversion is performed on the codepage-invariant + * subset ("invariant characters") of the platform encoding. See utypes.h. + * If target is NULL, then the number of bytes required for + * target is returned. It is assumed that the target is big enough + * to fit all of the characters. + * @return the output string length, not including the terminating NUL + * @stable ICU 2.0 + */ + inline int32_t extract(int32_t start, + int32_t startLength, + char *target, + const char *codepage = 0) const; + + /** + * Copy the characters in the range + * [start, start + length) into an array of characters + * in a specified codepage. + * This function does not write any more than targetLength + * characters but returns the length of the entire output string + * so that one can allocate a larger buffer and call the function again + * if necessary. + * The output string is NUL-terminated if possible. + * + * Recommendation: For invariant-character strings use + * extract(int32_t start, int32_t length, char *target, int32_t targetCapacity, enum EInvariant inv) const + * because it avoids object code dependencies of UnicodeString on + * the conversion code. + * + * @param start offset of first character which will be copied + * @param startLength the number of characters to extract + * @param target the target buffer for extraction + * @param targetLength the length of the target buffer + * @param codepage the desired codepage for the characters. 0 has + * the special meaning of the default codepage + * If codepage is an empty string (""), + * then a simple conversion is performed on the codepage-invariant + * subset ("invariant characters") of the platform encoding. See utypes.h. + * If target is NULL, then the number of bytes required for + * target is returned. + * @return the output string length, not including the terminating NUL + * @stable ICU 2.0 + */ + int32_t extract(int32_t start, + int32_t startLength, + char *target, + uint32_t targetLength, + const char *codepage = 0) const; + + /** + * Convert the UnicodeString into a codepage string using an existing UConverter. + * The output string is NUL-terminated if possible. + * + * This function avoids the overhead of opening and closing a converter if + * multiple strings are extracted. + * + * @param dest destination string buffer, can be NULL if destCapacity==0 + * @param destCapacity the number of chars available at dest + * @param cnv the converter object to be used (ucnv_resetFromUnicode() will be called), + * or NULL for the default converter + * @param errorCode normal ICU error code + * @return the length of the output string, not counting the terminating NUL; + * if the length is greater than destCapacity, then the string will not fit + * and a buffer of the indicated length would need to be passed in + * @stable ICU 2.0 + */ + int32_t extract(char *dest, int32_t destCapacity, + UConverter *cnv, + UErrorCode &errorCode) const; + +#endif + + /* Length operations */ + + /** + * Return the length of the UnicodeString object. + * The length is the number of UChar code units are in the UnicodeString. + * If you want the number of code points, please use countChar32(). + * @return the length of the UnicodeString object + * @see countChar32 + * @stable ICU 2.0 + */ + inline int32_t length(void) const; + + /** + * Count Unicode code points in the length UChar code units of the string. + * A code point may occupy either one or two UChar code units. + * Counting code points involves reading all code units. + * + * This functions is basically the inverse of moveIndex32(). + * + * @param start the index of the first code unit to check + * @param length the number of UChar code units to check + * @return the number of code points in the specified code units + * @see length + * @stable ICU 2.0 + */ + int32_t + countChar32(int32_t start=0, int32_t length=INT32_MAX) const; + + /** + * Check if the length UChar code units of the string + * contain more Unicode code points than a certain number. + * This is more efficient than counting all code points in this part of the string + * and comparing that number with a threshold. + * This function may not need to scan the string at all if the length + * falls within a certain range, and + * never needs to count more than 'number+1' code points. + * Logically equivalent to (countChar32(start, length)>number). + * A Unicode code point may occupy either one or two UChar code units. + * + * @param start the index of the first code unit to check (0 for the entire string) + * @param length the number of UChar code units to check + * (use INT32_MAX for the entire string; remember that start/length + * values are pinned) + * @param number The number of code points in the (sub)string is compared against + * the 'number' parameter. + * @return Boolean value for whether the string contains more Unicode code points + * than 'number'. Same as (u_countChar32(s, length)>number). + * @see countChar32 + * @see u_strHasMoreChar32Than + * @stable ICU 2.4 + */ + UBool + hasMoreChar32Than(int32_t start, int32_t length, int32_t number) const; + + /** + * Determine if this string is empty. + * @return TRUE if this string contains 0 characters, FALSE otherwise. + * @stable ICU 2.0 + */ + inline UBool isEmpty(void) const; + + /** + * Return the capacity of the internal buffer of the UnicodeString object. + * This is useful together with the getBuffer functions. + * See there for details. + * + * @return the number of UChars available in the internal buffer + * @see getBuffer + * @stable ICU 2.0 + */ + inline int32_t getCapacity(void) const; + + /* Other operations */ + + /** + * Generate a hash code for this object. + * @return The hash code of this UnicodeString. + * @stable ICU 2.0 + */ + inline int32_t hashCode(void) const; + + /** + * Determine if this object contains a valid string. + * A bogus string has no value. It is different from an empty string. + * It can be used to indicate that no string value is available. + * getBuffer() and getTerminatedBuffer() return NULL, and + * length() returns 0. + * + * @return TRUE if the string is valid, FALSE otherwise + * @see setToBogus() + * @stable ICU 2.0 + */ + inline UBool isBogus(void) const; + + + //======================================== + // Write operations + //======================================== + + /* Assignment operations */ + + /** + * Assignment operator. Replace the characters in this UnicodeString + * with the characters from srcText. + * @param srcText The text containing the characters to replace + * @return a reference to this + * @stable ICU 2.0 + */ + UnicodeString &operator=(const UnicodeString &srcText); + + /** + * Almost the same as the assignment operator. + * Replace the characters in this UnicodeString + * with the characters from srcText. + * + * This function works the same for all strings except for ones that + * are readonly aliases. + * Starting with ICU 2.4, the assignment operator and the copy constructor + * allocate a new buffer and copy the buffer contents even for readonly aliases. + * This function implements the old, more efficient but less safe behavior + * of making this string also a readonly alias to the same buffer. + * The fastCopyFrom function must be used only if it is known that the lifetime of + * this UnicodeString is at least as long as the lifetime of the aliased buffer + * including its contents, for example for strings from resource bundles + * or aliases to string contents. + * + * @param src The text containing the characters to replace. + * @return a reference to this + * @stable ICU 2.4 + */ + UnicodeString &fastCopyFrom(const UnicodeString &src); + + /** + * Assignment operator. Replace the characters in this UnicodeString + * with the code unit ch. + * @param ch the code unit to replace + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& operator= (UChar ch); + + /** + * Assignment operator. Replace the characters in this UnicodeString + * with the code point ch. + * @param ch the code point to replace + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& operator= (UChar32 ch); + + /** + * Set the text in the UnicodeString object to the characters + * in srcText in the range + * [srcStart, srcText.length()). + * srcText is not modified. + * @param srcText the source for the new characters + * @param srcStart the offset into srcText where new characters + * will be obtained + * @return a reference to this + * @stable ICU 2.2 + */ + inline UnicodeString& setTo(const UnicodeString& srcText, + int32_t srcStart); + + /** + * Set the text in the UnicodeString object to the characters + * in srcText in the range + * [srcStart, srcStart + srcLength). + * srcText is not modified. + * @param srcText the source for the new characters + * @param srcStart the offset into srcText where new characters + * will be obtained + * @param srcLength the number of characters in srcText in the + * replace string. + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& setTo(const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLength); + + /** + * Set the text in the UnicodeString object to the characters in + * srcText. + * srcText is not modified. + * @param srcText the source for the new characters + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& setTo(const UnicodeString& srcText); + + /** + * Set the characters in the UnicodeString object to the characters + * in srcChars. srcChars is not modified. + * @param srcChars the source for the new characters + * @param srcLength the number of Unicode characters in srcChars. + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& setTo(const UChar *srcChars, + int32_t srcLength); + + /** + * Set the characters in the UnicodeString object to the code unit + * srcChar. + * @param srcChar the code unit which becomes the UnicodeString's character + * content + * @return a reference to this + * @stable ICU 2.0 + */ + UnicodeString& setTo(UChar srcChar); + + /** + * Set the characters in the UnicodeString object to the code point + * srcChar. + * @param srcChar the code point which becomes the UnicodeString's character + * content + * @return a reference to this + * @stable ICU 2.0 + */ + UnicodeString& setTo(UChar32 srcChar); + + /** + * Aliasing setTo() function, analogous to the readonly-aliasing UChar* constructor. + * The text will be used for the UnicodeString object, but + * it will not be released when the UnicodeString is destroyed. + * This has copy-on-write semantics: + * When the string is modified, then the buffer is first copied into + * newly allocated memory. + * The aliased buffer is never modified. + * In an assignment to another UnicodeString, the text will be aliased again, + * so that both strings then alias the same readonly-text. + * + * @param isTerminated specifies if text is NUL-terminated. + * This must be true if textLength==-1. + * @param text The characters to alias for the UnicodeString. + * @param textLength The number of Unicode characters in text to alias. + * If -1, then this constructor will determine the length + * by calling u_strlen(). + * @return a reference to this + * @stable ICU 2.0 + */ + UnicodeString &setTo(UBool isTerminated, + const UChar *text, + int32_t textLength); + + /** + * Aliasing setTo() function, analogous to the writable-aliasing UChar* constructor. + * The text will be used for the UnicodeString object, but + * it will not be released when the UnicodeString is destroyed. + * This has write-through semantics: + * For as long as the capacity of the buffer is sufficient, write operations + * will directly affect the buffer. When more capacity is necessary, then + * a new buffer will be allocated and the contents copied as with regularly + * constructed strings. + * In an assignment to another UnicodeString, the buffer will be copied. + * The extract(UChar *dst) function detects whether the dst pointer is the same + * as the string buffer itself and will in this case not copy the contents. + * + * @param buffer The characters to alias for the UnicodeString. + * @param buffLength The number of Unicode characters in buffer to alias. + * @param buffCapacity The size of buffer in UChars. + * @return a reference to this + * @stable ICU 2.0 + */ + UnicodeString &setTo(UChar *buffer, + int32_t buffLength, + int32_t buffCapacity); + + /** + * Make this UnicodeString object invalid. + * The string will test TRUE with isBogus(). + * + * A bogus string has no value. It is different from an empty string. + * It can be used to indicate that no string value is available. + * getBuffer() and getTerminatedBuffer() return NULL, and + * length() returns 0. + * + * This utility function is used throughout the UnicodeString + * implementation to indicate that a UnicodeString operation failed, + * and may be used in other functions, + * especially but not exclusively when such functions do not + * take a UErrorCode for simplicity. + * + * The following methods, and no others, will clear a string object's bogus flag: + * - remove() + * - remove(0, INT32_MAX) + * - truncate(0) + * - operator=() (assignment operator) + * - setTo(...) + * + * The simplest ways to turn a bogus string into an empty one + * is to use the remove() function. + * Examples for other functions that are equivalent to "set to empty string": + * \code + * if(s.isBogus()) { + * s.remove(); // set to an empty string (remove all), or + * s.remove(0, INT32_MAX); // set to an empty string (remove all), or + * s.truncate(0); // set to an empty string (complete truncation), or + * s=UnicodeString(); // assign an empty string, or + * s.setTo((UChar32)-1); // set to a pseudo code point that is out of range, or + * static const UChar nul=0; + * s.setTo(&nul, 0); // set to an empty C Unicode string + * } + * \endcode + * + * @see isBogus() + * @stable ICU 2.0 + */ + void setToBogus(); + + /** + * Set the character at the specified offset to the specified character. + * @param offset A valid offset into the text of the character to set + * @param ch The new character + * @return A reference to this + * @stable ICU 2.0 + */ + UnicodeString& setCharAt(int32_t offset, + UChar ch); + + + /* Append operations */ + + /** + * Append operator. Append the code unit ch to the UnicodeString + * object. + * @param ch the code unit to be appended + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& operator+= (UChar ch); + + /** + * Append operator. Append the code point ch to the UnicodeString + * object. + * @param ch the code point to be appended + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& operator+= (UChar32 ch); + + /** + * Append operator. Append the characters in srcText to the + * UnicodeString object at offset start. srcText is + * not modified. + * @param srcText the source for the new characters + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& operator+= (const UnicodeString& srcText); + + /** + * Append the characters + * in srcText in the range + * [srcStart, srcStart + srcLength) to the + * UnicodeString object at offset start. srcText + * is not modified. + * @param srcText the source for the new characters + * @param srcStart the offset into srcText where new characters + * will be obtained + * @param srcLength the number of characters in srcText in + * the append string + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& append(const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLength); + + /** + * Append the characters in srcText to the UnicodeString object at + * offset start. srcText is not modified. + * @param srcText the source for the new characters + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& append(const UnicodeString& srcText); + + /** + * Append the characters in srcChars in the range + * [srcStart, srcStart + srcLength) to the UnicodeString + * object at offset + * start. srcChars is not modified. + * @param srcChars the source for the new characters + * @param srcStart the offset into srcChars where new characters + * will be obtained + * @param srcLength the number of characters in srcChars in + * the append string + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& append(const UChar *srcChars, + int32_t srcStart, + int32_t srcLength); + + /** + * Append the characters in srcChars to the UnicodeString object + * at offset start. srcChars is not modified. + * @param srcChars the source for the new characters + * @param srcLength the number of Unicode characters in srcChars + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& append(const UChar *srcChars, + int32_t srcLength); + + /** + * Append the code unit srcChar to the UnicodeString object. + * @param srcChar the code unit to append + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& append(UChar srcChar); + + /** + * Append the code point srcChar to the UnicodeString object. + * @param srcChar the code point to append + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& append(UChar32 srcChar); + + + /* Insert operations */ + + /** + * Insert the characters in srcText in the range + * [srcStart, srcStart + srcLength) into the UnicodeString + * object at offset start. srcText is not modified. + * @param start the offset where the insertion begins + * @param srcText the source for the new characters + * @param srcStart the offset into srcText where new characters + * will be obtained + * @param srcLength the number of characters in srcText in + * the insert string + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& insert(int32_t start, + const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLength); + + /** + * Insert the characters in srcText into the UnicodeString object + * at offset start. srcText is not modified. + * @param start the offset where the insertion begins + * @param srcText the source for the new characters + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& insert(int32_t start, + const UnicodeString& srcText); + + /** + * Insert the characters in srcChars in the range + * [srcStart, srcStart + srcLength) into the UnicodeString + * object at offset start. srcChars is not modified. + * @param start the offset at which the insertion begins + * @param srcChars the source for the new characters + * @param srcStart the offset into srcChars where new characters + * will be obtained + * @param srcLength the number of characters in srcChars + * in the insert string + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& insert(int32_t start, + const UChar *srcChars, + int32_t srcStart, + int32_t srcLength); + + /** + * Insert the characters in srcChars into the UnicodeString object + * at offset start. srcChars is not modified. + * @param start the offset where the insertion begins + * @param srcChars the source for the new characters + * @param srcLength the number of Unicode characters in srcChars. + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& insert(int32_t start, + const UChar *srcChars, + int32_t srcLength); + + /** + * Insert the code unit srcChar into the UnicodeString object at + * offset start. + * @param start the offset at which the insertion occurs + * @param srcChar the code unit to insert + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& insert(int32_t start, + UChar srcChar); + + /** + * Insert the code point srcChar into the UnicodeString object at + * offset start. + * @param start the offset at which the insertion occurs + * @param srcChar the code point to insert + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& insert(int32_t start, + UChar32 srcChar); + + + /* Replace operations */ + + /** + * Replace the characters in the range + * [start, start + length) with the characters in + * srcText in the range + * [srcStart, srcStart + srcLength). + * srcText is not modified. + * @param start the offset at which the replace operation begins + * @param length the number of characters to replace. The character at + * start + length is not modified. + * @param srcText the source for the new characters + * @param srcStart the offset into srcText where new characters + * will be obtained + * @param srcLength the number of characters in srcText in + * the replace string + * @return a reference to this + * @stable ICU 2.0 + */ + UnicodeString& replace(int32_t start, + int32_t length, + const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLength); + + /** + * Replace the characters in the range + * [start, start + length) + * with the characters in srcText. srcText is + * not modified. + * @param start the offset at which the replace operation begins + * @param length the number of characters to replace. The character at + * start + length is not modified. + * @param srcText the source for the new characters + * @return a reference to this + * @stable ICU 2.0 + */ + UnicodeString& replace(int32_t start, + int32_t length, + const UnicodeString& srcText); + + /** + * Replace the characters in the range + * [start, start + length) with the characters in + * srcChars in the range + * [srcStart, srcStart + srcLength). srcChars + * is not modified. + * @param start the offset at which the replace operation begins + * @param length the number of characters to replace. The character at + * start + length is not modified. + * @param srcChars the source for the new characters + * @param srcStart the offset into srcChars where new characters + * will be obtained + * @param srcLength the number of characters in srcChars + * in the replace string + * @return a reference to this + * @stable ICU 2.0 + */ + UnicodeString& replace(int32_t start, + int32_t length, + const UChar *srcChars, + int32_t srcStart, + int32_t srcLength); + + /** + * Replace the characters in the range + * [start, start + length) with the characters in + * srcChars. srcChars is not modified. + * @param start the offset at which the replace operation begins + * @param length number of characters to replace. The character at + * start + length is not modified. + * @param srcChars the source for the new characters + * @param srcLength the number of Unicode characters in srcChars + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& replace(int32_t start, + int32_t length, + const UChar *srcChars, + int32_t srcLength); + + /** + * Replace the characters in the range + * [start, start + length) with the code unit + * srcChar. + * @param start the offset at which the replace operation begins + * @param length the number of characters to replace. The character at + * start + length is not modified. + * @param srcChar the new code unit + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& replace(int32_t start, + int32_t length, + UChar srcChar); + + /** + * Replace the characters in the range + * [start, start + length) with the code point + * srcChar. + * @param start the offset at which the replace operation begins + * @param length the number of characters to replace. The character at + * start + length is not modified. + * @param srcChar the new code point + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& replace(int32_t start, + int32_t length, + UChar32 srcChar); + + /** + * Replace the characters in the range [start, limit) + * with the characters in srcText. srcText is not modified. + * @param start the offset at which the replace operation begins + * @param limit the offset immediately following the replace range + * @param srcText the source for the new characters + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& replaceBetween(int32_t start, + int32_t limit, + const UnicodeString& srcText); + + /** + * Replace the characters in the range [start, limit) + * with the characters in srcText in the range + * [srcStart, srcLimit). srcText is not modified. + * @param start the offset at which the replace operation begins + * @param limit the offset immediately following the replace range + * @param srcText the source for the new characters + * @param srcStart the offset into srcChars where new characters + * will be obtained + * @param srcLimit the offset immediately following the range to copy + * in srcText + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& replaceBetween(int32_t start, + int32_t limit, + const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLimit); + + /** + * Replace a substring of this object with the given text. + * @param start the beginning index, inclusive; 0 <= start + * <= limit. + * @param limit the ending index, exclusive; start <= limit + * <= length(). + * @param text the text to replace characters start + * to limit - 1 + * @stable ICU 2.0 + */ + virtual void handleReplaceBetween(int32_t start, + int32_t limit, + const UnicodeString& text); + + /** + * Replaceable API + * @return TRUE if it has MetaData + * @stable ICU 2.4 + */ + virtual UBool hasMetaData() const; + + /** + * Copy a substring of this object, retaining attribute (out-of-band) + * information. This method is used to duplicate or reorder substrings. + * The destination index must not overlap the source range. + * + * @param start the beginning index, inclusive; 0 <= start <= + * limit. + * @param limit the ending index, exclusive; start <= limit <= + * length(). + * @param dest the destination index. The characters from + * start..limit-1 will be copied to dest. + * Implementations of this method may assume that dest <= start || + * dest >= limit. + * @stable ICU 2.0 + */ + virtual void copy(int32_t start, int32_t limit, int32_t dest); + + /* Search and replace operations */ + + /** + * Replace all occurrences of characters in oldText with the characters + * in newText + * @param oldText the text containing the search text + * @param newText the text containing the replacement text + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& findAndReplace(const UnicodeString& oldText, + const UnicodeString& newText); + + /** + * Replace all occurrences of characters in oldText with characters + * in newText + * in the range [start, start + length). + * @param start the start of the range in which replace will performed + * @param length the length of the range in which replace will be performed + * @param oldText the text containing the search text + * @param newText the text containing the replacement text + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& findAndReplace(int32_t start, + int32_t length, + const UnicodeString& oldText, + const UnicodeString& newText); + + /** + * Replace all occurrences of characters in oldText in the range + * [oldStart, oldStart + oldLength) with the characters + * in newText in the range + * [newStart, newStart + newLength) + * in the range [start, start + length). + * @param start the start of the range in which replace will performed + * @param length the length of the range in which replace will be performed + * @param oldText the text containing the search text + * @param oldStart the start of the search range in oldText + * @param oldLength the length of the search range in oldText + * @param newText the text containing the replacement text + * @param newStart the start of the replacement range in newText + * @param newLength the length of the replacement range in newText + * @return a reference to this + * @stable ICU 2.0 + */ + UnicodeString& findAndReplace(int32_t start, + int32_t length, + const UnicodeString& oldText, + int32_t oldStart, + int32_t oldLength, + const UnicodeString& newText, + int32_t newStart, + int32_t newLength); + + + /* Remove operations */ + + /** + * Remove all characters from the UnicodeString object. + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& remove(void); + + /** + * Remove the characters in the range + * [start, start + length) from the UnicodeString object. + * @param start the offset of the first character to remove + * @param length the number of characters to remove + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& remove(int32_t start, + int32_t length = (int32_t)INT32_MAX); + + /** + * Remove the characters in the range + * [start, limit) from the UnicodeString object. + * @param start the offset of the first character to remove + * @param limit the offset immediately following the range to remove + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& removeBetween(int32_t start, + int32_t limit = (int32_t)INT32_MAX); + + + /* Length operations */ + + /** + * Pad the start of this UnicodeString with the character padChar. + * If the length of this UnicodeString is less than targetLength, + * length() - targetLength copies of padChar will be added to the + * beginning of this UnicodeString. + * @param targetLength the desired length of the string + * @param padChar the character to use for padding. Defaults to + * space (U+0020) + * @return TRUE if the text was padded, FALSE otherwise. + * @stable ICU 2.0 + */ + UBool padLeading(int32_t targetLength, + UChar padChar = 0x0020); + + /** + * Pad the end of this UnicodeString with the character padChar. + * If the length of this UnicodeString is less than targetLength, + * length() - targetLength copies of padChar will be added to the + * end of this UnicodeString. + * @param targetLength the desired length of the string + * @param padChar the character to use for padding. Defaults to + * space (U+0020) + * @return TRUE if the text was padded, FALSE otherwise. + * @stable ICU 2.0 + */ + UBool padTrailing(int32_t targetLength, + UChar padChar = 0x0020); + + /** + * Truncate this UnicodeString to the targetLength. + * @param targetLength the desired length of this UnicodeString. + * @return TRUE if the text was truncated, FALSE otherwise + * @stable ICU 2.0 + */ + inline UBool truncate(int32_t targetLength); + + /** + * Trims leading and trailing whitespace from this UnicodeString. + * @return a reference to this + * @stable ICU 2.0 + */ + UnicodeString& trim(void); + + + /* Miscellaneous operations */ + + /** + * Reverse this UnicodeString in place. + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& reverse(void); + + /** + * Reverse the range [start, start + length) in + * this UnicodeString. + * @param start the start of the range to reverse + * @param length the number of characters to to reverse + * @return a reference to this + * @stable ICU 2.0 + */ + inline UnicodeString& reverse(int32_t start, + int32_t length); + + /** + * Convert the characters in this to UPPER CASE following the conventions of + * the default locale. + * @return A reference to this. + * @stable ICU 2.0 + */ + UnicodeString& toUpper(void); + + /** + * Convert the characters in this to UPPER CASE following the conventions of + * a specific locale. + * @param locale The locale containing the conventions to use. + * @return A reference to this. + * @stable ICU 2.0 + */ + UnicodeString& toUpper(const Locale& locale); + + /** + * Convert the characters in this to lower case following the conventions of + * the default locale. + * @return A reference to this. + * @stable ICU 2.0 + */ + UnicodeString& toLower(void); + + /** + * Convert the characters in this to lower case following the conventions of + * a specific locale. + * @param locale The locale containing the conventions to use. + * @return A reference to this. + * @stable ICU 2.0 + */ + UnicodeString& toLower(const Locale& locale); + +#if !UCONFIG_NO_BREAK_ITERATION + + /** + * Titlecase this string, convenience function using the default locale. + * + * Casing is locale-dependent and context-sensitive. + * Titlecasing uses a break iterator to find the first characters of words + * that are to be titlecased. It titlecases those characters and lowercases + * all others. + * + * The titlecase break iterator can be provided to customize for arbitrary + * styles, using rules and dictionaries beyond the standard iterators. + * It may be more efficient to always provide an iterator to avoid + * opening and closing one for each string. + * The standard titlecase iterator for the root locale implements the + * algorithm of Unicode TR 21. + * + * This function uses only the setText(), first() and next() methods of the + * provided break iterator. + * + * @param titleIter A break iterator to find the first characters of words + * that are to be titlecased. + * If none is provided (0), then a standard titlecase + * break iterator is opened. + * Otherwise the provided iterator is set to the string's text. + * @return A reference to this. + * @stable ICU 2.1 + */ + UnicodeString &toTitle(BreakIterator *titleIter); + + /** + * Titlecase this string. + * + * Casing is locale-dependent and context-sensitive. + * Titlecasing uses a break iterator to find the first characters of words + * that are to be titlecased. It titlecases those characters and lowercases + * all others. + * + * The titlecase break iterator can be provided to customize for arbitrary + * styles, using rules and dictionaries beyond the standard iterators. + * It may be more efficient to always provide an iterator to avoid + * opening and closing one for each string. + * The standard titlecase iterator for the root locale implements the + * algorithm of Unicode TR 21. + * + * This function uses only the setText(), first() and next() methods of the + * provided break iterator. + * + * @param titleIter A break iterator to find the first characters of words + * that are to be titlecased. + * If none is provided (0), then a standard titlecase + * break iterator is opened. + * Otherwise the provided iterator is set to the string's text. + * @param locale The locale to consider. + * @return A reference to this. + * @stable ICU 2.1 + */ + UnicodeString &toTitle(BreakIterator *titleIter, const Locale &locale); + + /** + * Titlecase this string, with options. + * + * Casing is locale-dependent and context-sensitive. + * Titlecasing uses a break iterator to find the first characters of words + * that are to be titlecased. It titlecases those characters and lowercases + * all others. (This can be modified with options.) + * + * The titlecase break iterator can be provided to customize for arbitrary + * styles, using rules and dictionaries beyond the standard iterators. + * It may be more efficient to always provide an iterator to avoid + * opening and closing one for each string. + * The standard titlecase iterator for the root locale implements the + * algorithm of Unicode TR 21. + * + * This function uses only the setText(), first() and next() methods of the + * provided break iterator. + * + * @param titleIter A break iterator to find the first characters of words + * that are to be titlecased. + * If none is provided (0), then a standard titlecase + * break iterator is opened. + * Otherwise the provided iterator is set to the string's text. + * @param locale The locale to consider. + * @param options Options bit set, see ucasemap_open(). + * @return A reference to this. + * @see U_TITLECASE_NO_LOWERCASE + * @see U_TITLECASE_NO_BREAK_ADJUSTMENT + * @see ucasemap_open + * @draft ICU 3.8 + */ + UnicodeString &toTitle(BreakIterator *titleIter, const Locale &locale, uint32_t options); + +#endif + + /** + * Case-fold the characters in this string. + * Case-folding is locale-independent and not context-sensitive, + * but there is an option for whether to include or exclude mappings for dotted I + * and dotless i that are marked with 'I' in CaseFolding.txt. + * The result may be longer or shorter than the original. + * + * @param options Either U_FOLD_CASE_DEFAULT or U_FOLD_CASE_EXCLUDE_SPECIAL_I + * @return A reference to this. + * @stable ICU 2.0 + */ + UnicodeString &foldCase(uint32_t options=0 /*U_FOLD_CASE_DEFAULT*/); + + //======================================== + // Access to the internal buffer + //======================================== + + /** + * Get a read/write pointer to the internal buffer. + * The buffer is guaranteed to be large enough for at least minCapacity UChars, + * writable, and is still owned by the UnicodeString object. + * Calls to getBuffer(minCapacity) must not be nested, and + * must be matched with calls to releaseBuffer(newLength). + * If the string buffer was read-only or shared, + * then it will be reallocated and copied. + * + * An attempted nested call will return 0, and will not further modify the + * state of the UnicodeString object. + * It also returns 0 if the string is bogus. + * + * The actual capacity of the string buffer may be larger than minCapacity. + * getCapacity() returns the actual capacity. + * For many operations, the full capacity should be used to avoid reallocations. + * + * While the buffer is "open" between getBuffer(minCapacity) + * and releaseBuffer(newLength), the following applies: + * - The string length is set to 0. + * - Any read API call on the UnicodeString object will behave like on a 0-length string. + * - Any write API call on the UnicodeString object is disallowed and will have no effect. + * - You can read from and write to the returned buffer. + * - The previous string contents will still be in the buffer; + * if you want to use it, then you need to call length() before getBuffer(minCapacity). + * If the length() was greater than minCapacity, then any contents after minCapacity + * may be lost. + * The buffer contents is not NUL-terminated by getBuffer(). + * If length()(s.length(). + * (See getTerminatedBuffer().) + * + * The buffer may reside in read-only memory. Its contents must not + * be modified. + * + * @return a read-only pointer to the internal string buffer, + * or 0 if the string is empty or bogus + * + * @see getBuffer(int32_t minCapacity) + * @see getTerminatedBuffer() + * @stable ICU 2.0 + */ + inline const UChar *getBuffer() const; + + /** + * Get a read-only pointer to the internal buffer, + * making sure that it is NUL-terminated. + * This can be called at any time on a valid UnicodeString. + * + * It returns 0 if the string is bogus, or + * during an "open" getBuffer(minCapacity), or if the buffer cannot + * be NUL-terminated (because memory allocation failed). + * + * It can be called as many times as desired. + * The pointer that it returns will remain valid until the UnicodeString object is modified, + * at which time the pointer is semantically invalidated and must not be used any more. + * + * The capacity of the buffer can be determined with getCapacity(). + * The part after length()+1 may or may not be initialized and valid, + * depending on the history of the UnicodeString object. + * + * The buffer contents is guaranteed to be NUL-terminated. + * getTerminatedBuffer() may reallocate the buffer if a terminating NUL + * is written. + * For this reason, this function is not const, unlike getBuffer(). + * Note that a UnicodeString may also contain NUL characters as part of its contents. + * + * The buffer may reside in read-only memory. Its contents must not + * be modified. + * + * @return a read-only pointer to the internal string buffer, + * or 0 if the string is empty or bogus + * + * @see getBuffer(int32_t minCapacity) + * @see getBuffer() + * @stable ICU 2.2 + */ + inline const UChar *getTerminatedBuffer(); + + //======================================== + // Constructors + //======================================== + + /** Construct an empty UnicodeString. + * @stable ICU 2.0 + */ + UnicodeString(); + + /** + * Construct a UnicodeString with capacity to hold capacity UChars + * @param capacity the number of UChars this UnicodeString should hold + * before a resize is necessary; if count is greater than 0 and count + * code points c take up more space than capacity, then capacity is adjusted + * accordingly. + * @param c is used to initially fill the string + * @param count specifies how many code points c are to be written in the + * string + * @stable ICU 2.0 + */ + UnicodeString(int32_t capacity, UChar32 c, int32_t count); + + /** + * Single UChar (code unit) constructor. + * @param ch the character to place in the UnicodeString + * @stable ICU 2.0 + */ + UnicodeString(UChar ch); + + /** + * Single UChar32 (code point) constructor. + * @param ch the character to place in the UnicodeString + * @stable ICU 2.0 + */ + UnicodeString(UChar32 ch); + + /** + * UChar* constructor. + * @param text The characters to place in the UnicodeString. text + * must be NULL (U+0000) terminated. + * @stable ICU 2.0 + */ + UnicodeString(const UChar *text); + + /** + * UChar* constructor. + * @param text The characters to place in the UnicodeString. + * @param textLength The number of Unicode characters in text + * to copy. + * @stable ICU 2.0 + */ + UnicodeString(const UChar *text, + int32_t textLength); + + /** + * Readonly-aliasing UChar* constructor. + * The text will be used for the UnicodeString object, but + * it will not be released when the UnicodeString is destroyed. + * This has copy-on-write semantics: + * When the string is modified, then the buffer is first copied into + * newly allocated memory. + * The aliased buffer is never modified. + * In an assignment to another UnicodeString, the text will be aliased again, + * so that both strings then alias the same readonly-text. + * + * @param isTerminated specifies if text is NUL-terminated. + * This must be true if textLength==-1. + * @param text The characters to alias for the UnicodeString. + * @param textLength The number of Unicode characters in text to alias. + * If -1, then this constructor will determine the length + * by calling u_strlen(). + * @stable ICU 2.0 + */ + UnicodeString(UBool isTerminated, + const UChar *text, + int32_t textLength); + + /** + * Writable-aliasing UChar* constructor. + * The text will be used for the UnicodeString object, but + * it will not be released when the UnicodeString is destroyed. + * This has write-through semantics: + * For as long as the capacity of the buffer is sufficient, write operations + * will directly affect the buffer. When more capacity is necessary, then + * a new buffer will be allocated and the contents copied as with regularly + * constructed strings. + * In an assignment to another UnicodeString, the buffer will be copied. + * The extract(UChar *dst) function detects whether the dst pointer is the same + * as the string buffer itself and will in this case not copy the contents. + * + * @param buffer The characters to alias for the UnicodeString. + * @param buffLength The number of Unicode characters in buffer to alias. + * @param buffCapacity The size of buffer in UChars. + * @stable ICU 2.0 + */ + UnicodeString(UChar *buffer, int32_t buffLength, int32_t buffCapacity); + +#if !UCONFIG_NO_CONVERSION + + /** + * char* constructor. + * @param codepageData an array of bytes, null-terminated + * @param codepage the encoding of codepageData. The special + * value 0 for codepage indicates that the text is in the + * platform's default codepage. + * + * If codepage is an empty string (""), + * then a simple conversion is performed on the codepage-invariant + * subset ("invariant characters") of the platform encoding. See utypes.h. + * Recommendation: For invariant-character strings use the constructor + * UnicodeString(const char *src, int32_t length, enum EInvariant inv) + * because it avoids object code dependencies of UnicodeString on + * the conversion code. + * + * @stable ICU 2.0 + */ + UnicodeString(const char *codepageData, + const char *codepage = 0); + + /** + * char* constructor. + * @param codepageData an array of bytes. + * @param dataLength The number of bytes in codepageData. + * @param codepage the encoding of codepageData. The special + * value 0 for codepage indicates that the text is in the + * platform's default codepage. + * If codepage is an empty string (""), + * then a simple conversion is performed on the codepage-invariant + * subset ("invariant characters") of the platform encoding. See utypes.h. + * Recommendation: For invariant-character strings use the constructor + * UnicodeString(const char *src, int32_t length, enum EInvariant inv) + * because it avoids object code dependencies of UnicodeString on + * the conversion code. + * + * @stable ICU 2.0 + */ + UnicodeString(const char *codepageData, + int32_t dataLength, + const char *codepage = 0); + + /** + * char * / UConverter constructor. + * This constructor uses an existing UConverter object to + * convert the codepage string to Unicode and construct a UnicodeString + * from that. + * + * The converter is reset at first. + * If the error code indicates a failure before this constructor is called, + * or if an error occurs during conversion or construction, + * then the string will be bogus. + * + * This function avoids the overhead of opening and closing a converter if + * multiple strings are constructed. + * + * @param src input codepage string + * @param srcLength length of the input string, can be -1 for NUL-terminated strings + * @param cnv converter object (ucnv_resetToUnicode() will be called), + * can be NULL for the default converter + * @param errorCode normal ICU error code + * @stable ICU 2.0 + */ + UnicodeString( + const char *src, int32_t srcLength, + UConverter *cnv, + UErrorCode &errorCode); + +#endif + + /** + * Constructs a Unicode string from an invariant-character char * string. + * About invariant characters see utypes.h. + * This constructor has no runtime dependency on conversion code and is + * therefore recommended over ones taking a charset name string + * (where the empty string "" indicates invariant-character conversion). + * + * Use the macro US_INV as the third, signature-distinguishing parameter. + * + * For example: + * \code + * void fn(const char *s) { + * UnicodeString ustr(s, -1, US_INV); + * // use ustr ... + * } + * \endcode + * + * @param src String using only invariant characters. + * @param length Length of src, or -1 if NUL-terminated. + * @param inv Signature-distinguishing paramater, use US_INV. + * + * @see US_INV + * @stable ICU 3.2 + */ + UnicodeString(const char *src, int32_t length, enum EInvariant inv); + + + /** + * Copy constructor. + * @param that The UnicodeString object to copy. + * @stable ICU 2.0 + */ + UnicodeString(const UnicodeString& that); + + /** + * 'Substring' constructor from tail of source string. + * @param src The UnicodeString object to copy. + * @param srcStart The offset into src at which to start copying. + * @stable ICU 2.2 + */ + UnicodeString(const UnicodeString& src, int32_t srcStart); + + /** + * 'Substring' constructor from subrange of source string. + * @param src The UnicodeString object to copy. + * @param srcStart The offset into src at which to start copying. + * @param srcLength The number of characters from src to copy. + * @stable ICU 2.2 + */ + UnicodeString(const UnicodeString& src, int32_t srcStart, int32_t srcLength); + + /** + * Clone this object, an instance of a subclass of Replaceable. + * Clones can be used concurrently in multiple threads. + * If a subclass does not implement clone(), or if an error occurs, + * then NULL is returned. + * The clone functions in all subclasses return a pointer to a Replaceable + * because some compilers do not support covariant (same-as-this) + * return types; cast to the appropriate subclass if necessary. + * The caller must delete the clone. + * + * @return a clone of this object + * + * @see Replaceable::clone + * @see getDynamicClassID + * @stable ICU 2.6 + */ + virtual Replaceable *clone() const; + + /** Destructor. + * @stable ICU 2.0 + */ + virtual ~UnicodeString(); + + + /* Miscellaneous operations */ + + /** + * Unescape a string of characters and return a string containing + * the result. The following escape sequences are recognized: + * + * \\uhhhh 4 hex digits; h in [0-9A-Fa-f] + * \\Uhhhhhhhh 8 hex digits + * \\xhh 1-2 hex digits + * \\ooo 1-3 octal digits; o in [0-7] + * \\cX control-X; X is masked with 0x1F + * + * as well as the standard ANSI C escapes: + * + * \\a => U+0007, \\b => U+0008, \\t => U+0009, \\n => U+000A, + * \\v => U+000B, \\f => U+000C, \\r => U+000D, \\e => U+001B, + * \\" => U+0022, \\' => U+0027, \\? => U+003F, \\\\ => U+005C + * + * Anything else following a backslash is generically escaped. For + * example, "[a\\-z]" returns "[a-z]". + * + * If an escape sequence is ill-formed, this method returns an empty + * string. An example of an ill-formed sequence is "\\u" followed by + * fewer than 4 hex digits. + * + * This function is similar to u_unescape() but not identical to it. + * The latter takes a source char*, so it does escape recognition + * and also invariant conversion. + * + * @return a string with backslash escapes interpreted, or an + * empty string on error. + * @see UnicodeString#unescapeAt() + * @see u_unescape() + * @see u_unescapeAt() + * @stable ICU 2.0 + */ + UnicodeString unescape() const; + + /** + * Unescape a single escape sequence and return the represented + * character. See unescape() for a listing of the recognized escape + * sequences. The character at offset-1 is assumed (without + * checking) to be a backslash. If the escape sequence is + * ill-formed, or the offset is out of range, (UChar32)0xFFFFFFFF is + * returned. + * + * @param offset an input output parameter. On input, it is the + * offset into this string where the escape sequence is located, + * after the initial backslash. On output, it is advanced after the + * last character parsed. On error, it is not advanced at all. + * @return the character represented by the escape sequence at + * offset, or (UChar32)0xFFFFFFFF on error. + * @see UnicodeString#unescape() + * @see u_unescape() + * @see u_unescapeAt() + * @stable ICU 2.0 + */ + UChar32 unescapeAt(int32_t &offset) const; + + /** + * ICU "poor man's RTTI", returns a UClassID for this class. + * + * @stable ICU 2.2 + */ + static UClassID U_EXPORT2 getStaticClassID(); + + /** + * ICU "poor man's RTTI", returns a UClassID for the actual class. + * + * @stable ICU 2.2 + */ + virtual UClassID getDynamicClassID() const; + + //======================================== + // Implementation methods + //======================================== + +protected: + /** + * Implement Replaceable::getLength() (see jitterbug 1027). + * @stable ICU 2.4 + */ + virtual int32_t getLength() const; + + /** + * The change in Replaceable to use virtual getCharAt() allows + * UnicodeString::charAt() to be inline again (see jitterbug 709). + * @stable ICU 2.4 + */ + virtual UChar getCharAt(int32_t offset) const; + + /** + * The change in Replaceable to use virtual getChar32At() allows + * UnicodeString::char32At() to be inline again (see jitterbug 709). + * @stable ICU 2.4 + */ + virtual UChar32 getChar32At(int32_t offset) const; + +private: + + inline int8_t + doCompare(int32_t start, + int32_t length, + const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLength) const; + + int8_t doCompare(int32_t start, + int32_t length, + const UChar *srcChars, + int32_t srcStart, + int32_t srcLength) const; + + inline int8_t + doCompareCodePointOrder(int32_t start, + int32_t length, + const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLength) const; + + int8_t doCompareCodePointOrder(int32_t start, + int32_t length, + const UChar *srcChars, + int32_t srcStart, + int32_t srcLength) const; + + inline int8_t + doCaseCompare(int32_t start, + int32_t length, + const UnicodeString &srcText, + int32_t srcStart, + int32_t srcLength, + uint32_t options) const; + + int8_t + doCaseCompare(int32_t start, + int32_t length, + const UChar *srcChars, + int32_t srcStart, + int32_t srcLength, + uint32_t options) const; + + int32_t doIndexOf(UChar c, + int32_t start, + int32_t length) const; + + int32_t doIndexOf(UChar32 c, + int32_t start, + int32_t length) const; + + int32_t doLastIndexOf(UChar c, + int32_t start, + int32_t length) const; + + int32_t doLastIndexOf(UChar32 c, + int32_t start, + int32_t length) const; + + void doExtract(int32_t start, + int32_t length, + UChar *dst, + int32_t dstStart) const; + + inline void doExtract(int32_t start, + int32_t length, + UnicodeString& target) const; + + inline UChar doCharAt(int32_t offset) const; + + UnicodeString& doReplace(int32_t start, + int32_t length, + const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLength); + + UnicodeString& doReplace(int32_t start, + int32_t length, + const UChar *srcChars, + int32_t srcStart, + int32_t srcLength); + + UnicodeString& doReverse(int32_t start, + int32_t length); + + // calculate hash code + int32_t doHashCode(void) const; + + // get pointer to start of array + inline UChar* getArrayStart(void); + inline const UChar* getArrayStart(void) const; + + // allocate the array; result may be fStackBuffer + // sets refCount to 1 if appropriate + // sets fArray, fCapacity, and fFlags + // returns boolean for success or failure + UBool allocate(int32_t capacity); + + // release the array if owned + void releaseArray(void); + + // turn a bogus string into an empty one + void unBogus(); + + // implements assigment operator, copy constructor, and fastCopyFrom() + UnicodeString ©From(const UnicodeString &src, UBool fastCopy=FALSE); + + // Pin start and limit to acceptable values. + inline void pinIndex(int32_t& start) const; + inline void pinIndices(int32_t& start, + int32_t& length) const; + +#if !UCONFIG_NO_CONVERSION + + /* Internal extract() using UConverter. */ + int32_t doExtract(int32_t start, int32_t length, + char *dest, int32_t destCapacity, + UConverter *cnv, + UErrorCode &errorCode) const; + + /* + * Real constructor for converting from codepage data. + * It assumes that it is called with !fRefCounted. + * + * If codepage==0, then the default converter + * is used for the platform encoding. + * If codepage is an empty string (""), + * then a simple conversion is performed on the codepage-invariant + * subset ("invariant characters") of the platform encoding. See utypes.h. + */ + void doCodepageCreate(const char *codepageData, + int32_t dataLength, + const char *codepage); + + /* + * Worker function for creating a UnicodeString from + * a codepage string using a UConverter. + */ + void + doCodepageCreate(const char *codepageData, + int32_t dataLength, + UConverter *converter, + UErrorCode &status); + +#endif + + /* + * This function is called when write access to the array + * is necessary. + * + * We need to make a copy of the array if + * the buffer is read-only, or + * the buffer is refCounted (shared), and refCount>1, or + * the buffer is too small. + * + * Return FALSE if memory could not be allocated. + */ + UBool cloneArrayIfNeeded(int32_t newCapacity = -1, + int32_t growCapacity = -1, + UBool doCopyArray = TRUE, + int32_t **pBufferToDelete = 0, + UBool forceClone = FALSE); + + // common function for case mappings + UnicodeString & + caseMap(BreakIterator *titleIter, + const char *locale, + uint32_t options, + int32_t toWhichCase); + + // ref counting + void addRef(void); + int32_t removeRef(void); + int32_t refCount(void) const; + + // constants + enum { + US_STACKBUF_SIZE=7, // Size of stack buffer for small strings + kInvalidUChar=0xffff, // invalid UChar index + kGrowSize=128, // grow size for this buffer + kInvalidHashCode=0, // invalid hash code + kEmptyHashCode=1, // hash code for empty string + + // bit flag values for fFlags + kIsBogus=1, // this string is bogus, i.e., not valid or NULL + kUsingStackBuffer=2,// fArray==fStackBuffer + kRefCounted=4, // there is a refCount field before the characters in fArray + kBufferIsReadonly=8,// do not write to this buffer + kOpenGetBuffer=16, // getBuffer(minCapacity) was called (is "open"), + // and releaseBuffer(newLength) must be called + + // combined values for convenience + kShortString=kUsingStackBuffer, + kLongString=kRefCounted, + kReadonlyAlias=kBufferIsReadonly, + kWritableAlias=0 + }; + + friend class StringCharacterIterator; + friend class StringThreadTest; + + /* + * The following are all the class fields that are stored + * in each UnicodeString object. + * Note that UnicodeString has virtual functions, + * therefore there is an implicit vtable pointer + * as the first real field. + * The fields should be aligned such that no padding is + * necessary, mostly by having larger types first. + * On 32-bit machines, the size should be 32 bytes, + * on 64-bit machines (8-byte pointers), it should be 40 bytes. + */ + // (implicit) *vtable; + int32_t fLength; // number of characters in fArray + int32_t fCapacity; // sizeof fArray + UChar *fArray; // the Unicode data + uint16_t fFlags; // bit flags: see constants above + UChar fStackBuffer [ US_STACKBUF_SIZE ]; // buffer for small strings + +}; + +/** + * Create a new UnicodeString with the concatenation of two others. + * + * @param s1 The first string to be copied to the new one. + * @param s2 The second string to be copied to the new one, after s1. + * @return UnicodeString(s1).append(s2) + * @stable ICU 2.8 + */ +U_COMMON_API UnicodeString U_EXPORT2 +operator+ (const UnicodeString &s1, const UnicodeString &s2); + +//======================================== +// Inline members +//======================================== + +//======================================== +// Privates +//======================================== + +inline void +UnicodeString::pinIndex(int32_t& start) const +{ + // pin index + if(start < 0) { + start = 0; + } else if(start > fLength) { + start = fLength; + } +} + +inline void +UnicodeString::pinIndices(int32_t& start, + int32_t& _length) const +{ + // pin indices + if(start < 0) { + start = 0; + } else if(start > fLength) { + start = fLength; + } + if(_length < 0) { + _length = 0; + } else if(_length > (fLength - start)) { + _length = (fLength - start); + } +} + +inline UChar* +UnicodeString::getArrayStart() +{ return fArray; } + +inline const UChar* +UnicodeString::getArrayStart() const +{ return fArray; } + +//======================================== +// Read-only implementation methods +//======================================== +inline int32_t +UnicodeString::length() const +{ return fLength; } + +inline int32_t +UnicodeString::getCapacity() const +{ return fCapacity; } + +inline int32_t +UnicodeString::hashCode() const +{ return doHashCode(); } + +inline UBool +UnicodeString::isBogus() const +{ return (UBool)(fFlags & kIsBogus); } + +inline const UChar * +UnicodeString::getBuffer() const { + if(!(fFlags&(kIsBogus|kOpenGetBuffer))) { + return fArray; + } else { + return 0; + } +} + +//======================================== +// Read-only alias methods +//======================================== +inline int8_t +UnicodeString::doCompare(int32_t start, + int32_t length, + const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLength) const +{ + if(srcText.isBogus()) { + return (int8_t)!isBogus(); // 0 if both are bogus, 1 otherwise + } else { + srcText.pinIndices(srcStart, srcLength); + return doCompare(start, length, srcText.fArray, srcStart, srcLength); + } +} + +inline UBool +UnicodeString::operator== (const UnicodeString& text) const +{ + if(isBogus()) { + return text.isBogus(); + } else { + return + !text.isBogus() && + fLength == text.fLength && + doCompare(0, fLength, text, 0, text.fLength) == 0; + } +} + +inline UBool +UnicodeString::operator!= (const UnicodeString& text) const +{ return (! operator==(text)); } + +inline UBool +UnicodeString::operator> (const UnicodeString& text) const +{ return doCompare(0, fLength, text, 0, text.fLength) == 1; } + +inline UBool +UnicodeString::operator< (const UnicodeString& text) const +{ return doCompare(0, fLength, text, 0, text.fLength) == -1; } + +inline UBool +UnicodeString::operator>= (const UnicodeString& text) const +{ return doCompare(0, fLength, text, 0, text.fLength) != -1; } + +inline UBool +UnicodeString::operator<= (const UnicodeString& text) const +{ return doCompare(0, fLength, text, 0, text.fLength) != 1; } + +inline int8_t +UnicodeString::compare(const UnicodeString& text) const +{ return doCompare(0, fLength, text, 0, text.fLength); } + +inline int8_t +UnicodeString::compare(int32_t start, + int32_t _length, + const UnicodeString& srcText) const +{ return doCompare(start, _length, srcText, 0, srcText.fLength); } + +inline int8_t +UnicodeString::compare(const UChar *srcChars, + int32_t srcLength) const +{ return doCompare(0, fLength, srcChars, 0, srcLength); } + +inline int8_t +UnicodeString::compare(int32_t start, + int32_t _length, + const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLength) const +{ return doCompare(start, _length, srcText, srcStart, srcLength); } + +inline int8_t +UnicodeString::compare(int32_t start, + int32_t _length, + const UChar *srcChars) const +{ return doCompare(start, _length, srcChars, 0, _length); } + +inline int8_t +UnicodeString::compare(int32_t start, + int32_t _length, + const UChar *srcChars, + int32_t srcStart, + int32_t srcLength) const +{ return doCompare(start, _length, srcChars, srcStart, srcLength); } + +inline int8_t +UnicodeString::compareBetween(int32_t start, + int32_t limit, + const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLimit) const +{ return doCompare(start, limit - start, + srcText, srcStart, srcLimit - srcStart); } + +inline int8_t +UnicodeString::doCompareCodePointOrder(int32_t start, + int32_t length, + const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLength) const +{ + if(srcText.isBogus()) { + return (int8_t)!isBogus(); // 0 if both are bogus, 1 otherwise + } else { + srcText.pinIndices(srcStart, srcLength); + return doCompareCodePointOrder(start, length, srcText.fArray, srcStart, srcLength); + } +} + +inline int8_t +UnicodeString::compareCodePointOrder(const UnicodeString& text) const +{ return doCompareCodePointOrder(0, fLength, text, 0, text.fLength); } + +inline int8_t +UnicodeString::compareCodePointOrder(int32_t start, + int32_t _length, + const UnicodeString& srcText) const +{ return doCompareCodePointOrder(start, _length, srcText, 0, srcText.fLength); } + +inline int8_t +UnicodeString::compareCodePointOrder(const UChar *srcChars, + int32_t srcLength) const +{ return doCompareCodePointOrder(0, fLength, srcChars, 0, srcLength); } + +inline int8_t +UnicodeString::compareCodePointOrder(int32_t start, + int32_t _length, + const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLength) const +{ return doCompareCodePointOrder(start, _length, srcText, srcStart, srcLength); } + +inline int8_t +UnicodeString::compareCodePointOrder(int32_t start, + int32_t _length, + const UChar *srcChars) const +{ return doCompareCodePointOrder(start, _length, srcChars, 0, _length); } + +inline int8_t +UnicodeString::compareCodePointOrder(int32_t start, + int32_t _length, + const UChar *srcChars, + int32_t srcStart, + int32_t srcLength) const +{ return doCompareCodePointOrder(start, _length, srcChars, srcStart, srcLength); } + +inline int8_t +UnicodeString::compareCodePointOrderBetween(int32_t start, + int32_t limit, + const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLimit) const +{ return doCompareCodePointOrder(start, limit - start, + srcText, srcStart, srcLimit - srcStart); } + +inline int8_t +UnicodeString::doCaseCompare(int32_t start, + int32_t length, + const UnicodeString &srcText, + int32_t srcStart, + int32_t srcLength, + uint32_t options) const +{ + if(srcText.isBogus()) { + return (int8_t)!isBogus(); // 0 if both are bogus, 1 otherwise + } else { + srcText.pinIndices(srcStart, srcLength); + return doCaseCompare(start, length, srcText.fArray, srcStart, srcLength, options); + } +} + +inline int8_t +UnicodeString::caseCompare(const UnicodeString &text, uint32_t options) const { + return doCaseCompare(0, fLength, text, 0, text.fLength, options); +} + +inline int8_t +UnicodeString::caseCompare(int32_t start, + int32_t _length, + const UnicodeString &srcText, + uint32_t options) const { + return doCaseCompare(start, _length, srcText, 0, srcText.fLength, options); +} + +inline int8_t +UnicodeString::caseCompare(const UChar *srcChars, + int32_t srcLength, + uint32_t options) const { + return doCaseCompare(0, fLength, srcChars, 0, srcLength, options); +} + +inline int8_t +UnicodeString::caseCompare(int32_t start, + int32_t _length, + const UnicodeString &srcText, + int32_t srcStart, + int32_t srcLength, + uint32_t options) const { + return doCaseCompare(start, _length, srcText, srcStart, srcLength, options); +} + +inline int8_t +UnicodeString::caseCompare(int32_t start, + int32_t _length, + const UChar *srcChars, + uint32_t options) const { + return doCaseCompare(start, _length, srcChars, 0, _length, options); +} + +inline int8_t +UnicodeString::caseCompare(int32_t start, + int32_t _length, + const UChar *srcChars, + int32_t srcStart, + int32_t srcLength, + uint32_t options) const { + return doCaseCompare(start, _length, srcChars, srcStart, srcLength, options); +} + +inline int8_t +UnicodeString::caseCompareBetween(int32_t start, + int32_t limit, + const UnicodeString &srcText, + int32_t srcStart, + int32_t srcLimit, + uint32_t options) const { + return doCaseCompare(start, limit - start, srcText, srcStart, srcLimit - srcStart, options); +} + +inline int32_t +UnicodeString::indexOf(const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLength, + int32_t start, + int32_t _length) const +{ + if(!srcText.isBogus()) { + srcText.pinIndices(srcStart, srcLength); + if(srcLength > 0) { + return indexOf(srcText.getArrayStart(), srcStart, srcLength, start, _length); + } + } + return -1; +} + +inline int32_t +UnicodeString::indexOf(const UnicodeString& text) const +{ return indexOf(text, 0, text.fLength, 0, fLength); } + +inline int32_t +UnicodeString::indexOf(const UnicodeString& text, + int32_t start) const { + pinIndex(start); + return indexOf(text, 0, text.fLength, start, fLength - start); +} + +inline int32_t +UnicodeString::indexOf(const UnicodeString& text, + int32_t start, + int32_t _length) const +{ return indexOf(text, 0, text.fLength, start, _length); } + +inline int32_t +UnicodeString::indexOf(const UChar *srcChars, + int32_t srcLength, + int32_t start) const { + pinIndex(start); + return indexOf(srcChars, 0, srcLength, start, fLength - start); +} + +inline int32_t +UnicodeString::indexOf(const UChar *srcChars, + int32_t srcLength, + int32_t start, + int32_t _length) const +{ return indexOf(srcChars, 0, srcLength, start, _length); } + +inline int32_t +UnicodeString::indexOf(UChar c, + int32_t start, + int32_t _length) const +{ return doIndexOf(c, start, _length); } + +inline int32_t +UnicodeString::indexOf(UChar32 c, + int32_t start, + int32_t _length) const +{ return doIndexOf(c, start, _length); } + +inline int32_t +UnicodeString::indexOf(UChar c) const +{ return doIndexOf(c, 0, fLength); } + +inline int32_t +UnicodeString::indexOf(UChar32 c) const +{ return indexOf(c, 0, fLength); } + +inline int32_t +UnicodeString::indexOf(UChar c, + int32_t start) const { + pinIndex(start); + return doIndexOf(c, start, fLength - start); +} + +inline int32_t +UnicodeString::indexOf(UChar32 c, + int32_t start) const { + pinIndex(start); + return indexOf(c, start, fLength - start); +} + +inline int32_t +UnicodeString::lastIndexOf(const UChar *srcChars, + int32_t srcLength, + int32_t start, + int32_t _length) const +{ return lastIndexOf(srcChars, 0, srcLength, start, _length); } + +inline int32_t +UnicodeString::lastIndexOf(const UChar *srcChars, + int32_t srcLength, + int32_t start) const { + pinIndex(start); + return lastIndexOf(srcChars, 0, srcLength, start, fLength - start); +} + +inline int32_t +UnicodeString::lastIndexOf(const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLength, + int32_t start, + int32_t _length) const +{ + if(!srcText.isBogus()) { + srcText.pinIndices(srcStart, srcLength); + if(srcLength > 0) { + return lastIndexOf(srcText.getArrayStart(), srcStart, srcLength, start, _length); + } + } + return -1; +} + +inline int32_t +UnicodeString::lastIndexOf(const UnicodeString& text, + int32_t start, + int32_t _length) const +{ return lastIndexOf(text, 0, text.fLength, start, _length); } + +inline int32_t +UnicodeString::lastIndexOf(const UnicodeString& text, + int32_t start) const { + pinIndex(start); + return lastIndexOf(text, 0, text.fLength, start, fLength - start); +} + +inline int32_t +UnicodeString::lastIndexOf(const UnicodeString& text) const +{ return lastIndexOf(text, 0, text.fLength, 0, fLength); } + +inline int32_t +UnicodeString::lastIndexOf(UChar c, + int32_t start, + int32_t _length) const +{ return doLastIndexOf(c, start, _length); } + +inline int32_t +UnicodeString::lastIndexOf(UChar32 c, + int32_t start, + int32_t _length) const { + return doLastIndexOf(c, start, _length); +} + +inline int32_t +UnicodeString::lastIndexOf(UChar c) const +{ return doLastIndexOf(c, 0, fLength); } + +inline int32_t +UnicodeString::lastIndexOf(UChar32 c) const { + return lastIndexOf(c, 0, fLength); +} + +inline int32_t +UnicodeString::lastIndexOf(UChar c, + int32_t start) const { + pinIndex(start); + return doLastIndexOf(c, start, fLength - start); +} + +inline int32_t +UnicodeString::lastIndexOf(UChar32 c, + int32_t start) const { + pinIndex(start); + return lastIndexOf(c, start, fLength - start); +} + +inline UBool +UnicodeString::startsWith(const UnicodeString& text) const +{ return compare(0, text.fLength, text, 0, text.fLength) == 0; } + +inline UBool +UnicodeString::startsWith(const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLength) const +{ return doCompare(0, srcLength, srcText, srcStart, srcLength) == 0; } + +inline UBool +UnicodeString::startsWith(const UChar *srcChars, + int32_t srcLength) const +{ return doCompare(0, srcLength, srcChars, 0, srcLength) == 0; } + +inline UBool +UnicodeString::startsWith(const UChar *srcChars, + int32_t srcStart, + int32_t srcLength) const +{ return doCompare(0, srcLength, srcChars, srcStart, srcLength) == 0;} + +inline UBool +UnicodeString::endsWith(const UnicodeString& text) const +{ return doCompare(fLength - text.fLength, text.fLength, + text, 0, text.fLength) == 0; } + +inline UBool +UnicodeString::endsWith(const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLength) const { + srcText.pinIndices(srcStart, srcLength); + return doCompare(fLength - srcLength, srcLength, + srcText, srcStart, srcLength) == 0; +} + +inline UBool +UnicodeString::endsWith(const UChar *srcChars, + int32_t srcLength) const { + if(srcLength < 0) { + srcLength = u_strlen(srcChars); + } + return doCompare(fLength - srcLength, srcLength, + srcChars, 0, srcLength) == 0; +} + +inline UBool +UnicodeString::endsWith(const UChar *srcChars, + int32_t srcStart, + int32_t srcLength) const { + if(srcLength < 0) { + srcLength = u_strlen(srcChars + srcStart); + } + return doCompare(fLength - srcLength, srcLength, + srcChars, srcStart, srcLength) == 0; +} + +//======================================== +// replace +//======================================== +inline UnicodeString& +UnicodeString::replace(int32_t start, + int32_t _length, + const UnicodeString& srcText) +{ return doReplace(start, _length, srcText, 0, srcText.fLength); } + +inline UnicodeString& +UnicodeString::replace(int32_t start, + int32_t _length, + const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLength) +{ return doReplace(start, _length, srcText, srcStart, srcLength); } + +inline UnicodeString& +UnicodeString::replace(int32_t start, + int32_t _length, + const UChar *srcChars, + int32_t srcLength) +{ return doReplace(start, _length, srcChars, 0, srcLength); } + +inline UnicodeString& +UnicodeString::replace(int32_t start, + int32_t _length, + const UChar *srcChars, + int32_t srcStart, + int32_t srcLength) +{ return doReplace(start, _length, srcChars, srcStart, srcLength); } + +inline UnicodeString& +UnicodeString::replace(int32_t start, + int32_t _length, + UChar srcChar) +{ return doReplace(start, _length, &srcChar, 0, 1); } + +inline UnicodeString& +UnicodeString::replace(int32_t start, + int32_t _length, + UChar32 srcChar) { + UChar buffer[U16_MAX_LENGTH]; + int32_t count = 0; + UBool isError = FALSE; + U16_APPEND(buffer, count, U16_MAX_LENGTH, srcChar, isError); + return doReplace(start, _length, buffer, 0, count); +} + +inline UnicodeString& +UnicodeString::replaceBetween(int32_t start, + int32_t limit, + const UnicodeString& srcText) +{ return doReplace(start, limit - start, srcText, 0, srcText.fLength); } + +inline UnicodeString& +UnicodeString::replaceBetween(int32_t start, + int32_t limit, + const UnicodeString& srcText, + int32_t srcStart, + int32_t srcLimit) +{ return doReplace(start, limit - start, srcText, srcStart, srcLimit - srcStart); } + +inline UnicodeString& +UnicodeString::findAndReplace(const UnicodeString& oldText, + const UnicodeString& newText) +{ return findAndReplace(0, fLength, oldText, 0, oldText.fLength, + newText, 0, newText.fLength); } + +inline UnicodeString& +UnicodeString::findAndReplace(int32_t start, + int32_t _length, + const UnicodeString& oldText, + const UnicodeString& newText) +{ return findAndReplace(start, _length, oldText, 0, oldText.fLength, + newText, 0, newText.fLength); } + +// ============================ +// extract +// ============================ +inline void +UnicodeString::doExtract(int32_t start, + int32_t _length, + UnicodeString& target) const +{ target.replace(0, target.fLength, *this, start, _length); } + +inline void +UnicodeString::extract(int32_t start, + int32_t _length, + UChar *target, + int32_t targetStart) const +{ doExtract(start, _length, target, targetStart); } + +inline void +UnicodeString::extract(int32_t start, + int32_t _length, + UnicodeString& target) const +{ doExtract(start, _length, target); } + +#if !UCONFIG_NO_CONVERSION + +inline int32_t +UnicodeString::extract(int32_t start, + int32_t _length, + char *dst, + const char *codepage) const + +{ + // This dstSize value will be checked explicitly + return extract(start, _length, dst, dst!=0 ? 0xffffffff : 0, codepage); +} + +#endif + +inline void +UnicodeString::extractBetween(int32_t start, + int32_t limit, + UChar *dst, + int32_t dstStart) const { + pinIndex(start); + pinIndex(limit); + doExtract(start, limit - start, dst, dstStart); +} + +inline UChar +UnicodeString::doCharAt(int32_t offset) const +{ + if((uint32_t)offset < (uint32_t)fLength) { + return fArray[offset]; + } else { + return kInvalidUChar; + } +} + +inline UChar +UnicodeString::charAt(int32_t offset) const +{ return doCharAt(offset); } + +inline UChar +UnicodeString::operator[] (int32_t offset) const +{ return doCharAt(offset); } + +inline UChar32 +UnicodeString::char32At(int32_t offset) const +{ + if((uint32_t)offset < (uint32_t)fLength) { + UChar32 c; + U16_GET(fArray, 0, offset, fLength, c); + return c; + } else { + return kInvalidUChar; + } +} + +inline int32_t +UnicodeString::getChar32Start(int32_t offset) const { + if((uint32_t)offset < (uint32_t)fLength) { + U16_SET_CP_START(fArray, 0, offset); + return offset; + } else { + return 0; + } +} + +inline int32_t +UnicodeString::getChar32Limit(int32_t offset) const { + if((uint32_t)offset < (uint32_t)fLength) { + U16_SET_CP_LIMIT(fArray, 0, offset, fLength); + return offset; + } else { + return fLength; + } +} + +inline UBool +UnicodeString::isEmpty() const { + return fLength == 0; +} + +//======================================== +// Write implementation methods +//======================================== +inline const UChar * +UnicodeString::getTerminatedBuffer() { + if(fFlags&(kIsBogus|kOpenGetBuffer)) { + return 0; + } else if(fLengthUnicode normalization API + * + * unorm_normalize transforms Unicode text into an equivalent composed or + * decomposed form, allowing for easier sorting and searching of text. + * unorm_normalize supports the standard normalization forms described in + * + * Unicode Standard Annex #15: Unicode Normalization Forms. + * + * Characters with accents or other adornments can be encoded in + * several different ways in Unicode. For example, take the character A-acute. + * In Unicode, this can be encoded as a single character (the + * "composed" form): + * + * \code + * 00C1 LATIN CAPITAL LETTER A WITH ACUTE + * \endcode + * + * or as two separate characters (the "decomposed" form): + * + * \code + * 0041 LATIN CAPITAL LETTER A + * 0301 COMBINING ACUTE ACCENT + * \endcode + * + * To a user of your program, however, both of these sequences should be + * treated as the same "user-level" character "A with acute accent". When you are searching or + * comparing text, you must ensure that these two sequences are treated + * equivalently. In addition, you must handle characters with more than one + * accent. Sometimes the order of a character's combining accents is + * significant, while in other cases accent sequences in different orders are + * really equivalent. + * + * Similarly, the string "ffi" can be encoded as three separate letters: + * + * \code + * 0066 LATIN SMALL LETTER F + * 0066 LATIN SMALL LETTER F + * 0069 LATIN SMALL LETTER I + * \endcode + * + * or as the single character + * + * \code + * FB03 LATIN SMALL LIGATURE FFI + * \endcode + * + * The ffi ligature is not a distinct semantic character, and strictly speaking + * it shouldn't be in Unicode at all, but it was included for compatibility + * with existing character sets that already provided it. The Unicode standard + * identifies such characters by giving them "compatibility" decompositions + * into the corresponding semantic characters. When sorting and searching, you + * will often want to use these mappings. + * + * unorm_normalize helps solve these problems by transforming text into the + * canonical composed and decomposed forms as shown in the first example above. + * In addition, you can have it perform compatibility decompositions so that + * you can treat compatibility characters the same as their equivalents. + * Finally, unorm_normalize rearranges accents into the proper canonical + * order, so that you do not have to worry about accent rearrangement on your + * own. + * + * Form FCD, "Fast C or D", is also designed for collation. + * It allows to work on strings that are not necessarily normalized + * with an algorithm (like in collation) that works under "canonical closure", i.e., it treats precomposed + * characters and their decomposed equivalents the same. + * + * It is not a normalization form because it does not provide for uniqueness of representation. Multiple strings + * may be canonically equivalent (their NFDs are identical) and may all conform to FCD without being identical + * themselves. + * + * The form is defined such that the "raw decomposition", the recursive canonical decomposition of each character, + * results in a string that is canonically ordered. This means that precomposed characters are allowed for as long + * as their decompositions do not need canonical reordering. + * + * Its advantage for a process like collation is that all NFD and most NFC texts - and many unnormalized texts - + * already conform to FCD and do not need to be normalized (NFD) for such a process. The FCD quick check will + * return UNORM_YES for most strings in practice. + * + * unorm_normalize(UNORM_FCD) may be implemented with UNORM_NFD. + * + * For more details on FCD see the collation design document: + * http://source.icu-project.org/repos/icu/icuhtml/trunk/design/collation/ICU_collation_design.htm + * + * ICU collation performs either NFD or FCD normalization automatically if normalization + * is turned on for the collator object. + * Beyond collation and string search, normalized strings may be useful for string equivalence comparisons, + * transliteration/transcription, unique representations, etc. + * + * The W3C generally recommends to exchange texts in NFC. + * Note also that most legacy character encodings use only precomposed forms and often do not + * encode any combining marks by themselves. For conversion to such character encodings the + * Unicode text needs to be normalized to NFC. + * For more usage examples, see the Unicode Standard Annex. + */ + +/** + * Constants for normalization modes. + * @stable ICU 2.0 + */ +typedef enum { + /** No decomposition/composition. @stable ICU 2.0 */ + UNORM_NONE = 1, + /** Canonical decomposition. @stable ICU 2.0 */ + UNORM_NFD = 2, + /** Compatibility decomposition. @stable ICU 2.0 */ + UNORM_NFKD = 3, + /** Canonical decomposition followed by canonical composition. @stable ICU 2.0 */ + UNORM_NFC = 4, + /** Default normalization. @stable ICU 2.0 */ + UNORM_DEFAULT = UNORM_NFC, + /** Compatibility decomposition followed by canonical composition. @stable ICU 2.0 */ + UNORM_NFKC =5, + /** "Fast C or D" form. @stable ICU 2.0 */ + UNORM_FCD = 6, + + /** One more than the highest normalization mode constant. @stable ICU 2.0 */ + UNORM_MODE_COUNT +} UNormalizationMode; + +/** + * Constants for options flags for normalization. + * Use 0 for default options, + * including normalization according to the Unicode version + * that is currently supported by ICU (see u_getUnicodeVersion). + * @stable ICU 2.6 + */ +enum { + /** + * Options bit set value to select Unicode 3.2 normalization + * (except NormalizationCorrections). + * At most one Unicode version can be selected at a time. + * @stable ICU 2.6 + */ + UNORM_UNICODE_3_2=0x20 +}; + +/** + * Lowest-order bit number of unorm_compare() options bits corresponding to + * normalization options bits. + * + * The options parameter for unorm_compare() uses most bits for + * itself and for various comparison and folding flags. + * The most significant bits, however, are shifted down and passed on + * to the normalization implementation. + * (That is, from unorm_compare(..., options, ...), + * options>>UNORM_COMPARE_NORM_OPTIONS_SHIFT will be passed on to the + * internal normalization functions.) + * + * @see unorm_compare + * @stable ICU 2.6 + */ +#define UNORM_COMPARE_NORM_OPTIONS_SHIFT 20 + +/** + * Normalize a string. + * The string will be normalized according the specified normalization mode + * and options. + * The source and result buffers must not be the same, nor overlap. + * + * @param source The string to normalize. + * @param sourceLength The length of source, or -1 if NUL-terminated. + * @param mode The normalization mode; one of UNORM_NONE, + * UNORM_NFD, UNORM_NFC, UNORM_NFKC, UNORM_NFKD, UNORM_DEFAULT. + * @param options The normalization options, ORed together (0 for no options). + * @param result A pointer to a buffer to receive the result string. + * The result string is NUL-terminated if possible. + * @param resultLength The maximum size of result. + * @param status A pointer to a UErrorCode to receive any errors. + * @return The total buffer size needed; if greater than resultLength, + * the output was truncated, and the error code is set to U_BUFFER_OVERFLOW_ERROR. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +unorm_normalize(const UChar *source, int32_t sourceLength, + UNormalizationMode mode, int32_t options, + UChar *result, int32_t resultLength, + UErrorCode *status); +#endif +/** + * Result values for unorm_quickCheck(). + * For details see Unicode Technical Report 15. + * @stable ICU 2.0 + */ +typedef enum UNormalizationCheckResult { + /** + * Indicates that string is not in the normalized format + */ + UNORM_NO, + /** + * Indicates that string is in the normalized format + */ + UNORM_YES, + /** + * Indicates that string cannot be determined if it is in the normalized + * format without further thorough checks. + */ + UNORM_MAYBE +} UNormalizationCheckResult; +#if !UCONFIG_NO_NORMALIZATION +/** + * Performing quick check on a string, to quickly determine if the string is + * in a particular normalization format. + * Three types of result can be returned UNORM_YES, UNORM_NO or + * UNORM_MAYBE. Result UNORM_YES indicates that the argument + * string is in the desired normalized format, UNORM_NO determines that + * argument string is not in the desired normalized format. A + * UNORM_MAYBE result indicates that a more thorough check is required, + * the user may have to put the string in its normalized form and compare the + * results. + * + * @param source string for determining if it is in a normalized format + * @param sourcelength length of source to test, or -1 if NUL-terminated + * @param mode which normalization form to test for + * @param status a pointer to a UErrorCode to receive any errors + * @return UNORM_YES, UNORM_NO or UNORM_MAYBE + * + * @see unorm_isNormalized + * @stable ICU 2.0 + */ +U_STABLE UNormalizationCheckResult U_EXPORT2 +unorm_quickCheck(const UChar *source, int32_t sourcelength, + UNormalizationMode mode, + UErrorCode *status); + +/** + * Performing quick check on a string; same as unorm_quickCheck but + * takes an extra options parameter like most normalization functions. + * + * @param src String that is to be tested if it is in a normalization format. + * @param srcLength Length of source to test, or -1 if NUL-terminated. + * @param mode Which normalization form to test for. + * @param options The normalization options, ORed together (0 for no options). + * @param pErrorCode ICU error code in/out parameter. + * Must fulfill U_SUCCESS before the function call. + * @return UNORM_YES, UNORM_NO or UNORM_MAYBE + * + * @see unorm_quickCheck + * @see unorm_isNormalized + * @stable ICU 2.6 + */ +U_STABLE UNormalizationCheckResult U_EXPORT2 +unorm_quickCheckWithOptions(const UChar *src, int32_t srcLength, + UNormalizationMode mode, int32_t options, + UErrorCode *pErrorCode); + +/** + * Test if a string is in a given normalization form. + * This is semantically equivalent to source.equals(normalize(source, mode)) . + * + * Unlike unorm_quickCheck(), this function returns a definitive result, + * never a "maybe". + * For NFD, NFKD, and FCD, both functions work exactly the same. + * For NFC and NFKC where quickCheck may return "maybe", this function will + * perform further tests to arrive at a TRUE/FALSE result. + * + * @param src String that is to be tested if it is in a normalization format. + * @param srcLength Length of source to test, or -1 if NUL-terminated. + * @param mode Which normalization form to test for. + * @param pErrorCode ICU error code in/out parameter. + * Must fulfill U_SUCCESS before the function call. + * @return Boolean value indicating whether the source string is in the + * "mode" normalization form. + * + * @see unorm_quickCheck + * @stable ICU 2.2 + */ +U_STABLE UBool U_EXPORT2 +unorm_isNormalized(const UChar *src, int32_t srcLength, + UNormalizationMode mode, + UErrorCode *pErrorCode); + +/** + * Test if a string is in a given normalization form; same as unorm_isNormalized but + * takes an extra options parameter like most normalization functions. + * + * @param src String that is to be tested if it is in a normalization format. + * @param srcLength Length of source to test, or -1 if NUL-terminated. + * @param mode Which normalization form to test for. + * @param options The normalization options, ORed together (0 for no options). + * @param pErrorCode ICU error code in/out parameter. + * Must fulfill U_SUCCESS before the function call. + * @return Boolean value indicating whether the source string is in the + * "mode/options" normalization form. + * + * @see unorm_quickCheck + * @see unorm_isNormalized + * @stable ICU 2.6 + */ +U_STABLE UBool U_EXPORT2 +unorm_isNormalizedWithOptions(const UChar *src, int32_t srcLength, + UNormalizationMode mode, int32_t options, + UErrorCode *pErrorCode); + +/** + * Iterative normalization forward. + * This function (together with unorm_previous) is somewhat + * similar to the C++ Normalizer class (see its non-static functions). + * + * Iterative normalization is useful when only a small portion of a longer + * string/text needs to be processed. + * + * For example, the likelihood may be high that processing the first 10% of some + * text will be sufficient to find certain data. + * Another example: When one wants to concatenate two normalized strings and get a + * normalized result, it is much more efficient to normalize just a small part of + * the result around the concatenation place instead of re-normalizing everything. + * + * The input text is an instance of the C character iteration API UCharIterator. + * It may wrap around a simple string, a CharacterIterator, a Replaceable, or any + * other kind of text object. + * + * If a buffer overflow occurs, then the caller needs to reset the iterator to the + * old index and call the function again with a larger buffer - if the caller cares + * for the actual output. + * Regardless of the output buffer, the iterator will always be moved to the next + * normalization boundary. + * + * This function (like unorm_previous) serves two purposes: + * + * 1) To find the next boundary so that the normalization of the part of the text + * from the current position to that boundary does not affect and is not affected + * by the part of the text beyond that boundary. + * + * 2) To normalize the text up to the boundary. + * + * The second step is optional, per the doNormalize parameter. + * It is omitted for operations like string concatenation, where the two adjacent + * string ends need to be normalized together. + * In such a case, the output buffer will just contain a copy of the text up to the + * boundary. + * + * pNeededToNormalize is an output-only parameter. Its output value is only defined + * if normalization was requested (doNormalize) and successful (especially, no + * buffer overflow). + * It is useful for operations like a normalizing transliterator, where one would + * not want to replace a piece of text if it is not modified. + * + * If doNormalize==TRUE and pNeededToNormalize!=NULL then *pNeeded... is set TRUE + * if the normalization was necessary. + * + * If doNormalize==FALSE then *pNeededToNormalize will be set to FALSE. + * + * If the buffer overflows, then *pNeededToNormalize will be undefined; + * essentially, whenever U_FAILURE is true (like in buffer overflows), this result + * will be undefined. + * + * @param src The input text in the form of a C character iterator. + * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting. + * @param destCapacity The number of UChars that fit into dest. + * @param mode The normalization mode. + * @param options The normalization options, ORed together (0 for no options). + * @param doNormalize Indicates if the source text up to the next boundary + * is to be normalized (TRUE) or just copied (FALSE). + * @param pNeededToNormalize Output flag indicating if the normalization resulted in + * different text from the input. + * Not defined if an error occurs including buffer overflow. + * Always FALSE if !doNormalize. + * @param pErrorCode ICU error code in/out parameter. + * Must fulfill U_SUCCESS before the function call. + * @return Length of output (number of UChars) when successful or buffer overflow. + * + * @see unorm_previous + * @see unorm_normalize + * + * @stable ICU 2.1 + */ +U_STABLE int32_t U_EXPORT2 +unorm_next(UCharIterator *src, + UChar *dest, int32_t destCapacity, + UNormalizationMode mode, int32_t options, + UBool doNormalize, UBool *pNeededToNormalize, + UErrorCode *pErrorCode); + +/** + * Iterative normalization backward. + * This function (together with unorm_next) is somewhat + * similar to the C++ Normalizer class (see its non-static functions). + * For all details see unorm_next. + * + * @param src The input text in the form of a C character iterator. + * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting. + * @param destCapacity The number of UChars that fit into dest. + * @param mode The normalization mode. + * @param options The normalization options, ORed together (0 for no options). + * @param doNormalize Indicates if the source text up to the next boundary + * is to be normalized (TRUE) or just copied (FALSE). + * @param pNeededToNormalize Output flag indicating if the normalization resulted in + * different text from the input. + * Not defined if an error occurs including buffer overflow. + * Always FALSE if !doNormalize. + * @param pErrorCode ICU error code in/out parameter. + * Must fulfill U_SUCCESS before the function call. + * @return Length of output (number of UChars) when successful or buffer overflow. + * + * @see unorm_next + * @see unorm_normalize + * + * @stable ICU 2.1 + */ +U_STABLE int32_t U_EXPORT2 +unorm_previous(UCharIterator *src, + UChar *dest, int32_t destCapacity, + UNormalizationMode mode, int32_t options, + UBool doNormalize, UBool *pNeededToNormalize, + UErrorCode *pErrorCode); + +/** + * Concatenate normalized strings, making sure that the result is normalized as well. + * + * If both the left and the right strings are in + * the normalization form according to "mode/options", + * then the result will be + * + * \code + * dest=normalize(left+right, mode, options) + * \endcode + * + * With the input strings already being normalized, + * this function will use unorm_next() and unorm_previous() + * to find the adjacent end pieces of the input strings. + * Only the concatenation of these end pieces will be normalized and + * then concatenated with the remaining parts of the input strings. + * + * It is allowed to have dest==left to avoid copying the entire left string. + * + * @param left Left source string, may be same as dest. + * @param leftLength Length of left source string, or -1 if NUL-terminated. + * @param right Right source string. Must not be the same as dest, nor overlap. + * @param rightLength Length of right source string, or -1 if NUL-terminated. + * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting. + * @param destCapacity The number of UChars that fit into dest. + * @param mode The normalization mode. + * @param options The normalization options, ORed together (0 for no options). + * @param pErrorCode ICU error code in/out parameter. + * Must fulfill U_SUCCESS before the function call. + * @return Length of output (number of UChars) when successful or buffer overflow. + * + * @see unorm_normalize + * @see unorm_next + * @see unorm_previous + * + * @stable ICU 2.1 + */ +U_STABLE int32_t U_EXPORT2 +unorm_concatenate(const UChar *left, int32_t leftLength, + const UChar *right, int32_t rightLength, + UChar *dest, int32_t destCapacity, + UNormalizationMode mode, int32_t options, + UErrorCode *pErrorCode); + +/** + * Option bit for unorm_compare: + * Both input strings are assumed to fulfill FCD conditions. + * @stable ICU 2.2 + */ +#define UNORM_INPUT_IS_FCD 0x20000 + +/** + * Option bit for unorm_compare: + * Perform case-insensitive comparison. + * @stable ICU 2.2 + */ +#define U_COMPARE_IGNORE_CASE 0x10000 + +#ifndef U_COMPARE_CODE_POINT_ORDER +/* see also unistr.h and ustring.h */ +/** + * Option bit for u_strCaseCompare, u_strcasecmp, unorm_compare, etc: + * Compare strings in code point order instead of code unit order. + * @stable ICU 2.2 + */ +#define U_COMPARE_CODE_POINT_ORDER 0x8000 +#endif + +/** + * Compare two strings for canonical equivalence. + * Further options include case-insensitive comparison and + * code point order (as opposed to code unit order). + * + * Canonical equivalence between two strings is defined as their normalized + * forms (NFD or NFC) being identical. + * This function compares strings incrementally instead of normalizing + * (and optionally case-folding) both strings entirely, + * improving performance significantly. + * + * Bulk normalization is only necessary if the strings do not fulfill the FCD + * conditions. Only in this case, and only if the strings are relatively long, + * is memory allocated temporarily. + * For FCD strings and short non-FCD strings there is no memory allocation. + * + * Semantically, this is equivalent to + * strcmp[CodePointOrder](NFD(foldCase(NFD(s1))), NFD(foldCase(NFD(s2)))) + * where code point order and foldCase are all optional. + * + * UAX 21 2.5 Caseless Matching specifies that for a canonical caseless match + * the case folding must be performed first, then the normalization. + * + * @param s1 First source string. + * @param length1 Length of first source string, or -1 if NUL-terminated. + * + * @param s2 Second source string. + * @param length2 Length of second source string, or -1 if NUL-terminated. + * + * @param options A bit set of options: + * - U_FOLD_CASE_DEFAULT or 0 is used for default options: + * Case-sensitive comparison in code unit order, and the input strings + * are quick-checked for FCD. + * + * - UNORM_INPUT_IS_FCD + * Set if the caller knows that both s1 and s2 fulfill the FCD conditions. + * If not set, the function will quickCheck for FCD + * and normalize if necessary. + * + * - U_COMPARE_CODE_POINT_ORDER + * Set to choose code point order instead of code unit order + * (see u_strCompare for details). + * + * - U_COMPARE_IGNORE_CASE + * Set to compare strings case-insensitively using case folding, + * instead of case-sensitively. + * If set, then the following case folding options are used. + * + * - Options as used with case-insensitive comparisons, currently: + * + * - U_FOLD_CASE_EXCLUDE_SPECIAL_I + * (see u_strCaseCompare for details) + * + * - regular normalization options shifted left by UNORM_COMPARE_NORM_OPTIONS_SHIFT + * + * @param pErrorCode ICU error code in/out parameter. + * Must fulfill U_SUCCESS before the function call. + * @return <0 or 0 or >0 as usual for string comparisons + * + * @see unorm_normalize + * @see UNORM_FCD + * @see u_strCompare + * @see u_strCaseCompare + * + * @stable ICU 2.2 + */ +U_STABLE int32_t U_EXPORT2 +unorm_compare(const UChar *s1, int32_t length1, + const UChar *s2, int32_t length2, + uint32_t options, + UErrorCode *pErrorCode); + +#endif /* #if !UCONFIG_NO_NORMALIZATION */ + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/uobject.h b/Build/source/libs/icu/icu-xetex/common/unicode/uobject.h new file mode 100644 index 00000000000..3d8b96ebec2 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/uobject.h @@ -0,0 +1,308 @@ +/* +****************************************************************************** +* +* Copyright (C) 2002-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: uobject.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2002jun26 +* created by: Markus W. Scherer +*/ + +#ifndef __UOBJECT_H__ +#define __UOBJECT_H__ + +#include "unicode/utypes.h" + +U_NAMESPACE_BEGIN + +/** + * \file + * \brief C++ API: Common ICU base class UObject. + */ + +/** U_OVERRIDE_CXX_ALLOCATION - Define this to override operator new and + * delete in UMemory. Enabled by default for ICU. + * + * Enabling forces all allocation of ICU object types to use ICU's + * memory allocation. On Windows, this allows the ICU DLL to be used by + * applications that statically link the C Runtime library, meaning that + * the app and ICU will be using different heaps. + * + * @stable ICU 2.2 + */ +#ifndef U_OVERRIDE_CXX_ALLOCATION +#define U_OVERRIDE_CXX_ALLOCATION 1 +#endif + +/** U_HAVE_PLACEMENT_NEW - Define this to define the placement new and + * delete in UMemory for STL. + * + * @stable ICU 2.6 + */ +#ifndef U_HAVE_PLACEMENT_NEW +#define U_HAVE_PLACEMENT_NEW 1 +#endif + + +/** U_HAVE_DEBUG_LOCATION_NEW - Define this to define the MFC debug + * version of the operator new. + * + * @stable ICU 3.4 + */ +#ifndef U_HAVE_DEBUG_LOCATION_NEW +#define U_HAVE_DEBUG_LOCATION_NEW 0 +#endif + +/** + * UMemory is the common ICU base class. + * All other ICU C++ classes are derived from UMemory (starting with ICU 2.4). + * + * This is primarily to make it possible and simple to override the + * C++ memory management by adding new/delete operators to this base class. + * + * To override ALL ICU memory management, including that from plain C code, + * replace the allocation functions declared in cmemory.h + * + * UMemory does not contain any virtual functions. + * Common "boilerplate" functions are defined in UObject. + * + * @stable ICU 2.4 + */ +class U_COMMON_API UMemory { +public: + +#if U_OVERRIDE_CXX_ALLOCATION + /** + * Override for ICU4C C++ memory management. + * simple, non-class types are allocated using the macros in common/cmemory.h + * (uprv_malloc(), uprv_free(), uprv_realloc()); + * they or something else could be used here to implement C++ new/delete + * for ICU4C C++ classes + * @stable ICU 2.4 + */ + static void * U_EXPORT2 operator new(size_t size); + + /** + * Override for ICU4C C++ memory management. + * See new(). + * @stable ICU 2.4 + */ + static void * U_EXPORT2 operator new[](size_t size); + + /** + * Override for ICU4C C++ memory management. + * simple, non-class types are allocated using the macros in common/cmemory.h + * (uprv_malloc(), uprv_free(), uprv_realloc()); + * they or something else could be used here to implement C++ new/delete + * for ICU4C C++ classes + * @stable ICU 2.4 + */ + static void U_EXPORT2 operator delete(void *p); + + /** + * Override for ICU4C C++ memory management. + * See delete(). + * @stable ICU 2.4 + */ + static void U_EXPORT2 operator delete[](void *p); + +#if U_HAVE_PLACEMENT_NEW + /** + * Override for ICU4C C++ memory management for STL. + * See new(). + * @stable ICU 2.6 + */ + static inline void * U_EXPORT2 operator new(size_t, void *ptr) { return ptr; } + + /** + * Override for ICU4C C++ memory management for STL. + * See delete(). + * @stable ICU 2.6 + */ + static inline void U_EXPORT2 operator delete(void *, void *) {} +#endif /* U_HAVE_PLACEMENT_NEW */ +#if U_HAVE_DEBUG_LOCATION_NEW + /** + * This method overrides the MFC debug version of the operator new + * + * @param size The requested memory size + * @param file The file where the allocation was requested + * @param line The line where the allocation was requested + */ + static void * U_EXPORT2 operator new(size_t size, const char* file, int line); + /** + * This method provides a matching delete for the MFC debug new + * + * @param p The pointer to the allocated memory + * @param file The file where the allocation was requested + * @param line The line where the allocation was requested + */ + static void U_EXPORT2 operator delete(void* p, const char* file, int line); +#endif /* U_HAVE_DEBUG_LOCATION_NEW */ +#endif /* U_OVERRIDE_CXX_ALLOCATION */ + + /* + * Assignment operator not declared. The compiler will provide one + * which does nothing since this class does not contain any data members. + * API/code coverage may show the assignment operator as present and + * untested - ignore. + * Subclasses need this assignment operator if they use compiler-provided + * assignment operators of their own. An alternative to not declaring one + * here would be to declare and empty-implement a protected or public one. + UMemory &UMemory::operator=(const UMemory &); + */ +}; + +/** + * UObject is the common ICU "boilerplate" class. + * UObject inherits UMemory (starting with ICU 2.4), + * and all other public ICU C++ classes + * are derived from UObject (starting with ICU 2.2). + * + * UObject contains common virtual functions like for ICU's "poor man's RTTI". + * It does not contain default implementations of virtual methods + * like getDynamicClassID to allow derived classes such as Format + * to declare these as pure virtual. + * + * The clone() function is not available in UObject because it is not + * implemented by all ICU classes. + * Many ICU services provide a clone() function for their class trees, + * defined on the service's C++ base class, and all subclasses within that + * service class tree return a pointer to the service base class + * (which itself is a subclass of UObject). + * This is because some compilers do not support covariant (same-as-this) + * return types; cast to the appropriate subclass if necessary. + * + * @stable ICU 2.2 + */ +class U_COMMON_API UObject : public UMemory { +public: + /** + * Destructor. + * + * @stable ICU 2.2 + */ + virtual ~UObject(); + + /** + * ICU4C "poor man's RTTI", returns a UClassID for the actual ICU class. + * + * @stable ICU 2.2 + */ + virtual UClassID getDynamicClassID() const = 0; + +protected: + // the following functions are protected to prevent instantiation and + // direct use of UObject itself + + // default constructor + // commented out because UObject is abstract (see getDynamicClassID) + // inline UObject() {} + + // copy constructor + // commented out because UObject is abstract (see getDynamicClassID) + // inline UObject(const UObject &other) {} + +#if 0 + // TODO Sometime in the future. Implement operator==(). + // (This comment inserted in 2.2) + // some or all of the following "boilerplate" functions may be made public + // in a future ICU4C release when all subclasses implement them + + // assignment operator + // (not virtual, see "Taligent's Guide to Designing Programs" pp.73..74) + // commented out because the implementation is the same as a compiler's default + // UObject &operator=(const UObject &other) { return *this; } + + // comparison operators + virtual inline UBool operator==(const UObject &other) const { return this==&other; } + inline UBool operator!=(const UObject &other) const { return !operator==(other); } + + // clone() commented out from the base class: + // some compilers do not support co-variant return types + // (i.e., subclasses would have to return UObject * as well, instead of SubClass *) + // see also UObject class documentation. + // virtual UObject *clone() const; +#endif + + /* + * Assignment operator not declared. The compiler will provide one + * which does nothing since this class does not contain any data members. + * API/code coverage may show the assignment operator as present and + * untested - ignore. + * Subclasses need this assignment operator if they use compiler-provided + * assignment operators of their own. An alternative to not declaring one + * here would be to declare and empty-implement a protected or public one. + UObject &UObject::operator=(const UObject &); + */ + +// Future implementation for RTTI that support subtyping. [alan] +// +// public: +// /** +// * @internal +// */ +// static UClassID getStaticClassID(); +// +// /** +// * @internal +// */ +// UBool instanceOf(UClassID type) const; +}; + +/** + * This is a simple macro to add ICU RTTI to an ICU object implementation. + * This does not go into the header. This should only be used in *.cpp files. + * + * @param myClass The name of the class that needs RTTI defined. + * @internal + */ +#define UOBJECT_DEFINE_RTTI_IMPLEMENTATION(myClass) \ + UClassID U_EXPORT2 myClass::getStaticClassID() { \ + static char classID = 0; \ + return (UClassID)&classID; \ + } \ + UClassID myClass::getDynamicClassID() const \ + { return myClass::getStaticClassID(); } + + +/** + * This macro adds ICU RTTI to an ICU abstract class implementation. + * This macro should be invoked in *.cpp files. The corresponding + * header should declare getStaticClassID. + * + * @param myClass The name of the class that needs RTTI defined. + * @internal + */ +#define UOBJECT_DEFINE_ABSTRACT_RTTI_IMPLEMENTATION(myClass) \ + UClassID U_EXPORT2 myClass::getStaticClassID() { \ + static char classID = 0; \ + return (UClassID)&classID; \ + } + +// /** +// * This macro adds ICU RTTI to an ICU concrete class implementation. +// * This macro should be invoked in *.cpp files. The corresponding +// * header should declare getDynamicClassID and getStaticClassID. +// * +// * @param myClass The name of the class that needs RTTI defined. +// * @param myParent The name of the myClass's parent. +// * @internal +// */ +/*#define UOBJECT_DEFINE_RTTI_IMPLEMENTATION(myClass, myParent) \ + UOBJECT_DEFINE_ABSTRACT_RTTI_IMPLEMENTATION(myClass, myParent) \ + UClassID myClass::getDynamicClassID() const { \ + return myClass::getStaticClassID(); \ + } +*/ + + +U_NAMESPACE_END + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/uobslete.h b/Build/source/libs/icu/icu-xetex/common/unicode/uobslete.h new file mode 100644 index 00000000000..489753311cd --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/uobslete.h @@ -0,0 +1,32 @@ +/* +******************************************************************************* +* Copyright (C) 2004-2007, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* +* file name: +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* Created by: genheaders.pl, a perl script written by Ram Viswanadha +* +* Contains data for commenting out APIs. +* Gets included by umachine.h +* +* THIS FILE IS MACHINE-GENERATED, DON'T PLAY WITH IT IF YOU DON'T KNOW WHAT +* YOU ARE DOING, OTHERWISE VERY BAD THINGS WILL HAPPEN! +*/ + +#ifndef UOBSLETE_H +#define UOBSLETE_H + +#ifdef U_HIDE_OBSOLETE_API + +# if U_DISABLE_RENAMING +# else +# endif /* U_DISABLE_RENAMING */ + +#endif /* U_HIDE_OBSOLETE_API */ +#endif /* UOBSLETE_H */ + diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/urename.h b/Build/source/libs/icu/icu-xetex/common/unicode/urename.h new file mode 100644 index 00000000000..8a2b5c344c8 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/urename.h @@ -0,0 +1,1730 @@ +/* +******************************************************************************* +* Copyright (C) 2002-2007, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* +* file name: urename.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* Created by: Perl script written by Vladimir Weinstein +* +* Contains data for renaming ICU exports. +* Gets included by umachine.h +* +* THIS FILE IS MACHINE-GENERATED, DON'T PLAY WITH IT IF YOU DON'T KNOW WHAT +* YOU ARE DOING, OTHERWISE VERY BAD THINGS WILL HAPPEN! +*/ + +#ifndef URENAME_H +#define URENAME_H + +/* Uncomment the following line to disable renaming on platforms + that do not use Autoconf. */ +/* #define U_DISABLE_RENAMING 1 */ + +#if !U_DISABLE_RENAMING + +/* C exports renaming data */ + +#define T_CString_int64ToString T_CString_int64ToString_3_8 +#define T_CString_integerToString T_CString_integerToString_3_8 +#define T_CString_stricmp T_CString_stricmp_3_8 +#define T_CString_stringToInteger T_CString_stringToInteger_3_8 +#define T_CString_strnicmp T_CString_strnicmp_3_8 +#define T_CString_toLowerCase T_CString_toLowerCase_3_8 +#define T_CString_toUpperCase T_CString_toUpperCase_3_8 +#define UCNV_FROM_U_CALLBACK_ESCAPE UCNV_FROM_U_CALLBACK_ESCAPE_3_8 +#define UCNV_FROM_U_CALLBACK_SKIP UCNV_FROM_U_CALLBACK_SKIP_3_8 +#define UCNV_FROM_U_CALLBACK_STOP UCNV_FROM_U_CALLBACK_STOP_3_8 +#define UCNV_FROM_U_CALLBACK_SUBSTITUTE UCNV_FROM_U_CALLBACK_SUBSTITUTE_3_8 +#define UCNV_TO_U_CALLBACK_ESCAPE UCNV_TO_U_CALLBACK_ESCAPE_3_8 +#define UCNV_TO_U_CALLBACK_SKIP UCNV_TO_U_CALLBACK_SKIP_3_8 +#define UCNV_TO_U_CALLBACK_STOP UCNV_TO_U_CALLBACK_STOP_3_8 +#define UCNV_TO_U_CALLBACK_SUBSTITUTE UCNV_TO_U_CALLBACK_SUBSTITUTE_3_8 +#define UDataMemory_createNewInstance UDataMemory_createNewInstance_3_8 +#define UDataMemory_init UDataMemory_init_3_8 +#define UDataMemory_isLoaded UDataMemory_isLoaded_3_8 +#define UDataMemory_normalizeDataPointer UDataMemory_normalizeDataPointer_3_8 +#define UDataMemory_setData UDataMemory_setData_3_8 +#define UDatamemory_assign UDatamemory_assign_3_8 +#define _ASCIIData _ASCIIData_3_8 +#define _Bocu1Data _Bocu1Data_3_8 +#define _CESU8Data _CESU8Data_3_8 +#define _HZData _HZData_3_8 +#define _IMAPData _IMAPData_3_8 +#define _ISCIIData _ISCIIData_3_8 +#define _ISO2022Data _ISO2022Data_3_8 +#define _LMBCSData1 _LMBCSData1_3_8 +#define _LMBCSData11 _LMBCSData11_3_8 +#define _LMBCSData16 _LMBCSData16_3_8 +#define _LMBCSData17 _LMBCSData17_3_8 +#define _LMBCSData18 _LMBCSData18_3_8 +#define _LMBCSData19 _LMBCSData19_3_8 +#define _LMBCSData2 _LMBCSData2_3_8 +#define _LMBCSData3 _LMBCSData3_3_8 +#define _LMBCSData4 _LMBCSData4_3_8 +#define _LMBCSData5 _LMBCSData5_3_8 +#define _LMBCSData6 _LMBCSData6_3_8 +#define _LMBCSData8 _LMBCSData8_3_8 +#define _Latin1Data _Latin1Data_3_8 +#define _MBCSData _MBCSData_3_8 +#define _SCSUData _SCSUData_3_8 +#define _UTF16BEData _UTF16BEData_3_8 +#define _UTF16Data _UTF16Data_3_8 +#define _UTF16LEData _UTF16LEData_3_8 +#define _UTF32BEData _UTF32BEData_3_8 +#define _UTF32Data _UTF32Data_3_8 +#define _UTF32LEData _UTF32LEData_3_8 +#define _UTF7Data _UTF7Data_3_8 +#define _UTF8Data _UTF8Data_3_8 +#define cmemory_cleanup cmemory_cleanup_3_8 +#define cmemory_inUse cmemory_inUse_3_8 +#define le_close le_close_3_8 +#define le_create le_create_3_8 +#define le_getCharIndices le_getCharIndices_3_8 +#define le_getCharIndicesWithBase le_getCharIndicesWithBase_3_8 +#define le_getGlyphCount le_getGlyphCount_3_8 +#define le_getGlyphPosition le_getGlyphPosition_3_8 +#define le_getGlyphPositions le_getGlyphPositions_3_8 +#define le_getGlyphs le_getGlyphs_3_8 +#define le_layoutChars le_layoutChars_3_8 +#define le_reset le_reset_3_8 +#define locale_getKeywords locale_getKeywords_3_8 +#define locale_get_default locale_get_default_3_8 +#define locale_set_default locale_set_default_3_8 +#define pl_addFontRun pl_addFontRun_3_8 +#define pl_addLocaleRun pl_addLocaleRun_3_8 +#define pl_addValueRun pl_addValueRun_3_8 +#define pl_close pl_close_3_8 +#define pl_closeFontRuns pl_closeFontRuns_3_8 +#define pl_closeLine pl_closeLine_3_8 +#define pl_closeLocaleRuns pl_closeLocaleRuns_3_8 +#define pl_closeValueRuns pl_closeValueRuns_3_8 +#define pl_countLineRuns pl_countLineRuns_3_8 +#define pl_create pl_create_3_8 +#define pl_getAscent pl_getAscent_3_8 +#define pl_getDescent pl_getDescent_3_8 +#define pl_getFontRunCount pl_getFontRunCount_3_8 +#define pl_getFontRunFont pl_getFontRunFont_3_8 +#define pl_getFontRunLastLimit pl_getFontRunLastLimit_3_8 +#define pl_getFontRunLimit pl_getFontRunLimit_3_8 +#define pl_getLeading pl_getLeading_3_8 +#define pl_getLineAscent pl_getLineAscent_3_8 +#define pl_getLineDescent pl_getLineDescent_3_8 +#define pl_getLineLeading pl_getLineLeading_3_8 +#define pl_getLineVisualRun pl_getLineVisualRun_3_8 +#define pl_getLineWidth pl_getLineWidth_3_8 +#define pl_getLocaleRunCount pl_getLocaleRunCount_3_8 +#define pl_getLocaleRunLastLimit pl_getLocaleRunLastLimit_3_8 +#define pl_getLocaleRunLimit pl_getLocaleRunLimit_3_8 +#define pl_getLocaleRunLocale pl_getLocaleRunLocale_3_8 +#define pl_getParagraphLevel pl_getParagraphLevel_3_8 +#define pl_getTextDirection pl_getTextDirection_3_8 +#define pl_getValueRunCount pl_getValueRunCount_3_8 +#define pl_getValueRunLastLimit pl_getValueRunLastLimit_3_8 +#define pl_getValueRunLimit pl_getValueRunLimit_3_8 +#define pl_getValueRunValue pl_getValueRunValue_3_8 +#define pl_getVisualRunAscent pl_getVisualRunAscent_3_8 +#define pl_getVisualRunDescent pl_getVisualRunDescent_3_8 +#define pl_getVisualRunDirection pl_getVisualRunDirection_3_8 +#define pl_getVisualRunFont pl_getVisualRunFont_3_8 +#define pl_getVisualRunGlyphCount pl_getVisualRunGlyphCount_3_8 +#define pl_getVisualRunGlyphToCharMap pl_getVisualRunGlyphToCharMap_3_8 +#define pl_getVisualRunGlyphs pl_getVisualRunGlyphs_3_8 +#define pl_getVisualRunLeading pl_getVisualRunLeading_3_8 +#define pl_getVisualRunPositions pl_getVisualRunPositions_3_8 +#define pl_isComplex pl_isComplex_3_8 +#define pl_nextLine pl_nextLine_3_8 +#define pl_openEmptyFontRuns pl_openEmptyFontRuns_3_8 +#define pl_openEmptyLocaleRuns pl_openEmptyLocaleRuns_3_8 +#define pl_openEmptyValueRuns pl_openEmptyValueRuns_3_8 +#define pl_openFontRuns pl_openFontRuns_3_8 +#define pl_openLocaleRuns pl_openLocaleRuns_3_8 +#define pl_openValueRuns pl_openValueRuns_3_8 +#define pl_reflow pl_reflow_3_8 +#define pl_resetFontRuns pl_resetFontRuns_3_8 +#define pl_resetLocaleRuns pl_resetLocaleRuns_3_8 +#define pl_resetValueRuns pl_resetValueRuns_3_8 +#define res_countArrayItems res_countArrayItems_3_8 +#define res_findResource res_findResource_3_8 +#define res_getAlias res_getAlias_3_8 +#define res_getArrayItem res_getArrayItem_3_8 +#define res_getBinary res_getBinary_3_8 +#define res_getIntVector res_getIntVector_3_8 +#define res_getResource res_getResource_3_8 +#define res_getString res_getString_3_8 +#define res_getTableItemByIndex res_getTableItemByIndex_3_8 +#define res_getTableItemByKey res_getTableItemByKey_3_8 +#define res_load res_load_3_8 +#define res_unload res_unload_3_8 +#define transliterator_cleanup transliterator_cleanup_3_8 +#define triedict_swap triedict_swap_3_8 +#define u_UCharsToChars u_UCharsToChars_3_8 +#define u_austrcpy u_austrcpy_3_8 +#define u_austrncpy u_austrncpy_3_8 +#define u_catclose u_catclose_3_8 +#define u_catgets u_catgets_3_8 +#define u_catopen u_catopen_3_8 +#define u_charAge u_charAge_3_8 +#define u_charDigitValue u_charDigitValue_3_8 +#define u_charDirection u_charDirection_3_8 +#define u_charFromName u_charFromName_3_8 +#define u_charMirror u_charMirror_3_8 +#define u_charName u_charName_3_8 +#define u_charType u_charType_3_8 +#define u_charsToUChars u_charsToUChars_3_8 +#define u_cleanup u_cleanup_3_8 +#define u_countChar32 u_countChar32_3_8 +#define u_digit u_digit_3_8 +#define u_enumCharNames u_enumCharNames_3_8 +#define u_enumCharTypes u_enumCharTypes_3_8 +#define u_errorName u_errorName_3_8 +#define u_fclose u_fclose_3_8 +#define u_feof u_feof_3_8 +#define u_fflush u_fflush_3_8 +#define u_fgetConverter u_fgetConverter_3_8 +#define u_fgetc u_fgetc_3_8 +#define u_fgetcodepage u_fgetcodepage_3_8 +#define u_fgetcx u_fgetcx_3_8 +#define u_fgetfile u_fgetfile_3_8 +#define u_fgetlocale u_fgetlocale_3_8 +#define u_fgets u_fgets_3_8 +#define u_file_read u_file_read_3_8 +#define u_file_write u_file_write_3_8 +#define u_file_write_flush u_file_write_flush_3_8 +#define u_finit u_finit_3_8 +#define u_foldCase u_foldCase_3_8 +#define u_fopen u_fopen_3_8 +#define u_forDigit u_forDigit_3_8 +#define u_formatMessage u_formatMessage_3_8 +#define u_formatMessageWithError u_formatMessageWithError_3_8 +#define u_fprintf u_fprintf_3_8 +#define u_fprintf_u u_fprintf_u_3_8 +#define u_fputc u_fputc_3_8 +#define u_fputs u_fputs_3_8 +#define u_frewind u_frewind_3_8 +#define u_fscanf u_fscanf_3_8 +#define u_fscanf_u u_fscanf_u_3_8 +#define u_fsetcodepage u_fsetcodepage_3_8 +#define u_fsetlocale u_fsetlocale_3_8 +#define u_fsettransliterator u_fsettransliterator_3_8 +#define u_fstropen u_fstropen_3_8 +#define u_fungetc u_fungetc_3_8 +#define u_getCombiningClass u_getCombiningClass_3_8 +#define u_getDataDirectory u_getDataDirectory_3_8 +#define u_getDefaultConverter u_getDefaultConverter_3_8 +#define u_getFC_NFKC_Closure u_getFC_NFKC_Closure_3_8 +#define u_getISOComment u_getISOComment_3_8 +#define u_getIntPropertyMaxValue u_getIntPropertyMaxValue_3_8 +#define u_getIntPropertyMinValue u_getIntPropertyMinValue_3_8 +#define u_getIntPropertyValue u_getIntPropertyValue_3_8 +#define u_getNumericValue u_getNumericValue_3_8 +#define u_getPropertyEnum u_getPropertyEnum_3_8 +#define u_getPropertyName u_getPropertyName_3_8 +#define u_getPropertyValueEnum u_getPropertyValueEnum_3_8 +#define u_getPropertyValueName u_getPropertyValueName_3_8 +#define u_getUnicodeProperties u_getUnicodeProperties_3_8 +#define u_getUnicodeVersion u_getUnicodeVersion_3_8 +#define u_getVersion u_getVersion_3_8 +#define u_growBufferFromStatic u_growBufferFromStatic_3_8 +#define u_hasBinaryProperty u_hasBinaryProperty_3_8 +#define u_init u_init_3_8 +#define u_isIDIgnorable u_isIDIgnorable_3_8 +#define u_isIDPart u_isIDPart_3_8 +#define u_isIDStart u_isIDStart_3_8 +#define u_isISOControl u_isISOControl_3_8 +#define u_isJavaIDPart u_isJavaIDPart_3_8 +#define u_isJavaIDStart u_isJavaIDStart_3_8 +#define u_isJavaSpaceChar u_isJavaSpaceChar_3_8 +#define u_isMirrored u_isMirrored_3_8 +#define u_isUAlphabetic u_isUAlphabetic_3_8 +#define u_isULowercase u_isULowercase_3_8 +#define u_isUUppercase u_isUUppercase_3_8 +#define u_isUWhiteSpace u_isUWhiteSpace_3_8 +#define u_isWhitespace u_isWhitespace_3_8 +#define u_isalnum u_isalnum_3_8 +#define u_isalnumPOSIX u_isalnumPOSIX_3_8 +#define u_isalpha u_isalpha_3_8 +#define u_isbase u_isbase_3_8 +#define u_isblank u_isblank_3_8 +#define u_iscntrl u_iscntrl_3_8 +#define u_isdefined u_isdefined_3_8 +#define u_isdigit u_isdigit_3_8 +#define u_isgraph u_isgraph_3_8 +#define u_isgraphPOSIX u_isgraphPOSIX_3_8 +#define u_islower u_islower_3_8 +#define u_isprint u_isprint_3_8 +#define u_isprintPOSIX u_isprintPOSIX_3_8 +#define u_ispunct u_ispunct_3_8 +#define u_isspace u_isspace_3_8 +#define u_istitle u_istitle_3_8 +#define u_isupper u_isupper_3_8 +#define u_isxdigit u_isxdigit_3_8 +#define u_lengthOfIdenticalLevelRun u_lengthOfIdenticalLevelRun_3_8 +#define u_locbund_close u_locbund_close_3_8 +#define u_locbund_getNumberFormat u_locbund_getNumberFormat_3_8 +#define u_locbund_init u_locbund_init_3_8 +#define u_memcasecmp u_memcasecmp_3_8 +#define u_memchr u_memchr_3_8 +#define u_memchr32 u_memchr32_3_8 +#define u_memcmp u_memcmp_3_8 +#define u_memcmpCodePointOrder u_memcmpCodePointOrder_3_8 +#define u_memcpy u_memcpy_3_8 +#define u_memmove u_memmove_3_8 +#define u_memrchr u_memrchr_3_8 +#define u_memrchr32 u_memrchr32_3_8 +#define u_memset u_memset_3_8 +#define u_parseMessage u_parseMessage_3_8 +#define u_parseMessageWithError u_parseMessageWithError_3_8 +#define u_printf_parse u_printf_parse_3_8 +#define u_releaseDefaultConverter u_releaseDefaultConverter_3_8 +#define u_scanf_parse u_scanf_parse_3_8 +#define u_setAtomicIncDecFunctions u_setAtomicIncDecFunctions_3_8 +#define u_setDataDirectory u_setDataDirectory_3_8 +#define u_setMemoryFunctions u_setMemoryFunctions_3_8 +#define u_setMutexFunctions u_setMutexFunctions_3_8 +#define u_shapeArabic u_shapeArabic_3_8 +#define u_snprintf u_snprintf_3_8 +#define u_snprintf_u u_snprintf_u_3_8 +#define u_sprintf u_sprintf_3_8 +#define u_sprintf_u u_sprintf_u_3_8 +#define u_sscanf u_sscanf_3_8 +#define u_sscanf_u u_sscanf_u_3_8 +#define u_strCaseCompare u_strCaseCompare_3_8 +#define u_strCompare u_strCompare_3_8 +#define u_strCompareIter u_strCompareIter_3_8 +#define u_strFindFirst u_strFindFirst_3_8 +#define u_strFindLast u_strFindLast_3_8 +#define u_strFoldCase u_strFoldCase_3_8 +#define u_strFromPunycode u_strFromPunycode_3_8 +#define u_strFromUTF32 u_strFromUTF32_3_8 +#define u_strFromUTF8 u_strFromUTF8_3_8 +#define u_strFromUTF8Lenient u_strFromUTF8Lenient_3_8 +#define u_strFromUTF8WithSub u_strFromUTF8WithSub_3_8 +#define u_strFromWCS u_strFromWCS_3_8 +#define u_strHasMoreChar32Than u_strHasMoreChar32Than_3_8 +#define u_strToLower u_strToLower_3_8 +#define u_strToPunycode u_strToPunycode_3_8 +#define u_strToTitle u_strToTitle_3_8 +#define u_strToUTF32 u_strToUTF32_3_8 +#define u_strToUTF8 u_strToUTF8_3_8 +#define u_strToUTF8WithSub u_strToUTF8WithSub_3_8 +#define u_strToUpper u_strToUpper_3_8 +#define u_strToWCS u_strToWCS_3_8 +#define u_strcasecmp u_strcasecmp_3_8 +#define u_strcat u_strcat_3_8 +#define u_strchr u_strchr_3_8 +#define u_strchr32 u_strchr32_3_8 +#define u_strcmp u_strcmp_3_8 +#define u_strcmpCodePointOrder u_strcmpCodePointOrder_3_8 +#define u_strcmpFold u_strcmpFold_3_8 +#define u_strcpy u_strcpy_3_8 +#define u_strcspn u_strcspn_3_8 +#define u_strlen u_strlen_3_8 +#define u_strncasecmp u_strncasecmp_3_8 +#define u_strncat u_strncat_3_8 +#define u_strncmp u_strncmp_3_8 +#define u_strncmpCodePointOrder u_strncmpCodePointOrder_3_8 +#define u_strncpy u_strncpy_3_8 +#define u_strpbrk u_strpbrk_3_8 +#define u_strrchr u_strrchr_3_8 +#define u_strrchr32 u_strrchr32_3_8 +#define u_strrstr u_strrstr_3_8 +#define u_strspn u_strspn_3_8 +#define u_strstr u_strstr_3_8 +#define u_strtok_r u_strtok_r_3_8 +#define u_terminateChars u_terminateChars_3_8 +#define u_terminateUChar32s u_terminateUChar32s_3_8 +#define u_terminateUChars u_terminateUChars_3_8 +#define u_terminateWChars u_terminateWChars_3_8 +#define u_tolower u_tolower_3_8 +#define u_totitle u_totitle_3_8 +#define u_toupper u_toupper_3_8 +#define u_uastrcpy u_uastrcpy_3_8 +#define u_uastrncpy u_uastrncpy_3_8 +#define u_unescape u_unescape_3_8 +#define u_unescapeAt u_unescapeAt_3_8 +#define u_versionFromString u_versionFromString_3_8 +#define u_versionToString u_versionToString_3_8 +#define u_vformatMessage u_vformatMessage_3_8 +#define u_vformatMessageWithError u_vformatMessageWithError_3_8 +#define u_vfprintf u_vfprintf_3_8 +#define u_vfprintf_u u_vfprintf_u_3_8 +#define u_vfscanf u_vfscanf_3_8 +#define u_vfscanf_u u_vfscanf_u_3_8 +#define u_vparseMessage u_vparseMessage_3_8 +#define u_vparseMessageWithError u_vparseMessageWithError_3_8 +#define u_vsnprintf u_vsnprintf_3_8 +#define u_vsnprintf_u u_vsnprintf_u_3_8 +#define u_vsprintf u_vsprintf_3_8 +#define u_vsprintf_u u_vsprintf_u_3_8 +#define u_vsscanf u_vsscanf_3_8 +#define u_vsscanf_u u_vsscanf_u_3_8 +#define u_writeDiff u_writeDiff_3_8 +#define u_writeIdenticalLevelRun u_writeIdenticalLevelRun_3_8 +#define u_writeIdenticalLevelRunTwoChars u_writeIdenticalLevelRunTwoChars_3_8 +#define ubidi_addPropertyStarts ubidi_addPropertyStarts_3_8 +#define ubidi_close ubidi_close_3_8 +#define ubidi_closeProps ubidi_closeProps_3_8 +#define ubidi_countParagraphs ubidi_countParagraphs_3_8 +#define ubidi_countRuns ubidi_countRuns_3_8 +#define ubidi_getClass ubidi_getClass_3_8 +#define ubidi_getClassCallback ubidi_getClassCallback_3_8 +#define ubidi_getCustomizedClass ubidi_getCustomizedClass_3_8 +#define ubidi_getDirection ubidi_getDirection_3_8 +#define ubidi_getDummy ubidi_getDummy_3_8 +#define ubidi_getJoiningGroup ubidi_getJoiningGroup_3_8 +#define ubidi_getJoiningType ubidi_getJoiningType_3_8 +#define ubidi_getLength ubidi_getLength_3_8 +#define ubidi_getLevelAt ubidi_getLevelAt_3_8 +#define ubidi_getLevels ubidi_getLevels_3_8 +#define ubidi_getLogicalIndex ubidi_getLogicalIndex_3_8 +#define ubidi_getLogicalMap ubidi_getLogicalMap_3_8 +#define ubidi_getLogicalRun ubidi_getLogicalRun_3_8 +#define ubidi_getMaxValue ubidi_getMaxValue_3_8 +#define ubidi_getMemory ubidi_getMemory_3_8 +#define ubidi_getMirror ubidi_getMirror_3_8 +#define ubidi_getParaLevel ubidi_getParaLevel_3_8 +#define ubidi_getParagraph ubidi_getParagraph_3_8 +#define ubidi_getParagraphByIndex ubidi_getParagraphByIndex_3_8 +#define ubidi_getProcessedLength ubidi_getProcessedLength_3_8 +#define ubidi_getReorderingMode ubidi_getReorderingMode_3_8 +#define ubidi_getReorderingOptions ubidi_getReorderingOptions_3_8 +#define ubidi_getResultLength ubidi_getResultLength_3_8 +#define ubidi_getRuns ubidi_getRuns_3_8 +#define ubidi_getSingleton ubidi_getSingleton_3_8 +#define ubidi_getText ubidi_getText_3_8 +#define ubidi_getVisualIndex ubidi_getVisualIndex_3_8 +#define ubidi_getVisualMap ubidi_getVisualMap_3_8 +#define ubidi_getVisualRun ubidi_getVisualRun_3_8 +#define ubidi_invertMap ubidi_invertMap_3_8 +#define ubidi_isBidiControl ubidi_isBidiControl_3_8 +#define ubidi_isInverse ubidi_isInverse_3_8 +#define ubidi_isJoinControl ubidi_isJoinControl_3_8 +#define ubidi_isMirrored ubidi_isMirrored_3_8 +#define ubidi_isOrderParagraphsLTR ubidi_isOrderParagraphsLTR_3_8 +#define ubidi_open ubidi_open_3_8 +#define ubidi_openSized ubidi_openSized_3_8 +#define ubidi_orderParagraphsLTR ubidi_orderParagraphsLTR_3_8 +#define ubidi_reorderLogical ubidi_reorderLogical_3_8 +#define ubidi_reorderVisual ubidi_reorderVisual_3_8 +#define ubidi_setClassCallback ubidi_setClassCallback_3_8 +#define ubidi_setInverse ubidi_setInverse_3_8 +#define ubidi_setLine ubidi_setLine_3_8 +#define ubidi_setPara ubidi_setPara_3_8 +#define ubidi_setReorderingMode ubidi_setReorderingMode_3_8 +#define ubidi_setReorderingOptions ubidi_setReorderingOptions_3_8 +#define ubidi_writeReordered ubidi_writeReordered_3_8 +#define ubidi_writeReverse ubidi_writeReverse_3_8 +#define ublock_getCode ublock_getCode_3_8 +#define ubrk_close ubrk_close_3_8 +#define ubrk_countAvailable ubrk_countAvailable_3_8 +#define ubrk_current ubrk_current_3_8 +#define ubrk_first ubrk_first_3_8 +#define ubrk_following ubrk_following_3_8 +#define ubrk_getAvailable ubrk_getAvailable_3_8 +#define ubrk_getLocaleByType ubrk_getLocaleByType_3_8 +#define ubrk_getRuleStatus ubrk_getRuleStatus_3_8 +#define ubrk_getRuleStatusVec ubrk_getRuleStatusVec_3_8 +#define ubrk_isBoundary ubrk_isBoundary_3_8 +#define ubrk_last ubrk_last_3_8 +#define ubrk_next ubrk_next_3_8 +#define ubrk_open ubrk_open_3_8 +#define ubrk_openRules ubrk_openRules_3_8 +#define ubrk_preceding ubrk_preceding_3_8 +#define ubrk_previous ubrk_previous_3_8 +#define ubrk_safeClone ubrk_safeClone_3_8 +#define ubrk_setText ubrk_setText_3_8 +#define ubrk_setUText ubrk_setUText_3_8 +#define ubrk_swap ubrk_swap_3_8 +#define ucal_add ucal_add_3_8 +#define ucal_clear ucal_clear_3_8 +#define ucal_clearField ucal_clearField_3_8 +#define ucal_close ucal_close_3_8 +#define ucal_countAvailable ucal_countAvailable_3_8 +#define ucal_equivalentTo ucal_equivalentTo_3_8 +#define ucal_get ucal_get_3_8 +#define ucal_getAttribute ucal_getAttribute_3_8 +#define ucal_getAvailable ucal_getAvailable_3_8 +#define ucal_getDSTSavings ucal_getDSTSavings_3_8 +#define ucal_getDefaultTimeZone ucal_getDefaultTimeZone_3_8 +#define ucal_getGregorianChange ucal_getGregorianChange_3_8 +#define ucal_getLimit ucal_getLimit_3_8 +#define ucal_getLocaleByType ucal_getLocaleByType_3_8 +#define ucal_getMillis ucal_getMillis_3_8 +#define ucal_getNow ucal_getNow_3_8 +#define ucal_getTZDataVersion ucal_getTZDataVersion_3_8 +#define ucal_getTimeZoneDisplayName ucal_getTimeZoneDisplayName_3_8 +#define ucal_inDaylightTime ucal_inDaylightTime_3_8 +#define ucal_isSet ucal_isSet_3_8 +#define ucal_open ucal_open_3_8 +#define ucal_openCountryTimeZones ucal_openCountryTimeZones_3_8 +#define ucal_openTimeZones ucal_openTimeZones_3_8 +#define ucal_roll ucal_roll_3_8 +#define ucal_set ucal_set_3_8 +#define ucal_setAttribute ucal_setAttribute_3_8 +#define ucal_setDate ucal_setDate_3_8 +#define ucal_setDateTime ucal_setDateTime_3_8 +#define ucal_setDefaultTimeZone ucal_setDefaultTimeZone_3_8 +#define ucal_setGregorianChange ucal_setGregorianChange_3_8 +#define ucal_setMillis ucal_setMillis_3_8 +#define ucal_setTimeZone ucal_setTimeZone_3_8 +#define ucase_addCaseClosure ucase_addCaseClosure_3_8 +#define ucase_addPropertyStarts ucase_addPropertyStarts_3_8 +#define ucase_addStringCaseClosure ucase_addStringCaseClosure_3_8 +#define ucase_close ucase_close_3_8 +#define ucase_fold ucase_fold_3_8 +#define ucase_getCaseLocale ucase_getCaseLocale_3_8 +#define ucase_getDummy ucase_getDummy_3_8 +#define ucase_getSingleton ucase_getSingleton_3_8 +#define ucase_getType ucase_getType_3_8 +#define ucase_getTypeOrIgnorable ucase_getTypeOrIgnorable_3_8 +#define ucase_hasBinaryProperty ucase_hasBinaryProperty_3_8 +#define ucase_isCaseSensitive ucase_isCaseSensitive_3_8 +#define ucase_isSoftDotted ucase_isSoftDotted_3_8 +#define ucase_toFullFolding ucase_toFullFolding_3_8 +#define ucase_toFullLower ucase_toFullLower_3_8 +#define ucase_toFullTitle ucase_toFullTitle_3_8 +#define ucase_toFullUpper ucase_toFullUpper_3_8 +#define ucase_tolower ucase_tolower_3_8 +#define ucase_totitle ucase_totitle_3_8 +#define ucase_toupper ucase_toupper_3_8 +#define ucasemap_close ucasemap_close_3_8 +#define ucasemap_getBreakIterator ucasemap_getBreakIterator_3_8 +#define ucasemap_getLocale ucasemap_getLocale_3_8 +#define ucasemap_getOptions ucasemap_getOptions_3_8 +#define ucasemap_open ucasemap_open_3_8 +#define ucasemap_setBreakIterator ucasemap_setBreakIterator_3_8 +#define ucasemap_setLocale ucasemap_setLocale_3_8 +#define ucasemap_setOptions ucasemap_setOptions_3_8 +#define ucasemap_toTitle ucasemap_toTitle_3_8 +#define ucasemap_utf8FoldCase ucasemap_utf8FoldCase_3_8 +#define ucasemap_utf8ToLower ucasemap_utf8ToLower_3_8 +#define ucasemap_utf8ToTitle ucasemap_utf8ToTitle_3_8 +#define ucasemap_utf8ToUpper ucasemap_utf8ToUpper_3_8 +#define uchar_addPropertyStarts uchar_addPropertyStarts_3_8 +#define uchar_getHST uchar_getHST_3_8 +#define uchar_swapNames uchar_swapNames_3_8 +#define ucln_common_registerCleanup ucln_common_registerCleanup_3_8 +#define ucln_i18n_registerCleanup ucln_i18n_registerCleanup_3_8 +#define ucln_io_registerCleanup ucln_io_registerCleanup_3_8 +#define ucln_lib_cleanup ucln_lib_cleanup_3_8 +#define ucln_registerCleanup ucln_registerCleanup_3_8 +#define ucnv_MBCSFromUChar32 ucnv_MBCSFromUChar32_3_8 +#define ucnv_MBCSFromUnicodeWithOffsets ucnv_MBCSFromUnicodeWithOffsets_3_8 +#define ucnv_MBCSGetType ucnv_MBCSGetType_3_8 +#define ucnv_MBCSGetUnicodeSetForBytes ucnv_MBCSGetUnicodeSetForBytes_3_8 +#define ucnv_MBCSGetUnicodeSetForUnicode ucnv_MBCSGetUnicodeSetForUnicode_3_8 +#define ucnv_MBCSIsLeadByte ucnv_MBCSIsLeadByte_3_8 +#define ucnv_MBCSSimpleGetNextUChar ucnv_MBCSSimpleGetNextUChar_3_8 +#define ucnv_MBCSToUnicodeWithOffsets ucnv_MBCSToUnicodeWithOffsets_3_8 +#define ucnv_bld_countAvailableConverters ucnv_bld_countAvailableConverters_3_8 +#define ucnv_bld_getAvailableConverter ucnv_bld_getAvailableConverter_3_8 +#define ucnv_cbFromUWriteBytes ucnv_cbFromUWriteBytes_3_8 +#define ucnv_cbFromUWriteSub ucnv_cbFromUWriteSub_3_8 +#define ucnv_cbFromUWriteUChars ucnv_cbFromUWriteUChars_3_8 +#define ucnv_cbToUWriteSub ucnv_cbToUWriteSub_3_8 +#define ucnv_cbToUWriteUChars ucnv_cbToUWriteUChars_3_8 +#define ucnv_close ucnv_close_3_8 +#define ucnv_compareNames ucnv_compareNames_3_8 +#define ucnv_convert ucnv_convert_3_8 +#define ucnv_convertEx ucnv_convertEx_3_8 +#define ucnv_countAliases ucnv_countAliases_3_8 +#define ucnv_countAvailable ucnv_countAvailable_3_8 +#define ucnv_countStandards ucnv_countStandards_3_8 +#define ucnv_createAlgorithmicConverter ucnv_createAlgorithmicConverter_3_8 +#define ucnv_createConverter ucnv_createConverter_3_8 +#define ucnv_createConverterFromPackage ucnv_createConverterFromPackage_3_8 +#define ucnv_createConverterFromSharedData ucnv_createConverterFromSharedData_3_8 +#define ucnv_detectUnicodeSignature ucnv_detectUnicodeSignature_3_8 +#define ucnv_extContinueMatchFromU ucnv_extContinueMatchFromU_3_8 +#define ucnv_extContinueMatchToU ucnv_extContinueMatchToU_3_8 +#define ucnv_extGetUnicodeSet ucnv_extGetUnicodeSet_3_8 +#define ucnv_extInitialMatchFromU ucnv_extInitialMatchFromU_3_8 +#define ucnv_extInitialMatchToU ucnv_extInitialMatchToU_3_8 +#define ucnv_extSimpleMatchFromU ucnv_extSimpleMatchFromU_3_8 +#define ucnv_extSimpleMatchToU ucnv_extSimpleMatchToU_3_8 +#define ucnv_fixFileSeparator ucnv_fixFileSeparator_3_8 +#define ucnv_flushCache ucnv_flushCache_3_8 +#define ucnv_fromAlgorithmic ucnv_fromAlgorithmic_3_8 +#define ucnv_fromUChars ucnv_fromUChars_3_8 +#define ucnv_fromUCountPending ucnv_fromUCountPending_3_8 +#define ucnv_fromUWriteBytes ucnv_fromUWriteBytes_3_8 +#define ucnv_fromUnicode ucnv_fromUnicode_3_8 +#define ucnv_fromUnicode_UTF8 ucnv_fromUnicode_UTF8_3_8 +#define ucnv_fromUnicode_UTF8_OFFSETS_LOGIC ucnv_fromUnicode_UTF8_OFFSETS_LOGIC_3_8 +#define ucnv_getAlias ucnv_getAlias_3_8 +#define ucnv_getAliases ucnv_getAliases_3_8 +#define ucnv_getAvailableName ucnv_getAvailableName_3_8 +#define ucnv_getCCSID ucnv_getCCSID_3_8 +#define ucnv_getCanonicalName ucnv_getCanonicalName_3_8 +#define ucnv_getCompleteUnicodeSet ucnv_getCompleteUnicodeSet_3_8 +#define ucnv_getDefaultName ucnv_getDefaultName_3_8 +#define ucnv_getDisplayName ucnv_getDisplayName_3_8 +#define ucnv_getFromUCallBack ucnv_getFromUCallBack_3_8 +#define ucnv_getInvalidChars ucnv_getInvalidChars_3_8 +#define ucnv_getInvalidUChars ucnv_getInvalidUChars_3_8 +#define ucnv_getMaxCharSize ucnv_getMaxCharSize_3_8 +#define ucnv_getMinCharSize ucnv_getMinCharSize_3_8 +#define ucnv_getName ucnv_getName_3_8 +#define ucnv_getNextUChar ucnv_getNextUChar_3_8 +#define ucnv_getNonSurrogateUnicodeSet ucnv_getNonSurrogateUnicodeSet_3_8 +#define ucnv_getPlatform ucnv_getPlatform_3_8 +#define ucnv_getStandard ucnv_getStandard_3_8 +#define ucnv_getStandardName ucnv_getStandardName_3_8 +#define ucnv_getStarters ucnv_getStarters_3_8 +#define ucnv_getSubstChars ucnv_getSubstChars_3_8 +#define ucnv_getToUCallBack ucnv_getToUCallBack_3_8 +#define ucnv_getType ucnv_getType_3_8 +#define ucnv_getUnicodeSet ucnv_getUnicodeSet_3_8 +#define ucnv_incrementRefCount ucnv_incrementRefCount_3_8 +#define ucnv_io_countKnownConverters ucnv_io_countKnownConverters_3_8 +#define ucnv_io_getConverterName ucnv_io_getConverterName_3_8 +#define ucnv_io_stripASCIIForCompare ucnv_io_stripASCIIForCompare_3_8 +#define ucnv_io_stripEBCDICForCompare ucnv_io_stripEBCDICForCompare_3_8 +#define ucnv_isAmbiguous ucnv_isAmbiguous_3_8 +#define ucnv_load ucnv_load_3_8 +#define ucnv_loadSharedData ucnv_loadSharedData_3_8 +#define ucnv_open ucnv_open_3_8 +#define ucnv_openAllNames ucnv_openAllNames_3_8 +#define ucnv_openCCSID ucnv_openCCSID_3_8 +#define ucnv_openPackage ucnv_openPackage_3_8 +#define ucnv_openStandardNames ucnv_openStandardNames_3_8 +#define ucnv_openU ucnv_openU_3_8 +#define ucnv_reset ucnv_reset_3_8 +#define ucnv_resetFromUnicode ucnv_resetFromUnicode_3_8 +#define ucnv_resetToUnicode ucnv_resetToUnicode_3_8 +#define ucnv_safeClone ucnv_safeClone_3_8 +#define ucnv_setDefaultName ucnv_setDefaultName_3_8 +#define ucnv_setFallback ucnv_setFallback_3_8 +#define ucnv_setFromUCallBack ucnv_setFromUCallBack_3_8 +#define ucnv_setSubstChars ucnv_setSubstChars_3_8 +#define ucnv_setSubstString ucnv_setSubstString_3_8 +#define ucnv_setToUCallBack ucnv_setToUCallBack_3_8 +#define ucnv_swap ucnv_swap_3_8 +#define ucnv_swapAliases ucnv_swapAliases_3_8 +#define ucnv_toAlgorithmic ucnv_toAlgorithmic_3_8 +#define ucnv_toUChars ucnv_toUChars_3_8 +#define ucnv_toUCountPending ucnv_toUCountPending_3_8 +#define ucnv_toUWriteCodePoint ucnv_toUWriteCodePoint_3_8 +#define ucnv_toUWriteUChars ucnv_toUWriteUChars_3_8 +#define ucnv_toUnicode ucnv_toUnicode_3_8 +#define ucnv_unload ucnv_unload_3_8 +#define ucnv_unloadSharedDataIfReady ucnv_unloadSharedDataIfReady_3_8 +#define ucnv_usesFallback ucnv_usesFallback_3_8 +#define ucol_allocWeights ucol_allocWeights_3_8 +#define ucol_assembleTailoringTable ucol_assembleTailoringTable_3_8 +#define ucol_calcSortKey ucol_calcSortKey_3_8 +#define ucol_calcSortKeySimpleTertiary ucol_calcSortKeySimpleTertiary_3_8 +#define ucol_cloneBinary ucol_cloneBinary_3_8 +#define ucol_cloneRuleData ucol_cloneRuleData_3_8 +#define ucol_close ucol_close_3_8 +#define ucol_closeElements ucol_closeElements_3_8 +#define ucol_collatorToIdentifier ucol_collatorToIdentifier_3_8 +#define ucol_countAvailable ucol_countAvailable_3_8 +#define ucol_createElements ucol_createElements_3_8 +#define ucol_doCE ucol_doCE_3_8 +#define ucol_equal ucol_equal_3_8 +#define ucol_equals ucol_equals_3_8 +#define ucol_forgetUCA ucol_forgetUCA_3_8 +#define ucol_getAttribute ucol_getAttribute_3_8 +#define ucol_getAttributeOrDefault ucol_getAttributeOrDefault_3_8 +#define ucol_getAvailable ucol_getAvailable_3_8 +#define ucol_getBound ucol_getBound_3_8 +#define ucol_getCEGenerator ucol_getCEGenerator_3_8 +#define ucol_getCEStrengthDifference ucol_getCEStrengthDifference_3_8 +#define ucol_getContractions ucol_getContractions_3_8 +#define ucol_getContractionsAndExpansions ucol_getContractionsAndExpansions_3_8 +#define ucol_getDisplayName ucol_getDisplayName_3_8 +#define ucol_getFirstCE ucol_getFirstCE_3_8 +#define ucol_getFunctionalEquivalent ucol_getFunctionalEquivalent_3_8 +#define ucol_getKeywordValues ucol_getKeywordValues_3_8 +#define ucol_getKeywords ucol_getKeywords_3_8 +#define ucol_getLocale ucol_getLocale_3_8 +#define ucol_getLocaleByType ucol_getLocaleByType_3_8 +#define ucol_getMaxExpansion ucol_getMaxExpansion_3_8 +#define ucol_getNextCE ucol_getNextCE_3_8 +#define ucol_getNextGenerated ucol_getNextGenerated_3_8 +#define ucol_getOffset ucol_getOffset_3_8 +#define ucol_getPrevCE ucol_getPrevCE_3_8 +#define ucol_getRules ucol_getRules_3_8 +#define ucol_getRulesEx ucol_getRulesEx_3_8 +#define ucol_getShortDefinitionString ucol_getShortDefinitionString_3_8 +#define ucol_getSimpleCEGenerator ucol_getSimpleCEGenerator_3_8 +#define ucol_getSortKey ucol_getSortKey_3_8 +#define ucol_getSortKeySize ucol_getSortKeySize_3_8 +#define ucol_getSortKeyWithAllocation ucol_getSortKeyWithAllocation_3_8 +#define ucol_getStrength ucol_getStrength_3_8 +#define ucol_getTailoredSet ucol_getTailoredSet_3_8 +#define ucol_getUCAVersion ucol_getUCAVersion_3_8 +#define ucol_getUnsafeSet ucol_getUnsafeSet_3_8 +#define ucol_getVariableTop ucol_getVariableTop_3_8 +#define ucol_getVersion ucol_getVersion_3_8 +#define ucol_greater ucol_greater_3_8 +#define ucol_greaterOrEqual ucol_greaterOrEqual_3_8 +#define ucol_identifierToShortString ucol_identifierToShortString_3_8 +#define ucol_initBuffers ucol_initBuffers_3_8 +#define ucol_initCollator ucol_initCollator_3_8 +#define ucol_initInverseUCA ucol_initInverseUCA_3_8 +#define ucol_initUCA ucol_initUCA_3_8 +#define ucol_inv_getGapPositions ucol_inv_getGapPositions_3_8 +#define ucol_inv_getNextCE ucol_inv_getNextCE_3_8 +#define ucol_inv_getPrevCE ucol_inv_getPrevCE_3_8 +#define ucol_isTailored ucol_isTailored_3_8 +#define ucol_keyHashCode ucol_keyHashCode_3_8 +#define ucol_mergeSortkeys ucol_mergeSortkeys_3_8 +#define ucol_next ucol_next_3_8 +#define ucol_nextSortKeyPart ucol_nextSortKeyPart_3_8 +#define ucol_nextWeight ucol_nextWeight_3_8 +#define ucol_normalizeShortDefinitionString ucol_normalizeShortDefinitionString_3_8 +#define ucol_open ucol_open_3_8 +#define ucol_openAvailableLocales ucol_openAvailableLocales_3_8 +#define ucol_openBinary ucol_openBinary_3_8 +#define ucol_openElements ucol_openElements_3_8 +#define ucol_openFromIdentifier ucol_openFromIdentifier_3_8 +#define ucol_openFromShortString ucol_openFromShortString_3_8 +#define ucol_openRules ucol_openRules_3_8 +#define ucol_open_internal ucol_open_internal_3_8 +#define ucol_prepareShortStringOpen ucol_prepareShortStringOpen_3_8 +#define ucol_previous ucol_previous_3_8 +#define ucol_primaryOrder ucol_primaryOrder_3_8 +#define ucol_prv_getSpecialCE ucol_prv_getSpecialCE_3_8 +#define ucol_prv_getSpecialPrevCE ucol_prv_getSpecialPrevCE_3_8 +#define ucol_reset ucol_reset_3_8 +#define ucol_restoreVariableTop ucol_restoreVariableTop_3_8 +#define ucol_safeClone ucol_safeClone_3_8 +#define ucol_secondaryOrder ucol_secondaryOrder_3_8 +#define ucol_setAttribute ucol_setAttribute_3_8 +#define ucol_setOffset ucol_setOffset_3_8 +#define ucol_setOptionsFromHeader ucol_setOptionsFromHeader_3_8 +#define ucol_setReqValidLocales ucol_setReqValidLocales_3_8 +#define ucol_setStrength ucol_setStrength_3_8 +#define ucol_setText ucol_setText_3_8 +#define ucol_setVariableTop ucol_setVariableTop_3_8 +#define ucol_shortStringToIdentifier ucol_shortStringToIdentifier_3_8 +#define ucol_strcoll ucol_strcoll_3_8 +#define ucol_strcollIter ucol_strcollIter_3_8 +#define ucol_swap ucol_swap_3_8 +#define ucol_swapBinary ucol_swapBinary_3_8 +#define ucol_swapInverseUCA ucol_swapInverseUCA_3_8 +#define ucol_tertiaryOrder ucol_tertiaryOrder_3_8 +#define ucol_tok_assembleTokenList ucol_tok_assembleTokenList_3_8 +#define ucol_tok_closeTokenList ucol_tok_closeTokenList_3_8 +#define ucol_tok_getNextArgument ucol_tok_getNextArgument_3_8 +#define ucol_tok_initTokenList ucol_tok_initTokenList_3_8 +#define ucol_tok_parseNextToken ucol_tok_parseNextToken_3_8 +#define ucol_updateInternalState ucol_updateInternalState_3_8 +#define ucsdet_close ucsdet_close_3_8 +#define ucsdet_detect ucsdet_detect_3_8 +#define ucsdet_detectAll ucsdet_detectAll_3_8 +#define ucsdet_enableInputFilter ucsdet_enableInputFilter_3_8 +#define ucsdet_getAllDetectableCharsets ucsdet_getAllDetectableCharsets_3_8 +#define ucsdet_getConfidence ucsdet_getConfidence_3_8 +#define ucsdet_getLanguage ucsdet_getLanguage_3_8 +#define ucsdet_getName ucsdet_getName_3_8 +#define ucsdet_getUChars ucsdet_getUChars_3_8 +#define ucsdet_isInputFilterEnabled ucsdet_isInputFilterEnabled_3_8 +#define ucsdet_open ucsdet_open_3_8 +#define ucsdet_setDeclaredEncoding ucsdet_setDeclaredEncoding_3_8 +#define ucsdet_setText ucsdet_setText_3_8 +#define ucurr_forLocale ucurr_forLocale_3_8 +#define ucurr_getDefaultFractionDigits ucurr_getDefaultFractionDigits_3_8 +#define ucurr_getName ucurr_getName_3_8 +#define ucurr_getRoundingIncrement ucurr_getRoundingIncrement_3_8 +#define ucurr_openISOCurrencies ucurr_openISOCurrencies_3_8 +#define ucurr_register ucurr_register_3_8 +#define ucurr_unregister ucurr_unregister_3_8 +#define udat_applyPattern udat_applyPattern_3_8 +#define udat_clone udat_clone_3_8 +#define udat_close udat_close_3_8 +#define udat_countAvailable udat_countAvailable_3_8 +#define udat_countSymbols udat_countSymbols_3_8 +#define udat_format udat_format_3_8 +#define udat_get2DigitYearStart udat_get2DigitYearStart_3_8 +#define udat_getAvailable udat_getAvailable_3_8 +#define udat_getCalendar udat_getCalendar_3_8 +#define udat_getLocaleByType udat_getLocaleByType_3_8 +#define udat_getNumberFormat udat_getNumberFormat_3_8 +#define udat_getSymbols udat_getSymbols_3_8 +#define udat_isLenient udat_isLenient_3_8 +#define udat_open udat_open_3_8 +#define udat_parse udat_parse_3_8 +#define udat_parseCalendar udat_parseCalendar_3_8 +#define udat_set2DigitYearStart udat_set2DigitYearStart_3_8 +#define udat_setCalendar udat_setCalendar_3_8 +#define udat_setLenient udat_setLenient_3_8 +#define udat_setNumberFormat udat_setNumberFormat_3_8 +#define udat_setSymbols udat_setSymbols_3_8 +#define udat_toPattern udat_toPattern_3_8 +#define udata_checkCommonData udata_checkCommonData_3_8 +#define udata_close udata_close_3_8 +#define udata_closeSwapper udata_closeSwapper_3_8 +#define udata_getHeaderSize udata_getHeaderSize_3_8 +#define udata_getInfo udata_getInfo_3_8 +#define udata_getInfoSize udata_getInfoSize_3_8 +#define udata_getLength udata_getLength_3_8 +#define udata_getMemory udata_getMemory_3_8 +#define udata_getRawMemory udata_getRawMemory_3_8 +#define udata_open udata_open_3_8 +#define udata_openChoice udata_openChoice_3_8 +#define udata_openSwapper udata_openSwapper_3_8 +#define udata_openSwapperForInputData udata_openSwapperForInputData_3_8 +#define udata_printError udata_printError_3_8 +#define udata_readInt16 udata_readInt16_3_8 +#define udata_readInt32 udata_readInt32_3_8 +#define udata_setAppData udata_setAppData_3_8 +#define udata_setCommonData udata_setCommonData_3_8 +#define udata_setFileAccess udata_setFileAccess_3_8 +#define udata_swapDataHeader udata_swapDataHeader_3_8 +#define udata_swapInvStringBlock udata_swapInvStringBlock_3_8 +#define udatpg_addPattern udatpg_addPattern_3_8 +#define udatpg_clone udatpg_clone_3_8 +#define udatpg_close udatpg_close_3_8 +#define udatpg_getAppendItemFormat udatpg_getAppendItemFormat_3_8 +#define udatpg_getAppendItemName udatpg_getAppendItemName_3_8 +#define udatpg_getBaseSkeleton udatpg_getBaseSkeleton_3_8 +#define udatpg_getBestPattern udatpg_getBestPattern_3_8 +#define udatpg_getDateTimeFormat udatpg_getDateTimeFormat_3_8 +#define udatpg_getDecimal udatpg_getDecimal_3_8 +#define udatpg_getPatternForSkeleton udatpg_getPatternForSkeleton_3_8 +#define udatpg_getSkeleton udatpg_getSkeleton_3_8 +#define udatpg_open udatpg_open_3_8 +#define udatpg_openBaseSkeletons udatpg_openBaseSkeletons_3_8 +#define udatpg_openEmpty udatpg_openEmpty_3_8 +#define udatpg_openSkeletons udatpg_openSkeletons_3_8 +#define udatpg_replaceFieldTypes udatpg_replaceFieldTypes_3_8 +#define udatpg_setAppendItemFormat udatpg_setAppendItemFormat_3_8 +#define udatpg_setAppendItemName udatpg_setAppendItemName_3_8 +#define udatpg_setDateTimeFormat udatpg_setDateTimeFormat_3_8 +#define udatpg_setDecimal udatpg_setDecimal_3_8 +#define uenum_close uenum_close_3_8 +#define uenum_count uenum_count_3_8 +#define uenum_next uenum_next_3_8 +#define uenum_nextDefault uenum_nextDefault_3_8 +#define uenum_openCharStringsEnumeration uenum_openCharStringsEnumeration_3_8 +#define uenum_openStringEnumeration uenum_openStringEnumeration_3_8 +#define uenum_reset uenum_reset_3_8 +#define uenum_unext uenum_unext_3_8 +#define uenum_unextDefault uenum_unextDefault_3_8 +#define ufile_close_translit ufile_close_translit_3_8 +#define ufile_fill_uchar_buffer ufile_fill_uchar_buffer_3_8 +#define ufile_flush_translit ufile_flush_translit_3_8 +#define ufile_getch ufile_getch_3_8 +#define ufile_getch32 ufile_getch32_3_8 +#define ufmt_64tou ufmt_64tou_3_8 +#define ufmt_defaultCPToUnicode ufmt_defaultCPToUnicode_3_8 +#define ufmt_digitvalue ufmt_digitvalue_3_8 +#define ufmt_isdigit ufmt_isdigit_3_8 +#define ufmt_ptou ufmt_ptou_3_8 +#define ufmt_uto64 ufmt_uto64_3_8 +#define ufmt_utop ufmt_utop_3_8 +#define uhash_close uhash_close_3_8 +#define uhash_compareCaselessUnicodeString uhash_compareCaselessUnicodeString_3_8 +#define uhash_compareChars uhash_compareChars_3_8 +#define uhash_compareIChars uhash_compareIChars_3_8 +#define uhash_compareLong uhash_compareLong_3_8 +#define uhash_compareUChars uhash_compareUChars_3_8 +#define uhash_compareUnicodeString uhash_compareUnicodeString_3_8 +#define uhash_count uhash_count_3_8 +#define uhash_deleteHashtable uhash_deleteHashtable_3_8 +#define uhash_deleteUVector uhash_deleteUVector_3_8 +#define uhash_deleteUnicodeString uhash_deleteUnicodeString_3_8 +#define uhash_equals uhash_equals_3_8 +#define uhash_find uhash_find_3_8 +#define uhash_freeBlock uhash_freeBlock_3_8 +#define uhash_get uhash_get_3_8 +#define uhash_geti uhash_geti_3_8 +#define uhash_hashCaselessUnicodeString uhash_hashCaselessUnicodeString_3_8 +#define uhash_hashChars uhash_hashChars_3_8 +#define uhash_hashIChars uhash_hashIChars_3_8 +#define uhash_hashLong uhash_hashLong_3_8 +#define uhash_hashUChars uhash_hashUChars_3_8 +#define uhash_hashUCharsN uhash_hashUCharsN_3_8 +#define uhash_hashUnicodeString uhash_hashUnicodeString_3_8 +#define uhash_iget uhash_iget_3_8 +#define uhash_igeti uhash_igeti_3_8 +#define uhash_init uhash_init_3_8 +#define uhash_iput uhash_iput_3_8 +#define uhash_iputi uhash_iputi_3_8 +#define uhash_iremove uhash_iremove_3_8 +#define uhash_iremovei uhash_iremovei_3_8 +#define uhash_nextElement uhash_nextElement_3_8 +#define uhash_open uhash_open_3_8 +#define uhash_openSize uhash_openSize_3_8 +#define uhash_put uhash_put_3_8 +#define uhash_puti uhash_puti_3_8 +#define uhash_remove uhash_remove_3_8 +#define uhash_removeAll uhash_removeAll_3_8 +#define uhash_removeElement uhash_removeElement_3_8 +#define uhash_removei uhash_removei_3_8 +#define uhash_setKeyComparator uhash_setKeyComparator_3_8 +#define uhash_setKeyDeleter uhash_setKeyDeleter_3_8 +#define uhash_setKeyHasher uhash_setKeyHasher_3_8 +#define uhash_setResizePolicy uhash_setResizePolicy_3_8 +#define uhash_setValueComparator uhash_setValueComparator_3_8 +#define uhash_setValueDeleter uhash_setValueDeleter_3_8 +#define uhst_addPropertyStarts uhst_addPropertyStarts_3_8 +#define uidna_IDNToASCII uidna_IDNToASCII_3_8 +#define uidna_IDNToUnicode uidna_IDNToUnicode_3_8 +#define uidna_compare uidna_compare_3_8 +#define uidna_toASCII uidna_toASCII_3_8 +#define uidna_toUnicode uidna_toUnicode_3_8 +#define uiter_current32 uiter_current32_3_8 +#define uiter_getState uiter_getState_3_8 +#define uiter_next32 uiter_next32_3_8 +#define uiter_previous32 uiter_previous32_3_8 +#define uiter_setCharacterIterator uiter_setCharacterIterator_3_8 +#define uiter_setReplaceable uiter_setReplaceable_3_8 +#define uiter_setState uiter_setState_3_8 +#define uiter_setString uiter_setString_3_8 +#define uiter_setUTF16BE uiter_setUTF16BE_3_8 +#define uiter_setUTF8 uiter_setUTF8_3_8 +#define uloc_acceptLanguage uloc_acceptLanguage_3_8 +#define uloc_acceptLanguageFromHTTP uloc_acceptLanguageFromHTTP_3_8 +#define uloc_canonicalize uloc_canonicalize_3_8 +#define uloc_countAvailable uloc_countAvailable_3_8 +#define uloc_getAvailable uloc_getAvailable_3_8 +#define uloc_getBaseName uloc_getBaseName_3_8 +#define uloc_getCountry uloc_getCountry_3_8 +#define uloc_getDefault uloc_getDefault_3_8 +#define uloc_getDisplayCountry uloc_getDisplayCountry_3_8 +#define uloc_getDisplayKeyword uloc_getDisplayKeyword_3_8 +#define uloc_getDisplayKeywordValue uloc_getDisplayKeywordValue_3_8 +#define uloc_getDisplayLanguage uloc_getDisplayLanguage_3_8 +#define uloc_getDisplayName uloc_getDisplayName_3_8 +#define uloc_getDisplayScript uloc_getDisplayScript_3_8 +#define uloc_getDisplayVariant uloc_getDisplayVariant_3_8 +#define uloc_getISO3Country uloc_getISO3Country_3_8 +#define uloc_getISO3Language uloc_getISO3Language_3_8 +#define uloc_getISOCountries uloc_getISOCountries_3_8 +#define uloc_getISOLanguages uloc_getISOLanguages_3_8 +#define uloc_getKeywordValue uloc_getKeywordValue_3_8 +#define uloc_getLCID uloc_getLCID_3_8 +#define uloc_getLanguage uloc_getLanguage_3_8 +#define uloc_getLocaleForLCID uloc_getLocaleForLCID_3_8 +#define uloc_getName uloc_getName_3_8 +#define uloc_getParent uloc_getParent_3_8 +#define uloc_getScript uloc_getScript_3_8 +#define uloc_getVariant uloc_getVariant_3_8 +#define uloc_openKeywordList uloc_openKeywordList_3_8 +#define uloc_openKeywords uloc_openKeywords_3_8 +#define uloc_setDefault uloc_setDefault_3_8 +#define uloc_setKeywordValue uloc_setKeywordValue_3_8 +#define ulocdata_close ulocdata_close_3_8 +#define ulocdata_getDelimiter ulocdata_getDelimiter_3_8 +#define ulocdata_getExemplarSet ulocdata_getExemplarSet_3_8 +#define ulocdata_getMeasurementSystem ulocdata_getMeasurementSystem_3_8 +#define ulocdata_getNoSubstitute ulocdata_getNoSubstitute_3_8 +#define ulocdata_getPaperSize ulocdata_getPaperSize_3_8 +#define ulocdata_open ulocdata_open_3_8 +#define ulocdata_setNoSubstitute ulocdata_setNoSubstitute_3_8 +#define umsg_applyPattern umsg_applyPattern_3_8 +#define umsg_autoQuoteApostrophe umsg_autoQuoteApostrophe_3_8 +#define umsg_clone umsg_clone_3_8 +#define umsg_close umsg_close_3_8 +#define umsg_format umsg_format_3_8 +#define umsg_getLocale umsg_getLocale_3_8 +#define umsg_open umsg_open_3_8 +#define umsg_parse umsg_parse_3_8 +#define umsg_setLocale umsg_setLocale_3_8 +#define umsg_toPattern umsg_toPattern_3_8 +#define umsg_vformat umsg_vformat_3_8 +#define umsg_vparse umsg_vparse_3_8 +#define umtx_atomic_dec umtx_atomic_dec_3_8 +#define umtx_atomic_inc umtx_atomic_inc_3_8 +#define umtx_cleanup umtx_cleanup_3_8 +#define umtx_destroy umtx_destroy_3_8 +#define umtx_init umtx_init_3_8 +#define umtx_lock umtx_lock_3_8 +#define umtx_unlock umtx_unlock_3_8 +#define unorm_addPropertyStarts unorm_addPropertyStarts_3_8 +#define unorm_closeIter unorm_closeIter_3_8 +#define unorm_compare unorm_compare_3_8 +#define unorm_compose unorm_compose_3_8 +#define unorm_concatenate unorm_concatenate_3_8 +#define unorm_decompose unorm_decompose_3_8 +#define unorm_getCanonStartSet unorm_getCanonStartSet_3_8 +#define unorm_getCanonicalDecomposition unorm_getCanonicalDecomposition_3_8 +#define unorm_getDecomposition unorm_getDecomposition_3_8 +#define unorm_getFCD16FromCodePoint unorm_getFCD16FromCodePoint_3_8 +#define unorm_getFCDTrie unorm_getFCDTrie_3_8 +#define unorm_getNX unorm_getNX_3_8 +#define unorm_getQuickCheck unorm_getQuickCheck_3_8 +#define unorm_getUnicodeVersion unorm_getUnicodeVersion_3_8 +#define unorm_haveData unorm_haveData_3_8 +#define unorm_internalIsFullCompositionExclusion unorm_internalIsFullCompositionExclusion_3_8 +#define unorm_internalNormalize unorm_internalNormalize_3_8 +#define unorm_internalNormalizeWithNX unorm_internalNormalizeWithNX_3_8 +#define unorm_internalQuickCheck unorm_internalQuickCheck_3_8 +#define unorm_isCanonSafeStart unorm_isCanonSafeStart_3_8 +#define unorm_isNFSkippable unorm_isNFSkippable_3_8 +#define unorm_isNormalized unorm_isNormalized_3_8 +#define unorm_isNormalizedWithOptions unorm_isNormalizedWithOptions_3_8 +#define unorm_next unorm_next_3_8 +#define unorm_normalize unorm_normalize_3_8 +#define unorm_openIter unorm_openIter_3_8 +#define unorm_previous unorm_previous_3_8 +#define unorm_quickCheck unorm_quickCheck_3_8 +#define unorm_quickCheckWithOptions unorm_quickCheckWithOptions_3_8 +#define unorm_setIter unorm_setIter_3_8 +#define unum_applyPattern unum_applyPattern_3_8 +#define unum_clone unum_clone_3_8 +#define unum_close unum_close_3_8 +#define unum_countAvailable unum_countAvailable_3_8 +#define unum_format unum_format_3_8 +#define unum_formatDouble unum_formatDouble_3_8 +#define unum_formatDoubleCurrency unum_formatDoubleCurrency_3_8 +#define unum_formatInt64 unum_formatInt64_3_8 +#define unum_getAttribute unum_getAttribute_3_8 +#define unum_getAvailable unum_getAvailable_3_8 +#define unum_getDoubleAttribute unum_getDoubleAttribute_3_8 +#define unum_getLocaleByType unum_getLocaleByType_3_8 +#define unum_getSymbol unum_getSymbol_3_8 +#define unum_getTextAttribute unum_getTextAttribute_3_8 +#define unum_open unum_open_3_8 +#define unum_parse unum_parse_3_8 +#define unum_parseDouble unum_parseDouble_3_8 +#define unum_parseDoubleCurrency unum_parseDoubleCurrency_3_8 +#define unum_parseInt64 unum_parseInt64_3_8 +#define unum_setAttribute unum_setAttribute_3_8 +#define unum_setDoubleAttribute unum_setDoubleAttribute_3_8 +#define unum_setSymbol unum_setSymbol_3_8 +#define unum_setTextAttribute unum_setTextAttribute_3_8 +#define unum_toPattern unum_toPattern_3_8 +#define upname_swap upname_swap_3_8 +#define uprops_getSource uprops_getSource_3_8 +#define upropsvec_addPropertyStarts upropsvec_addPropertyStarts_3_8 +#define uprv_asciiFromEbcdic uprv_asciiFromEbcdic_3_8 +#define uprv_asciitolower uprv_asciitolower_3_8 +#define uprv_ceil uprv_ceil_3_8 +#define uprv_cnttab_addContraction uprv_cnttab_addContraction_3_8 +#define uprv_cnttab_changeContraction uprv_cnttab_changeContraction_3_8 +#define uprv_cnttab_changeLastCE uprv_cnttab_changeLastCE_3_8 +#define uprv_cnttab_clone uprv_cnttab_clone_3_8 +#define uprv_cnttab_close uprv_cnttab_close_3_8 +#define uprv_cnttab_constructTable uprv_cnttab_constructTable_3_8 +#define uprv_cnttab_findCE uprv_cnttab_findCE_3_8 +#define uprv_cnttab_findCP uprv_cnttab_findCP_3_8 +#define uprv_cnttab_getCE uprv_cnttab_getCE_3_8 +#define uprv_cnttab_insertContraction uprv_cnttab_insertContraction_3_8 +#define uprv_cnttab_isTailored uprv_cnttab_isTailored_3_8 +#define uprv_cnttab_open uprv_cnttab_open_3_8 +#define uprv_cnttab_setContraction uprv_cnttab_setContraction_3_8 +#define uprv_compareASCIIPropertyNames uprv_compareASCIIPropertyNames_3_8 +#define uprv_compareEBCDICPropertyNames uprv_compareEBCDICPropertyNames_3_8 +#define uprv_compareInvAscii uprv_compareInvAscii_3_8 +#define uprv_compareInvEbcdic uprv_compareInvEbcdic_3_8 +#define uprv_convertToLCID uprv_convertToLCID_3_8 +#define uprv_convertToPosix uprv_convertToPosix_3_8 +#define uprv_copyAscii uprv_copyAscii_3_8 +#define uprv_copyEbcdic uprv_copyEbcdic_3_8 +#define uprv_ebcdicFromAscii uprv_ebcdicFromAscii_3_8 +#define uprv_ebcdictolower uprv_ebcdictolower_3_8 +#define uprv_fabs uprv_fabs_3_8 +#define uprv_floor uprv_floor_3_8 +#define uprv_fmax uprv_fmax_3_8 +#define uprv_fmin uprv_fmin_3_8 +#define uprv_fmod uprv_fmod_3_8 +#define uprv_free uprv_free_3_8 +#define uprv_getCharNameCharacters uprv_getCharNameCharacters_3_8 +#define uprv_getDefaultCodepage uprv_getDefaultCodepage_3_8 +#define uprv_getDefaultLocaleID uprv_getDefaultLocaleID_3_8 +#define uprv_getInfinity uprv_getInfinity_3_8 +#define uprv_getMaxCharNameLength uprv_getMaxCharNameLength_3_8 +#define uprv_getMaxValues uprv_getMaxValues_3_8 +#define uprv_getNaN uprv_getNaN_3_8 +#define uprv_getStaticCurrencyName uprv_getStaticCurrencyName_3_8 +#define uprv_getUTCtime uprv_getUTCtime_3_8 +#define uprv_haveProperties uprv_haveProperties_3_8 +#define uprv_init_collIterate uprv_init_collIterate_3_8 +#define uprv_int32Comparator uprv_int32Comparator_3_8 +#define uprv_isInfinite uprv_isInfinite_3_8 +#define uprv_isInvariantString uprv_isInvariantString_3_8 +#define uprv_isInvariantUString uprv_isInvariantUString_3_8 +#define uprv_isNaN uprv_isNaN_3_8 +#define uprv_isNegativeInfinity uprv_isNegativeInfinity_3_8 +#define uprv_isPositiveInfinity uprv_isPositiveInfinity_3_8 +#define uprv_isRuleWhiteSpace uprv_isRuleWhiteSpace_3_8 +#define uprv_itou uprv_itou_3_8 +#define uprv_log uprv_log_3_8 +#define uprv_malloc uprv_malloc_3_8 +#define uprv_mapFile uprv_mapFile_3_8 +#define uprv_max uprv_max_3_8 +#define uprv_maximumPtr uprv_maximumPtr_3_8 +#define uprv_maxMantissa uprv_maxMantissa_3_8 +#define uprv_min uprv_min_3_8 +#define uprv_modf uprv_modf_3_8 +#define uprv_openRuleWhiteSpaceSet uprv_openRuleWhiteSpaceSet_3_8 +#define uprv_parseCurrency uprv_parseCurrency_3_8 +#define uprv_pathIsAbsolute uprv_pathIsAbsolute_3_8 +#define uprv_pow uprv_pow_3_8 +#define uprv_pow10 uprv_pow10_3_8 +#define uprv_realloc uprv_realloc_3_8 +#define uprv_round uprv_round_3_8 +#define uprv_sortArray uprv_sortArray_3_8 +#define uprv_strCompare uprv_strCompare_3_8 +#define uprv_strdup uprv_strdup_3_8 +#define uprv_strndup uprv_strndup_3_8 +#define uprv_syntaxError uprv_syntaxError_3_8 +#define uprv_timezone uprv_timezone_3_8 +#define uprv_toupper uprv_toupper_3_8 +#define uprv_trunc uprv_trunc_3_8 +#define uprv_tzname uprv_tzname_3_8 +#define uprv_tzset uprv_tzset_3_8 +#define uprv_uca_addAnElement uprv_uca_addAnElement_3_8 +#define uprv_uca_assembleTable uprv_uca_assembleTable_3_8 +#define uprv_uca_canonicalClosure uprv_uca_canonicalClosure_3_8 +#define uprv_uca_cloneTempTable uprv_uca_cloneTempTable_3_8 +#define uprv_uca_closeTempTable uprv_uca_closeTempTable_3_8 +#define uprv_uca_getCodePointFromRaw uprv_uca_getCodePointFromRaw_3_8 +#define uprv_uca_getImplicitFromRaw uprv_uca_getImplicitFromRaw_3_8 +#define uprv_uca_getRawFromCodePoint uprv_uca_getRawFromCodePoint_3_8 +#define uprv_uca_getRawFromImplicit uprv_uca_getRawFromImplicit_3_8 +#define uprv_uca_initImplicitConstants uprv_uca_initImplicitConstants_3_8 +#define uprv_uca_initTempTable uprv_uca_initTempTable_3_8 +#define uprv_uint16Comparator uprv_uint16Comparator_3_8 +#define uprv_uint32Comparator uprv_uint32Comparator_3_8 +#define uprv_unmapFile uprv_unmapFile_3_8 +#define uregex_appendReplacement uregex_appendReplacement_3_8 +#define uregex_appendTail uregex_appendTail_3_8 +#define uregex_clone uregex_clone_3_8 +#define uregex_close uregex_close_3_8 +#define uregex_end uregex_end_3_8 +#define uregex_find uregex_find_3_8 +#define uregex_findNext uregex_findNext_3_8 +#define uregex_flags uregex_flags_3_8 +#define uregex_getText uregex_getText_3_8 +#define uregex_group uregex_group_3_8 +#define uregex_groupCount uregex_groupCount_3_8 +#define uregex_lookingAt uregex_lookingAt_3_8 +#define uregex_matches uregex_matches_3_8 +#define uregex_open uregex_open_3_8 +#define uregex_openC uregex_openC_3_8 +#define uregex_pattern uregex_pattern_3_8 +#define uregex_replaceAll uregex_replaceAll_3_8 +#define uregex_replaceFirst uregex_replaceFirst_3_8 +#define uregex_reset uregex_reset_3_8 +#define uregex_setText uregex_setText_3_8 +#define uregex_split uregex_split_3_8 +#define uregex_start uregex_start_3_8 +#define ures_clone ures_clone_3_8 +#define ures_close ures_close_3_8 +#define ures_copyResb ures_copyResb_3_8 +#define ures_countArrayItems ures_countArrayItems_3_8 +#define ures_equal ures_equal_3_8 +#define ures_findResource ures_findResource_3_8 +#define ures_findSubResource ures_findSubResource_3_8 +#define ures_getBinary ures_getBinary_3_8 +#define ures_getByIndex ures_getByIndex_3_8 +#define ures_getByKey ures_getByKey_3_8 +#define ures_getByKeyWithFallback ures_getByKeyWithFallback_3_8 +#define ures_getFunctionalEquivalent ures_getFunctionalEquivalent_3_8 +#define ures_getInt ures_getInt_3_8 +#define ures_getIntVector ures_getIntVector_3_8 +#define ures_getKey ures_getKey_3_8 +#define ures_getKeywordValues ures_getKeywordValues_3_8 +#define ures_getLocale ures_getLocale_3_8 +#define ures_getLocaleByType ures_getLocaleByType_3_8 +#define ures_getName ures_getName_3_8 +#define ures_getNextResource ures_getNextResource_3_8 +#define ures_getNextString ures_getNextString_3_8 +#define ures_getParentBundle ures_getParentBundle_3_8 +#define ures_getSize ures_getSize_3_8 +#define ures_getString ures_getString_3_8 +#define ures_getStringByIndex ures_getStringByIndex_3_8 +#define ures_getStringByKey ures_getStringByKey_3_8 +#define ures_getStringByKeyWithFallback ures_getStringByKeyWithFallback_3_8 +#define ures_getType ures_getType_3_8 +#define ures_getUInt ures_getUInt_3_8 +#define ures_getUTF8String ures_getUTF8String_3_8 +#define ures_getUTF8StringByIndex ures_getUTF8StringByIndex_3_8 +#define ures_getUTF8StringByKey ures_getUTF8StringByKey_3_8 +#define ures_getVersion ures_getVersion_3_8 +#define ures_getVersionNumber ures_getVersionNumber_3_8 +#define ures_hasNext ures_hasNext_3_8 +#define ures_initStackObject ures_initStackObject_3_8 +#define ures_open ures_open_3_8 +#define ures_openAvailableLocales ures_openAvailableLocales_3_8 +#define ures_openDirect ures_openDirect_3_8 +#define ures_openFillIn ures_openFillIn_3_8 +#define ures_openU ures_openU_3_8 +#define ures_resetIterator ures_resetIterator_3_8 +#define ures_swap ures_swap_3_8 +#define uscript_closeRun uscript_closeRun_3_8 +#define uscript_getCode uscript_getCode_3_8 +#define uscript_getName uscript_getName_3_8 +#define uscript_getScript uscript_getScript_3_8 +#define uscript_getShortName uscript_getShortName_3_8 +#define uscript_nextRun uscript_nextRun_3_8 +#define uscript_openRun uscript_openRun_3_8 +#define uscript_resetRun uscript_resetRun_3_8 +#define uscript_setRunText uscript_setRunText_3_8 +#define usearch_close usearch_close_3_8 +#define usearch_first usearch_first_3_8 +#define usearch_following usearch_following_3_8 +#define usearch_getAttribute usearch_getAttribute_3_8 +#define usearch_getBreakIterator usearch_getBreakIterator_3_8 +#define usearch_getCollator usearch_getCollator_3_8 +#define usearch_getMatchedLength usearch_getMatchedLength_3_8 +#define usearch_getMatchedStart usearch_getMatchedStart_3_8 +#define usearch_getMatchedText usearch_getMatchedText_3_8 +#define usearch_getOffset usearch_getOffset_3_8 +#define usearch_getPattern usearch_getPattern_3_8 +#define usearch_getText usearch_getText_3_8 +#define usearch_handleNextCanonical usearch_handleNextCanonical_3_8 +#define usearch_handleNextExact usearch_handleNextExact_3_8 +#define usearch_handlePreviousCanonical usearch_handlePreviousCanonical_3_8 +#define usearch_handlePreviousExact usearch_handlePreviousExact_3_8 +#define usearch_last usearch_last_3_8 +#define usearch_next usearch_next_3_8 +#define usearch_open usearch_open_3_8 +#define usearch_openFromCollator usearch_openFromCollator_3_8 +#define usearch_preceding usearch_preceding_3_8 +#define usearch_previous usearch_previous_3_8 +#define usearch_reset usearch_reset_3_8 +#define usearch_setAttribute usearch_setAttribute_3_8 +#define usearch_setBreakIterator usearch_setBreakIterator_3_8 +#define usearch_setCollator usearch_setCollator_3_8 +#define usearch_setOffset usearch_setOffset_3_8 +#define usearch_setPattern usearch_setPattern_3_8 +#define usearch_setText usearch_setText_3_8 +#define uset_add uset_add_3_8 +#define uset_addAll uset_addAll_3_8 +#define uset_addAllCodePoints uset_addAllCodePoints_3_8 +#define uset_addRange uset_addRange_3_8 +#define uset_addString uset_addString_3_8 +#define uset_applyIntPropertyValue uset_applyIntPropertyValue_3_8 +#define uset_applyPattern uset_applyPattern_3_8 +#define uset_applyPropertyAlias uset_applyPropertyAlias_3_8 +#define uset_charAt uset_charAt_3_8 +#define uset_clear uset_clear_3_8 +#define uset_clone uset_clone_3_8 +#define uset_cloneAsThawed uset_cloneAsThawed_3_8 +#define uset_close uset_close_3_8 +#define uset_compact uset_compact_3_8 +#define uset_complement uset_complement_3_8 +#define uset_complementAll uset_complementAll_3_8 +#define uset_contains uset_contains_3_8 +#define uset_containsAll uset_containsAll_3_8 +#define uset_containsAllCodePoints uset_containsAllCodePoints_3_8 +#define uset_containsNone uset_containsNone_3_8 +#define uset_containsRange uset_containsRange_3_8 +#define uset_containsSome uset_containsSome_3_8 +#define uset_containsString uset_containsString_3_8 +#define uset_equals uset_equals_3_8 +#define uset_freeze uset_freeze_3_8 +#define uset_getItem uset_getItem_3_8 +#define uset_getItemCount uset_getItemCount_3_8 +#define uset_getSerializedRange uset_getSerializedRange_3_8 +#define uset_getSerializedRangeCount uset_getSerializedRangeCount_3_8 +#define uset_getSerializedSet uset_getSerializedSet_3_8 +#define uset_indexOf uset_indexOf_3_8 +#define uset_isEmpty uset_isEmpty_3_8 +#define uset_isFrozen uset_isFrozen_3_8 +#define uset_open uset_open_3_8 +#define uset_openPattern uset_openPattern_3_8 +#define uset_openPatternOptions uset_openPatternOptions_3_8 +#define uset_remove uset_remove_3_8 +#define uset_removeAll uset_removeAll_3_8 +#define uset_removeRange uset_removeRange_3_8 +#define uset_removeString uset_removeString_3_8 +#define uset_resemblesPattern uset_resemblesPattern_3_8 +#define uset_retain uset_retain_3_8 +#define uset_retainAll uset_retainAll_3_8 +#define uset_serialize uset_serialize_3_8 +#define uset_serializedContains uset_serializedContains_3_8 +#define uset_set uset_set_3_8 +#define uset_setSerializedToOne uset_setSerializedToOne_3_8 +#define uset_size uset_size_3_8 +#define uset_span uset_span_3_8 +#define uset_spanBack uset_spanBack_3_8 +#define uset_spanBackUTF8 uset_spanBackUTF8_3_8 +#define uset_spanUTF8 uset_spanUTF8_3_8 +#define uset_toPattern uset_toPattern_3_8 +#define usprep_close usprep_close_3_8 +#define usprep_open usprep_open_3_8 +#define usprep_prepare usprep_prepare_3_8 +#define usprep_swap usprep_swap_3_8 +#define ustr_foldCase ustr_foldCase_3_8 +#define ustr_toLower ustr_toLower_3_8 +#define ustr_toTitle ustr_toTitle_3_8 +#define ustr_toUpper ustr_toUpper_3_8 +#define utext_char32At utext_char32At_3_8 +#define utext_clone utext_clone_3_8 +#define utext_close utext_close_3_8 +#define utext_copy utext_copy_3_8 +#define utext_current32 utext_current32_3_8 +#define utext_equals utext_equals_3_8 +#define utext_extract utext_extract_3_8 +#define utext_freeze utext_freeze_3_8 +#define utext_getNativeIndex utext_getNativeIndex_3_8 +#define utext_getPreviousNativeIndex utext_getPreviousNativeIndex_3_8 +#define utext_hasMetaData utext_hasMetaData_3_8 +#define utext_isLengthExpensive utext_isLengthExpensive_3_8 +#define utext_isWritable utext_isWritable_3_8 +#define utext_moveIndex32 utext_moveIndex32_3_8 +#define utext_nativeLength utext_nativeLength_3_8 +#define utext_next32 utext_next32_3_8 +#define utext_next32From utext_next32From_3_8 +#define utext_openCharacterIterator utext_openCharacterIterator_3_8 +#define utext_openConstUnicodeString utext_openConstUnicodeString_3_8 +#define utext_openReplaceable utext_openReplaceable_3_8 +#define utext_openUChars utext_openUChars_3_8 +#define utext_openUTF8 utext_openUTF8_3_8 +#define utext_openUnicodeString utext_openUnicodeString_3_8 +#define utext_previous32 utext_previous32_3_8 +#define utext_previous32From utext_previous32From_3_8 +#define utext_replace utext_replace_3_8 +#define utext_setNativeIndex utext_setNativeIndex_3_8 +#define utext_setup utext_setup_3_8 +#define utf8_appendCharSafeBody utf8_appendCharSafeBody_3_8 +#define utf8_back1SafeBody utf8_back1SafeBody_3_8 +#define utf8_countTrailBytes utf8_countTrailBytes_3_8 +#define utf8_nextCharSafeBody utf8_nextCharSafeBody_3_8 +#define utf8_prevCharSafeBody utf8_prevCharSafeBody_3_8 +#define utmscale_fromInt64 utmscale_fromInt64_3_8 +#define utmscale_getTimeScaleValue utmscale_getTimeScaleValue_3_8 +#define utmscale_toInt64 utmscale_toInt64_3_8 +#define utrace_cleanup utrace_cleanup_3_8 +#define utrace_data utrace_data_3_8 +#define utrace_entry utrace_entry_3_8 +#define utrace_exit utrace_exit_3_8 +#define utrace_format utrace_format_3_8 +#define utrace_functionName utrace_functionName_3_8 +#define utrace_getFunctions utrace_getFunctions_3_8 +#define utrace_getLevel utrace_getLevel_3_8 +#define utrace_level utrace_level_3_8 +#define utrace_setFunctions utrace_setFunctions_3_8 +#define utrace_setLevel utrace_setLevel_3_8 +#define utrace_vformat utrace_vformat_3_8 +#define utrans_clone utrans_clone_3_8 +#define utrans_close utrans_close_3_8 +#define utrans_countAvailableIDs utrans_countAvailableIDs_3_8 +#define utrans_getAvailableID utrans_getAvailableID_3_8 +#define utrans_getID utrans_getID_3_8 +#define utrans_getUnicodeID utrans_getUnicodeID_3_8 +#define utrans_open utrans_open_3_8 +#define utrans_openIDs utrans_openIDs_3_8 +#define utrans_openInverse utrans_openInverse_3_8 +#define utrans_openU utrans_openU_3_8 +#define utrans_register utrans_register_3_8 +#define utrans_rep_caseContextIterator utrans_rep_caseContextIterator_3_8 +#define utrans_setFilter utrans_setFilter_3_8 +#define utrans_stripRules utrans_stripRules_3_8 +#define utrans_trans utrans_trans_3_8 +#define utrans_transIncremental utrans_transIncremental_3_8 +#define utrans_transIncrementalUChars utrans_transIncrementalUChars_3_8 +#define utrans_transUChars utrans_transUChars_3_8 +#define utrans_unregister utrans_unregister_3_8 +#define utrans_unregisterID utrans_unregisterID_3_8 +#define utrie_clone utrie_clone_3_8 +#define utrie_close utrie_close_3_8 +#define utrie_defaultGetFoldingOffset utrie_defaultGetFoldingOffset_3_8 +#define utrie_enum utrie_enum_3_8 +#define utrie_get32 utrie_get32_3_8 +#define utrie_getData utrie_getData_3_8 +#define utrie_open utrie_open_3_8 +#define utrie_serialize utrie_serialize_3_8 +#define utrie_set32 utrie_set32_3_8 +#define utrie_setRange32 utrie_setRange32_3_8 +#define utrie_swap utrie_swap_3_8 +#define utrie_unserialize utrie_unserialize_3_8 +#define utrie_unserializeDummy utrie_unserializeDummy_3_8 +/* C++ class names renaming defines */ + +#ifdef XP_CPLUSPLUS +#if !U_HAVE_NAMESPACE + +#define AbsoluteValueSubstitution AbsoluteValueSubstitution_3_8 +#define AlternateSubstitutionSubtable AlternateSubstitutionSubtable_3_8 +#define AnchorTable AnchorTable_3_8 +#define AnnualTimeZoneRule AnnualTimeZoneRule_3_8 +#define AnyTransliterator AnyTransliterator_3_8 +#define ArabicOpenTypeLayoutEngine ArabicOpenTypeLayoutEngine_3_8 +#define ArabicShaping ArabicShaping_3_8 +#define BMPSet BMPSet_3_8 +#define BasicCalendarFactory BasicCalendarFactory_3_8 +#define BasicTimeZone BasicTimeZone_3_8 +#define BinarySearchLookupTable BinarySearchLookupTable_3_8 +#define BreakIterator BreakIterator_3_8 +#define BuddhistCalendar BuddhistCalendar_3_8 +#define BuildCompactTrieHorizontalNode BuildCompactTrieHorizontalNode_3_8 +#define BuildCompactTrieNode BuildCompactTrieNode_3_8 +#define BuildCompactTrieVerticalNode BuildCompactTrieVerticalNode_3_8 +#define CFactory CFactory_3_8 +#define Calendar Calendar_3_8 +#define CalendarAstronomer CalendarAstronomer_3_8 +#define CalendarCache CalendarCache_3_8 +#define CalendarData CalendarData_3_8 +#define CalendarService CalendarService_3_8 +#define CanonMarkFilter CanonMarkFilter_3_8 +#define CanonShaping CanonShaping_3_8 +#define CanonicalIterator CanonicalIterator_3_8 +#define CaseMapTransliterator CaseMapTransliterator_3_8 +#define ChainingContextualSubstitutionFormat1Subtable ChainingContextualSubstitutionFormat1Subtable_3_8 +#define ChainingContextualSubstitutionFormat2Subtable ChainingContextualSubstitutionFormat2Subtable_3_8 +#define ChainingContextualSubstitutionFormat3Subtable ChainingContextualSubstitutionFormat3Subtable_3_8 +#define ChainingContextualSubstitutionSubtable ChainingContextualSubstitutionSubtable_3_8 +#define CharSubstitutionFilter CharSubstitutionFilter_3_8 +#define CharacterIterator CharacterIterator_3_8 +#define CharsetDetector CharsetDetector_3_8 +#define CharsetMatch CharsetMatch_3_8 +#define CharsetRecog_2022 CharsetRecog_2022_3_8 +#define CharsetRecog_2022CN CharsetRecog_2022CN_3_8 +#define CharsetRecog_2022JP CharsetRecog_2022JP_3_8 +#define CharsetRecog_2022KR CharsetRecog_2022KR_3_8 +#define CharsetRecog_8859_1 CharsetRecog_8859_1_3_8 +#define CharsetRecog_8859_1_da CharsetRecog_8859_1_da_3_8 +#define CharsetRecog_8859_1_de CharsetRecog_8859_1_de_3_8 +#define CharsetRecog_8859_1_en CharsetRecog_8859_1_en_3_8 +#define CharsetRecog_8859_1_es CharsetRecog_8859_1_es_3_8 +#define CharsetRecog_8859_1_fr CharsetRecog_8859_1_fr_3_8 +#define CharsetRecog_8859_1_it CharsetRecog_8859_1_it_3_8 +#define CharsetRecog_8859_1_nl CharsetRecog_8859_1_nl_3_8 +#define CharsetRecog_8859_1_no CharsetRecog_8859_1_no_3_8 +#define CharsetRecog_8859_1_pt CharsetRecog_8859_1_pt_3_8 +#define CharsetRecog_8859_1_sv CharsetRecog_8859_1_sv_3_8 +#define CharsetRecog_8859_2 CharsetRecog_8859_2_3_8 +#define CharsetRecog_8859_2_cs CharsetRecog_8859_2_cs_3_8 +#define CharsetRecog_8859_2_hu CharsetRecog_8859_2_hu_3_8 +#define CharsetRecog_8859_2_pl CharsetRecog_8859_2_pl_3_8 +#define CharsetRecog_8859_2_ro CharsetRecog_8859_2_ro_3_8 +#define CharsetRecog_8859_5 CharsetRecog_8859_5_3_8 +#define CharsetRecog_8859_5_ru CharsetRecog_8859_5_ru_3_8 +#define CharsetRecog_8859_6 CharsetRecog_8859_6_3_8 +#define CharsetRecog_8859_6_ar CharsetRecog_8859_6_ar_3_8 +#define CharsetRecog_8859_7 CharsetRecog_8859_7_3_8 +#define CharsetRecog_8859_7_el CharsetRecog_8859_7_el_3_8 +#define CharsetRecog_8859_8 CharsetRecog_8859_8_3_8 +#define CharsetRecog_8859_8_I_he CharsetRecog_8859_8_I_he_3_8 +#define CharsetRecog_8859_8_he CharsetRecog_8859_8_he_3_8 +#define CharsetRecog_8859_9 CharsetRecog_8859_9_3_8 +#define CharsetRecog_8859_9_tr CharsetRecog_8859_9_tr_3_8 +#define CharsetRecog_KOI8_R CharsetRecog_KOI8_R_3_8 +#define CharsetRecog_UTF8 CharsetRecog_UTF8_3_8 +#define CharsetRecog_UTF_16_BE CharsetRecog_UTF_16_BE_3_8 +#define CharsetRecog_UTF_16_LE CharsetRecog_UTF_16_LE_3_8 +#define CharsetRecog_UTF_32 CharsetRecog_UTF_32_3_8 +#define CharsetRecog_UTF_32_BE CharsetRecog_UTF_32_BE_3_8 +#define CharsetRecog_UTF_32_LE CharsetRecog_UTF_32_LE_3_8 +#define CharsetRecog_Unicode CharsetRecog_Unicode_3_8 +#define CharsetRecog_big5 CharsetRecog_big5_3_8 +#define CharsetRecog_euc CharsetRecog_euc_3_8 +#define CharsetRecog_euc_jp CharsetRecog_euc_jp_3_8 +#define CharsetRecog_euc_kr CharsetRecog_euc_kr_3_8 +#define CharsetRecog_gb_18030 CharsetRecog_gb_18030_3_8 +#define CharsetRecog_mbcs CharsetRecog_mbcs_3_8 +#define CharsetRecog_sbcs CharsetRecog_sbcs_3_8 +#define CharsetRecog_sjis CharsetRecog_sjis_3_8 +#define CharsetRecog_windows_1251 CharsetRecog_windows_1251_3_8 +#define CharsetRecog_windows_1256 CharsetRecog_windows_1256_3_8 +#define CharsetRecognizer CharsetRecognizer_3_8 +#define ChoiceFormat ChoiceFormat_3_8 +#define ClassDefFormat1Table ClassDefFormat1Table_3_8 +#define ClassDefFormat2Table ClassDefFormat2Table_3_8 +#define ClassDefinitionTable ClassDefinitionTable_3_8 +#define CollationElementIterator CollationElementIterator_3_8 +#define CollationKey CollationKey_3_8 +#define CollationLocaleListEnumeration CollationLocaleListEnumeration_3_8 +#define Collator Collator_3_8 +#define CollatorFactory CollatorFactory_3_8 +#define CompactTrieDictionary CompactTrieDictionary_3_8 +#define CompactTrieEnumeration CompactTrieEnumeration_3_8 +#define CompoundTransliterator CompoundTransliterator_3_8 +#define ContextualGlyphSubstitutionProcessor ContextualGlyphSubstitutionProcessor_3_8 +#define ContextualSubstitutionBase ContextualSubstitutionBase_3_8 +#define ContextualSubstitutionFormat1Subtable ContextualSubstitutionFormat1Subtable_3_8 +#define ContextualSubstitutionFormat2Subtable ContextualSubstitutionFormat2Subtable_3_8 +#define ContextualSubstitutionFormat3Subtable ContextualSubstitutionFormat3Subtable_3_8 +#define ContextualSubstitutionSubtable ContextualSubstitutionSubtable_3_8 +#define CoverageFormat1Table CoverageFormat1Table_3_8 +#define CoverageFormat2Table CoverageFormat2Table_3_8 +#define CoverageTable CoverageTable_3_8 +#define CurrencyAmount CurrencyAmount_3_8 +#define CurrencyFormat CurrencyFormat_3_8 +#define CurrencyUnit CurrencyUnit_3_8 +#define CursiveAttachmentSubtable CursiveAttachmentSubtable_3_8 +#define DTRedundantEnumeration DTRedundantEnumeration_3_8 +#define DTSkeletonEnumeration DTSkeletonEnumeration_3_8 +#define DateFormat DateFormat_3_8 +#define DateFormatSymbols DateFormatSymbols_3_8 +#define DateTimeMatcher DateTimeMatcher_3_8 +#define DateTimePatternGenerator DateTimePatternGenerator_3_8 +#define DateTimeRule DateTimeRule_3_8 +#define DecimalFormat DecimalFormat_3_8 +#define DecimalFormatSymbols DecimalFormatSymbols_3_8 +#define DefaultCalendarFactory DefaultCalendarFactory_3_8 +#define DefaultCharMapper DefaultCharMapper_3_8 +#define DeviceTable DeviceTable_3_8 +#define DictionaryBreakEngine DictionaryBreakEngine_3_8 +#define DigitList DigitList_3_8 +#define DistanceInfo DistanceInfo_3_8 +#define Entry Entry_3_8 +#define EnumToOffset EnumToOffset_3_8 +#define EscapeTransliterator EscapeTransliterator_3_8 +#define EventListener EventListener_3_8 +#define ExtensionSubtable ExtensionSubtable_3_8 +#define FeatureListTable FeatureListTable_3_8 +#define FieldPosition FieldPosition_3_8 +#define FontRuns FontRuns_3_8 +#define Format Format_3_8 +#define Format1AnchorTable Format1AnchorTable_3_8 +#define Format2AnchorTable Format2AnchorTable_3_8 +#define Format3AnchorTable Format3AnchorTable_3_8 +#define FormatParser FormatParser_3_8 +#define Formattable Formattable_3_8 +#define ForwardCharacterIterator ForwardCharacterIterator_3_8 +#define FractionalPartSubstitution FractionalPartSubstitution_3_8 +#define FunctionReplacer FunctionReplacer_3_8 +#define GDEFMarkFilter GDEFMarkFilter_3_8 +#define GXLayoutEngine GXLayoutEngine_3_8 +#define GlyphDefinitionTableHeader GlyphDefinitionTableHeader_3_8 +#define GlyphIterator GlyphIterator_3_8 +#define GlyphLookupTableHeader GlyphLookupTableHeader_3_8 +#define GlyphPositionAdjustments GlyphPositionAdjustments_3_8 +#define GlyphPositioningLookupProcessor GlyphPositioningLookupProcessor_3_8 +#define GlyphPositioningTableHeader GlyphPositioningTableHeader_3_8 +#define GlyphSubstitutionLookupProcessor GlyphSubstitutionLookupProcessor_3_8 +#define GlyphSubstitutionTableHeader GlyphSubstitutionTableHeader_3_8 +#define Grego Grego_3_8 +#define GregorianCalendar GregorianCalendar_3_8 +#define HanOpenTypeLayoutEngine HanOpenTypeLayoutEngine_3_8 +#define HangulOpenTypeLayoutEngine HangulOpenTypeLayoutEngine_3_8 +#define HebrewCalendar HebrewCalendar_3_8 +#define ICUBreakIteratorFactory ICUBreakIteratorFactory_3_8 +#define ICUBreakIteratorService ICUBreakIteratorService_3_8 +#define ICUCollatorFactory ICUCollatorFactory_3_8 +#define ICUCollatorService ICUCollatorService_3_8 +#define ICULanguageBreakFactory ICULanguageBreakFactory_3_8 +#define ICULocaleService ICULocaleService_3_8 +#define ICUNotifier ICUNotifier_3_8 +#define ICUNumberFormatFactory ICUNumberFormatFactory_3_8 +#define ICUNumberFormatService ICUNumberFormatService_3_8 +#define ICUResourceBundleFactory ICUResourceBundleFactory_3_8 +#define ICUService ICUService_3_8 +#define ICUServiceFactory ICUServiceFactory_3_8 +#define ICUServiceKey ICUServiceKey_3_8 +#define ICU_Utility ICU_Utility_3_8 +#define IndianCalendar IndianCalendar_3_8 +#define IndicClassTable IndicClassTable_3_8 +#define IndicOpenTypeLayoutEngine IndicOpenTypeLayoutEngine_3_8 +#define IndicRearrangementProcessor IndicRearrangementProcessor_3_8 +#define IndicReordering IndicReordering_3_8 +#define InitialTimeZoneRule InitialTimeZoneRule_3_8 +#define InputText InputText_3_8 +#define IntegralPartSubstitution IntegralPartSubstitution_3_8 +#define IslamicCalendar IslamicCalendar_3_8 +#define IteratedChar IteratedChar_3_8 +#define JapaneseCalendar JapaneseCalendar_3_8 +#define KernTable KernTable_3_8 +#define KeywordEnumeration KeywordEnumeration_3_8 +#define KhmerClassTable KhmerClassTable_3_8 +#define KhmerOpenTypeLayoutEngine KhmerOpenTypeLayoutEngine_3_8 +#define KhmerReordering KhmerReordering_3_8 +#define LECharMapper LECharMapper_3_8 +#define LEFontInstance LEFontInstance_3_8 +#define LEGlyphFilter LEGlyphFilter_3_8 +#define LEGlyphStorage LEGlyphStorage_3_8 +#define LEInsertionCallback LEInsertionCallback_3_8 +#define LEInsertionList LEInsertionList_3_8 +#define LXUtilities LXUtilities_3_8 +#define LanguageBreakEngine LanguageBreakEngine_3_8 +#define LanguageBreakFactory LanguageBreakFactory_3_8 +#define LayoutEngine LayoutEngine_3_8 +#define LigatureSubstitutionProcessor LigatureSubstitutionProcessor_3_8 +#define LigatureSubstitutionSubtable LigatureSubstitutionSubtable_3_8 +#define LocDataParser LocDataParser_3_8 +#define Locale Locale_3_8 +#define LocaleBased LocaleBased_3_8 +#define LocaleKey LocaleKey_3_8 +#define LocaleKeyFactory LocaleKeyFactory_3_8 +#define LocaleRuns LocaleRuns_3_8 +#define LocaleUtility LocaleUtility_3_8 +#define LocalizationInfo LocalizationInfo_3_8 +#define LookupListTable LookupListTable_3_8 +#define LookupProcessor LookupProcessor_3_8 +#define LookupSubtable LookupSubtable_3_8 +#define LookupTable LookupTable_3_8 +#define LowercaseTransliterator LowercaseTransliterator_3_8 +#define MPreFixups MPreFixups_3_8 +#define MarkArray MarkArray_3_8 +#define MarkToBasePositioningSubtable MarkToBasePositioningSubtable_3_8 +#define MarkToLigaturePositioningSubtable MarkToLigaturePositioningSubtable_3_8 +#define MarkToMarkPositioningSubtable MarkToMarkPositioningSubtable_3_8 +#define Math Math_3_8 +#define Measure Measure_3_8 +#define MeasureFormat MeasureFormat_3_8 +#define MeasureUnit MeasureUnit_3_8 +#define MessageFormat MessageFormat_3_8 +#define MessageFormatAdapter MessageFormatAdapter_3_8 +#define ModulusSubstitution ModulusSubstitution_3_8 +#define MoonRiseSetCoordFunc MoonRiseSetCoordFunc_3_8 +#define MoonTimeAngleFunc MoonTimeAngleFunc_3_8 +#define MorphSubtableHeader MorphSubtableHeader_3_8 +#define MorphTableHeader MorphTableHeader_3_8 +#define MultipleSubstitutionSubtable MultipleSubstitutionSubtable_3_8 +#define MultiplierSubstitution MultiplierSubstitution_3_8 +#define MutableTrieDictionary MutableTrieDictionary_3_8 +#define MutableTrieEnumeration MutableTrieEnumeration_3_8 +#define NFFactory NFFactory_3_8 +#define NFRule NFRule_3_8 +#define NFRuleSet NFRuleSet_3_8 +#define NFSubstitution NFSubstitution_3_8 +#define NGramParser NGramParser_3_8 +#define NameToEnum NameToEnum_3_8 +#define NameUnicodeTransliterator NameUnicodeTransliterator_3_8 +#define NonContextualGlyphSubstitutionProcessor NonContextualGlyphSubstitutionProcessor_3_8 +#define NonContiguousEnumToOffset NonContiguousEnumToOffset_3_8 +#define NormalizationTransliterator NormalizationTransliterator_3_8 +#define Normalizer Normalizer_3_8 +#define NullSubstitution NullSubstitution_3_8 +#define NullTransliterator NullTransliterator_3_8 +#define NumberFormat NumberFormat_3_8 +#define NumberFormatFactory NumberFormatFactory_3_8 +#define NumeratorSubstitution NumeratorSubstitution_3_8 +#define OlsonTimeZone OlsonTimeZone_3_8 +#define OpenTypeLayoutEngine OpenTypeLayoutEngine_3_8 +#define OpenTypeUtilities OpenTypeUtilities_3_8 +#define PairPositioningFormat1Subtable PairPositioningFormat1Subtable_3_8 +#define PairPositioningFormat2Subtable PairPositioningFormat2Subtable_3_8 +#define PairPositioningSubtable PairPositioningSubtable_3_8 +#define ParagraphLayout ParagraphLayout_3_8 +#define ParseData ParseData_3_8 +#define ParsePosition ParsePosition_3_8 +#define PatternMap PatternMap_3_8 +#define PatternMapIterator PatternMapIterator_3_8 +#define PersianCalendar PersianCalendar_3_8 +#define PropertyAliases PropertyAliases_3_8 +#define PtnElem PtnElem_3_8 +#define PtnSkeleton PtnSkeleton_3_8 +#define Quantifier Quantifier_3_8 +#define RBBIDataWrapper RBBIDataWrapper_3_8 +#define RBBINode RBBINode_3_8 +#define RBBIRuleBuilder RBBIRuleBuilder_3_8 +#define RBBIRuleScanner RBBIRuleScanner_3_8 +#define RBBISetBuilder RBBISetBuilder_3_8 +#define RBBIStateDescriptor RBBIStateDescriptor_3_8 +#define RBBISymbolTable RBBISymbolTable_3_8 +#define RBBISymbolTableEntry RBBISymbolTableEntry_3_8 +#define RBBITableBuilder RBBITableBuilder_3_8 +#define RangeDescriptor RangeDescriptor_3_8 +#define RegexCompile RegexCompile_3_8 +#define RegexMatcher RegexMatcher_3_8 +#define RegexPattern RegexPattern_3_8 +#define RegexStaticSets RegexStaticSets_3_8 +#define RelativeDateFormat RelativeDateFormat_3_8 +#define RemoveTransliterator RemoveTransliterator_3_8 +#define Replaceable Replaceable_3_8 +#define ReplaceableGlue ReplaceableGlue_3_8 +#define ResourceBundle ResourceBundle_3_8 +#define RiseSetCoordFunc RiseSetCoordFunc_3_8 +#define RuleBasedBreakIterator RuleBasedBreakIterator_3_8 +#define RuleBasedCollator RuleBasedCollator_3_8 +#define RuleBasedNumberFormat RuleBasedNumberFormat_3_8 +#define RuleBasedTimeZone RuleBasedTimeZone_3_8 +#define RuleBasedTransliterator RuleBasedTransliterator_3_8 +#define RuleCharacterIterator RuleCharacterIterator_3_8 +#define RuleHalf RuleHalf_3_8 +#define RunArray RunArray_3_8 +#define SameValueSubstitution SameValueSubstitution_3_8 +#define ScriptListTable ScriptListTable_3_8 +#define ScriptRunIterator ScriptRunIterator_3_8 +#define ScriptTable ScriptTable_3_8 +#define SearchIterator SearchIterator_3_8 +#define SegmentArrayProcessor SegmentArrayProcessor_3_8 +#define SegmentSingleProcessor SegmentSingleProcessor_3_8 +#define ServiceEnumeration ServiceEnumeration_3_8 +#define ServiceListener ServiceListener_3_8 +#define SimpleArrayProcessor SimpleArrayProcessor_3_8 +#define SimpleDateFormat SimpleDateFormat_3_8 +#define SimpleFactory SimpleFactory_3_8 +#define SimpleLocaleKeyFactory SimpleLocaleKeyFactory_3_8 +#define SimpleNumberFormatFactory SimpleNumberFormatFactory_3_8 +#define SimpleTimeZone SimpleTimeZone_3_8 +#define SinglePositioningFormat1Subtable SinglePositioningFormat1Subtable_3_8 +#define SinglePositioningFormat2Subtable SinglePositioningFormat2Subtable_3_8 +#define SinglePositioningSubtable SinglePositioningSubtable_3_8 +#define SingleSubstitutionFormat1Subtable SingleSubstitutionFormat1Subtable_3_8 +#define SingleSubstitutionFormat2Subtable SingleSubstitutionFormat2Subtable_3_8 +#define SingleSubstitutionSubtable SingleSubstitutionSubtable_3_8 +#define SingleTableProcessor SingleTableProcessor_3_8 +#define Spec Spec_3_8 +#define StateTableProcessor StateTableProcessor_3_8 +#define StringCharacterIterator StringCharacterIterator_3_8 +#define StringEnumeration StringEnumeration_3_8 +#define StringLocalizationInfo StringLocalizationInfo_3_8 +#define StringMatcher StringMatcher_3_8 +#define StringPair StringPair_3_8 +#define StringReplacer StringReplacer_3_8 +#define StringSearch StringSearch_3_8 +#define StyleRuns StyleRuns_3_8 +#define SubstitutionLookup SubstitutionLookup_3_8 +#define SubtableProcessor SubtableProcessor_3_8 +#define SunTimeAngleFunc SunTimeAngleFunc_3_8 +#define SymbolTable SymbolTable_3_8 +#define TZEnumeration TZEnumeration_3_8 +#define TaiwanCalendar TaiwanCalendar_3_8 +#define TernaryNode TernaryNode_3_8 +#define ThaiBreakEngine ThaiBreakEngine_3_8 +#define ThaiLayoutEngine ThaiLayoutEngine_3_8 +#define ThaiShaping ThaiShaping_3_8 +#define TibetanClassTable TibetanClassTable_3_8 +#define TibetanOpenTypeLayoutEngine TibetanOpenTypeLayoutEngine_3_8 +#define TibetanReordering TibetanReordering_3_8 +#define TimeArrayTimeZoneRule TimeArrayTimeZoneRule_3_8 +#define TimeZone TimeZone_3_8 +#define TimeZoneKeysEnumeration TimeZoneKeysEnumeration_3_8 +#define TimeZoneRule TimeZoneRule_3_8 +#define TimeZoneTransition TimeZoneTransition_3_8 +#define TitlecaseTransliterator TitlecaseTransliterator_3_8 +#define TransliterationRule TransliterationRule_3_8 +#define TransliterationRuleData TransliterationRuleData_3_8 +#define TransliterationRuleSet TransliterationRuleSet_3_8 +#define Transliterator Transliterator_3_8 +#define TransliteratorAlias TransliteratorAlias_3_8 +#define TransliteratorIDParser TransliteratorIDParser_3_8 +#define TransliteratorParser TransliteratorParser_3_8 +#define TransliteratorRegistry TransliteratorRegistry_3_8 +#define TrieWordDictionary TrieWordDictionary_3_8 +#define TrimmedArrayProcessor TrimmedArrayProcessor_3_8 +#define UCharCharacterIterator UCharCharacterIterator_3_8 +#define ULocRuns ULocRuns_3_8 +#define UMemory UMemory_3_8 +#define UObject UObject_3_8 +#define URegularExpression URegularExpression_3_8 +#define UStack UStack_3_8 +#define UStringEnumeration UStringEnumeration_3_8 +#define UVector UVector_3_8 +#define UVector32 UVector32_3_8 +#define UnescapeTransliterator UnescapeTransliterator_3_8 +#define UnhandledEngine UnhandledEngine_3_8 +#define UnicodeArabicOpenTypeLayoutEngine UnicodeArabicOpenTypeLayoutEngine_3_8 +#define UnicodeFilter UnicodeFilter_3_8 +#define UnicodeFunctor UnicodeFunctor_3_8 +#define UnicodeMatcher UnicodeMatcher_3_8 +#define UnicodeNameTransliterator UnicodeNameTransliterator_3_8 +#define UnicodeReplacer UnicodeReplacer_3_8 +#define UnicodeSet UnicodeSet_3_8 +#define UnicodeSetIterator UnicodeSetIterator_3_8 +#define UnicodeSetStringSpan UnicodeSetStringSpan_3_8 +#define UnicodeString UnicodeString_3_8 +#define UppercaseTransliterator UppercaseTransliterator_3_8 +#define VTZReader VTZReader_3_8 +#define VTZWriter VTZWriter_3_8 +#define VTimeZone VTimeZone_3_8 +#define ValueRecord ValueRecord_3_8 +#define ValueRuns ValueRuns_3_8 +#define locale_set_default_internal locale_set_default_internal_3_8 +#define util64_fromDouble util64_fromDouble_3_8 +#define util64_pow util64_pow_3_8 +#define util64_tou util64_tou_3_8 + +#endif +#endif + +#endif + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/urep.h b/Build/source/libs/icu/icu-xetex/common/unicode/urep.h new file mode 100644 index 00000000000..57b547c8783 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/urep.h @@ -0,0 +1,155 @@ +/* +****************************************************************************** +* Copyright (C) 1997-2005, International Business Machines +* Corporation and others. All Rights Reserved. +****************************************************************************** +* Date Name Description +* 06/23/00 aliu Creation. +****************************************************************************** +*/ + +#ifndef __UREP_H +#define __UREP_H + +#include "unicode/utypes.h" + +U_CDECL_BEGIN + +/******************************************************************** + * General Notes + ******************************************************************** + * TODO + * Add usage scenario + * Add test code + * Talk about pinning + * Talk about "can truncate result if out of memory" + */ + +/******************************************************************** + * Data Structures + ********************************************************************/ +/** + * \file + * \brief C API: Callbacks for UReplacebale + */ +/** + * An opaque replaceable text object. This will be manipulated only + * through the caller-supplied UReplaceableFunctor struct. Related + * to the C++ class Replaceable. + * This is currently only used in the Transliterator C API, see utrans.h . + * @stable ICU 2.0 + */ +typedef void* UReplaceable; + +/** + * A set of function pointers that transliterators use to manipulate a + * UReplaceable. The caller should supply the required functions to + * manipulate their text appropriately. Related to the C++ class + * Replaceable. + * @stable ICU 2.0 + */ +typedef struct UReplaceableCallbacks { + + /** + * Function pointer that returns the number of UChar code units in + * this text. + * + * @param rep A pointer to "this" UReplaceable object. + * @return The length of the text. + * @stable ICU 2.0 + */ + int32_t (*length)(const UReplaceable* rep); + + /** + * Function pointer that returns a UChar code units at the given + * offset into this text; 0 <= offset < n, where n is the value + * returned by (*length)(rep). See unistr.h for a description of + * charAt() vs. char32At(). + * + * @param rep A pointer to "this" UReplaceable object. + * @param offset The index at which to fetch the UChar (code unit). + * @return The UChar (code unit) at offset, or U+FFFF if the offset is out of bounds. + * @stable ICU 2.0 + */ + UChar (*charAt)(const UReplaceable* rep, + int32_t offset); + + /** + * Function pointer that returns a UChar32 code point at the given + * offset into this text. See unistr.h for a description of + * charAt() vs. char32At(). + * + * @param rep A pointer to "this" UReplaceable object. + * @param offset The index at which to fetch the UChar32 (code point). + * @return The UChar32 (code point) at offset, or U+FFFF if the offset is out of bounds. + * @stable ICU 2.0 + */ + UChar32 (*char32At)(const UReplaceable* rep, + int32_t offset); + + /** + * Function pointer that replaces text between start and limit in + * this text with the given text. Attributes (out of band info) + * should be retained. + * + * @param rep A pointer to "this" UReplaceable object. + * @param start the starting index of the text to be replaced, + * inclusive. + * @param limit the ending index of the text to be replaced, + * exclusive. + * @param text the new text to replace the UChars from + * start..limit-1. + * @param textLength the number of UChars at text, or -1 if text + * is null-terminated. + * @stable ICU 2.0 + */ + void (*replace)(UReplaceable* rep, + int32_t start, + int32_t limit, + const UChar* text, + int32_t textLength); + + /** + * Function pointer that copies the characters in the range + * [start, limit) into the array dst. + * + * @param rep A pointer to "this" UReplaceable object. + * @param start offset of first character which will be copied + * into the array + * @param limit offset immediately following the last character to + * be copied + * @param dst array in which to copy characters. The length of + * dst must be at least (limit - start). + * @stable ICU 2.1 + */ + void (*extract)(UReplaceable* rep, + int32_t start, + int32_t limit, + UChar* dst); + + /** + * Function pointer that copies text between start and limit in + * this text to another index in the text. Attributes (out of + * band info) should be retained. After this call, there will be + * (at least) two copies of the characters originally located at + * start..limit-1. + * + * @param rep A pointer to "this" UReplaceable object. + * @param start the starting index of the text to be copied, + * inclusive. + * @param limit the ending index of the text to be copied, + * exclusive. + * @param dest the index at which the copy of the UChars should be + * inserted. + * @stable ICU 2.0 + */ + void (*copy)(UReplaceable* rep, + int32_t start, + int32_t limit, + int32_t dest); + +} UReplaceableCallbacks; + +U_CDECL_END + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/ures.h b/Build/source/libs/icu/icu-xetex/common/unicode/ures.h new file mode 100644 index 00000000000..9cc2e89cf68 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/ures.h @@ -0,0 +1,871 @@ +/* +********************************************************************** +* Copyright (C) 1997-2007, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* +* File URES.H (formerly CRESBUND.H) +* +* Modification History: +* +* Date Name Description +* 04/01/97 aliu Creation. +* 02/22/99 damiba overhaul. +* 04/04/99 helena Fixed internal header inclusion. +* 04/15/99 Madhu Updated Javadoc +* 06/14/99 stephen Removed functions taking a filename suffix. +* 07/20/99 stephen Language-independent ypedef to void* +* 11/09/99 weiv Added ures_getLocale() +* 06/24/02 weiv Added support for resource sharing +****************************************************************************** +*/ + +#ifndef URES_H +#define URES_H + +#include "unicode/utypes.h" +#include "unicode/uloc.h" + +/** + * \file + * \brief C API: Resource Bundle + * + *

C API: Resource Bundle

+ * + * C API representing a collection of resource information pertaining to a given + * locale. A resource bundle provides a way of accessing locale- specific information in + * a data file. You create a resource bundle that manages the resources for a given + * locale and then ask it for individual resources. + *

+ * Resource bundles in ICU4C are currently defined using text files which conform to the following + * BNF definition. + * More on resource bundle concepts and syntax can be found in the + * Users Guide. + *

+ */ + +/** + * UResourceBundle is an opaque type for handles for resource bundles in C APIs. + * @stable ICU 2.0 + */ +struct UResourceBundle; + +/** + * @stable ICU 2.0 + */ +typedef struct UResourceBundle UResourceBundle; + +/** + * Numeric constants for types of resource items. + * @see ures_getType + * @stable ICU 2.0 + */ +typedef enum { + /** Resource type constant for "no resource". @stable ICU 2.6 */ + URES_NONE=-1, + + /** Resource type constant for 16-bit Unicode strings. @stable ICU 2.6 */ + URES_STRING=0, + + /** Resource type constant for binary data. @stable ICU 2.6 */ + URES_BINARY=1, + + /** Resource type constant for tables of key-value pairs. @stable ICU 2.6 */ + URES_TABLE=2, + + /** + * Resource type constant for aliases; + * internally stores a string which identifies the actual resource + * storing the data (can be in a different resource bundle). + * Resolved internally before delivering the actual resource through the API. + * @stable ICU 2.6 + */ + URES_ALIAS=3, + +#ifndef U_HIDE_INTERNAL_API + + /** + * Internal use only. + * Alternative resource type constant for tables of key-value pairs. + * Never returned by ures_getType(). + * @internal + */ + URES_TABLE32=4, + +#endif /* U_HIDE_INTERNAL_API */ + + /** + * Resource type constant for a single 28-bit integer, interpreted as + * signed or unsigned by the ures_getInt() or ures_getUInt() function. + * @see ures_getInt + * @see ures_getUInt + * @stable ICU 2.6 + */ + URES_INT=7, + + /** Resource type constant for arrays of resources. @stable ICU 2.6 */ + URES_ARRAY=8, + + /** + * Resource type constant for vectors of 32-bit integers. + * @see ures_getIntVector + * @stable ICU 2.6 + */ + URES_INT_VECTOR = 14, +#ifndef U_HIDE_DEPRECATED_API + /** @deprecated ICU 2.6 Use the URES_ constant instead. */ + RES_NONE=URES_NONE, + /** @deprecated ICU 2.6 Use the URES_ constant instead. */ + RES_STRING=URES_STRING, + /** @deprecated ICU 2.6 Use the URES_ constant instead. */ + RES_BINARY=URES_BINARY, + /** @deprecated ICU 2.6 Use the URES_ constant instead. */ + RES_TABLE=URES_TABLE, + /** @deprecated ICU 2.6 Use the URES_ constant instead. */ + RES_ALIAS=URES_ALIAS, + /** @deprecated ICU 2.6 Use the URES_ constant instead. */ + RES_INT=URES_INT, + /** @deprecated ICU 2.6 Use the URES_ constant instead. */ + RES_ARRAY=URES_ARRAY, + /** @deprecated ICU 2.6 Use the URES_ constant instead. */ + RES_INT_VECTOR=URES_INT_VECTOR, + /** @deprecated ICU 2.6 Not used. */ + RES_RESERVED=15, +#endif /* U_HIDE_DEPRECATED_API */ + + URES_LIMIT = 16 +} UResType; + +/* + * Functions to create and destroy resource bundles. + */ + +/** + * Opens a UResourceBundle, from which users can extract strings by using + * their corresponding keys. + * Note that the caller is responsible of calling ures_close on each succesfully + * opened resource bundle. + * @param packageName The packageName and locale together point to an ICU udata object, + * as defined by udata_open( packageName, "res", locale, err) + * or equivalent. Typically, packageName will refer to a (.dat) file, or to + * a package registered with udata_setAppData(). Using a full file or directory + * pathname for packageName is deprecated. If NULL, ICU data will be used. + * @param locale specifies the locale for which we want to open the resource + * if NULL, the default locale will be used. If strlen(locale) == 0 + * root locale will be used. + * + * @param status fills in the outgoing error code. + * The UErrorCode err parameter is used to return status information to the user. To + * check whether the construction succeeded or not, you should check the value of + * U_SUCCESS(err). If you wish more detailed information, you can check for + * informational status results which still indicate success. U_USING_FALLBACK_WARNING + * indicates that a fall back locale was used. For example, 'de_CH' was requested, + * but nothing was found there, so 'de' was used. U_USING_DEFAULT_WARNING indicates that + * the default locale data or root locale data was used; neither the requested locale + * nor any of its fall back locales could be found. Please see the users guide for more + * information on this topic. + * @return a newly allocated resource bundle. + * @see ures_close + * @stable ICU 2.0 + */ +U_STABLE UResourceBundle* U_EXPORT2 +ures_open(const char* packageName, + const char* locale, + UErrorCode* status); + + +/** This function does not care what kind of localeID is passed in. It simply opens a bundle with + * that name. Fallback mechanism is disabled for the new bundle. If the requested bundle contains + * an %%ALIAS directive, the results are undefined. + * @param packageName The packageName and locale together point to an ICU udata object, + * as defined by udata_open( packageName, "res", locale, err) + * or equivalent. Typically, packageName will refer to a (.dat) file, or to + * a package registered with udata_setAppData(). Using a full file or directory + * pathname for packageName is deprecated. If NULL, ICU data will be used. + * @param locale specifies the locale for which we want to open the resource + * if NULL, the default locale will be used. If strlen(locale) == 0 + * root locale will be used. + * + * @param status fills in the outgoing error code. Either U_ZERO_ERROR or U_MISSING_RESOURCE_ERROR + * @return a newly allocated resource bundle or NULL if it doesn't exist. + * @see ures_close + * @stable ICU 2.0 + */ +U_STABLE UResourceBundle* U_EXPORT2 +ures_openDirect(const char* packageName, + const char* locale, + UErrorCode* status); + +/** + * Same as ures_open() but takes a const UChar *path. + * This path will be converted to char * using the default converter, + * then ures_open() is called. + * + * @param packageName The packageName and locale together point to an ICU udata object, + * as defined by udata_open( packageName, "res", locale, err) + * or equivalent. Typically, packageName will refer to a (.dat) file, or to + * a package registered with udata_setAppData(). Using a full file or directory + * pathname for packageName is deprecated. If NULL, ICU data will be used. + * @param locale specifies the locale for which we want to open the resource + * if NULL, the default locale will be used. If strlen(locale) == 0 + * root locale will be used. + * @param status fills in the outgoing error code. + * @return a newly allocated resource bundle. + * @see ures_open + * @stable ICU 2.0 + */ +U_STABLE UResourceBundle* U_EXPORT2 +ures_openU(const UChar* packageName, + const char* locale, + UErrorCode* status); + +/** + * Returns the number of strings/arrays in resource bundles. + * Better to use ures_getSize, as this function will be deprecated. + * + *@param resourceBundle resource bundle containing the desired strings + *@param resourceKey key tagging the resource + *@param err fills in the outgoing error code + * could be U_MISSING_RESOURCE_ERROR if the key is not found + * could be a non-failing error + * e.g.: U_USING_FALLBACK_WARNING,U_USING_FALLBACK_WARNING + *@return: for Arrays: returns the number of resources in the array + * Tables: returns the number of resources in the table + * single string: returns 1 + *@see ures_getSize + * @deprecated ICU 2.8 User ures_getSize instead + */ +U_DEPRECATED int32_t U_EXPORT2 +ures_countArrayItems(const UResourceBundle* resourceBundle, + const char* resourceKey, + UErrorCode* err); +/** + * Close a resource bundle, all pointers returned from the various ures_getXXX calls + * on this particular bundle should be considered invalid henceforth. + * + * @param resourceBundle a pointer to a resourceBundle struct. Can be NULL. + * @see ures_open + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ures_close(UResourceBundle* resourceBundle); + +/** + * Return the version number associated with this ResourceBundle as a string. Please + * use ures_getVersion as this function is going to be deprecated. + * + * @param resourceBundle The resource bundle for which the version is checked. + * @return A version number string as specified in the resource bundle or its parent. + * The caller does not own this string. + * @see ures_getVersion + * @deprecated ICU 2.8 Use ures_getVersion instead. + */ +U_DEPRECATED const char* U_EXPORT2 +ures_getVersionNumber(const UResourceBundle* resourceBundle); + +/** + * Return the version number associated with this ResourceBundle as an + * UVersionInfo array. + * + * @param resB The resource bundle for which the version is checked. + * @param versionInfo A UVersionInfo array that is filled with the version number + * as specified in the resource bundle or its parent. + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ures_getVersion(const UResourceBundle* resB, + UVersionInfo versionInfo); + +/** + * Return the name of the Locale associated with this ResourceBundle. This API allows + * you to query for the real locale of the resource. For example, if you requested + * "en_US_CALIFORNIA" and only "en_US" bundle exists, "en_US" will be returned. + * For subresources, the locale where this resource comes from will be returned. + * If fallback has occured, getLocale will reflect this. + * + * @param resourceBundle resource bundle in question + * @param status just for catching illegal arguments + * @return A Locale name + * @deprecated ICU 2.8 Use ures_getLocaleByType instead. + */ +U_DEPRECATED const char* U_EXPORT2 +ures_getLocale(const UResourceBundle* resourceBundle, + UErrorCode* status); + + +/** + * Return the name of the Locale associated with this ResourceBundle. + * You can choose between requested, valid and real locale. + * + * @param resourceBundle resource bundle in question + * @param type You can choose between requested, valid and actual + * locale. For description see the definition of + * ULocDataLocaleType in uloc.h + * @param status just for catching illegal arguments + * @return A Locale name + * @stable ICU 2.8 + */ +U_STABLE const char* U_EXPORT2 +ures_getLocaleByType(const UResourceBundle* resourceBundle, + ULocDataLocaleType type, + UErrorCode* status); + + +/** + * Same as ures_open() but uses the fill-in parameter instead of allocating + * a bundle, if r!=NULL. + * TODO need to revisit usefulness of this function + * and usage model for fillIn parameters without knowing sizeof(UResourceBundle) + * @param r The resourcebundle to open + * @param packageName The packageName and locale together point to an ICU udata object, + * as defined by udata_open( packageName, "res", locale, err) + * or equivalent. Typically, packageName will refer to a (.dat) file, or to + * a package registered with udata_setAppData(). Using a full file or directory + * pathname for packageName is deprecated. If NULL, ICU data will be used. + * @param localeID specifies the locale for which we want to open the resource + * @param status The error code + * @return a newly allocated resource bundle or NULL if it doesn't exist. + * @internal + */ +U_INTERNAL void U_EXPORT2 +ures_openFillIn(UResourceBundle *r, + const char* packageName, + const char* localeID, + UErrorCode* status); + +/** + * Returns a string from a string resource type + * + * @param resourceBundle a string resource + * @param len fills in the length of resulting string + * @param status fills in the outgoing error code + * could be U_MISSING_RESOURCE_ERROR if the key is not found + * Always check the value of status. Don't count on returning NULL. + * could be a non-failing error + * e.g.: U_USING_FALLBACK_WARNING,U_USING_DEFAULT_WARNING + * @return a pointer to a zero-terminated UChar array which lives in a memory mapped/DLL file. + * @see ures_getBinary + * @see ures_getIntVector + * @see ures_getInt + * @see ures_getUInt + * @stable ICU 2.0 + */ +U_STABLE const UChar* U_EXPORT2 +ures_getString(const UResourceBundle* resourceBundle, + int32_t* len, + UErrorCode* status); + +/** + * Returns a UTF-8 string from a string resource. + * The UTF-8 string may be returnable directly as a pointer, or + * it may need to be copied, or transformed from UTF-16 using u_strToUTF8() + * or equivalent. + * + * If forceCopy==TRUE, then the string is always written to the dest buffer + * and dest is returned. + * + * If forceCopy==FALSE, then the string is returned as a pointer if possible, + * without needing a dest buffer (it can be NULL). If the string needs to be + * copied or transformed, then it may be placed into dest at an arbitrary offset. + * + * If the string is to be written to dest, then U_BUFFER_OVERFLOW_ERROR and + * U_STRING_NOT_TERMINATED_WARNING are set if appropriate, as usual. + * + * If the string is transformed from UTF-16, then a conversion error may occur + * if an unpaired surrogate is encountered. If the function is successful, then + * the output UTF-8 string is always well-formed. + * + * @param resB Resource bundle. + * @param dest Destination buffer. Can be NULL only if capacity=*length==0. + * @param length Input: Capacity of destination buffer. + * Output: Actual length of the UTF-8 string, not counting the + * terminating NUL, even in case of U_BUFFER_OVERFLOW_ERROR. + * Can be NULL, meaning capacity=0 and the string length is not + * returned to the caller. + * @param forceCopy If TRUE, then the output string will always be written to + * dest, with U_BUFFER_OVERFLOW_ERROR and + * U_STRING_NOT_TERMINATED_WARNING set if appropriate. + * If FALSE, then the dest buffer may or may not contain a + * copy of the string. dest may or may not be modified. + * If a copy needs to be written, then the UErrorCode parameter + * indicates overflow etc. as usual. + * @param status Pointer to a standard ICU error code. Its input value must + * pass the U_SUCCESS() test, or else the function returns + * immediately. Check for U_FAILURE() on output or use with + * function chaining. (See User Guide for details.) + * @return The pointer to the UTF-8 string. It may be dest, or at some offset + * from dest (only if !forceCopy), or in unrelated memory. + * Always NUL-terminated unless the string was written to dest and + * length==capacity (in which case U_STRING_NOT_TERMINATED_WARNING is set). + * + * @see ures_getString + * @see u_strToUTF8 + * @stable ICU 3.6 + */ +U_STABLE const char * U_EXPORT2 +ures_getUTF8String(const UResourceBundle *resB, + char *dest, int32_t *length, + UBool forceCopy, + UErrorCode *status); + +/** + * Returns a binary data from a binary resource. + * + * @param resourceBundle a string resource + * @param len fills in the length of resulting byte chunk + * @param status fills in the outgoing error code + * could be U_MISSING_RESOURCE_ERROR if the key is not found + * Always check the value of status. Don't count on returning NULL. + * could be a non-failing error + * e.g.: U_USING_FALLBACK_WARNING,U_USING_DEFAULT_WARNING + * @return a pointer to a chuck of unsigned bytes which live in a memory mapped/DLL file. + * @see ures_getString + * @see ures_getIntVector + * @see ures_getInt + * @see ures_getUInt + * @stable ICU 2.0 + */ +U_STABLE const uint8_t* U_EXPORT2 +ures_getBinary(const UResourceBundle* resourceBundle, + int32_t* len, + UErrorCode* status); + +/** + * Returns a 32 bit integer array from a resource. + * + * @param resourceBundle an int vector resource + * @param len fills in the length of resulting byte chunk + * @param status fills in the outgoing error code + * could be U_MISSING_RESOURCE_ERROR if the key is not found + * Always check the value of status. Don't count on returning NULL. + * could be a non-failing error + * e.g.: U_USING_FALLBACK_WARNING,U_USING_DEFAULT_WARNING + * @return a pointer to a chunk of unsigned bytes which live in a memory mapped/DLL file. + * @see ures_getBinary + * @see ures_getString + * @see ures_getInt + * @see ures_getUInt + * @stable ICU 2.0 + */ +U_STABLE const int32_t* U_EXPORT2 +ures_getIntVector(const UResourceBundle* resourceBundle, + int32_t* len, + UErrorCode* status); + +/** + * Returns an unsigned integer from a resource. + * This integer is originally 28 bits. + * + * @param resourceBundle a string resource + * @param status fills in the outgoing error code + * could be U_MISSING_RESOURCE_ERROR if the key is not found + * could be a non-failing error + * e.g.: U_USING_FALLBACK_WARNING,U_USING_DEFAULT_WARNING + * @return an integer value + * @see ures_getInt + * @see ures_getIntVector + * @see ures_getBinary + * @see ures_getString + * @stable ICU 2.0 + */ +U_STABLE uint32_t U_EXPORT2 +ures_getUInt(const UResourceBundle* resourceBundle, + UErrorCode *status); + +/** + * Returns a signed integer from a resource. + * This integer is originally 28 bit and the sign gets propagated. + * + * @param resourceBundle a string resource + * @param status fills in the outgoing error code + * could be U_MISSING_RESOURCE_ERROR if the key is not found + * could be a non-failing error + * e.g.: U_USING_FALLBACK_WARNING,U_USING_DEFAULT_WARNING + * @return an integer value + * @see ures_getUInt + * @see ures_getIntVector + * @see ures_getBinary + * @see ures_getString + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +ures_getInt(const UResourceBundle* resourceBundle, + UErrorCode *status); + +/** + * Returns the size of a resource. Size for scalar types is always 1, + * and for vector/table types is the number of child resources. + * @warning Integer array is treated as a scalar type. There are no + * APIs to access individual members of an integer array. It + * is always returned as a whole. + * @param resourceBundle a resource + * @return number of resources in a given resource. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +ures_getSize(const UResourceBundle *resourceBundle); + +/** + * Returns the type of a resource. Available types are defined in enum UResType + * + * @param resourceBundle a resource + * @return type of the given resource. + * @see UResType + * @stable ICU 2.0 + */ +U_STABLE UResType U_EXPORT2 +ures_getType(const UResourceBundle *resourceBundle); + +/** + * Returns the key associated with a given resource. Not all the resources have a key - only + * those that are members of a table. + * + * @param resourceBundle a resource + * @return a key associated to this resource, or NULL if it doesn't have a key + * @stable ICU 2.0 + */ +U_STABLE const char * U_EXPORT2 +ures_getKey(const UResourceBundle *resourceBundle); + +/* ITERATION API + This API provides means for iterating through a resource +*/ + +/** + * Resets the internal context of a resource so that iteration starts from the first element. + * + * @param resourceBundle a resource + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +ures_resetIterator(UResourceBundle *resourceBundle); + +/** + * Checks whether the given resource has another element to iterate over. + * + * @param resourceBundle a resource + * @return TRUE if there are more elements, FALSE if there is no more elements + * @stable ICU 2.0 + */ +U_STABLE UBool U_EXPORT2 +ures_hasNext(const UResourceBundle *resourceBundle); + +/** + * Returns the next resource in a given resource or NULL if there are no more resources + * to iterate over. Features a fill-in parameter. + * + * @param resourceBundle a resource + * @param fillIn if NULL a new UResourceBundle struct is allocated and must be closed by the caller. + * Alternatively, you can supply a struct to be filled by this function. + * @param status fills in the outgoing error code. You may still get a non NULL result even if an + * error occured. Check status instead. + * @return a pointer to a UResourceBundle struct. If fill in param was NULL, caller must close it + * @stable ICU 2.0 + */ +U_STABLE UResourceBundle* U_EXPORT2 +ures_getNextResource(UResourceBundle *resourceBundle, + UResourceBundle *fillIn, + UErrorCode *status); + +/** + * Returns the next string in a given resource or NULL if there are no more resources + * to iterate over. + * + * @param resourceBundle a resource + * @param len fill in length of the string + * @param key fill in for key associated with this string. NULL if no key + * @param status fills in the outgoing error code. If an error occured, we may return NULL, but don't + * count on it. Check status instead! + * @return a pointer to a zero-terminated UChar array which lives in a memory mapped/DLL file. + * @stable ICU 2.0 + */ +U_STABLE const UChar* U_EXPORT2 +ures_getNextString(UResourceBundle *resourceBundle, + int32_t* len, + const char ** key, + UErrorCode *status); + +/** + * Returns the resource in a given resource at the specified index. Features a fill-in parameter. + * + * @param resourceBundle the resource bundle from which to get a sub-resource + * @param indexR an index to the wanted resource. + * @param fillIn if NULL a new UResourceBundle struct is allocated and must be closed by the caller. + * Alternatively, you can supply a struct to be filled by this function. + * @param status fills in the outgoing error code. Don't count on NULL being returned if an error has + * occured. Check status instead. + * @return a pointer to a UResourceBundle struct. If fill in param was NULL, caller must close it + * @stable ICU 2.0 + */ +U_STABLE UResourceBundle* U_EXPORT2 +ures_getByIndex(const UResourceBundle *resourceBundle, + int32_t indexR, + UResourceBundle *fillIn, + UErrorCode *status); + +/** + * Returns the string in a given resource at the specified index. + * + * @param resourceBundle a resource + * @param indexS an index to the wanted string. + * @param len fill in length of the string + * @param status fills in the outgoing error code. If an error occured, we may return NULL, but don't + * count on it. Check status instead! + * @return a pointer to a zero-terminated UChar array which lives in a memory mapped/DLL file. + * @stable ICU 2.0 + */ +U_STABLE const UChar* U_EXPORT2 +ures_getStringByIndex(const UResourceBundle *resourceBundle, + int32_t indexS, + int32_t* len, + UErrorCode *status); + +/** + * Returns a UTF-8 string from a resource at the specified index. + * The UTF-8 string may be returnable directly as a pointer, or + * it may need to be copied, or transformed from UTF-16 using u_strToUTF8() + * or equivalent. + * + * If forceCopy==TRUE, then the string is always written to the dest buffer + * and dest is returned. + * + * If forceCopy==FALSE, then the string is returned as a pointer if possible, + * without needing a dest buffer (it can be NULL). If the string needs to be + * copied or transformed, then it may be placed into dest at an arbitrary offset. + * + * If the string is to be written to dest, then U_BUFFER_OVERFLOW_ERROR and + * U_STRING_NOT_TERMINATED_WARNING are set if appropriate, as usual. + * + * If the string is transformed from UTF-16, then a conversion error may occur + * if an unpaired surrogate is encountered. If the function is successful, then + * the output UTF-8 string is always well-formed. + * + * @param resB Resource bundle. + * @param index An index to the wanted string. + * @param dest Destination buffer. Can be NULL only if capacity=*length==0. + * @param pLength Input: Capacity of destination buffer. + * Output: Actual length of the UTF-8 string, not counting the + * terminating NUL, even in case of U_BUFFER_OVERFLOW_ERROR. + * Can be NULL, meaning capacity=0 and the string length is not + * returned to the caller. + * @param forceCopy If TRUE, then the output string will always be written to + * dest, with U_BUFFER_OVERFLOW_ERROR and + * U_STRING_NOT_TERMINATED_WARNING set if appropriate. + * If FALSE, then the dest buffer may or may not contain a + * copy of the string. dest may or may not be modified. + * If a copy needs to be written, then the UErrorCode parameter + * indicates overflow etc. as usual. + * @param status Pointer to a standard ICU error code. Its input value must + * pass the U_SUCCESS() test, or else the function returns + * immediately. Check for U_FAILURE() on output or use with + * function chaining. (See User Guide for details.) + * @return The pointer to the UTF-8 string. It may be dest, or at some offset + * from dest (only if !forceCopy), or in unrelated memory. + * Always NUL-terminated unless the string was written to dest and + * length==capacity (in which case U_STRING_NOT_TERMINATED_WARNING is set). + * + * @see ures_getStringByIndex + * @see u_strToUTF8 + * @stable ICU 3.6 + */ +U_STABLE const char * U_EXPORT2 +ures_getUTF8StringByIndex(const UResourceBundle *resB, + int32_t index, + char *dest, int32_t *pLength, + UBool forceCopy, + UErrorCode *status); + +/** + * Returns a resource in a given resource that has a given key. This procedure works only with table + * resources. Features a fill-in parameter. + * + * @param resourceBundle a resource + * @param key a key associated with the wanted resource + * @param fillIn if NULL a new UResourceBundle struct is allocated and must be closed by the caller. + * Alternatively, you can supply a struct to be filled by this function. + * @param status fills in the outgoing error code. + * @return a pointer to a UResourceBundle struct. If fill in param was NULL, caller must close it + * @stable ICU 2.0 + */ +U_STABLE UResourceBundle* U_EXPORT2 +ures_getByKey(const UResourceBundle *resourceBundle, + const char* key, + UResourceBundle *fillIn, + UErrorCode *status); + +/** + * Returns a string in a given resource that has a given key. This procedure works only with table + * resources. + * + * @param resB a resource + * @param key a key associated with the wanted string + * @param len fill in length of the string + * @param status fills in the outgoing error code. If an error occured, we may return NULL, but don't + * count on it. Check status instead! + * @return a pointer to a zero-terminated UChar array which lives in a memory mapped/DLL file. + * @stable ICU 2.0 + */ +U_STABLE const UChar* U_EXPORT2 +ures_getStringByKey(const UResourceBundle *resB, + const char* key, + int32_t* len, + UErrorCode *status); + +/** + * Returns a UTF-8 string from a resource and a key. + * This function works only with table resources. + * + * The UTF-8 string may be returnable directly as a pointer, or + * it may need to be copied, or transformed from UTF-16 using u_strToUTF8() + * or equivalent. + * + * If forceCopy==TRUE, then the string is always written to the dest buffer + * and dest is returned. + * + * If forceCopy==FALSE, then the string is returned as a pointer if possible, + * without needing a dest buffer (it can be NULL). If the string needs to be + * copied or transformed, then it may be placed into dest at an arbitrary offset. + * + * If the string is to be written to dest, then U_BUFFER_OVERFLOW_ERROR and + * U_STRING_NOT_TERMINATED_WARNING are set if appropriate, as usual. + * + * If the string is transformed from UTF-16, then a conversion error may occur + * if an unpaired surrogate is encountered. If the function is successful, then + * the output UTF-8 string is always well-formed. + * + * @param resB Resource bundle. + * @param key A key associated with the wanted resource + * @param dest Destination buffer. Can be NULL only if capacity=*length==0. + * @param pLength Input: Capacity of destination buffer. + * Output: Actual length of the UTF-8 string, not counting the + * terminating NUL, even in case of U_BUFFER_OVERFLOW_ERROR. + * Can be NULL, meaning capacity=0 and the string length is not + * returned to the caller. + * @param forceCopy If TRUE, then the output string will always be written to + * dest, with U_BUFFER_OVERFLOW_ERROR and + * U_STRING_NOT_TERMINATED_WARNING set if appropriate. + * If FALSE, then the dest buffer may or may not contain a + * copy of the string. dest may or may not be modified. + * If a copy needs to be written, then the UErrorCode parameter + * indicates overflow etc. as usual. + * @param status Pointer to a standard ICU error code. Its input value must + * pass the U_SUCCESS() test, or else the function returns + * immediately. Check for U_FAILURE() on output or use with + * function chaining. (See User Guide for details.) + * @return The pointer to the UTF-8 string. It may be dest, or at some offset + * from dest (only if !forceCopy), or in unrelated memory. + * Always NUL-terminated unless the string was written to dest and + * length==capacity (in which case U_STRING_NOT_TERMINATED_WARNING is set). + * + * @see ures_getStringByKey + * @see u_strToUTF8 + * @stable ICU 3.6 + */ +U_STABLE const char * U_EXPORT2 +ures_getUTF8StringByKey(const UResourceBundle *resB, + const char *key, + char *dest, int32_t *pLength, + UBool forceCopy, + UErrorCode *status); + +#ifdef XP_CPLUSPLUS +#include "unicode/unistr.h" + +U_NAMESPACE_BEGIN +/** + * returns a string from a string resource type + * + * @param resB a resource + * @param status: fills in the outgoing error code + * could be U_MISSING_RESOURCE_ERROR if the key is not found + * could be a non-failing error + * e.g.: U_USING_FALLBACK_WARNING,U_USING_DEFAULT_WARNING + * @return a UnicodeString object. If there is an error, string is bogus + * @stable ICU 2.0 + */ +inline UnicodeString +ures_getUnicodeString(const UResourceBundle *resB, + UErrorCode* status) +{ + int32_t len = 0; + const UChar *r = ures_getString(resB, &len, status); + return UnicodeString(TRUE, r, len); +} + +/** + * Returns the next string in a resource or NULL if there are no more resources + * to iterate over. + * + * @param resB a resource + * @param key fill in for key associated with this string + * @param status fills in the outgoing error code + * @return an UnicodeString object. + * @stable ICU 2.0 + */ +inline UnicodeString +ures_getNextUnicodeString(UResourceBundle *resB, + const char ** key, + UErrorCode* status) +{ + int32_t len = 0; + const UChar* r = ures_getNextString(resB, &len, key, status); + return UnicodeString(TRUE, r, len); +} + +/** + * Returns the string in a given resource at the specified index. + * + * @param resB a resource + * @param index an index to the wanted string. + * @param status fills in the outgoing error code + * @return an UnicodeString object. If there is an error, string is bogus + * @stable ICU 2.0 + */ +inline UnicodeString +ures_getUnicodeStringByIndex(const UResourceBundle *resB, + int32_t indexS, + UErrorCode* status) +{ + int32_t len = 0; + const UChar* r = ures_getStringByIndex(resB, indexS, &len, status); + return UnicodeString(TRUE, r, len); +} + +/** + * Returns a string in a resource that has a given key. This procedure works only with table + * resources. + * + * @param resB a resource + * @param key a key associated with the wanted string + * @param status fills in the outgoing error code + * @return an UnicodeString object. If there is an error, string is bogus + * @stable ICU 2.0 + */ +inline UnicodeString +ures_getUnicodeStringByKey(const UResourceBundle *resB, + const char* key, + UErrorCode* status) +{ + int32_t len = 0; + const UChar* r = ures_getStringByKey(resB, key, &len, status); + return UnicodeString(TRUE, r, len); +} + +U_NAMESPACE_END + +#endif + +/** + * Create a string enumerator, owned by the caller, of all locales located within + * the specified resource tree. + * @param packageName name of the tree, such as (NULL) or U_ICUDATA_ALIAS or or "ICUDATA-coll" + * This call is similar to uloc_getAvailable(). + * @param status error code + * @stable ICU 3.2 + */ +U_STABLE UEnumeration* U_EXPORT2 +ures_openAvailableLocales(const char *packageName, UErrorCode *status); + + +#endif /*_URES*/ +/*eof*/ diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/uscript.h b/Build/source/libs/icu/icu-xetex/common/unicode/uscript.h new file mode 100644 index 00000000000..6aebc68b3ca --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/uscript.h @@ -0,0 +1,238 @@ +/* + ********************************************************************** + * Copyright (C) 1997-2007, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + * + * File USCRIPT.H + * + * Modification History: + * + * Date Name Description + * 07/06/2001 Ram Creation. + ****************************************************************************** + */ + +#ifndef USCRIPT_H +#define USCRIPT_H +#include "unicode/utypes.h" + +/** + * \file + * \brief C API: Unicode Script Information + */ + +/** + * Constants for ISO 15924 script codes. + * + * Many of these script codes - those from Unicode's ScriptNames.txt - + * are character property values for Unicode's Script property. + * See UAX #24 Script Names (http://www.unicode.org/reports/tr24/). + * + * Starting with ICU 3.6, constants for most ISO 15924 script codes + * are included (currently excluding private-use codes Qaaa..Qabx). + * For scripts for which there are codes in ISO 15924 but which are not + * used in the Unicode Character Database (UCD), there are no Unicode characters + * associated with those scripts. + * + * For example, there are no characters that have a UCD script code of + * Hans or Hant. All Han ideographs have the Hani script code. + * The Hans and Hant script codes are used with CLDR data. + * + * ISO 15924 script codes are included for use with CLDR and similar. + * + * @stable ICU 2.2 + */ +typedef enum UScriptCode { + USCRIPT_INVALID_CODE = -1, + USCRIPT_COMMON = 0 , /* Zyyy */ + USCRIPT_INHERITED = 1, /* Qaai */ + USCRIPT_ARABIC = 2, /* Arab */ + USCRIPT_ARMENIAN = 3, /* Armn */ + USCRIPT_BENGALI = 4, /* Beng */ + USCRIPT_BOPOMOFO = 5, /* Bopo */ + USCRIPT_CHEROKEE = 6, /* Cher */ + USCRIPT_COPTIC = 7, /* Copt */ + USCRIPT_CYRILLIC = 8, /* Cyrl */ + USCRIPT_DESERET = 9, /* Dsrt */ + USCRIPT_DEVANAGARI = 10, /* Deva */ + USCRIPT_ETHIOPIC = 11, /* Ethi */ + USCRIPT_GEORGIAN = 12, /* Geor */ + USCRIPT_GOTHIC = 13, /* Goth */ + USCRIPT_GREEK = 14, /* Grek */ + USCRIPT_GUJARATI = 15, /* Gujr */ + USCRIPT_GURMUKHI = 16, /* Guru */ + USCRIPT_HAN = 17, /* Hani */ + USCRIPT_HANGUL = 18, /* Hang */ + USCRIPT_HEBREW = 19, /* Hebr */ + USCRIPT_HIRAGANA = 20, /* Hira */ + USCRIPT_KANNADA = 21, /* Knda */ + USCRIPT_KATAKANA = 22, /* Kana */ + USCRIPT_KHMER = 23, /* Khmr */ + USCRIPT_LAO = 24, /* Laoo */ + USCRIPT_LATIN = 25, /* Latn */ + USCRIPT_MALAYALAM = 26, /* Mlym */ + USCRIPT_MONGOLIAN = 27, /* Mong */ + USCRIPT_MYANMAR = 28, /* Mymr */ + USCRIPT_OGHAM = 29, /* Ogam */ + USCRIPT_OLD_ITALIC = 30, /* Ital */ + USCRIPT_ORIYA = 31, /* Orya */ + USCRIPT_RUNIC = 32, /* Runr */ + USCRIPT_SINHALA = 33, /* Sinh */ + USCRIPT_SYRIAC = 34, /* Syrc */ + USCRIPT_TAMIL = 35, /* Taml */ + USCRIPT_TELUGU = 36, /* Telu */ + USCRIPT_THAANA = 37, /* Thaa */ + USCRIPT_THAI = 38, /* Thai */ + USCRIPT_TIBETAN = 39, /* Tibt */ + /** Canadian_Aboriginal script. @stable ICU 2.6 */ + USCRIPT_CANADIAN_ABORIGINAL = 40, /* Cans */ + /** Canadian_Aboriginal script (alias). @stable ICU 2.2 */ + USCRIPT_UCAS = USCRIPT_CANADIAN_ABORIGINAL, + USCRIPT_YI = 41, /* Yiii */ + USCRIPT_TAGALOG = 42, /* Tglg */ + USCRIPT_HANUNOO = 43, /* Hano */ + USCRIPT_BUHID = 44, /* Buhd */ + USCRIPT_TAGBANWA = 45, /* Tagb */ + + /* New scripts in Unicode 4 @stable ICU 2.6 */ + USCRIPT_BRAILLE = 46, /* Brai */ + USCRIPT_CYPRIOT = 47, /* Cprt */ + USCRIPT_LIMBU = 48, /* Limb */ + USCRIPT_LINEAR_B = 49, /* Linb */ + USCRIPT_OSMANYA = 50, /* Osma */ + USCRIPT_SHAVIAN = 51, /* Shaw */ + USCRIPT_TAI_LE = 52, /* Tale */ + USCRIPT_UGARITIC = 53, /* Ugar */ + + /** New script code in Unicode 4.0.1 @stable ICU 3.0 */ + USCRIPT_KATAKANA_OR_HIRAGANA = 54,/*Hrkt */ + + /* New scripts in Unicode 4.1 @stable ICU 3.4 */ + USCRIPT_BUGINESE = 55, /* Bugi */ + USCRIPT_GLAGOLITIC = 56, /* Glag */ + USCRIPT_KHAROSHTHI = 57, /* Khar */ + USCRIPT_SYLOTI_NAGRI = 58, /* Sylo */ + USCRIPT_NEW_TAI_LUE = 59, /* Talu */ + USCRIPT_TIFINAGH = 60, /* Tfng */ + USCRIPT_OLD_PERSIAN = 61, /* Xpeo */ + + /* New script codes from ISO 15924 @stable ICU 3.6 */ + USCRIPT_BALINESE = 62, /* Bali */ + USCRIPT_BATAK = 63, /* Batk */ + USCRIPT_BLISSYMBOLS = 64, /* Blis */ + USCRIPT_BRAHMI = 65, /* Brah */ + USCRIPT_CHAM = 66, /* Cham */ + USCRIPT_CIRTH = 67, /* Cirt */ + USCRIPT_OLD_CHURCH_SLAVONIC_CYRILLIC = 68, /* Cyrs */ + USCRIPT_DEMOTIC_EGYPTIAN = 69, /* Egyd */ + USCRIPT_HIERATIC_EGYPTIAN = 70, /* Egyh */ + USCRIPT_EGYPTIAN_HIEROGLYPHS = 71, /* Egyp */ + USCRIPT_KHUTSURI = 72, /* Geok */ + USCRIPT_SIMPLIFIED_HAN = 73, /* Hans */ + USCRIPT_TRADITIONAL_HAN = 74, /* Hant */ + USCRIPT_PAHAWH_HMONG = 75, /* Hmng */ + USCRIPT_OLD_HUNGARIAN = 76, /* Hung */ + USCRIPT_HARAPPAN_INDUS = 77, /* Inds */ + USCRIPT_JAVANESE = 78, /* Java */ + USCRIPT_KAYAH_LI = 79, /* Kali */ + USCRIPT_LATIN_FRAKTUR = 80, /* Latf */ + USCRIPT_LATIN_GAELIC = 81, /* Latg */ + USCRIPT_LEPCHA = 82, /* Lepc */ + USCRIPT_LINEAR_A = 83, /* Lina */ + USCRIPT_MANDAEAN = 84, /* Mand */ + USCRIPT_MAYAN_HIEROGLYPHS = 85, /* Maya */ + USCRIPT_MEROITIC = 86, /* Mero */ + USCRIPT_NKO = 87, /* Nkoo */ + USCRIPT_ORKHON = 88, /* Orkh */ + USCRIPT_OLD_PERMIC = 89, /* Perm */ + USCRIPT_PHAGS_PA = 90, /* Phag */ + USCRIPT_PHOENICIAN = 91, /* Phnx */ + USCRIPT_PHONETIC_POLLARD = 92, /* Plrd */ + USCRIPT_RONGORONGO = 93, /* Roro */ + USCRIPT_SARATI = 94, /* Sara */ + USCRIPT_ESTRANGELO_SYRIAC = 95, /* Syre */ + USCRIPT_WESTERN_SYRIAC = 96, /* Syrj */ + USCRIPT_EASTERN_SYRIAC = 97, /* Syrn */ + USCRIPT_TENGWAR = 98, /* Teng */ + USCRIPT_VAI = 99, /* Vaii */ + USCRIPT_VISIBLE_SPEECH = 100, /* Visp */ + USCRIPT_CUNEIFORM = 101,/* Xsux */ + USCRIPT_UNWRITTEN_LANGUAGES = 102,/* Zxxx */ + USCRIPT_UNKNOWN = 103,/* Zzzz */ /* Unknown="Code for uncoded script", for unassigned code points */ + + /* New script codes from ISO 15924 @draft ICU 3.8 */ + USCRIPT_CARIAN = 104,/* Cari */ + USCRIPT_JAPANESE = 105,/* Jpan */ + USCRIPT_LANNA = 106,/* Lana */ + USCRIPT_LYCIAN = 107,/* Lyci */ + USCRIPT_LYDIAN = 108,/* Lydi */ + USCRIPT_OL_CHIKI = 109,/* Olck */ + USCRIPT_REJANG = 110,/* Rjng */ + USCRIPT_SAURASHTRA = 111,/* Saur */ + USCRIPT_SIGN_WRITING = 112,/* Sgnw */ + USCRIPT_SUNDANESE = 113,/* Sund */ + USCRIPT_MOON = 114,/* Moon */ + USCRIPT_MEITEI_MAYEK = 115,/* Mtei */ + + /* Private use codes from Qaaa - Qabx are not supported*/ + USCRIPT_CODE_LIMIT = 116 +} UScriptCode; + +/** + * Gets script codes associated with the given locale or ISO 15924 abbreviation or name. + * Fills in USCRIPT_MALAYALAM given "Malayam" OR "Mlym". + * Fills in USCRIPT_LATIN given "en" OR "en_US" + * If required capacity is greater than capacity of the destination buffer then the error code + * is set to U_BUFFER_OVERFLOW_ERROR and the required capacity is returned + * + *

Note: To search by short or long script alias only, use + * u_getPropertyValueEnum(UCHAR_SCRIPT, alias) instead. This does + * a fast lookup with no access of the locale data. + * @param nameOrAbbrOrLocale name of the script, as given in + * PropertyValueAliases.txt, or ISO 15924 code or locale + * @param fillIn the UScriptCode buffer to fill in the script code + * @param capacity the capacity (size) fo UScriptCode buffer passed in. + * @param err the error status code. + * @return The number of script codes filled in the buffer passed in + * @stable ICU 2.4 + */ +U_STABLE int32_t U_EXPORT2 +uscript_getCode(const char* nameOrAbbrOrLocale,UScriptCode* fillIn,int32_t capacity,UErrorCode *err); + +/** + * Gets a script name associated with the given script code. + * Returns "Malayam" given USCRIPT_MALAYALAM + * @param scriptCode UScriptCode enum + * @return script long name as given in + * PropertyValueAliases.txt, or NULL if scriptCode is invalid + * @stable ICU 2.4 + */ +U_STABLE const char* U_EXPORT2 +uscript_getName(UScriptCode scriptCode); + +/** + * Gets a script name associated with the given script code. + * Returns "Mlym" given USCRIPT_MALAYALAM + * @param scriptCode UScriptCode enum + * @return script abbreviated name as given in + * PropertyValueAliases.txt, or NULL if scriptCode is invalid + * @stable ICU 2.4 + */ +U_STABLE const char* U_EXPORT2 +uscript_getShortName(UScriptCode scriptCode); + +/** + * Gets the script code associated with the given codepoint. + * Returns USCRIPT_MALAYALAM given 0x0D02 + * @param codepoint UChar32 codepoint + * @param err the error status code. + * @return The UScriptCode, or 0 if codepoint is invalid + * @stable ICU 2.4 + */ +U_STABLE UScriptCode U_EXPORT2 +uscript_getScript(UChar32 codepoint, UErrorCode *err); + +#endif + + diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/uset.h b/Build/source/libs/icu/icu-xetex/common/unicode/uset.h new file mode 100644 index 00000000000..8232ff9f3ec --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/uset.h @@ -0,0 +1,1055 @@ +/* +******************************************************************************* +* +* Copyright (C) 2002-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: uset.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2002mar07 +* created by: Markus W. Scherer +* +* C version of UnicodeSet. +*/ + + +/** + * \file + * \brief C API: Unicode Set + * + *

This is a C wrapper around the C++ UnicodeSet class.

+ */ + +#ifndef __USET_H__ +#define __USET_H__ + +#include "unicode/utypes.h" +#include "unicode/uchar.h" + +#ifndef UCNV_H +struct USet; +/** + * A UnicodeSet. Use the uset_* API to manipulate. Create with + * uset_open*, and destroy with uset_close. + * @stable ICU 2.4 + */ +typedef struct USet USet; +#endif + +/** + * Bitmask values to be passed to uset_openPatternOptions() or + * uset_applyPattern() taking an option parameter. + * @stable ICU 2.4 + */ +enum { + /** + * Ignore white space within patterns unless quoted or escaped. + * @stable ICU 2.4 + */ + USET_IGNORE_SPACE = 1, + + /** + * Enable case insensitive matching. E.g., "[ab]" with this flag + * will match 'a', 'A', 'b', and 'B'. "[^ab]" with this flag will + * match all except 'a', 'A', 'b', and 'B'. This performs a full + * closure over case mappings, e.g. U+017F for s. + * + * The resulting set is a superset of the input for the code points but + * not for the strings. + * It performs a case mapping closure of the code points and adds + * full case folding strings for the code points, and reduces strings of + * the original set to their full case folding equivalents. + * + * This is designed for case-insensitive matches, for example + * in regular expressions. The full code point case closure allows checking of + * an input character directly against the closure set. + * Strings are matched by comparing the case-folded form from the closure + * set with an incremental case folding of the string in question. + * + * The closure set will also contain single code points if the original + * set contained case-equivalent strings (like U+00DF for "ss" or "Ss" etc.). + * This is not necessary (that is, redundant) for the above matching method + * but results in the same closure sets regardless of whether the original + * set contained the code point or a string. + * + * @stable ICU 2.4 + */ + USET_CASE_INSENSITIVE = 2, + + /** + * Enable case insensitive matching. E.g., "[ab]" with this flag + * will match 'a', 'A', 'b', and 'B'. "[^ab]" with this flag will + * match all except 'a', 'A', 'b', and 'B'. This adds the lower-, + * title-, and uppercase mappings as well as the case folding + * of each existing element in the set. + * @stable ICU 3.2 + */ + USET_ADD_CASE_MAPPINGS = 4, + + /** + * Enough for any single-code point set + * @internal + */ + USET_SERIALIZED_STATIC_ARRAY_CAPACITY=8 +}; + +#ifndef U_HIDE_DRAFT_API + +/** + * Argument values for whether span() and similar functions continue while + * the current character is contained vs. not contained in the set. + * + * The functionality is straightforward for sets with only single code points, + * without strings (which is the common case): + * - USET_SPAN_CONTAINED and USET_SPAN_SIMPLE + * work the same. + * - span() and spanBack() partition any string the same way when + * alternating between span(USET_SPAN_NOT_CONTAINED) and + * span(either "contained" condition). + * - Using a complemented (inverted) set and the opposite span conditions + * yields the same results. + * + * When a set contains multi-code point strings, then these statements may not + * be true, depending on the strings in the set (for example, whether they + * overlap with each other) and the string that is processed. + * For a set with strings: + * - The complement of the set contains the opposite set of code points, + * but the same set of strings. + * Therefore, complementing both the set and the span conditions + * may yield different results. + * - When starting spans at different positions in a string + * (span(s, ...) vs. span(s+1, ...)) the ends of the spans may be different + * because a set string may start before the later position. + * - span(USET_SPAN_SIMPLE) may be shorter than + * span(USET_SPAN_CONTAINED) because it will not recursively try + * all possible paths. + * For example, with a set which contains the three strings "xy", "xya" and "ax", + * span("xyax", USET_SPAN_CONTAINED) will return 4 but + * span("xyax", USET_SPAN_SIMPLE) will return 3. + * span(USET_SPAN_SIMPLE) will never be longer than + * span(USET_SPAN_CONTAINED). + * - With either "contained" condition, span() and spanBack() may partition + * a string in different ways. + * For example, with a set which contains the two strings "ab" and "ba", + * and when processing the string "aba", + * span() will yield contained/not-contained boundaries of { 0, 2, 3 } + * while spanBack() will yield boundaries of { 0, 1, 3 }. + * + * Note: If it is important to get the same boundaries whether iterating forward + * or backward through a string, then either only span() should be used and + * the boundaries cached for backward operation, or an ICU BreakIterator + * could be used. + * + * Note: Unpaired surrogates are treated like surrogate code points. + * Similarly, set strings match only on code point boundaries, + * never in the middle of a surrogate pair. + * Illegal UTF-8 sequences are treated like U+FFFD. + * When processing UTF-8 strings, malformed set strings + * (strings with unpaired surrogates which cannot be converted to UTF-8) + * are ignored. + * + * @draft ICU 3.8 + */ +typedef enum USetSpanCondition { + /** + * Continue a span() while there is no set element at the current position. + * Stops before the first set element (character or string). + * (For code points only, this is like while contains(current)==FALSE). + * + * When span() returns, the substring between where it started and the position + * it returned consists only of characters that are not in the set, + * and none of its strings overlap with the span. + * + * @draft ICU 3.8 + */ + USET_SPAN_NOT_CONTAINED = 0, + /** + * Continue a span() while there is a set element at the current position. + * (For characters only, this is like while contains(current)==TRUE). + * + * When span() returns, the substring between where it started and the position + * it returned consists only of set elements (characters or strings) that are in the set. + * + * If a set contains strings, then the span will be the longest substring + * matching any of the possible concatenations of set elements (characters or strings). + * (There must be a single, non-overlapping concatenation of characters or strings.) + * This is equivalent to a POSIX regular expression for (OR of each set element)*. + * + * @draft ICU 3.8 + */ + USET_SPAN_CONTAINED = 1, + /** + * Continue a span() while there is a set element at the current position. + * (For characters only, this is like while contains(current)==TRUE). + * + * When span() returns, the substring between where it started and the position + * it returned consists only of set elements (characters or strings) that are in the set. + * + * If a set only contains single characters, then this is the same + * as USET_SPAN_CONTAINED. + * + * If a set contains strings, then the span will be the longest substring + * with a match at each position with the longest single set element (character or string). + * + * Use this span condition together with other longest-match algorithms, + * such as ICU converters (ucnv_getUnicodeSet()). + * + * @draft ICU 3.8 + */ + USET_SPAN_SIMPLE = 2, + /** + * One more than the last span condition. + * @draft ICU 3.8 + */ + USET_SPAN_CONDITION_COUNT +} USetSpanCondition; + +#endif /* U_HIDE_DRAFT_API */ + +/** + * A serialized form of a Unicode set. Limited manipulations are + * possible directly on a serialized set. See below. + * @stable ICU 2.4 + */ +typedef struct USerializedSet { + /** + * The serialized Unicode Set. + * @stable ICU 2.4 + */ + const uint16_t *array; + /** + * The length of the array that contains BMP characters. + * @stable ICU 2.4 + */ + int32_t bmpLength; + /** + * The total length of the array. + * @stable ICU 2.4 + */ + int32_t length; + /** + * A small buffer for the array to reduce memory allocations. + * @stable ICU 2.4 + */ + uint16_t staticArray[USET_SERIALIZED_STATIC_ARRAY_CAPACITY]; +} USerializedSet; + +/********************************************************************* + * USet API + *********************************************************************/ + +/** + * Creates a USet object that contains the range of characters + * start..end, inclusive. + * @param start first character of the range, inclusive + * @param end last character of the range, inclusive + * @return a newly created USet. The caller must call uset_close() on + * it when done. + * @stable ICU 2.4 + */ +U_STABLE USet* U_EXPORT2 +uset_open(UChar32 start, UChar32 end); + +/** + * Creates a set from the given pattern. See the UnicodeSet class + * description for the syntax of the pattern language. + * @param pattern a string specifying what characters are in the set + * @param patternLength the length of the pattern, or -1 if null + * terminated + * @param ec the error code + * @stable ICU 2.4 + */ +U_STABLE USet* U_EXPORT2 +uset_openPattern(const UChar* pattern, int32_t patternLength, + UErrorCode* ec); + +/** + * Creates a set from the given pattern. See the UnicodeSet class + * description for the syntax of the pattern language. + * @param pattern a string specifying what characters are in the set + * @param patternLength the length of the pattern, or -1 if null + * terminated + * @param options bitmask for options to apply to the pattern. + * Valid options are USET_IGNORE_SPACE and USET_CASE_INSENSITIVE. + * @param ec the error code + * @stable ICU 2.4 + */ +U_STABLE USet* U_EXPORT2 +uset_openPatternOptions(const UChar* pattern, int32_t patternLength, + uint32_t options, + UErrorCode* ec); + +/** + * Disposes of the storage used by a USet object. This function should + * be called exactly once for objects returned by uset_open(). + * @param set the object to dispose of + * @stable ICU 2.4 + */ +U_STABLE void U_EXPORT2 +uset_close(USet* set); + +/** + * Returns a copy of this object. + * If this set is frozen, then the clone will be frozen as well. + * Use uset_cloneAsThawed() for a mutable clone of a frozen set. + * @param set the original set + * @return the newly allocated copy of the set + * @see uset_cloneAsThawed + * @draft ICU 3.8 + */ +U_DRAFT USet * U_EXPORT2 +uset_clone(const USet *set); + +/** + * Determines whether the set has been frozen (made immutable) or not. + * See the ICU4J Freezable interface for details. + * @param set the set + * @return TRUE/FALSE for whether the set has been frozen + * @see uset_freeze + * @see uset_cloneAsThawed + * @draft ICU 3.8 + */ +U_DRAFT UBool U_EXPORT2 +uset_isFrozen(const USet *set); + +/** + * Freeze the set (make it immutable). + * Once frozen, it cannot be unfrozen and is therefore thread-safe + * until it is deleted. + * See the ICU4J Freezable interface for details. + * Freezing the set may also make some operations faster, for example + * uset_contains() and uset_span(). + * A frozen set will not be modified. (It remains frozen.) + * @param set the set + * @return the same set, now frozen + * @see uset_isFrozen + * @see uset_cloneAsThawed + * @draft ICU 3.8 + */ +U_DRAFT void U_EXPORT2 +uset_freeze(USet *set); + +/** + * Clone the set and make the clone mutable. + * See the ICU4J Freezable interface for details. + * @param set the set + * @return the mutable clone + * @see uset_freeze + * @see uset_isFrozen + * @see uset_clone + * @draft ICU 3.8 + */ +U_DRAFT USet * U_EXPORT2 +uset_cloneAsThawed(const USet *set); + +/** + * Causes the USet object to represent the range start - end. + * If start > end then this USet is set to an empty range. + * A frozen set will not be modified. + * @param set the object to set to the given range + * @param start first character in the set, inclusive + * @param end last character in the set, inclusive + * @stable ICU 3.2 + */ +U_STABLE void U_EXPORT2 +uset_set(USet* set, + UChar32 start, UChar32 end); + +/** + * Modifies the set to represent the set specified by the given + * pattern. See the UnicodeSet class description for the syntax of + * the pattern language. See also the User Guide chapter about UnicodeSet. + * Empties the set passed before applying the pattern. + * A frozen set will not be modified. + * @param set The set to which the pattern is to be applied. + * @param pattern A pointer to UChar string specifying what characters are in the set. + * The character at pattern[0] must be a '['. + * @param patternLength The length of the UChar string. -1 if NUL terminated. + * @param options A bitmask for options to apply to the pattern. + * Valid options are USET_IGNORE_SPACE and USET_CASE_INSENSITIVE. + * @param status Returns an error if the pattern cannot be parsed. + * @return Upon successful parse, the value is either + * the index of the character after the closing ']' + * of the parsed pattern. + * If the status code indicates failure, then the return value + * is the index of the error in the source. + * + * @stable ICU 2.8 + */ +U_STABLE int32_t U_EXPORT2 +uset_applyPattern(USet *set, + const UChar *pattern, int32_t patternLength, + uint32_t options, + UErrorCode *status); + +/** + * Modifies the set to contain those code points which have the given value + * for the given binary or enumerated property, as returned by + * u_getIntPropertyValue. Prior contents of this set are lost. + * A frozen set will not be modified. + * + * @param set the object to contain the code points defined by the property + * + * @param prop a property in the range UCHAR_BIN_START..UCHAR_BIN_LIMIT-1 + * or UCHAR_INT_START..UCHAR_INT_LIMIT-1 + * or UCHAR_MASK_START..UCHAR_MASK_LIMIT-1. + * + * @param value a value in the range u_getIntPropertyMinValue(prop).. + * u_getIntPropertyMaxValue(prop), with one exception. If prop is + * UCHAR_GENERAL_CATEGORY_MASK, then value should not be a UCharCategory, but + * rather a mask value produced by U_GET_GC_MASK(). This allows grouped + * categories such as [:L:] to be represented. + * + * @param ec error code input/output parameter + * + * @stable ICU 3.2 + */ +U_STABLE void U_EXPORT2 +uset_applyIntPropertyValue(USet* set, + UProperty prop, int32_t value, UErrorCode* ec); + +/** + * Modifies the set to contain those code points which have the + * given value for the given property. Prior contents of this + * set are lost. + * A frozen set will not be modified. + * + * @param set the object to contain the code points defined by the given + * property and value alias + * + * @param prop a string specifying a property alias, either short or long. + * The name is matched loosely. See PropertyAliases.txt for names and a + * description of loose matching. If the value string is empty, then this + * string is interpreted as either a General_Category value alias, a Script + * value alias, a binary property alias, or a special ID. Special IDs are + * matched loosely and correspond to the following sets: + * + * "ANY" = [\\u0000-\\U0010FFFF], + * "ASCII" = [\\u0000-\\u007F], + * "Assigned" = [:^Cn:]. + * + * @param propLength the length of the prop, or -1 if NULL + * + * @param value a string specifying a value alias, either short or long. + * The name is matched loosely. See PropertyValueAliases.txt for names + * and a description of loose matching. In addition to aliases listed, + * numeric values and canonical combining classes may be expressed + * numerically, e.g., ("nv", "0.5") or ("ccc", "220"). The value string + * may also be empty. + * + * @param valueLength the length of the value, or -1 if NULL + * + * @param ec error code input/output parameter + * + * @stable ICU 3.2 + */ +U_STABLE void U_EXPORT2 +uset_applyPropertyAlias(USet* set, + const UChar *prop, int32_t propLength, + const UChar *value, int32_t valueLength, + UErrorCode* ec); + +/** + * Return true if the given position, in the given pattern, appears + * to be the start of a UnicodeSet pattern. + * + * @param pattern a string specifying the pattern + * @param patternLength the length of the pattern, or -1 if NULL + * @param pos the given position + * @stable ICU 3.2 + */ +U_STABLE UBool U_EXPORT2 +uset_resemblesPattern(const UChar *pattern, int32_t patternLength, + int32_t pos); + +/** + * Returns a string representation of this set. If the result of + * calling this function is passed to a uset_openPattern(), it + * will produce another set that is equal to this one. + * @param set the set + * @param result the string to receive the rules, may be NULL + * @param resultCapacity the capacity of result, may be 0 if result is NULL + * @param escapeUnprintable if TRUE then convert unprintable + * character to their hex escape representations, \\uxxxx or + * \\Uxxxxxxxx. Unprintable characters are those other than + * U+000A, U+0020..U+007E. + * @param ec error code. + * @return length of string, possibly larger than resultCapacity + * @stable ICU 2.4 + */ +U_STABLE int32_t U_EXPORT2 +uset_toPattern(const USet* set, + UChar* result, int32_t resultCapacity, + UBool escapeUnprintable, + UErrorCode* ec); + +/** + * Adds the given character to the given USet. After this call, + * uset_contains(set, c) will return TRUE. + * A frozen set will not be modified. + * @param set the object to which to add the character + * @param c the character to add + * @stable ICU 2.4 + */ +U_STABLE void U_EXPORT2 +uset_add(USet* set, UChar32 c); + +/** + * Adds all of the elements in the specified set to this set if + * they're not already present. This operation effectively + * modifies this set so that its value is the union of the two + * sets. The behavior of this operation is unspecified if the specified + * collection is modified while the operation is in progress. + * A frozen set will not be modified. + * + * @param set the object to which to add the set + * @param additionalSet the source set whose elements are to be added to this set. + * @stable ICU 2.6 + */ +U_STABLE void U_EXPORT2 +uset_addAll(USet* set, const USet *additionalSet); + +/** + * Adds the given range of characters to the given USet. After this call, + * uset_contains(set, start, end) will return TRUE. + * A frozen set will not be modified. + * @param set the object to which to add the character + * @param start the first character of the range to add, inclusive + * @param end the last character of the range to add, inclusive + * @stable ICU 2.2 + */ +U_STABLE void U_EXPORT2 +uset_addRange(USet* set, UChar32 start, UChar32 end); + +/** + * Adds the given string to the given USet. After this call, + * uset_containsString(set, str, strLen) will return TRUE. + * A frozen set will not be modified. + * @param set the object to which to add the character + * @param str the string to add + * @param strLen the length of the string or -1 if null terminated. + * @stable ICU 2.4 + */ +U_STABLE void U_EXPORT2 +uset_addString(USet* set, const UChar* str, int32_t strLen); + +/** + * Adds each of the characters in this string to the set. Thus "ch" => {"c", "h"} + * If this set already any particular character, it has no effect on that character. + * A frozen set will not be modified. + * @param set the object to which to add the character + * @param str the source string + * @param strLen the length of the string or -1 if null terminated. + * @stable ICU 3.4 + */ +U_STABLE void U_EXPORT2 +uset_addAllCodePoints(USet* set, const UChar *str, int32_t strLen); + +/** + * Removes the given character from the given USet. After this call, + * uset_contains(set, c) will return FALSE. + * A frozen set will not be modified. + * @param set the object from which to remove the character + * @param c the character to remove + * @stable ICU 2.4 + */ +U_STABLE void U_EXPORT2 +uset_remove(USet* set, UChar32 c); + +/** + * Removes the given range of characters from the given USet. After this call, + * uset_contains(set, start, end) will return FALSE. + * A frozen set will not be modified. + * @param set the object to which to add the character + * @param start the first character of the range to remove, inclusive + * @param end the last character of the range to remove, inclusive + * @stable ICU 2.2 + */ +U_STABLE void U_EXPORT2 +uset_removeRange(USet* set, UChar32 start, UChar32 end); + +/** + * Removes the given string to the given USet. After this call, + * uset_containsString(set, str, strLen) will return FALSE. + * A frozen set will not be modified. + * @param set the object to which to add the character + * @param str the string to remove + * @param strLen the length of the string or -1 if null terminated. + * @stable ICU 2.4 + */ +U_STABLE void U_EXPORT2 +uset_removeString(USet* set, const UChar* str, int32_t strLen); + +/** + * Removes from this set all of its elements that are contained in the + * specified set. This operation effectively modifies this + * set so that its value is the asymmetric set difference of + * the two sets. + * A frozen set will not be modified. + * @param set the object from which the elements are to be removed + * @param removeSet the object that defines which elements will be + * removed from this set + * @stable ICU 3.2 + */ +U_STABLE void U_EXPORT2 +uset_removeAll(USet* set, const USet* removeSet); + +/** + * Retain only the elements in this set that are contained in the + * specified range. If start > end then an empty range is + * retained, leaving the set empty. This is equivalent to + * a boolean logic AND, or a set INTERSECTION. + * A frozen set will not be modified. + * + * @param set the object for which to retain only the specified range + * @param start first character, inclusive, of range to be retained + * to this set. + * @param end last character, inclusive, of range to be retained + * to this set. + * @stable ICU 3.2 + */ +U_STABLE void U_EXPORT2 +uset_retain(USet* set, UChar32 start, UChar32 end); + +/** + * Retains only the elements in this set that are contained in the + * specified set. In other words, removes from this set all of + * its elements that are not contained in the specified set. This + * operation effectively modifies this set so that its value is + * the intersection of the two sets. + * A frozen set will not be modified. + * + * @param set the object on which to perform the retain + * @param retain set that defines which elements this set will retain + * @stable ICU 3.2 + */ +U_STABLE void U_EXPORT2 +uset_retainAll(USet* set, const USet* retain); + +/** + * Reallocate this objects internal structures to take up the least + * possible space, without changing this object's value. + * A frozen set will not be modified. + * + * @param set the object on which to perfrom the compact + * @stable ICU 3.2 + */ +U_STABLE void U_EXPORT2 +uset_compact(USet* set); + +/** + * Inverts this set. This operation modifies this set so that + * its value is its complement. This operation does not affect + * the multicharacter strings, if any. + * A frozen set will not be modified. + * @param set the set + * @stable ICU 2.4 + */ +U_STABLE void U_EXPORT2 +uset_complement(USet* set); + +/** + * Complements in this set all elements contained in the specified + * set. Any character in the other set will be removed if it is + * in this set, or will be added if it is not in this set. + * A frozen set will not be modified. + * + * @param set the set with which to complement + * @param complement set that defines which elements will be xor'ed + * from this set. + * @stable ICU 3.2 + */ +U_STABLE void U_EXPORT2 +uset_complementAll(USet* set, const USet* complement); + +/** + * Removes all of the elements from this set. This set will be + * empty after this call returns. + * A frozen set will not be modified. + * @param set the set + * @stable ICU 2.4 + */ +U_STABLE void U_EXPORT2 +uset_clear(USet* set); + +/** + * Returns TRUE if the given USet contains no characters and no + * strings. + * @param set the set + * @return true if set is empty + * @stable ICU 2.4 + */ +U_STABLE UBool U_EXPORT2 +uset_isEmpty(const USet* set); + +/** + * Returns TRUE if the given USet contains the given character. + * This function works faster with a frozen set. + * @param set the set + * @param c The codepoint to check for within the set + * @return true if set contains c + * @stable ICU 2.4 + */ +U_STABLE UBool U_EXPORT2 +uset_contains(const USet* set, UChar32 c); + +/** + * Returns TRUE if the given USet contains all characters c + * where start <= c && c <= end. + * @param set the set + * @param start the first character of the range to test, inclusive + * @param end the last character of the range to test, inclusive + * @return TRUE if set contains the range + * @stable ICU 2.2 + */ +U_STABLE UBool U_EXPORT2 +uset_containsRange(const USet* set, UChar32 start, UChar32 end); + +/** + * Returns TRUE if the given USet contains the given string. + * @param set the set + * @param str the string + * @param strLen the length of the string or -1 if null terminated. + * @return true if set contains str + * @stable ICU 2.4 + */ +U_STABLE UBool U_EXPORT2 +uset_containsString(const USet* set, const UChar* str, int32_t strLen); + +/** + * Returns the index of the given character within this set, where + * the set is ordered by ascending code point. If the character + * is not in this set, return -1. The inverse of this method is + * charAt(). + * @param set the set + * @param c the character to obtain the index for + * @return an index from 0..size()-1, or -1 + * @stable ICU 3.2 + */ +U_STABLE int32_t U_EXPORT2 +uset_indexOf(const USet* set, UChar32 c); + +/** + * Returns the character at the given index within this set, where + * the set is ordered by ascending code point. If the index is + * out of range, return (UChar32)-1. The inverse of this method is + * indexOf(). + * @param set the set + * @param index an index from 0..size()-1 to obtain the char for + * @return the character at the given index, or (UChar32)-1. + * @stable ICU 3.2 + */ +U_STABLE UChar32 U_EXPORT2 +uset_charAt(const USet* set, int32_t index); + +/** + * Returns the number of characters and strings contained in the given + * USet. + * @param set the set + * @return a non-negative integer counting the characters and strings + * contained in set + * @stable ICU 2.4 + */ +U_STABLE int32_t U_EXPORT2 +uset_size(const USet* set); + +/** + * Returns the number of items in this set. An item is either a range + * of characters or a single multicharacter string. + * @param set the set + * @return a non-negative integer counting the character ranges + * and/or strings contained in set + * @stable ICU 2.4 + */ +U_STABLE int32_t U_EXPORT2 +uset_getItemCount(const USet* set); + +/** + * Returns an item of this set. An item is either a range of + * characters or a single multicharacter string. + * @param set the set + * @param itemIndex a non-negative integer in the range 0.. + * uset_getItemCount(set)-1 + * @param start pointer to variable to receive first character + * in range, inclusive + * @param end pointer to variable to receive last character in range, + * inclusive + * @param str buffer to receive the string, may be NULL + * @param strCapacity capacity of str, or 0 if str is NULL + * @param ec error code + * @return the length of the string (>= 2), or 0 if the item is a + * range, in which case it is the range *start..*end, or -1 if + * itemIndex is out of range + * @stable ICU 2.4 + */ +U_STABLE int32_t U_EXPORT2 +uset_getItem(const USet* set, int32_t itemIndex, + UChar32* start, UChar32* end, + UChar* str, int32_t strCapacity, + UErrorCode* ec); + +/** + * Returns true if set1 contains all the characters and strings + * of set2. It answers the question, 'Is set1 a superset of set2?' + * @param set1 set to be checked for containment + * @param set2 set to be checked for containment + * @return true if the test condition is met + * @stable ICU 3.2 + */ +U_STABLE UBool U_EXPORT2 +uset_containsAll(const USet* set1, const USet* set2); + +/** + * Returns true if this set contains all the characters + * of the given string. This is does not check containment of grapheme + * clusters, like uset_containsString. + * @param set set of characters to be checked for containment + * @param str string containing codepoints to be checked for containment + * @param strLen the length of the string or -1 if null terminated. + * @return true if the test condition is met + * @stable ICU 3.4 + */ +U_STABLE UBool U_EXPORT2 +uset_containsAllCodePoints(const USet* set, const UChar *str, int32_t strLen); + +/** + * Returns true if set1 contains none of the characters and strings + * of set2. It answers the question, 'Is set1 a disjoint set of set2?' + * @param set1 set to be checked for containment + * @param set2 set to be checked for containment + * @return true if the test condition is met + * @stable ICU 3.2 + */ +U_STABLE UBool U_EXPORT2 +uset_containsNone(const USet* set1, const USet* set2); + +/** + * Returns true if set1 contains some of the characters and strings + * of set2. It answers the question, 'Does set1 and set2 have an intersection?' + * @param set1 set to be checked for containment + * @param set2 set to be checked for containment + * @return true if the test condition is met + * @stable ICU 3.2 + */ +U_STABLE UBool U_EXPORT2 +uset_containsSome(const USet* set1, const USet* set2); + +/** + * Returns the length of the initial substring of the input string which + * consists only of characters and strings that are contained in this set + * (USET_SPAN_CONTAINED, USET_SPAN_SIMPLE), + * or only of characters and strings that are not contained + * in this set (USET_SPAN_NOT_CONTAINED). + * See USetSpanCondition for details. + * Similar to the strspn() C library function. + * Unpaired surrogates are treated according to contains() of their surrogate code points. + * This function works faster with a frozen set and with a non-negative string length argument. + * @param set the set + * @param s start of the string + * @param length of the string; can be -1 for NUL-terminated + * @param spanCondition specifies the containment condition + * @return the length of the initial substring according to the spanCondition; + * 0 if the start of the string does not fit the spanCondition + * @draft ICU 3.8 + * @see USetSpanCondition + */ +U_DRAFT int32_t U_EXPORT2 +uset_span(const USet *set, const UChar *s, int32_t length, USetSpanCondition spanCondition); + +/** + * Returns the start of the trailing substring of the input string which + * consists only of characters and strings that are contained in this set + * (USET_SPAN_CONTAINED, USET_SPAN_SIMPLE), + * or only of characters and strings that are not contained + * in this set (USET_SPAN_NOT_CONTAINED). + * See USetSpanCondition for details. + * Unpaired surrogates are treated according to contains() of their surrogate code points. + * This function works faster with a frozen set and with a non-negative string length argument. + * @param set the set + * @param s start of the string + * @param length of the string; can be -1 for NUL-terminated + * @param spanCondition specifies the containment condition + * @return the start of the trailing substring according to the spanCondition; + * the string length if the end of the string does not fit the spanCondition + * @draft ICU 3.8 + * @see USetSpanCondition + */ +U_DRAFT int32_t U_EXPORT2 +uset_spanBack(const USet *set, const UChar *s, int32_t length, USetSpanCondition spanCondition); + +/** + * Returns the length of the initial substring of the input string which + * consists only of characters and strings that are contained in this set + * (USET_SPAN_CONTAINED, USET_SPAN_SIMPLE), + * or only of characters and strings that are not contained + * in this set (USET_SPAN_NOT_CONTAINED). + * See USetSpanCondition for details. + * Similar to the strspn() C library function. + * Malformed byte sequences are treated according to contains(0xfffd). + * This function works faster with a frozen set and with a non-negative string length argument. + * @param set the set + * @param s start of the string (UTF-8) + * @param length of the string; can be -1 for NUL-terminated + * @param spanCondition specifies the containment condition + * @return the length of the initial substring according to the spanCondition; + * 0 if the start of the string does not fit the spanCondition + * @draft ICU 3.8 + * @see USetSpanCondition + */ +U_DRAFT int32_t U_EXPORT2 +uset_spanUTF8(const USet *set, const char *s, int32_t length, USetSpanCondition spanCondition); + +/** + * Returns the start of the trailing substring of the input string which + * consists only of characters and strings that are contained in this set + * (USET_SPAN_CONTAINED, USET_SPAN_SIMPLE), + * or only of characters and strings that are not contained + * in this set (USET_SPAN_NOT_CONTAINED). + * See USetSpanCondition for details. + * Malformed byte sequences are treated according to contains(0xfffd). + * This function works faster with a frozen set and with a non-negative string length argument. + * @param set the set + * @param s start of the string (UTF-8) + * @param length of the string; can be -1 for NUL-terminated + * @param spanCondition specifies the containment condition + * @return the start of the trailing substring according to the spanCondition; + * the string length if the end of the string does not fit the spanCondition + * @draft ICU 3.8 + * @see USetSpanCondition + */ +U_DRAFT int32_t U_EXPORT2 +uset_spanBackUTF8(const USet *set, const char *s, int32_t length, USetSpanCondition spanCondition); + +/** + * Returns true if set1 contains all of the characters and strings + * of set2, and vis versa. It answers the question, 'Is set1 equal to set2?' + * @param set1 set to be checked for containment + * @param set2 set to be checked for containment + * @return true if the test condition is met + * @stable ICU 3.2 + */ +U_STABLE UBool U_EXPORT2 +uset_equals(const USet* set1, const USet* set2); + +/********************************************************************* + * Serialized set API + *********************************************************************/ + +/** + * Serializes this set into an array of 16-bit integers. Serialization + * (currently) only records the characters in the set; multicharacter + * strings are ignored. + * + * The array + * has following format (each line is one 16-bit integer): + * + * length = (n+2*m) | (m!=0?0x8000:0) + * bmpLength = n; present if m!=0 + * bmp[0] + * bmp[1] + * ... + * bmp[n-1] + * supp-high[0] + * supp-low[0] + * supp-high[1] + * supp-low[1] + * ... + * supp-high[m-1] + * supp-low[m-1] + * + * The array starts with a header. After the header are n bmp + * code points, then m supplementary code points. Either n or m + * or both may be zero. n+2*m is always <= 0x7FFF. + * + * If there are no supplementary characters (if m==0) then the + * header is one 16-bit integer, 'length', with value n. + * + * If there are supplementary characters (if m!=0) then the header + * is two 16-bit integers. The first, 'length', has value + * (n+2*m)|0x8000. The second, 'bmpLength', has value n. + * + * After the header the code points are stored in ascending order. + * Supplementary code points are stored as most significant 16 + * bits followed by least significant 16 bits. + * + * @param set the set + * @param dest pointer to buffer of destCapacity 16-bit integers. + * May be NULL only if destCapacity is zero. + * @param destCapacity size of dest, or zero. Must not be negative. + * @param pErrorCode pointer to the error code. Will be set to + * U_INDEX_OUTOFBOUNDS_ERROR if n+2*m > 0x7FFF. Will be set to + * U_BUFFER_OVERFLOW_ERROR if n+2*m+(m!=0?2:1) > destCapacity. + * @return the total length of the serialized format, including + * the header, that is, n+2*m+(m!=0?2:1), or 0 on error other + * than U_BUFFER_OVERFLOW_ERROR. + * @stable ICU 2.4 + */ +U_STABLE int32_t U_EXPORT2 +uset_serialize(const USet* set, uint16_t* dest, int32_t destCapacity, UErrorCode* pErrorCode); + +/** + * Given a serialized array, fill in the given serialized set object. + * @param fillSet pointer to result + * @param src pointer to start of array + * @param srcLength length of array + * @return true if the given array is valid, otherwise false + * @stable ICU 2.4 + */ +U_STABLE UBool U_EXPORT2 +uset_getSerializedSet(USerializedSet* fillSet, const uint16_t* src, int32_t srcLength); + +/** + * Set the USerializedSet to contain the given character (and nothing + * else). + * @param fillSet pointer to result + * @param c The codepoint to set + * @stable ICU 2.4 + */ +U_STABLE void U_EXPORT2 +uset_setSerializedToOne(USerializedSet* fillSet, UChar32 c); + +/** + * Returns TRUE if the given USerializedSet contains the given + * character. + * @param set the serialized set + * @param c The codepoint to check for within the set + * @return true if set contains c + * @stable ICU 2.4 + */ +U_STABLE UBool U_EXPORT2 +uset_serializedContains(const USerializedSet* set, UChar32 c); + +/** + * Returns the number of disjoint ranges of characters contained in + * the given serialized set. Ignores any strings contained in the + * set. + * @param set the serialized set + * @return a non-negative integer counting the character ranges + * contained in set + * @stable ICU 2.4 + */ +U_STABLE int32_t U_EXPORT2 +uset_getSerializedRangeCount(const USerializedSet* set); + +/** + * Returns a range of characters contained in the given serialized + * set. + * @param set the serialized set + * @param rangeIndex a non-negative integer in the range 0.. + * uset_getSerializedRangeCount(set)-1 + * @param pStart pointer to variable to receive first character + * in range, inclusive + * @param pEnd pointer to variable to receive last character in range, + * inclusive + * @return true if rangeIndex is valid, otherwise false + * @stable ICU 2.4 + */ +U_STABLE UBool U_EXPORT2 +uset_getSerializedRange(const USerializedSet* set, int32_t rangeIndex, + UChar32* pStart, UChar32* pEnd); + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/usetiter.h b/Build/source/libs/icu/icu-xetex/common/unicode/usetiter.h new file mode 100644 index 00000000000..defa75cd7ed --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/usetiter.h @@ -0,0 +1,318 @@ +/* +********************************************************************** +* Copyright (c) 2002-2006, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +*/ +#ifndef USETITER_H +#define USETITER_H + +#include "unicode/utypes.h" +#include "unicode/uobject.h" +#include "unicode/unistr.h" + +/** + * \file + * \brief C++ API: UnicodeSetIterator iterates over the contents of a UnicodeSet. + */ + +U_NAMESPACE_BEGIN + +class UnicodeSet; +class UnicodeString; + +/** + * + * UnicodeSetIterator iterates over the contents of a UnicodeSet. It + * iterates over either code points or code point ranges. After all + * code points or ranges have been returned, it returns the + * multicharacter strings of the UnicodSet, if any. + * + * This class is not intended to be subclassed. Consider any fields + * or methods declared as "protected" to be private. The use of + * protected in this class is an artifact of history. + * + *

To iterate over code points and strings, use a loop like this: + *

+ * UnicodeSetIterator it(set);
+ * while (set.next()) {
+ *     processItem(set.getString());
+ * }
+ * 
+ *

Each item in the set is accessed as a string. Set elements + * consisting of single code points are returned as strings containing + * just the one code point. + * + *

To iterate over code point ranges, instead of individual code points, + * use a loop like this: + *

+ * UnicodeSetIterator it(set);
+ * while (it.nextRange()) {
+ *   if (it.isString()) {
+ *     processString(it.getString());
+ *   } else {
+ *     processCodepointRange(it.getCodepoint(), it.getCodepointEnd());
+ *   }
+ * }
+ * 
+ * @author M. Davis + * @stable ICU 2.4 + */ +class U_COMMON_API UnicodeSetIterator : public UObject { + + protected: + + /** + * Value of codepoint if the iterator points to a string. + * If codepoint == IS_STRING, then examine + * string for the current iteration result. + * @stable ICU 2.4 + */ + enum { IS_STRING = -1 }; + + /** + * Current code point, or the special value IS_STRING, if + * the iterator points to a string. + * @stable ICU 2.4 + */ + UChar32 codepoint; + + /** + * When iterating over ranges using nextRange(), + * codepointEnd contains the inclusive end of the + * iteration range, if codepoint != IS_STRING. If + * iterating over code points using next(), or if + * codepoint == IS_STRING, then the value of + * codepointEnd is undefined. + * @stable ICU 2.4 + */ + UChar32 codepointEnd; + + /** + * If codepoint == IS_STRING, then string points + * to the current string. If codepoint != IS_STRING, the + * value of string is undefined. + * @stable ICU 2.4 + */ + const UnicodeString* string; + + public: + + /** + * Create an iterator over the given set. The iterator is valid + * only so long as set is valid. + * @param set set to iterate over + * @stable ICU 2.4 + */ + UnicodeSetIterator(const UnicodeSet& set); + + /** + * Create an iterator over nothing. next() and + * nextRange() return false. This is a convenience + * constructor allowing the target to be set later. + * @stable ICU 2.4 + */ + UnicodeSetIterator(); + + /** + * Destructor. + * @stable ICU 2.4 + */ + virtual ~UnicodeSetIterator(); + + /** + * Returns true if the current element is a string. If so, the + * caller can retrieve it with getString(). If this + * method returns false, the current element is a code point or + * code point range, depending on whether next() or + * nextRange() was called. + * Elements of types string and codepoint can both be retrieved + * with the function getString(). + * Elements of type codepoint can also be retrieved with + * getCodepoint(). + * For ranges, getCodepoint() returns the starting codepoint + * of the range, and getCodepointEnd() returns the end + * of the range. + * @stable ICU 2.4 + */ + inline UBool isString() const; + + /** + * Returns the current code point, if isString() returned + * false. Otherwise returns an undefined result. + * @stable ICU 2.4 + */ + inline UChar32 getCodepoint() const; + + /** + * Returns the end of the current code point range, if + * isString() returned false and nextRange() was + * called. Otherwise returns an undefined result. + * @stable ICU 2.4 + */ + inline UChar32 getCodepointEnd() const; + + /** + * Returns the current string, if isString() returned + * true. If the current iteration item is a code point, a UnicodeString + * containing that single code point is returned. + * + * Ownership of the returned string remains with the iterator. + * The string is guaranteed to remain valid only until the iterator is + * advanced to the next item, or until the iterator is deleted. + * + * @stable ICU 2.4 + */ + const UnicodeString& getString(); + + /** + * Advances the iteration position to the next element in the set, + * which can be either a single code point or a string. + * If there are no more elements in the set, return false. + * + *

+ * If isString() == TRUE, the value is a + * string, otherwise the value is a + * single code point. Elements of either type can be retrieved + * with the function getString(), while elements of + * consisting of a single code point can be retrieved with + * getCodepoint() + * + *

The order of iteration is all code points in sorted order, + * followed by all strings sorted order. Do not mix + * calls to next() and nextRange() without + * calling reset() between them. The results of doing so + * are undefined. + * + * @return true if there was another element in the set. + * @stable ICU 2.4 + */ + UBool next(); + + /** + * Returns the next element in the set, either a code point range + * or a string. If there are no more elements in the set, return + * false. If isString() == TRUE, the value is a + * string and can be accessed with getString(). Otherwise the value is a + * range of one or more code points from getCodepoint() to + * getCodepointeEnd() inclusive. + * + *

The order of iteration is all code points ranges in sorted + * order, followed by all strings sorted order. Ranges are + * disjoint and non-contiguous. The value returned from getString() + * is undefined unless isString() == TRUE. Do not mix calls to + * next() and nextRange() without calling + * reset() between them. The results of doing so are + * undefined. + * + * @return true if there was another element in the set. + * @stable ICU 2.4 + */ + UBool nextRange(); + + /** + * Sets this iterator to visit the elements of the given set and + * resets it to the start of that set. The iterator is valid only + * so long as set is valid. + * @param set the set to iterate over. + * @stable ICU 2.4 + */ + void reset(const UnicodeSet& set); + + /** + * Resets this iterator to the start of the set. + * @stable ICU 2.4 + */ + void reset(); + + /** + * ICU "poor man's RTTI", returns a UClassID for this class. + * + * @stable ICU 2.4 + */ + static UClassID U_EXPORT2 getStaticClassID(); + + /** + * ICU "poor man's RTTI", returns a UClassID for the actual class. + * + * @stable ICU 2.4 + */ + virtual UClassID getDynamicClassID() const; + + // ======================= PRIVATES =========================== + + protected: + + // endElement and nextElements are really UChar32's, but we keep + // them as signed int32_t's so we can do comparisons with + // endElement set to -1. Leave them as int32_t's. + /** The set + * @stable ICU 2.4 + */ + const UnicodeSet* set; + /** End range + * @stable ICU 2.4 + */ + int32_t endRange; + /** Range + * @stable ICU 2.4 + */ + int32_t range; + /** End element + * @stable ICU 2.4 + */ + int32_t endElement; + /** Next element + * @stable ICU 2.4 + */ + int32_t nextElement; + //UBool abbreviated; + /** Next string + * @stable ICU 2.4 + */ + int32_t nextString; + /** String count + * @stable ICU 2.4 + */ + int32_t stringCount; + + /** + * Points to the string to use when the caller asks for a + * string and the current iteration item is a code point, not a string. + * @internal + */ + UnicodeString *cpString; + + /** Copy constructor. Disallowed. + * @stable ICU 2.4 + */ + UnicodeSetIterator(const UnicodeSetIterator&); // disallow + + /** Assignment operator. Disallowed. + * @stable ICU 2.4 + */ + UnicodeSetIterator& operator=(const UnicodeSetIterator&); // disallow + + /** Load range + * @stable ICU 2.4 + */ + virtual void loadRange(int32_t range); + +}; + +inline UBool UnicodeSetIterator::isString() const { + return codepoint == (UChar32)IS_STRING; +} + +inline UChar32 UnicodeSetIterator::getCodepoint() const { + return codepoint; +} + +inline UChar32 UnicodeSetIterator::getCodepointEnd() const { + return codepointEnd; +} + + +U_NAMESPACE_END + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/ushape.h b/Build/source/libs/icu/icu-xetex/common/unicode/ushape.h new file mode 100644 index 00000000000..f165e140310 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/ushape.h @@ -0,0 +1,263 @@ +/* +****************************************************************************** +* +* Copyright (C) 2000-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: ushape.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2000jun29 +* created by: Markus W. Scherer +*/ + +#ifndef __USHAPE_H__ +#define __USHAPE_H__ + +#include "unicode/utypes.h" + +/** + * \file + * \brief C API: Arabic shaping + * + */ + +/** + * Shape Arabic text on a character basis. + * + *

This function performs basic operations for "shaping" Arabic text. It is most + * useful for use with legacy data formats and legacy display technology + * (simple terminals). All operations are performed on Unicode characters.

+ * + *

Text-based shaping means that some character code points in the text are + * replaced by others depending on the context. It transforms one kind of text + * into another. In comparison, modern displays for Arabic text select + * appropriate, context-dependent font glyphs for each text element, which means + * that they transform text into a glyph vector.

+ * + *

Text transformations are necessary when modern display technology is not + * available or when text needs to be transformed to or from legacy formats that + * use "shaped" characters. Since the Arabic script is cursive, connecting + * adjacent letters to each other, computers select images for each letter based + * on the surrounding letters. This usually results in four images per Arabic + * letter: initial, middle, final, and isolated forms. In Unicode, on the other + * hand, letters are normally stored abstract, and a display system is expected + * to select the necessary glyphs. (This makes searching and other text + * processing easier because the same letter has only one code.) It is possible + * to mimic this with text transformations because there are characters in + * Unicode that are rendered as letters with a specific shape + * (or cursive connectivity). They were included for interoperability with + * legacy systems and codepages, and for unsophisticated display systems.

+ * + *

A second kind of text transformations is supported for Arabic digits: + * For compatibility with legacy codepages that only include European digits, + * it is possible to replace one set of digits by another, changing the + * character code points. These operations can be performed for either + * Arabic-Indic Digits (U+0660...U+0669) or Eastern (Extended) Arabic-Indic + * digits (U+06f0...U+06f9).

+ * + *

Some replacements may result in more or fewer characters (code points). + * By default, this means that the destination buffer may receive text with a + * length different from the source length. Some legacy systems rely on the + * length of the text to be constant. They expect extra spaces to be added + * or consumed either next to the affected character or at the end of the + * text.

+ * + *

For details about the available operations, see the description of the + * U_SHAPE_... options.

+ * + * @param source The input text. + * + * @param sourceLength The number of UChars in source. + * + * @param dest The destination buffer that will receive the results of the + * requested operations. It may be NULL only if + * destSize is 0. The source and destination must not + * overlap. + * + * @param destSize The size (capacity) of the destination buffer in UChars. + * If destSize is 0, then no output is produced, + * but the necessary buffer size is returned ("preflighting"). + * + * @param options This is a 32-bit set of flags that specify the operations + * that are performed on the input text. If no error occurs, + * then the result will always be written to the destination + * buffer. + * + * @param pErrorCode must be a valid pointer to an error code value, + * which must not indicate a failure before the function call. + * + * @return The number of UChars written to the destination buffer. + * If an error occured, then no output was written, or it may be + * incomplete. If U_BUFFER_OVERFLOW_ERROR is set, then + * the return value indicates the necessary destination buffer size. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +u_shapeArabic(const UChar *source, int32_t sourceLength, + UChar *dest, int32_t destSize, + uint32_t options, + UErrorCode *pErrorCode); + +/** + * Memory option: allow the result to have a different length than the source. + * @stable ICU 2.0 + */ +#define U_SHAPE_LENGTH_GROW_SHRINK 0 + +/** + * Memory option: the result must have the same length as the source. + * If more room is necessary, then try to consume spaces next to modified characters. + * @stable ICU 2.0 + */ +#define U_SHAPE_LENGTH_FIXED_SPACES_NEAR 1 + +/** + * Memory option: the result must have the same length as the source. + * If more room is necessary, then try to consume spaces at the end of the text. + * @stable ICU 2.0 + */ +#define U_SHAPE_LENGTH_FIXED_SPACES_AT_END 2 + +/** + * Memory option: the result must have the same length as the source. + * If more room is necessary, then try to consume spaces at the beginning of the text. + * @stable ICU 2.0 + */ +#define U_SHAPE_LENGTH_FIXED_SPACES_AT_BEGINNING 3 + +/** Bit mask for memory options. @stable ICU 2.0 */ +#define U_SHAPE_LENGTH_MASK 3 + + +/** Direction indicator: the source is in logical (keyboard) order. @stable ICU 2.0 */ +#define U_SHAPE_TEXT_DIRECTION_LOGICAL 0 + +/** + * Direction indicator: + * the source is in visual LTR order, + * the leftmost displayed character stored first. + * @stable ICU 2.0 + */ +#define U_SHAPE_TEXT_DIRECTION_VISUAL_LTR 4 + +/** Bit mask for direction indicators. @stable ICU 2.0 */ +#define U_SHAPE_TEXT_DIRECTION_MASK 4 + + +/** Letter shaping option: do not perform letter shaping. @stable ICU 2.0 */ +#define U_SHAPE_LETTERS_NOOP 0 + +/** Letter shaping option: replace abstract letter characters by "shaped" ones. @stable ICU 2.0 */ +#define U_SHAPE_LETTERS_SHAPE 8 + +/** Letter shaping option: replace "shaped" letter characters by abstract ones. @stable ICU 2.0 */ +#define U_SHAPE_LETTERS_UNSHAPE 0x10 + +/** + * Letter shaping option: replace abstract letter characters by "shaped" ones. + * The only difference with U_SHAPE_LETTERS_SHAPE is that Tashkeel letters + * are always "shaped" into the isolated form instead of the medial form + * (selecting code points from the Arabic Presentation Forms-B block). + * @stable ICU 2.0 + */ +#define U_SHAPE_LETTERS_SHAPE_TASHKEEL_ISOLATED 0x18 + +/** Bit mask for letter shaping options. @stable ICU 2.0 */ +#define U_SHAPE_LETTERS_MASK 0x18 + + +/** Digit shaping option: do not perform digit shaping. @stable ICU 2.0 */ +#define U_SHAPE_DIGITS_NOOP 0 + +/** + * Digit shaping option: + * Replace European digits (U+0030...) by Arabic-Indic digits. + * @stable ICU 2.0 + */ +#define U_SHAPE_DIGITS_EN2AN 0x20 + +/** + * Digit shaping option: + * Replace Arabic-Indic digits by European digits (U+0030...). + * @stable ICU 2.0 + */ +#define U_SHAPE_DIGITS_AN2EN 0x40 + +/** + * Digit shaping option: + * Replace European digits (U+0030...) by Arabic-Indic digits if the most recent + * strongly directional character is an Arabic letter + * (u_charDirection() result U_RIGHT_TO_LEFT_ARABIC [AL]).
+ * The direction of "preceding" depends on the direction indicator option. + * For the first characters, the preceding strongly directional character + * (initial state) is assumed to be not an Arabic letter + * (it is U_LEFT_TO_RIGHT [L] or U_RIGHT_TO_LEFT [R]). + * @stable ICU 2.0 + */ +#define U_SHAPE_DIGITS_ALEN2AN_INIT_LR 0x60 + +/** + * Digit shaping option: + * Replace European digits (U+0030...) by Arabic-Indic digits if the most recent + * strongly directional character is an Arabic letter + * (u_charDirection() result U_RIGHT_TO_LEFT_ARABIC [AL]).
+ * The direction of "preceding" depends on the direction indicator option. + * For the first characters, the preceding strongly directional character + * (initial state) is assumed to be an Arabic letter. + * @stable ICU 2.0 + */ +#define U_SHAPE_DIGITS_ALEN2AN_INIT_AL 0x80 + +/** Not a valid option value. May be replaced by a new option. @stable ICU 2.0 */ +#define U_SHAPE_DIGITS_RESERVED 0xa0 + +/** Bit mask for digit shaping options. @stable ICU 2.0 */ +#define U_SHAPE_DIGITS_MASK 0xe0 + + +/** Digit type option: Use Arabic-Indic digits (U+0660...U+0669). @stable ICU 2.0 */ +#define U_SHAPE_DIGIT_TYPE_AN 0 + +/** Digit type option: Use Eastern (Extended) Arabic-Indic digits (U+06f0...U+06f9). @stable ICU 2.0 */ +#define U_SHAPE_DIGIT_TYPE_AN_EXTENDED 0x100 + +/** Not a valid option value. May be replaced by a new option. @stable ICU 2.0 */ +#define U_SHAPE_DIGIT_TYPE_RESERVED 0x200 + +/** Bit mask for digit type options. @stable ICU 2.0 */ +#define U_SHAPE_DIGIT_TYPE_MASK 0x3f00 + +/** + * Tashkeel aggregation option: + * Replaces any combination of U+0651 with one of + * U+064C, U+064D, U+064E, U+064F, U+0650 with + * U+FC5E, U+FC5F, U+FC60, U+FC61, U+FC62 consecutively. + * @stable ICU 3.6 + */ +#define U_SHAPE_AGGREGATE_TASHKEEL 0x4000 +/** Tashkeel aggregation option: do not aggregate tashkeels. @stable ICU 3.6 */ +#define U_SHAPE_AGGREGATE_TASHKEEL_NOOP 0 +/** Bit mask for tashkeel aggregation. @stable ICU 3.6 */ +#define U_SHAPE_AGGREGATE_TASHKEEL_MASK 0x4000 + +/** + * Presentation form option: + * Don't replace Arabic Presentation Forms-A and Arabic Presentation Forms-B + * characters with 0+06xx characters, before shaping. + * @stable ICU 3.6 + */ +#define U_SHAPE_PRESERVE_PRESENTATION 0x8000 +/** Presentation form option: + * Replace Arabic Presentation Forms-A and Arabic Presentationo Forms-B with + * their unshaped correspondants in range 0+06xx, before shaping. + * @stable ICU 3.6 + */ +#define U_SHAPE_PRESERVE_PRESENTATION_NOOP 0 +/** Bit mask for preserve presentation form. @stable ICU 3.6 */ +#define U_SHAPE_PRESERVE_PRESENTATION_MASK 0x8000 + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/usprep.h b/Build/source/libs/icu/icu-xetex/common/unicode/usprep.h new file mode 100644 index 00000000000..c7e75a53fab --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/usprep.h @@ -0,0 +1,156 @@ +/* + ******************************************************************************* + * + * Copyright (C) 2003-2006, International Business Machines + * Corporation and others. All Rights Reserved. + * + ******************************************************************************* + * file name: usprep.h + * encoding: US-ASCII + * tab size: 8 (not used) + * indentation:4 + * + * created on: 2003jul2 + * created by: Ram Viswanadha + */ + +#ifndef __USPREP_H__ +#define __USPREP_H__ + +/** + * \file + * \brief C API: Implements the StringPrep algorithm. + */ + +#include "unicode/utypes.h" +/** + * + * StringPrep API implements the StingPrep framework as described by RFC 3454. + * StringPrep prepares Unicode strings for use in network protocols. + * Profiles of StingPrep are set of rules and data according to with the + * Unicode Strings are prepared. Each profiles contains tables which describe + * how a code point should be treated. The tables are broadly classied into + *
    + *
  • Unassinged Table: Contains code points that are unassigned + * in the Unicode Version supported by StringPrep. Currently + * RFC 3454 supports Unicode 3.2.
  • + *
  • Prohibited Table: Contains code points that are prohibted from + * the output of the StringPrep processing function.
  • + *
  • Mapping Table: Contains code ponts that are deleted from the output or case mapped.
  • + *
+ * + * The procedure for preparing Unicode strings: + *
    + *
  1. Map: For each character in the input, check if it has a mapping + * and, if so, replace it with its mapping.
  2. + *
  3. Normalize: Possibly normalize the result of step 1 using Unicode + * normalization.
  4. + *
  5. Prohibit: Check for any characters that are not allowed in the + * output. If any are found, return an error.
  6. + *
  7. Check bidi: Possibly check for right-to-left characters, and if + * any are found, make sure that the whole string satisfies the + * requirements for bidirectional strings. If the string does not + * satisfy the requirements for bidirectional strings, return an + * error.
  8. + *
+ * @author Ram Viswanadha + */ +#if !UCONFIG_NO_IDNA + +#include "unicode/parseerr.h" + +/** + * The StringPrep profile + * @stable ICU 2.8 + */ +typedef struct UStringPrepProfile UStringPrepProfile; + + +/** + * Option to prohibit processing of unassigned code points in the input + * + * @see usprep_prepare + * @stable ICU 2.8 + */ +#define USPREP_DEFAULT 0x0000 + +/** + * Option to allow processing of unassigned code points in the input + * + * @see usprep_prepare + * @stable ICU 2.8 + */ +#define USPREP_ALLOW_UNASSIGNED 0x0001 + + +/** + * Creates a StringPrep profile from the data file. + * + * @param path string containing the full path pointing to the directory + * where the profile reside followed by the package name + * e.g. "/usr/resource/my_app/profiles/mydata" on a Unix system. + * if NULL, ICU default data files will be used. + * @param fileName name of the profile file to be opened + * @param status ICU error code in/out parameter. Must not be NULL. + * Must fulfill U_SUCCESS before the function call. + * @return Pointer to UStringPrepProfile that is opened. Should be closed by + * calling usprep_close() + * @see usprep_close() + * @stable ICU 2.8 + */ +U_STABLE UStringPrepProfile* U_EXPORT2 +usprep_open(const char* path, + const char* fileName, + UErrorCode* status); + + +/** + * Closes the profile + * @param profile The profile to close + * @stable ICU 2.8 + */ +U_STABLE void U_EXPORT2 +usprep_close(UStringPrepProfile* profile); + + +/** + * Prepare the input buffer for use in applications with the given profile. This operation maps, normalizes(NFKC), + * checks for prohited and BiDi characters in the order defined by RFC 3454 + * depending on the options specified in the profile. + * + * @param prep The profile to use + * @param src Pointer to UChar buffer containing the string to prepare + * @param srcLength Number of characters in the source string + * @param dest Pointer to the destination buffer to receive the output + * @param destCapacity The capacity of destination array + * @param options A bit set of options: + * + * - USPREP_NONE Prohibit processing of unassigned code points in the input + * + * - USPREP_ALLOW_UNASSIGNED Treat the unassigned code points are in the input + * as normal Unicode code points. + * + * @param parseError Pointer to UParseError struct to receive information on position + * of error if an error is encountered. Can be NULL. + * @param status ICU in/out error code parameter. + * U_INVALID_CHAR_FOUND if src contains + * unmatched single surrogates. + * U_INDEX_OUTOFBOUNDS_ERROR if src contains + * too many code points. + * U_BUFFER_OVERFLOW_ERROR if destCapacity is not enough + * @return The number of UChars in the destination buffer + * @stable ICU 2.8 + */ + +U_STABLE int32_t U_EXPORT2 +usprep_prepare( const UStringPrepProfile* prep, + const UChar* src, int32_t srcLength, + UChar* dest, int32_t destCapacity, + int32_t options, + UParseError* parseError, + UErrorCode* status ); + + +#endif /* #if !UCONFIG_NO_IDNA */ + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/ustring.h b/Build/source/libs/icu/icu-xetex/common/unicode/ustring.h new file mode 100644 index 00000000000..f6a79478b60 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/ustring.h @@ -0,0 +1,1478 @@ +/* +********************************************************************** +* Copyright (C) 1998-2007, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* +* File ustring.h +* +* Modification History: +* +* Date Name Description +* 12/07/98 bertrand Creation. +****************************************************************************** +*/ + +#ifndef USTRING_H +#define USTRING_H + +#include "unicode/utypes.h" +#include "unicode/putil.h" +#include "unicode/uiter.h" + +/** Simple declaration for u_strToTitle() to avoid including unicode/ubrk.h. @stable ICU 2.1*/ +#ifndef UBRK_TYPEDEF_UBREAK_ITERATOR +# define UBRK_TYPEDEF_UBREAK_ITERATOR + typedef void UBreakIterator; +#endif + +/** + * \file + * \brief C API: Unicode string handling functions + * + * These C API functions provide general Unicode string handling. + * + * Some functions are equivalent in name, signature, and behavior to the ANSI C + * functions. (For example, they do not check for bad arguments like NULL string pointers.) + * In some cases, only the thread-safe variant of such a function is implemented here + * (see u_strtok_r()). + * + * Other functions provide more Unicode-specific functionality like locale-specific + * upper/lower-casing and string comparison in code point order. + * + * ICU uses 16-bit Unicode (UTF-16) in the form of arrays of UChar code units. + * UTF-16 encodes each Unicode code point with either one or two UChar code units. + * (This is the default form of Unicode, and a forward-compatible extension of the original, + * fixed-width form that was known as UCS-2. UTF-16 superseded UCS-2 with Unicode 2.0 + * in 1996.) + * + * Some APIs accept a 32-bit UChar32 value for a single code point. + * + * ICU also handles 16-bit Unicode text with unpaired surrogates. + * Such text is not well-formed UTF-16. + * Code-point-related functions treat unpaired surrogates as surrogate code points, + * i.e., as separate units. + * + * Although UTF-16 is a variable-width encoding form (like some legacy multi-byte encodings), + * it is much more efficient even for random access because the code unit values + * for single-unit characters vs. lead units vs. trail units are completely disjoint. + * This means that it is easy to determine character (code point) boundaries from + * random offsets in the string. + * + * Unicode (UTF-16) string processing is optimized for the single-unit case. + * Although it is important to support supplementary characters + * (which use pairs of lead/trail code units called "surrogates"), + * their occurrence is rare. Almost all characters in modern use require only + * a single UChar code unit (i.e., their code point values are <=0xffff). + * + * For more details see the User Guide Strings chapter (http://icu-project.org/userguide/strings.html). + * For a discussion of the handling of unpaired surrogates see also + * Jitterbug 2145 and its icu mailing list proposal on 2002-sep-18. + */ + +/** +* \defgroup ustring_ustrlen + */ +/*@{*/ +/** + * Determine the length of an array of UChar. + * + * @param s The array of UChars, NULL (U+0000) terminated. + * @return The number of UChars in chars, minus the terminator. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +u_strlen(const UChar *s); +/*@}*/ + +/** + * Count Unicode code points in the length UChar code units of the string. + * A code point may occupy either one or two UChar code units. + * Counting code points involves reading all code units. + * + * This functions is basically the inverse of the U16_FWD_N() macro (see utf.h). + * + * @param s The input string. + * @param length The number of UChar code units to be checked, or -1 to count all + * code points before the first NUL (U+0000). + * @return The number of code points in the specified code units. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +u_countChar32(const UChar *s, int32_t length); + +/** + * Check if the string contains more Unicode code points than a certain number. + * This is more efficient than counting all code points in the entire string + * and comparing that number with a threshold. + * This function may not need to scan the string at all if the length is known + * (not -1 for NUL-termination) and falls within a certain range, and + * never needs to count more than 'number+1' code points. + * Logically equivalent to (u_countChar32(s, length)>number). + * A Unicode code point may occupy either one or two UChar code units. + * + * @param s The input string. + * @param length The length of the string, or -1 if it is NUL-terminated. + * @param number The number of code points in the string is compared against + * the 'number' parameter. + * @return Boolean value for whether the string contains more Unicode code points + * than 'number'. Same as (u_countChar32(s, length)>number). + * @stable ICU 2.4 + */ +U_STABLE UBool U_EXPORT2 +u_strHasMoreChar32Than(const UChar *s, int32_t length, int32_t number); + +/** + * Concatenate two ustrings. Appends a copy of src, + * including the null terminator, to dst. The initial copied + * character from src overwrites the null terminator in dst. + * + * @param dst The destination string. + * @param src The source string. + * @return A pointer to dst. + * @stable ICU 2.0 + */ +U_STABLE UChar* U_EXPORT2 +u_strcat(UChar *dst, + const UChar *src); + +/** + * Concatenate two ustrings. + * Appends at most n characters from src to dst. + * Adds a terminating NUL. + * If src is too long, then only n-1 characters will be copied + * before the terminating NUL. + * If n<=0 then dst is not modified. + * + * @param dst The destination string. + * @param src The source string. + * @param n The maximum number of characters to compare. + * @return A pointer to dst. + * @stable ICU 2.0 + */ +U_STABLE UChar* U_EXPORT2 +u_strncat(UChar *dst, + const UChar *src, + int32_t n); + +/** + * Find the first occurrence of a substring in a string. + * The substring is found at code point boundaries. + * That means that if the substring begins with + * a trail surrogate or ends with a lead surrogate, + * then it is found only if these surrogates stand alone in the text. + * Otherwise, the substring edge units would be matched against + * halves of surrogate pairs. + * + * @param s The string to search (NUL-terminated). + * @param substring The substring to find (NUL-terminated). + * @return A pointer to the first occurrence of substring in s, + * or s itself if the substring is empty, + * or NULL if substring is not in s. + * @stable ICU 2.0 + * + * @see u_strrstr + * @see u_strFindFirst + * @see u_strFindLast + */ +U_STABLE UChar * U_EXPORT2 +u_strstr(const UChar *s, const UChar *substring); + +/** + * Find the first occurrence of a substring in a string. + * The substring is found at code point boundaries. + * That means that if the substring begins with + * a trail surrogate or ends with a lead surrogate, + * then it is found only if these surrogates stand alone in the text. + * Otherwise, the substring edge units would be matched against + * halves of surrogate pairs. + * + * @param s The string to search. + * @param length The length of s (number of UChars), or -1 if it is NUL-terminated. + * @param substring The substring to find (NUL-terminated). + * @param subLength The length of substring (number of UChars), or -1 if it is NUL-terminated. + * @return A pointer to the first occurrence of substring in s, + * or s itself if the substring is empty, + * or NULL if substring is not in s. + * @stable ICU 2.4 + * + * @see u_strstr + * @see u_strFindLast + */ +U_STABLE UChar * U_EXPORT2 +u_strFindFirst(const UChar *s, int32_t length, const UChar *substring, int32_t subLength); + +/** + * Find the first occurrence of a BMP code point in a string. + * A surrogate code point is found only if its match in the text is not + * part of a surrogate pair. + * A NUL character is found at the string terminator. + * + * @param s The string to search (NUL-terminated). + * @param c The BMP code point to find. + * @return A pointer to the first occurrence of c in s + * or NULL if c is not in s. + * @stable ICU 2.0 + * + * @see u_strchr32 + * @see u_memchr + * @see u_strstr + * @see u_strFindFirst + */ +U_STABLE UChar * U_EXPORT2 +u_strchr(const UChar *s, UChar c); + +/** + * Find the first occurrence of a code point in a string. + * A surrogate code point is found only if its match in the text is not + * part of a surrogate pair. + * A NUL character is found at the string terminator. + * + * @param s The string to search (NUL-terminated). + * @param c The code point to find. + * @return A pointer to the first occurrence of c in s + * or NULL if c is not in s. + * @stable ICU 2.0 + * + * @see u_strchr + * @see u_memchr32 + * @see u_strstr + * @see u_strFindFirst + */ +U_STABLE UChar * U_EXPORT2 +u_strchr32(const UChar *s, UChar32 c); + +/** + * Find the last occurrence of a substring in a string. + * The substring is found at code point boundaries. + * That means that if the substring begins with + * a trail surrogate or ends with a lead surrogate, + * then it is found only if these surrogates stand alone in the text. + * Otherwise, the substring edge units would be matched against + * halves of surrogate pairs. + * + * @param s The string to search (NUL-terminated). + * @param substring The substring to find (NUL-terminated). + * @return A pointer to the last occurrence of substring in s, + * or s itself if the substring is empty, + * or NULL if substring is not in s. + * @stable ICU 2.4 + * + * @see u_strstr + * @see u_strFindFirst + * @see u_strFindLast + */ +U_STABLE UChar * U_EXPORT2 +u_strrstr(const UChar *s, const UChar *substring); + +/** + * Find the last occurrence of a substring in a string. + * The substring is found at code point boundaries. + * That means that if the substring begins with + * a trail surrogate or ends with a lead surrogate, + * then it is found only if these surrogates stand alone in the text. + * Otherwise, the substring edge units would be matched against + * halves of surrogate pairs. + * + * @param s The string to search. + * @param length The length of s (number of UChars), or -1 if it is NUL-terminated. + * @param substring The substring to find (NUL-terminated). + * @param subLength The length of substring (number of UChars), or -1 if it is NUL-terminated. + * @return A pointer to the last occurrence of substring in s, + * or s itself if the substring is empty, + * or NULL if substring is not in s. + * @stable ICU 2.4 + * + * @see u_strstr + * @see u_strFindLast + */ +U_STABLE UChar * U_EXPORT2 +u_strFindLast(const UChar *s, int32_t length, const UChar *substring, int32_t subLength); + +/** + * Find the last occurrence of a BMP code point in a string. + * A surrogate code point is found only if its match in the text is not + * part of a surrogate pair. + * A NUL character is found at the string terminator. + * + * @param s The string to search (NUL-terminated). + * @param c The BMP code point to find. + * @return A pointer to the last occurrence of c in s + * or NULL if c is not in s. + * @stable ICU 2.4 + * + * @see u_strrchr32 + * @see u_memrchr + * @see u_strrstr + * @see u_strFindLast + */ +U_STABLE UChar * U_EXPORT2 +u_strrchr(const UChar *s, UChar c); + +/** + * Find the last occurrence of a code point in a string. + * A surrogate code point is found only if its match in the text is not + * part of a surrogate pair. + * A NUL character is found at the string terminator. + * + * @param s The string to search (NUL-terminated). + * @param c The code point to find. + * @return A pointer to the last occurrence of c in s + * or NULL if c is not in s. + * @stable ICU 2.4 + * + * @see u_strrchr + * @see u_memchr32 + * @see u_strrstr + * @see u_strFindLast + */ +U_STABLE UChar * U_EXPORT2 +u_strrchr32(const UChar *s, UChar32 c); + +/** + * Locates the first occurrence in the string string of any of the characters + * in the string matchSet. + * Works just like C's strpbrk but with Unicode. + * + * @param string The string in which to search, NUL-terminated. + * @param matchSet A NUL-terminated string defining a set of code points + * for which to search in the text string. + * @return A pointer to the character in string that matches one of the + * characters in matchSet, or NULL if no such character is found. + * @stable ICU 2.0 + */ +U_STABLE UChar * U_EXPORT2 +u_strpbrk(const UChar *string, const UChar *matchSet); + +/** + * Returns the number of consecutive characters in string, + * beginning with the first, that do not occur somewhere in matchSet. + * Works just like C's strcspn but with Unicode. + * + * @param string The string in which to search, NUL-terminated. + * @param matchSet A NUL-terminated string defining a set of code points + * for which to search in the text string. + * @return The number of initial characters in string that do not + * occur in matchSet. + * @see u_strspn + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +u_strcspn(const UChar *string, const UChar *matchSet); + +/** + * Returns the number of consecutive characters in string, + * beginning with the first, that occur somewhere in matchSet. + * Works just like C's strspn but with Unicode. + * + * @param string The string in which to search, NUL-terminated. + * @param matchSet A NUL-terminated string defining a set of code points + * for which to search in the text string. + * @return The number of initial characters in string that do + * occur in matchSet. + * @see u_strcspn + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +u_strspn(const UChar *string, const UChar *matchSet); + +/** + * The string tokenizer API allows an application to break a string into + * tokens. Unlike strtok(), the saveState (the current pointer within the + * original string) is maintained in saveState. In the first call, the + * argument src is a pointer to the string. In subsequent calls to + * return successive tokens of that string, src must be specified as + * NULL. The value saveState is set by this function to maintain the + * function's position within the string, and on each subsequent call + * you must give this argument the same variable. This function does + * handle surrogate pairs. This function is similar to the strtok_r() + * the POSIX Threads Extension (1003.1c-1995) version. + * + * @param src String containing token(s). This string will be modified. + * After the first call to u_strtok_r(), this argument must + * be NULL to get to the next token. + * @param delim Set of delimiter characters (Unicode code points). + * @param saveState The current pointer within the original string, + * which is set by this function. The saveState + * parameter should the address of a local variable of type + * UChar *. (i.e. defined "Uhar *myLocalSaveState" and use + * &myLocalSaveState for this parameter). + * @return A pointer to the next token found in src, or NULL + * when there are no more tokens. + * @stable ICU 2.0 + */ +U_STABLE UChar * U_EXPORT2 +u_strtok_r(UChar *src, + const UChar *delim, + UChar **saveState); + +/** + * Compare two Unicode strings for bitwise equality (code unit order). + * + * @param s1 A string to compare. + * @param s2 A string to compare. + * @return 0 if s1 and s2 are bitwise equal; a negative + * value if s1 is bitwise less than s2,; a positive + * value if s1 is bitwise greater than s2. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +u_strcmp(const UChar *s1, + const UChar *s2); + +/** + * Compare two Unicode strings in code point order. + * See u_strCompare for details. + * + * @param s1 A string to compare. + * @param s2 A string to compare. + * @return a negative/zero/positive integer corresponding to whether + * the first string is less than/equal to/greater than the second one + * in code point order + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +u_strcmpCodePointOrder(const UChar *s1, const UChar *s2); + +/** + * Compare two Unicode strings (binary order). + * + * The comparison can be done in code unit order or in code point order. + * They differ only in UTF-16 when + * comparing supplementary code points (U+10000..U+10ffff) + * to BMP code points near the end of the BMP (i.e., U+e000..U+ffff). + * In code unit order, high BMP code points sort after supplementary code points + * because they are stored as pairs of surrogates which are at U+d800..U+dfff. + * + * This functions works with strings of different explicitly specified lengths + * unlike the ANSI C-like u_strcmp() and u_memcmp() etc. + * NUL-terminated strings are possible with length arguments of -1. + * + * @param s1 First source string. + * @param length1 Length of first source string, or -1 if NUL-terminated. + * + * @param s2 Second source string. + * @param length2 Length of second source string, or -1 if NUL-terminated. + * + * @param codePointOrder Choose between code unit order (FALSE) + * and code point order (TRUE). + * + * @return <0 or 0 or >0 as usual for string comparisons + * + * @stable ICU 2.2 + */ +U_STABLE int32_t U_EXPORT2 +u_strCompare(const UChar *s1, int32_t length1, + const UChar *s2, int32_t length2, + UBool codePointOrder); + +/** + * Compare two Unicode strings (binary order) + * as presented by UCharIterator objects. + * Works otherwise just like u_strCompare(). + * + * Both iterators are reset to their start positions. + * When the function returns, it is undefined where the iterators + * have stopped. + * + * @param iter1 First source string iterator. + * @param iter2 Second source string iterator. + * @param codePointOrder Choose between code unit order (FALSE) + * and code point order (TRUE). + * + * @return <0 or 0 or >0 as usual for string comparisons + * + * @see u_strCompare + * + * @stable ICU 2.6 + */ +U_STABLE int32_t U_EXPORT2 +u_strCompareIter(UCharIterator *iter1, UCharIterator *iter2, UBool codePointOrder); + +#ifndef U_COMPARE_CODE_POINT_ORDER +/* see also unistr.h and unorm.h */ +/** + * Option bit for u_strCaseCompare, u_strcasecmp, unorm_compare, etc: + * Compare strings in code point order instead of code unit order. + * @stable ICU 2.2 + */ +#define U_COMPARE_CODE_POINT_ORDER 0x8000 +#endif + +/** + * Compare two strings case-insensitively using full case folding. + * This is equivalent to + * u_strCompare(u_strFoldCase(s1, options), + * u_strFoldCase(s2, options), + * (options&U_COMPARE_CODE_POINT_ORDER)!=0). + * + * The comparison can be done in UTF-16 code unit order or in code point order. + * They differ only when comparing supplementary code points (U+10000..U+10ffff) + * to BMP code points near the end of the BMP (i.e., U+e000..U+ffff). + * In code unit order, high BMP code points sort after supplementary code points + * because they are stored as pairs of surrogates which are at U+d800..U+dfff. + * + * This functions works with strings of different explicitly specified lengths + * unlike the ANSI C-like u_strcmp() and u_memcmp() etc. + * NUL-terminated strings are possible with length arguments of -1. + * + * @param s1 First source string. + * @param length1 Length of first source string, or -1 if NUL-terminated. + * + * @param s2 Second source string. + * @param length2 Length of second source string, or -1 if NUL-terminated. + * + * @param options A bit set of options: + * - U_FOLD_CASE_DEFAULT or 0 is used for default options: + * Comparison in code unit order with default case folding. + * + * - U_COMPARE_CODE_POINT_ORDER + * Set to choose code point order instead of code unit order + * (see u_strCompare for details). + * + * - U_FOLD_CASE_EXCLUDE_SPECIAL_I + * + * @param pErrorCode Must be a valid pointer to an error code value, + * which must not indicate a failure before the function call. + * + * @return <0 or 0 or >0 as usual for string comparisons + * + * @stable ICU 2.2 + */ +U_STABLE int32_t U_EXPORT2 +u_strCaseCompare(const UChar *s1, int32_t length1, + const UChar *s2, int32_t length2, + uint32_t options, + UErrorCode *pErrorCode); + +/** + * Compare two ustrings for bitwise equality. + * Compares at most n characters. + * + * @param ucs1 A string to compare. + * @param ucs2 A string to compare. + * @param n The maximum number of characters to compare. + * @return 0 if s1 and s2 are bitwise equal; a negative + * value if s1 is bitwise less than s2; a positive + * value if s1 is bitwise greater than s2. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +u_strncmp(const UChar *ucs1, + const UChar *ucs2, + int32_t n); + +/** + * Compare two Unicode strings in code point order. + * This is different in UTF-16 from u_strncmp() if supplementary characters are present. + * For details, see u_strCompare(). + * + * @param s1 A string to compare. + * @param s2 A string to compare. + * @param n The maximum number of characters to compare. + * @return a negative/zero/positive integer corresponding to whether + * the first string is less than/equal to/greater than the second one + * in code point order + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +u_strncmpCodePointOrder(const UChar *s1, const UChar *s2, int32_t n); + +/** + * Compare two strings case-insensitively using full case folding. + * This is equivalent to u_strcmp(u_strFoldCase(s1, options), u_strFoldCase(s2, options)). + * + * @param s1 A string to compare. + * @param s2 A string to compare. + * @param options A bit set of options: + * - U_FOLD_CASE_DEFAULT or 0 is used for default options: + * Comparison in code unit order with default case folding. + * + * - U_COMPARE_CODE_POINT_ORDER + * Set to choose code point order instead of code unit order + * (see u_strCompare for details). + * + * - U_FOLD_CASE_EXCLUDE_SPECIAL_I + * + * @return A negative, zero, or positive integer indicating the comparison result. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +u_strcasecmp(const UChar *s1, const UChar *s2, uint32_t options); + +/** + * Compare two strings case-insensitively using full case folding. + * This is equivalent to u_strcmp(u_strFoldCase(s1, at most n, options), + * u_strFoldCase(s2, at most n, options)). + * + * @param s1 A string to compare. + * @param s2 A string to compare. + * @param n The maximum number of characters each string to case-fold and then compare. + * @param options A bit set of options: + * - U_FOLD_CASE_DEFAULT or 0 is used for default options: + * Comparison in code unit order with default case folding. + * + * - U_COMPARE_CODE_POINT_ORDER + * Set to choose code point order instead of code unit order + * (see u_strCompare for details). + * + * - U_FOLD_CASE_EXCLUDE_SPECIAL_I + * + * @return A negative, zero, or positive integer indicating the comparison result. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +u_strncasecmp(const UChar *s1, const UChar *s2, int32_t n, uint32_t options); + +/** + * Compare two strings case-insensitively using full case folding. + * This is equivalent to u_strcmp(u_strFoldCase(s1, n, options), + * u_strFoldCase(s2, n, options)). + * + * @param s1 A string to compare. + * @param s2 A string to compare. + * @param length The number of characters in each string to case-fold and then compare. + * @param options A bit set of options: + * - U_FOLD_CASE_DEFAULT or 0 is used for default options: + * Comparison in code unit order with default case folding. + * + * - U_COMPARE_CODE_POINT_ORDER + * Set to choose code point order instead of code unit order + * (see u_strCompare for details). + * + * - U_FOLD_CASE_EXCLUDE_SPECIAL_I + * + * @return A negative, zero, or positive integer indicating the comparison result. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +u_memcasecmp(const UChar *s1, const UChar *s2, int32_t length, uint32_t options); + +/** + * Copy a ustring. Adds a null terminator. + * + * @param dst The destination string. + * @param src The source string. + * @return A pointer to dst. + * @stable ICU 2.0 + */ +U_STABLE UChar* U_EXPORT2 +u_strcpy(UChar *dst, + const UChar *src); + +/** + * Copy a ustring. + * Copies at most n characters. The result will be null terminated + * if the length of src is less than n. + * + * @param dst The destination string. + * @param src The source string. + * @param n The maximum number of characters to copy. + * @return A pointer to dst. + * @stable ICU 2.0 + */ +U_STABLE UChar* U_EXPORT2 +u_strncpy(UChar *dst, + const UChar *src, + int32_t n); + +#if !UCONFIG_NO_CONVERSION + +/** + * Copy a byte string encoded in the default codepage to a ustring. + * Adds a null terminator. + * Performs a host byte to UChar conversion + * + * @param dst The destination string. + * @param src The source string. + * @return A pointer to dst. + * @stable ICU 2.0 + */ +U_STABLE UChar* U_EXPORT2 u_uastrcpy(UChar *dst, + const char *src ); + +/** + * Copy a byte string encoded in the default codepage to a ustring. + * Copies at most n characters. The result will be null terminated + * if the length of src is less than n. + * Performs a host byte to UChar conversion + * + * @param dst The destination string. + * @param src The source string. + * @param n The maximum number of characters to copy. + * @return A pointer to dst. + * @stable ICU 2.0 + */ +U_STABLE UChar* U_EXPORT2 u_uastrncpy(UChar *dst, + const char *src, + int32_t n); + +/** + * Copy ustring to a byte string encoded in the default codepage. + * Adds a null terminator. + * Performs a UChar to host byte conversion + * + * @param dst The destination string. + * @param src The source string. + * @return A pointer to dst. + * @stable ICU 2.0 + */ +U_STABLE char* U_EXPORT2 u_austrcpy(char *dst, + const UChar *src ); + +/** + * Copy ustring to a byte string encoded in the default codepage. + * Copies at most n characters. The result will be null terminated + * if the length of src is less than n. + * Performs a UChar to host byte conversion + * + * @param dst The destination string. + * @param src The source string. + * @param n The maximum number of characters to copy. + * @return A pointer to dst. + * @stable ICU 2.0 + */ +U_STABLE char* U_EXPORT2 u_austrncpy(char *dst, + const UChar *src, + int32_t n ); + +#endif + +/** + * Synonym for memcpy(), but with UChars only. + * @param dest The destination string + * @param src The source string + * @param count The number of characters to copy + * @return A pointer to dest + * @stable ICU 2.0 + */ +U_STABLE UChar* U_EXPORT2 +u_memcpy(UChar *dest, const UChar *src, int32_t count); + +/** + * Synonym for memmove(), but with UChars only. + * @param dest The destination string + * @param src The source string + * @param count The number of characters to move + * @return A pointer to dest + * @stable ICU 2.0 + */ +U_STABLE UChar* U_EXPORT2 +u_memmove(UChar *dest, const UChar *src, int32_t count); + +/** + * Initialize count characters of dest to c. + * + * @param dest The destination string. + * @param c The character to initialize the string. + * @param count The maximum number of characters to set. + * @return A pointer to dest. + * @stable ICU 2.0 + */ +U_STABLE UChar* U_EXPORT2 +u_memset(UChar *dest, UChar c, int32_t count); + +/** + * Compare the first count UChars of each buffer. + * + * @param buf1 The first string to compare. + * @param buf2 The second string to compare. + * @param count The maximum number of UChars to compare. + * @return When buf1 < buf2, a negative number is returned. + * When buf1 == buf2, 0 is returned. + * When buf1 > buf2, a positive number is returned. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +u_memcmp(const UChar *buf1, const UChar *buf2, int32_t count); + +/** + * Compare two Unicode strings in code point order. + * This is different in UTF-16 from u_memcmp() if supplementary characters are present. + * For details, see u_strCompare(). + * + * @param s1 A string to compare. + * @param s2 A string to compare. + * @param count The maximum number of characters to compare. + * @return a negative/zero/positive integer corresponding to whether + * the first string is less than/equal to/greater than the second one + * in code point order + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +u_memcmpCodePointOrder(const UChar *s1, const UChar *s2, int32_t count); + +/** + * Find the first occurrence of a BMP code point in a string. + * A surrogate code point is found only if its match in the text is not + * part of a surrogate pair. + * A NUL character is found at the string terminator. + * + * @param s The string to search (contains count UChars). + * @param c The BMP code point to find. + * @param count The length of the string. + * @return A pointer to the first occurrence of c in s + * or NULL if c is not in s. + * @stable ICU 2.0 + * + * @see u_strchr + * @see u_memchr32 + * @see u_strFindFirst + */ +U_STABLE UChar* U_EXPORT2 +u_memchr(const UChar *s, UChar c, int32_t count); + +/** + * Find the first occurrence of a code point in a string. + * A surrogate code point is found only if its match in the text is not + * part of a surrogate pair. + * A NUL character is found at the string terminator. + * + * @param s The string to search (contains count UChars). + * @param c The code point to find. + * @param count The length of the string. + * @return A pointer to the first occurrence of c in s + * or NULL if c is not in s. + * @stable ICU 2.0 + * + * @see u_strchr32 + * @see u_memchr + * @see u_strFindFirst + */ +U_STABLE UChar* U_EXPORT2 +u_memchr32(const UChar *s, UChar32 c, int32_t count); + +/** + * Find the last occurrence of a BMP code point in a string. + * A surrogate code point is found only if its match in the text is not + * part of a surrogate pair. + * A NUL character is found at the string terminator. + * + * @param s The string to search (contains count UChars). + * @param c The BMP code point to find. + * @param count The length of the string. + * @return A pointer to the last occurrence of c in s + * or NULL if c is not in s. + * @stable ICU 2.4 + * + * @see u_strrchr + * @see u_memrchr32 + * @see u_strFindLast + */ +U_STABLE UChar* U_EXPORT2 +u_memrchr(const UChar *s, UChar c, int32_t count); + +/** + * Find the last occurrence of a code point in a string. + * A surrogate code point is found only if its match in the text is not + * part of a surrogate pair. + * A NUL character is found at the string terminator. + * + * @param s The string to search (contains count UChars). + * @param c The code point to find. + * @param count The length of the string. + * @return A pointer to the last occurrence of c in s + * or NULL if c is not in s. + * @stable ICU 2.4 + * + * @see u_strrchr32 + * @see u_memrchr + * @see u_strFindLast + */ +U_STABLE UChar* U_EXPORT2 +u_memrchr32(const UChar *s, UChar32 c, int32_t count); + +/** + * Unicode String literals in C. + * We need one macro to declare a variable for the string + * and to statically preinitialize it if possible, + * and a second macro to dynamically intialize such a string variable if necessary. + * + * The macros are defined for maximum performance. + * They work only for strings that contain "invariant characters", i.e., + * only latin letters, digits, and some punctuation. + * See utypes.h for details. + * + * A pair of macros for a single string must be used with the same + * parameters. + * The string parameter must be a C string literal. + * The length of the string, not including the terminating + * NUL, must be specified as a constant. + * The U_STRING_DECL macro should be invoked exactly once for one + * such string variable before it is used. + * + * Usage: + *
+ *    U_STRING_DECL(ustringVar1, "Quick-Fox 2", 11);
+ *    U_STRING_DECL(ustringVar2, "jumps 5%", 8);
+ *    static UBool didInit=FALSE;
+ * 
+ *    int32_t function() {
+ *        if(!didInit) {
+ *            U_STRING_INIT(ustringVar1, "Quick-Fox 2", 11);
+ *            U_STRING_INIT(ustringVar2, "jumps 5%", 8);
+ *            didInit=TRUE;
+ *        }
+ *        return u_strcmp(ustringVar1, ustringVar2);
+ *    }
+ * 
+ * @stable ICU 2.0 + */ +#if defined(U_DECLARE_UTF16) +# define U_STRING_DECL(var, cs, length) static const UChar var[(length)+1]=U_DECLARE_UTF16(cs) + /**@stable ICU 2.0 */ +# define U_STRING_INIT(var, cs, length) +#elif U_SIZEOF_WCHAR_T==U_SIZEOF_UCHAR && (U_CHARSET_FAMILY==U_ASCII_FAMILY || (U_SIZEOF_UCHAR == 2 && defined(U_WCHAR_IS_UTF16))) +# define U_STRING_DECL(var, cs, length) static const UChar var[(length)+1]=L ## cs + /**@stable ICU 2.0 */ +# define U_STRING_INIT(var, cs, length) +#elif U_SIZEOF_UCHAR==1 && U_CHARSET_FAMILY==U_ASCII_FAMILY +# define U_STRING_DECL(var, cs, length) static const UChar var[(length)+1]=cs + /**@stable ICU 2.0 */ +# define U_STRING_INIT(var, cs, length) +#else +# define U_STRING_DECL(var, cs, length) static UChar var[(length)+1] + /**@stable ICU 2.0 */ +# define U_STRING_INIT(var, cs, length) u_charsToUChars(cs, var, length+1) +#endif + +/** + * Unescape a string of characters and write the resulting + * Unicode characters to the destination buffer. The following escape + * sequences are recognized: + * + * \\uhhhh 4 hex digits; h in [0-9A-Fa-f] + * \\Uhhhhhhhh 8 hex digits + * \\xhh 1-2 hex digits + * \\x{h...} 1-8 hex digits + * \\ooo 1-3 octal digits; o in [0-7] + * \\cX control-X; X is masked with 0x1F + * + * as well as the standard ANSI C escapes: + * + * \\a => U+0007, \\b => U+0008, \\t => U+0009, \\n => U+000A, + * \\v => U+000B, \\f => U+000C, \\r => U+000D, \\e => U+001B, + * \\" => U+0022, \\' => U+0027, \\? => U+003F, \\\\ => U+005C + * + * Anything else following a backslash is generically escaped. For + * example, "[a\\-z]" returns "[a-z]". + * + * If an escape sequence is ill-formed, this method returns an empty + * string. An example of an ill-formed sequence is "\\u" followed by + * fewer than 4 hex digits. + * + * The above characters are recognized in the compiler's codepage, + * that is, they are coded as 'u', '\\', etc. Characters that are + * not parts of escape sequences are converted using u_charsToUChars(). + * + * This function is similar to UnicodeString::unescape() but not + * identical to it. The latter takes a source UnicodeString, so it + * does escape recognition but no conversion. + * + * @param src a zero-terminated string of invariant characters + * @param dest pointer to buffer to receive converted and unescaped + * text and, if there is room, a zero terminator. May be NULL for + * preflighting, in which case no UChars will be written, but the + * return value will still be valid. On error, an empty string is + * stored here (if possible). + * @param destCapacity the number of UChars that may be written at + * dest. Ignored if dest == NULL. + * @return the length of unescaped string. + * @see u_unescapeAt + * @see UnicodeString#unescape() + * @see UnicodeString#unescapeAt() + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +u_unescape(const char *src, + UChar *dest, int32_t destCapacity); + +U_CDECL_BEGIN +/** + * Callback function for u_unescapeAt() that returns a character of + * the source text given an offset and a context pointer. The context + * pointer will be whatever is passed into u_unescapeAt(). + * + * @param offset pointer to the offset that will be passed to u_unescapeAt(). + * @param context an opaque pointer passed directly into u_unescapeAt() + * @return the character represented by the escape sequence at + * offset + * @see u_unescapeAt + * @stable ICU 2.0 + */ +typedef UChar (U_CALLCONV *UNESCAPE_CHAR_AT)(int32_t offset, void *context); +U_CDECL_END + +/** + * Unescape a single sequence. The character at offset-1 is assumed + * (without checking) to be a backslash. This method takes a callback + * pointer to a function that returns the UChar at a given offset. By + * varying this callback, ICU functions are able to unescape char* + * strings, UnicodeString objects, and UFILE pointers. + * + * If offset is out of range, or if the escape sequence is ill-formed, + * (UChar32)0xFFFFFFFF is returned. See documentation of u_unescape() + * for a list of recognized sequences. + * + * @param charAt callback function that returns a UChar of the source + * text given an offset and a context pointer. + * @param offset pointer to the offset that will be passed to charAt. + * The offset value will be updated upon return to point after the + * last parsed character of the escape sequence. On error the offset + * is unchanged. + * @param length the number of characters in the source text. The + * last character of the source text is considered to be at offset + * length-1. + * @param context an opaque pointer passed directly into charAt. + * @return the character represented by the escape sequence at + * offset, or (UChar32)0xFFFFFFFF on error. + * @see u_unescape() + * @see UnicodeString#unescape() + * @see UnicodeString#unescapeAt() + * @stable ICU 2.0 + */ +U_STABLE UChar32 U_EXPORT2 +u_unescapeAt(UNESCAPE_CHAR_AT charAt, + int32_t *offset, + int32_t length, + void *context); + +/** + * Uppercase the characters in a string. + * Casing is locale-dependent and context-sensitive. + * The result may be longer or shorter than the original. + * The source string and the destination buffer are allowed to overlap. + * + * @param dest A buffer for the result string. The result will be zero-terminated if + * the buffer is large enough. + * @param destCapacity The size of the buffer (number of UChars). If it is 0, then + * dest may be NULL and the function will only return the length of the result + * without writing any of the result string. + * @param src The original string + * @param srcLength The length of the original string. If -1, then src must be zero-terminated. + * @param locale The locale to consider, or "" for the root locale or NULL for the default locale. + * @param pErrorCode Must be a valid pointer to an error code value, + * which must not indicate a failure before the function call. + * @return The length of the result string. It may be greater than destCapacity. In that case, + * only some of the result was written to the destination buffer. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +u_strToUpper(UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + const char *locale, + UErrorCode *pErrorCode); + +/** + * Lowercase the characters in a string. + * Casing is locale-dependent and context-sensitive. + * The result may be longer or shorter than the original. + * The source string and the destination buffer are allowed to overlap. + * + * @param dest A buffer for the result string. The result will be zero-terminated if + * the buffer is large enough. + * @param destCapacity The size of the buffer (number of UChars). If it is 0, then + * dest may be NULL and the function will only return the length of the result + * without writing any of the result string. + * @param src The original string + * @param srcLength The length of the original string. If -1, then src must be zero-terminated. + * @param locale The locale to consider, or "" for the root locale or NULL for the default locale. + * @param pErrorCode Must be a valid pointer to an error code value, + * which must not indicate a failure before the function call. + * @return The length of the result string. It may be greater than destCapacity. In that case, + * only some of the result was written to the destination buffer. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +u_strToLower(UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + const char *locale, + UErrorCode *pErrorCode); + +#if !UCONFIG_NO_BREAK_ITERATION + +/** + * Titlecase a string. + * Casing is locale-dependent and context-sensitive. + * Titlecasing uses a break iterator to find the first characters of words + * that are to be titlecased. It titlecases those characters and lowercases + * all others. + * + * The titlecase break iterator can be provided to customize for arbitrary + * styles, using rules and dictionaries beyond the standard iterators. + * It may be more efficient to always provide an iterator to avoid + * opening and closing one for each string. + * The standard titlecase iterator for the root locale implements the + * algorithm of Unicode TR 21. + * + * This function uses only the setText(), first() and next() methods of the + * provided break iterator. + * + * The result may be longer or shorter than the original. + * The source string and the destination buffer are allowed to overlap. + * + * @param dest A buffer for the result string. The result will be zero-terminated if + * the buffer is large enough. + * @param destCapacity The size of the buffer (number of UChars). If it is 0, then + * dest may be NULL and the function will only return the length of the result + * without writing any of the result string. + * @param src The original string + * @param srcLength The length of the original string. If -1, then src must be zero-terminated. + * @param titleIter A break iterator to find the first characters of words + * that are to be titlecased. + * If none is provided (NULL), then a standard titlecase + * break iterator is opened. + * @param locale The locale to consider, or "" for the root locale or NULL for the default locale. + * @param pErrorCode Must be a valid pointer to an error code value, + * which must not indicate a failure before the function call. + * @return The length of the result string. It may be greater than destCapacity. In that case, + * only some of the result was written to the destination buffer. + * @stable ICU 2.1 + */ +U_STABLE int32_t U_EXPORT2 +u_strToTitle(UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + UBreakIterator *titleIter, + const char *locale, + UErrorCode *pErrorCode); + +#endif + +/** + * Case-fold the characters in a string. + * Case-folding is locale-independent and not context-sensitive, + * but there is an option for whether to include or exclude mappings for dotted I + * and dotless i that are marked with 'I' in CaseFolding.txt. + * The result may be longer or shorter than the original. + * The source string and the destination buffer are allowed to overlap. + * + * @param dest A buffer for the result string. The result will be zero-terminated if + * the buffer is large enough. + * @param destCapacity The size of the buffer (number of UChars). If it is 0, then + * dest may be NULL and the function will only return the length of the result + * without writing any of the result string. + * @param src The original string + * @param srcLength The length of the original string. If -1, then src must be zero-terminated. + * @param options Either U_FOLD_CASE_DEFAULT or U_FOLD_CASE_EXCLUDE_SPECIAL_I + * @param pErrorCode Must be a valid pointer to an error code value, + * which must not indicate a failure before the function call. + * @return The length of the result string. It may be greater than destCapacity. In that case, + * only some of the result was written to the destination buffer. + * @stable ICU 2.0 + */ +U_STABLE int32_t U_EXPORT2 +u_strFoldCase(UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + uint32_t options, + UErrorCode *pErrorCode); + +#if defined(U_WCHAR_IS_UTF16) || defined(U_WCHAR_IS_UTF32) || !UCONFIG_NO_CONVERSION +/** + * Converts a sequence of UChars to wchar_t units. + * + * @param dest A buffer for the result string. The result will be zero-terminated if + * the buffer is large enough. + * @param destCapacity The size of the buffer (number of wchar_t's). If it is 0, then + * dest may be NULL and the function will only return the length of the + * result without writing any of the result string (pre-flighting). + * @param pDestLength A pointer to receive the number of units written to the destination. If + * pDestLength!=NULL then *pDestLength is always set to the + * number of output units corresponding to the transformation of + * all the input units, even in case of a buffer overflow. + * @param src The original source string + * @param srcLength The length of the original string. If -1, then src must be zero-terminated. + * @param pErrorCode Must be a valid pointer to an error code value, + * which must not indicate a failure before the function call. + * @return The pointer to destination buffer. + * @stable ICU 2.0 + */ +U_STABLE wchar_t* U_EXPORT2 +u_strToWCS(wchar_t *dest, + int32_t destCapacity, + int32_t *pDestLength, + const UChar *src, + int32_t srcLength, + UErrorCode *pErrorCode); +/** + * Converts a sequence of wchar_t units to UChars + * + * @param dest A buffer for the result string. The result will be zero-terminated if + * the buffer is large enough. + * @param destCapacity The size of the buffer (number of UChars). If it is 0, then + * dest may be NULL and the function will only return the length of the + * result without writing any of the result string (pre-flighting). + * @param pDestLength A pointer to receive the number of units written to the destination. If + * pDestLength!=NULL then *pDestLength is always set to the + * number of output units corresponding to the transformation of + * all the input units, even in case of a buffer overflow. + * @param src The original source string + * @param srcLength The length of the original string. If -1, then src must be zero-terminated. + * @param pErrorCode Must be a valid pointer to an error code value, + * which must not indicate a failure before the function call. + * @return The pointer to destination buffer. + * @stable ICU 2.0 + */ +U_STABLE UChar* U_EXPORT2 +u_strFromWCS(UChar *dest, + int32_t destCapacity, + int32_t *pDestLength, + const wchar_t *src, + int32_t srcLength, + UErrorCode *pErrorCode); +#endif /* defined(U_WCHAR_IS_UTF16) || defined(U_WCHAR_IS_UTF32) || !UCONFIG_NO_CONVERSION */ + +/** + * Converts a sequence of UChars (UTF-16) to UTF-8 bytes + * + * @param dest A buffer for the result string. The result will be zero-terminated if + * the buffer is large enough. + * @param destCapacity The size of the buffer (number of chars). If it is 0, then + * dest may be NULL and the function will only return the length of the + * result without writing any of the result string (pre-flighting). + * @param pDestLength A pointer to receive the number of units written to the destination. If + * pDestLength!=NULL then *pDestLength is always set to the + * number of output units corresponding to the transformation of + * all the input units, even in case of a buffer overflow. + * @param src The original source string + * @param srcLength The length of the original string. If -1, then src must be zero-terminated. + * @param pErrorCode Must be a valid pointer to an error code value, + * which must not indicate a failure before the function call. + * @return The pointer to destination buffer. + * @stable ICU 2.0 + * @see u_strToUTF8WithSub + * @see u_strFromUTF8 + */ +U_STABLE char* U_EXPORT2 +u_strToUTF8(char *dest, + int32_t destCapacity, + int32_t *pDestLength, + const UChar *src, + int32_t srcLength, + UErrorCode *pErrorCode); + +/** + * Converts a sequence of UTF-8 bytes to UChars (UTF-16). + * + * @param dest A buffer for the result string. The result will be zero-terminated if + * the buffer is large enough. + * @param destCapacity The size of the buffer (number of UChars). If it is 0, then + * dest may be NULL and the function will only return the length of the + * result without writing any of the result string (pre-flighting). + * @param pDestLength A pointer to receive the number of units written to the destination. If + * pDestLength!=NULL then *pDestLength is always set to the + * number of output units corresponding to the transformation of + * all the input units, even in case of a buffer overflow. + * @param src The original source string + * @param srcLength The length of the original string. If -1, then src must be zero-terminated. + * @param pErrorCode Must be a valid pointer to an error code value, + * which must not indicate a failure before the function call. + * @return The pointer to destination buffer. + * @stable ICU 2.0 + * @see u_strFromUTF8WithSub + * @see u_strFromUTF8Lenient + */ +U_STABLE UChar* U_EXPORT2 +u_strFromUTF8(UChar *dest, + int32_t destCapacity, + int32_t *pDestLength, + const char *src, + int32_t srcLength, + UErrorCode *pErrorCode); + +/** + * Converts a sequence of UChars (UTF-16) to UTF-8 bytes. + * Same as u_strToUTF8() except for the additional subchar which is output for + * illegal input sequences, instead of stopping with the U_INVALID_CHAR_FOUND error code. + * With subchar==U_SENTINEL, this function behaves exactly like u_strToUTF8(). + * + * @param dest A buffer for the result string. The result will be zero-terminated if + * the buffer is large enough. + * @param destCapacity The size of the buffer (number of chars). If it is 0, then + * dest may be NULL and the function will only return the length of the + * result without writing any of the result string (pre-flighting). + * @param pDestLength A pointer to receive the number of units written to the destination. If + * pDestLength!=NULL then *pDestLength is always set to the + * number of output units corresponding to the transformation of + * all the input units, even in case of a buffer overflow. + * @param src The original source string + * @param srcLength The length of the original string. If -1, then src must be zero-terminated. + * @param subchar The substitution character to use in place of an illegal input sequence, + * or U_SENTINEL if the function is to return with U_INVALID_CHAR_FOUND instead. + * A substitution character can be any valid Unicode code point (up to U+10FFFF) + * except for surrogate code points (U+D800..U+DFFF). + * The recommended value is U+FFFD "REPLACEMENT CHARACTER". + * @param pNumSubstitutions Output parameter receiving the number of substitutions if subchar>=0. + * Set to 0 if no substitutions occur or subchar<0. + * pNumSubstitutions can be NULL. + * @param pErrorCode Pointer to a standard ICU error code. Its input value must + * pass the U_SUCCESS() test, or else the function returns + * immediately. Check for U_FAILURE() on output or use with + * function chaining. (See User Guide for details.) + * @return The pointer to destination buffer. + * @see u_strToUTF8 + * @see u_strFromUTF8WithSub + * @stable ICU 3.6 + */ +U_STABLE char* U_EXPORT2 +u_strToUTF8WithSub(char *dest, + int32_t destCapacity, + int32_t *pDestLength, + const UChar *src, + int32_t srcLength, + UChar32 subchar, int32_t *pNumSubstitutions, + UErrorCode *pErrorCode); + +/** + * Converts a sequence of UTF-8 bytes to UChars (UTF-16). + * Same as u_strFromUTF8() except for the additional subchar which is output for + * illegal input sequences, instead of stopping with the U_INVALID_CHAR_FOUND error code. + * With subchar==U_SENTINEL, this function behaves exactly like u_strFromUTF8(). + * + * @param dest A buffer for the result string. The result will be zero-terminated if + * the buffer is large enough. + * @param destCapacity The size of the buffer (number of UChars). If it is 0, then + * dest may be NULL and the function will only return the length of the + * result without writing any of the result string (pre-flighting). + * @param pDestLength A pointer to receive the number of units written to the destination. If + * pDestLength!=NULL then *pDestLength is always set to the + * number of output units corresponding to the transformation of + * all the input units, even in case of a buffer overflow. + * @param src The original source string + * @param srcLength The length of the original string. If -1, then src must be zero-terminated. + * @param subchar The substitution character to use in place of an illegal input sequence, + * or U_SENTINEL if the function is to return with U_INVALID_CHAR_FOUND instead. + * A substitution character can be any valid Unicode code point (up to U+10FFFF) + * except for surrogate code points (U+D800..U+DFFF). + * The recommended value is U+FFFD "REPLACEMENT CHARACTER". + * @param pNumSubstitutions Output parameter receiving the number of substitutions if subchar>=0. + * Set to 0 if no substitutions occur or subchar<0. + * pNumSubstitutions can be NULL. + * @param pErrorCode Pointer to a standard ICU error code. Its input value must + * pass the U_SUCCESS() test, or else the function returns + * immediately. Check for U_FAILURE() on output or use with + * function chaining. (See User Guide for details.) + * @return The pointer to destination buffer. + * @see u_strFromUTF8 + * @see u_strFromUTF8Lenient + * @see u_strToUTF8WithSub + * @stable ICU 3.6 + */ +U_STABLE UChar* U_EXPORT2 +u_strFromUTF8WithSub(UChar *dest, + int32_t destCapacity, + int32_t *pDestLength, + const char *src, + int32_t srcLength, + UChar32 subchar, int32_t *pNumSubstitutions, + UErrorCode *pErrorCode); + +/** + * Converts a sequence of UTF-8 bytes to UChars (UTF-16). + * Same as u_strFromUTF8() except that this function is designed to be very fast, + * which it achieves by being lenient about malformed UTF-8 sequences. + * This function is intended for use in environments where UTF-8 text is + * expected to be well-formed. + * + * Its semantics are: + * - Well-formed UTF-8 text is correctly converted to well-formed UTF-16 text. + * - The function will not read beyond the input string, nor write beyond + * the destCapacity. + * - Malformed UTF-8 results in "garbage" 16-bit Unicode strings which may not + * be well-formed UTF-16. + * The function will resynchronize to valid code point boundaries + * within a small number of code points after an illegal sequence. + * - Non-shortest forms are not detected and will result in "spoofing" output. + * + * For further performance improvement, if srcLength is given (>=0), + * then it must be destCapacity>=srcLength. + * + * @param dest A buffer for the result string. The result will be zero-terminated if + * the buffer is large enough. + * @param destCapacity The size of the buffer (number of UChars). If it is 0, then + * dest may be NULL and the function will only return the length of the + * result without writing any of the result string (pre-flighting). + * Unlike for other ICU functions, if srcLength>=0 then it + * must be destCapacity>=srcLength. + * @param pDestLength A pointer to receive the number of units written to the destination. If + * pDestLength!=NULL then *pDestLength is always set to the + * number of output units corresponding to the transformation of + * all the input units, even in case of a buffer overflow. + * Unlike for other ICU functions, if srcLength>=0 but + * destCapacityIterating over text + * + * Here is sample code for a forward iteration over the contents of a UText + * + * \code + * UChar32 c; + * UText *ut = whatever(); + * + * for (c=utext_next32From(ut, 0); c>=0; c=utext_next32(ut)) { + * // do whatever with the codepoint c here. + * } + * \endcode + * + * And here is similar code to iterate in the reverse direction, from the end + * of the text towards the beginning. + * + * \code + * UChar32 c; + * UText *ut = whatever(); + * int textLength = utext_nativeLength(ut); + * for (c=utext_previous32From(ut, textLength); c>=0; c=utext_previous32(ut)) { + * // do whatever with the codepoint c here. + * } + * \endcode + * + * Characters and Indexing + * + * Indexing into text by UText functions is nearly always in terms of the native + * indexing of the underlying text storage. The storage format could be UTF-8 + * or UTF-32, for example. When coding to the UText access API, no assumptions + * can be made regarding the size of characters, or how far an index + * may move when iterating between characters. + * + * All indices supplied to UText functions are pinned to the length of the + * text. An out-of-bounds index is not considered to be an error, but is + * adjusted to be in the range 0 <= index <= length of input text. + * + * + * When an index position is returned from a UText function, it will be + * a native index to the underlying text. In the case of multi-unit characters, + * it will always refer to the first position of the character, + * never to the interior. This is essentially the same thing as saying that + * a returned index will always point to a boundary between characters. + * + * When a native index is supplied to a UText function, all indices that + * refer to any part of a multi-unit character representation are considered + * to be equivalent. In the case of multi-unit characters, an incoming index + * will be logically normalized to refer to the start of the character. + * + * It is possible to test whether a native index is on a code point boundary + * by doing a utext_setNativeIndex() followed by a utext_getNativeIndex(). + * If the index is returned unchanged, it was on a code point boundary. If + * an adjusted index is returned, the original index referred to the + * interior of a character. + * + * Conventions for calling UText functions + * + * Most UText access functions have as their first parameter a (UText *) pointer, + * which specifies the UText to be used. Unless otherwise noted, the + * pointer must refer to a valid, open UText. Attempting to + * use a closed UText or passing a NULL pointer is a programming error and + * will produce undefined results or NULL pointer exceptions. + * + * The UText_Open family of functions can either open an existing (closed) + * UText, or heap allocate a new UText. Here is sample code for creating + * a stack-allocated UText. + * + * \code + * char *s = whatever(); // A utf-8 string + * U_ErrorCode status = U_ZERO_ERROR; + * UText ut = UTEXT_INITIALIZER; + * utext_openUTF8(ut, s, -1, &status); + * if (U_FAILURE(status)) { + * // error handling + * } else { + * // work with the UText + * } + * \endcode + * + * Any existing UText passed to an open function _must_ have been initialized, + * either by the UTEXT_INITIALIZER, or by having been originally heap-allocated + * by an open function. Passing NULL will cause the open function to + * heap-allocate and fully initialize a new UText. + * + */ + + + +#include "unicode/utypes.h" +#ifdef XP_CPLUSPLUS +#include "unicode/rep.h" +#include "unicode/unistr.h" +#include "unicode/chariter.h" +#endif + + +U_CDECL_BEGIN + +struct UText; +typedef struct UText UText; /**< C typedef for struct UText. @stable ICU 3.6 */ + + +/*************************************************************************************** + * + * C Functions for creating UText wrappers around various kinds of text strings. + * + ****************************************************************************************/ + + +/** + * Close function for UText instances. + * Cleans up, releases any resources being held by an open UText. + *

+ * If the UText was originally allocated by one of the utext_open functions, + * the storage associated with the utext will also be freed. + * If the UText storage originated with the application, as it would with + * a local or static instance, the storage will not be deleted. + * + * An open UText can be reset to refer to new string by using one of the utext_open() + * functions without first closing the UText. + * + * @param ut The UText to be closed. + * @return NULL if the UText struct was deleted by the close. If the UText struct + * was originally provided by the caller to the open function, it is + * returned by this function, and may be safely used again in + * a subsequent utext_open. + * + * @stable ICU 3.4 + */ +U_STABLE UText * U_EXPORT2 +utext_close(UText *ut); + + +/** + * Open a read-only UText implementation for UTF-8 strings. + * + * \htmlonly + * Any invalid UTF-8 in the input will be handled in this way: + * a sequence of bytes that has the form of a truncated, but otherwise valid, + * UTF-8 sequence will be replaced by a single unicode replacement character, \uFFFD. + * Any other illegal bytes will each be replaced by a \uFFFD. + * \endhtmlonly + * + * @param ut Pointer to a UText struct. If NULL, a new UText will be created. + * If non-NULL, must refer to an initialized UText struct, which will then + * be reset to reference the specified UTF-8 string. + * @param s A UTF-8 string. Must not be NULL. + * @param length The length of the UTF-8 string in bytes, or -1 if the string is + * zero terminated. + * @param status Errors are returned here. + * @return A pointer to the UText. If a pre-allocated UText was provided, it + * will always be used and returned. + * @stable ICU 3.4 + */ +U_STABLE UText * U_EXPORT2 +utext_openUTF8(UText *ut, const char *s, int64_t length, UErrorCode *status); + + +/** + * Open a read-only UText for UChar * string. + * + * @param ut Pointer to a UText struct. If NULL, a new UText will be created. + * If non-NULL, must refer to an initialized UText struct, which will then + * be reset to reference the specified UChar string. + * @param s A UChar (UTF-16) string + * @param length The number of UChars in the input string, or -1 if the string is + * zero terminated. + * @param status Errors are returned here. + * @return A pointer to the UText. If a pre-allocated UText was provided, it + * will always be used and returned. + * @stable ICU 3.4 + */ +U_STABLE UText * U_EXPORT2 +utext_openUChars(UText *ut, const UChar *s, int64_t length, UErrorCode *status); + + +#ifdef XP_CPLUSPLUS +/** + * Open a writable UText for a non-const UnicodeString. + * + * @param ut Pointer to a UText struct. If NULL, a new UText will be created. + * If non-NULL, must refer to an initialized UText struct, which will then + * be reset to reference the specified input string. + * @param s A UnicodeString. + * @param status Errors are returned here. + * @return Pointer to the UText. If a UText was supplied as input, this + * will always be used and returned. + * @stable ICU 3.4 + */ +U_STABLE UText * U_EXPORT2 +utext_openUnicodeString(UText *ut, U_NAMESPACE_QUALIFIER UnicodeString *s, UErrorCode *status); + + +/** + * Open a UText for a const UnicodeString. The resulting UText will not be writable. + * + * @param ut Pointer to a UText struct. If NULL, a new UText will be created. + * If non-NULL, must refer to an initialized UText struct, which will then + * be reset to reference the specified input string. + * @param s A const UnicodeString to be wrapped. + * @param status Errors are returned here. + * @return Pointer to the UText. If a UText was supplied as input, this + * will always be used and returned. + * @stable ICU 3.4 + */ +U_STABLE UText * U_EXPORT2 +utext_openConstUnicodeString(UText *ut, const U_NAMESPACE_QUALIFIER UnicodeString *s, UErrorCode *status); + + +/** + * Open a writable UText implementation for an ICU Replaceable object. + * @param ut Pointer to a UText struct. If NULL, a new UText will be created. + * If non-NULL, must refer to an already existing UText, which will then + * be reset to reference the specified replaceable text. + * @param rep A Replaceable text object. + * @param status Errors are returned here. + * @return Pointer to the UText. If a UText was supplied as input, this + * will always be used and returned. + * @see Replaceable + * @stable ICU 3.4 + */ +U_STABLE UText * U_EXPORT2 +utext_openReplaceable(UText *ut, U_NAMESPACE_QUALIFIER Replaceable *rep, UErrorCode *status); + +/** + * Open a UText implementation over an ICU CharacterIterator. + * @param ut Pointer to a UText struct. If NULL, a new UText will be created. + * If non-NULL, must refer to an already existing UText, which will then + * be reset to reference the specified replaceable text. + * @param ci A Character Iterator. + * @param status Errors are returned here. + * @return Pointer to the UText. If a UText was supplied as input, this + * will always be used and returned. + * @see Replaceable + * @stable ICU 3.4 + */ +U_STABLE UText * U_EXPORT2 +utext_openCharacterIterator(UText *ut, U_NAMESPACE_QUALIFIER CharacterIterator *ic, UErrorCode *status); + +#endif + + +/** + * Clone a UText. This is much like opening a UText where the source text is itself + * another UText. + * + * A deep clone will copy both the UText data structures and the underlying text. + * The original and cloned UText will operate completely independently; modifications + * made to the text in one will not affect the other. Text providers are not + * required to support deep clones. The user of clone() must check the status return + * and be prepared to handle failures. + * + * The standard UText implementations for UTF8, UChar *, UnicodeString and + * Replaceable all support deep cloning. + * + * The UText returned from a deep clone will be writable, assuming that the text + * provider is able to support writing, even if the source UText had been made + * non-writable by means of UText_freeze(). + * + * A shallow clone replicates only the UText data structures; it does not make + * a copy of the underlying text. Shallow clones can be used as an efficient way to + * have multiple iterators active in a single text string that is not being + * modified. + * + * A shallow clone operation will not fail, barring truly exceptional conditions such + * as memory allocation failures. + * + * Shallow UText clones should be avoided if the UText functions that modify the + * text are expected to be used, either on the original or the cloned UText. + * Any such modifications can cause unpredictable behavior. Read Only + * shallow clones provide some protection against errors of this type by + * disabling text modification via the cloned UText. + * + * A shallow clone made with the readOnly parameter == FALSE will preserve the + * utext_isWritable() state of the source object. Note, however, that + * write operations must be avoided while more than one UText exists that refer + * to the same underlying text. + * + * A UText and its clone may be safely concurrently accessed by separate threads. + * This is true for read access only with shallow clones, and for both read and + * write access with deep clones. + * It is the responsibility of the Text Provider to ensure that this thread safety + * constraint is met. + * + * @param dest A UText struct to be filled in with the result of the clone operation, + * or NULL if the clone function should heap-allocate a new UText struct. + * If non-NULL, must refer to an already existing UText, which will then + * be reset to become the clone. + * @param src The UText to be cloned. + * @param deep TRUE to request a deep clone, FALSE for a shallow clone. + * @param readOnly TRUE to request that the cloned UText have read only access to the + * underlying text. + + * @param status Errors are returned here. For deep clones, U_UNSUPPORTED_ERROR + * will be returned if the text provider is unable to clone the + * original text. + * @return The newly created clone, or NULL if the clone operation failed. + * @stable ICU 3.4 + */ +U_STABLE UText * U_EXPORT2 +utext_clone(UText *dest, const UText *src, UBool deep, UBool readOnly, UErrorCode *status); + + +/** + * Compare two UText objects for equality. + * UTexts are equal if they are iterating over the same text, and + * have the same iteration position within the text. + * If either or both of the parameters are NULL, the comparison is FALSE. + * + * @param a The first of the two UTexts to compare. + * @param b The other UText to be compared. + * @return TRUE if the two UTexts are equal. + * @stable ICU 3.6 + */ +U_STABLE UBool U_EXPORT2 +utext_equals(const UText *a, const UText *b); + + +/***************************************************************************** + * + * Functions to work with the text represeted by a UText wrapper + * + *****************************************************************************/ + +/** + * Get the length of the text. Depending on the characteristics + * of the underlying text representation, this may be expensive. + * @see utext_isLengthExpensive() + * + * + * @param ut the text to be accessed. + * @return the length of the text, expressed in native units. + * + * @stable ICU 3.4 + */ +U_STABLE int64_t U_EXPORT2 +utext_nativeLength(UText *ut); + +/** + * Return TRUE if calculating the length of the text could be expensive. + * Finding the length of NUL terminated strings is considered to be expensive. + * + * Note that the value of this function may change + * as the result of other operations on a UText. + * Once the length of a string has been discovered, it will no longer + * be expensive to report it. + * + * @param ut the text to be accessed. + * @return TRUE if determining the length of the text could be time consuming. + * @stable ICU 3.4 + */ +U_STABLE UBool U_EXPORT2 +utext_isLengthExpensive(const UText *ut); + +/** + * Returns the code point at the requested index, + * or U_SENTINEL (-1) if it is out of bounds. + * + * If the specified index points to the interior of a multi-unit + * character - one of the trail bytes of a UTF-8 sequence, for example - + * the complete code point will be returned. + * + * The iteration position will be set to the start of the returned code point. + * + * This function is roughly equivalent to the the sequence + * utext_setNativeIndex(index); + * utext_current32(); + * (There is a subtle difference if the index is out of bounds by being less than zero - + * utext_setNativeIndex(negative value) sets the index to zero, after which utext_current() + * will return the char at zero. utext_char32At(negative index), on the other hand, will + * return the U_SENTINEL value of -1.) + * + * @param ut the text to be accessed + * @param nativeIndex the native index of the character to be accessed. If the index points + * to other than the first unit of a multi-unit character, it will be adjusted + * to the start of the character. + * @return the code point at the specified index. + * @stable ICU 3.4 + */ +U_STABLE UChar32 U_EXPORT2 +utext_char32At(UText *ut, int64_t nativeIndex); + + +/** + * + * Get the code point at the current iteration position, + * or U_SENTINEL (-1) if the iteration has reached the end of + * the input text. + * + * @param ut the text to be accessed. + * @return the Unicode code point at the current iterator position. + * @stable ICU 3.4 + */ +U_STABLE UChar32 U_EXPORT2 +utext_current32(UText *ut); + + +/** + * Get the code point at the current iteration position of the UText, and + * advance the position to the first index following the character. + * + * If the position is at the end of the text (the index following + * the last character, which is also the length of the text), + * return U_SENTINEL (-1) and do not advance the index. + * + * This is a post-increment operation. + * + * An inline macro version of this function, UTEXT_NEXT32(), + * is available for performance critical use. + * + * @param ut the text to be accessed. + * @return the Unicode code point at the iteration position. + * @see UTEXT_NEXT32 + * @stable ICU 3.4 + */ +U_STABLE UChar32 U_EXPORT2 +utext_next32(UText *ut); + + +/** + * Move the iterator position to the character (code point) whose + * index precedes the current position, and return that character. + * This is a pre-decrement operation. + * + * If the initial position is at the start of the text (index of 0) + * return U_SENTINEL (-1), and leave the position unchanged. + * + * An inline macro version of this function, UTEXT_PREVIOUS32(), + * is available for performance critical use. + * + * @param ut the text to be accessed. + * @return the previous UChar32 code point, or U_SENTINEL (-1) + * if the iteration has reached the start of the text. + * @see UTEXT_PREVIOUS32 + * @stable ICU 3.4 + */ +U_STABLE UChar32 U_EXPORT2 +utext_previous32(UText *ut); + + +/** + * Set the iteration index and return the code point at that index. + * Leave the iteration index at the start of the following code point. + * + * This function is the most efficient and convenient way to + * begin a forward iteration. The results are identical to the those + * from the sequence + * \code + * utext_setIndex(); + * utext_next32(); + * \endcode + * + * @param ut the text to be accessed. + * @param nativeIndex Iteration index, in the native units of the text provider. + * @return Code point which starts at or before index, + * or U_SENTINEL (-1) if it is out of bounds. + * @stable ICU 3.4 + */ +U_STABLE UChar32 U_EXPORT2 +utext_next32From(UText *ut, int64_t nativeIndex); + + + +/** + * Set the iteration index, and return the code point preceding the + * one specified by the initial index. Leave the iteration position + * at the start of the returned code point. + * + * This function is the most efficient and convenient way to + * begin a backwards iteration. + * + * @param ut the text to be accessed. + * @param nativeIndex Iteration index in the native units of the text provider. + * @return Code point preceding the one at the initial index, + * or U_SENTINEL (-1) if it is out of bounds. + * + * @stable ICU 3.4 + */ +U_STABLE UChar32 U_EXPORT2 +utext_previous32From(UText *ut, int64_t nativeIndex); + +/** + * Get the current iterator position, which can range from 0 to + * the length of the text. + * The position is a native index into the input text, in whatever format it + * may have (possibly UTF-8 for example), and may not always be the same as + * the corresponding UChar (UTF-16) index. + * The returned position will always be aligned to a code point boundary. + * + * @param ut the text to be accessed. + * @return the current index position, in the native units of the text provider. + * @stable ICU 3.4 + */ +U_STABLE int64_t U_EXPORT2 +utext_getNativeIndex(const UText *ut); + +/** + * Set the current iteration position to the nearest code point + * boundary at or preceding the specified index. + * The index is in the native units of the original input text. + * If the index is out of range, it will be pinned to be within + * the range of the input text. + *

+ * It will usually be more efficient to begin an iteration + * using the functions utext_next32From() or utext_previous32From() + * rather than setIndex(). + *

+ * Moving the index position to an adjacent character is best done + * with utext_next32(), utext_previous32() or utext_moveIndex32(). + * Attempting to do direct arithmetic on the index position is + * complicated by the fact that the size (in native units) of a + * character depends on the underlying representation of the character + * (UTF-8, UTF-16, UTF-32, arbitrary codepage), and is not + * easily knowable. + * + * @param ut the text to be accessed. + * @param nativeIndex the native unit index of the new iteration position. + * @stable ICU 3.4 + */ +U_STABLE void U_EXPORT2 +utext_setNativeIndex(UText *ut, int64_t nativeIndex); + +/** + * Move the iterator postion by delta code points. The number of code points + * is a signed number; a negative delta will move the iterator backwards, + * towards the start of the text. + *

+ * The index is moved by delta code points + * forward or backward, but no further backward than to 0 and + * no further forward than to utext_nativeLength(). + * The resulting index value will be in between 0 and length, inclusive. + * + * @param ut the text to be accessed. + * @param delta the signed number of code points to move the iteration position. + * @return TRUE if the position could be moved the requested number of positions while + * staying within the range [0 - text length]. + * @stable ICU 3.4 + */ +U_STABLE UBool U_EXPORT2 +utext_moveIndex32(UText *ut, int32_t delta); + +/** + * Get the native index of the character preceeding the current position. + * If the iteration position is already at the start of the text, zero + * is returned. + * The value returned is the same as that obtained from the following sequence, + * but without the side effect of changing the iteration position. + * + * \code + * UText *ut = whatever; + * ... + * utext_previous(ut) + * utext_getNativeIndex(ut); + * \endcode + * + * This function is most useful during forwards iteration, where it will get the + * native index of the character most recently returned from utext_next(). + * + * @param ut the text to be accessed + * @return the native index of the character preceeding the current index position, + * or zero if the current position is at the start of the text. + * @stable ICU 3.6 + */ +U_STABLE int64_t U_EXPORT2 +utext_getPreviousNativeIndex(UText *ut); + + +/** + * + * Extract text from a UText into a UChar buffer. The range of text to be extracted + * is specified in the native indices of the UText provider. These may not necessarily + * be UTF-16 indices. + *

+ * The size (number of 16 bit UChars) of the data to be extracted is returned. The + * full number of UChars is returned, even when the extracted text is truncated + * because the specified buffer size is too small. + *

+ * The extracted string will (if you are a user) / must (if you are a text provider) + * be NUL-terminated if there is sufficient space in the destination buffer. This + * terminating NUL is not included in the returned length. + *

+ * The iteration index is left at the position following the last extracted character. + * + * @param ut the UText from which to extract data. + * @param nativeStart the native index of the first character to extract.\ + * If the specified index is out of range, + * it will be pinned to to be within 0 <= index <= textLength + * @param nativeLimit the native string index of the position following the last + * character to extract. If the specified index is out of range, + * it will be pinned to to be within 0 <= index <= textLength. + * nativeLimit must be >= nativeStart. + * @param dest the UChar (UTF-16) buffer into which the extracted text is placed + * @param destCapacity The size, in UChars, of the destination buffer. May be zero + * for precomputing the required size. + * @param status receives any error status. + * U_BUFFER_OVERFLOW_ERROR: the extracted text was truncated because the + * buffer was too small. Returns number of UChars for preflighting. + * @return Number of UChars in the data to be extracted. Does not include a trailing NUL. + * + * @stable ICU 3.4 + */ +U_STABLE int32_t U_EXPORT2 +utext_extract(UText *ut, + int64_t nativeStart, int64_t nativeLimit, + UChar *dest, int32_t destCapacity, + UErrorCode *status); + + +/************************************************************************************ + * + * #define inline versions of selected performance-critical text access functions + * Caution: do not use auto increment++ or decrement-- expressions + * as parameters to these macros. + * + * For most use, where there is no extreme performance constraint, the + * normal, non-inline functions are a better choice. The resulting code + * will be smaller, and, if the need ever arises, easier to debug. + * + * These are implemented as #defines rather than real functions + * because there is no fully portable way to do inline functions in plain C. + * + ************************************************************************************/ + +/** + * inline version of utext_next32(), for performance-critical situations. + * + * Get the code point at the current iteration position of the UText, and + * advance the position to the first index following the character. + * This is a post-increment operation. + * Returns U_SENTINEL (-1) if the position is at the end of the + * text. + * + * @stable ICU 3.4 + */ +#define UTEXT_NEXT32(ut) \ + ((ut)->chunkOffset < (ut)->chunkLength && ((ut)->chunkContents)[(ut)->chunkOffset]<0xd800 ? \ + ((ut)->chunkContents)[((ut)->chunkOffset)++] : utext_next32(ut)) + +/** + * inline version of utext_previous32(), for performance-critical situations. + * + * Move the iterator position to the character (code point) whose + * index precedes the current position, and return that character. + * This is a pre-decrement operation. + * Returns U_SENTINEL (-1) if the position is at the start of the text. + * + * @stable ICU 3.4 + */ +#define UTEXT_PREVIOUS32(ut) \ + ((ut)->chunkOffset > 0 && \ + (ut)->chunkContents[(ut)->chunkOffset-1] < 0xd800 ? \ + (ut)->chunkContents[--((ut)->chunkOffset)] : utext_previous32(ut)) + +/** + * inline version of utext_getNativeIndex(), for performance-critical situations. + * + * Get the current iterator position, which can range from 0 to + * the length of the text. + * The position is a native index into the input text, in whatever format it + * may have (possibly UTF-8 for example), and may not always be the same as + * the corresponding UChar (UTF-16) index. + * The returned position will always be aligned to a code point boundary. + * + * @stable ICU 3.6 + */ +#define UTEXT_GETNATIVEINDEX(ut) \ + ((ut)->chunkOffset <= (ut)->nativeIndexingLimit? \ + (ut)->chunkNativeStart+(ut)->chunkOffset : \ + (ut)->pFuncs->mapOffsetToNative(ut)) + +/** + * inline version of utext_setNativeIndex(), for performance-critical situations. + * + * Set the current iteration position to the nearest code point + * boundary at or preceding the specified index. + * The index is in the native units of the original input text. + * If the index is out of range, it will be pinned to be within + * the range of the input text. + * + * @draft ICU 3.8 + */ +#define UTEXT_SETNATIVEINDEX(ut, ix) \ + { int64_t __offset = (ix) - (ut)->chunkNativeStart; \ + if (__offset>=0 && __offset<=(int64_t)(ut)->nativeIndexingLimit) { \ + (ut)->chunkOffset=(int32_t)__offset; \ + } else { \ + utext_setNativeIndex((ut), (ix)); } } + + + +/************************************************************************************ + * + * Functions related to writing or modifying the text. + * These will work only with modifiable UTexts. Attempting to + * modify a read-only UText will return an error status. + * + ************************************************************************************/ + + +/** + * Return TRUE if the text can be written (modified) with utext_replace() or + * utext_copy(). For the text to be writable, the text provider must + * be of a type that supports writing and the UText must not be frozen. + * + * Attempting to modify text when utext_isWriteable() is FALSE will fail - + * the text will not be modified, and an error will be returned from the function + * that attempted the modification. + * + * @param ut the UText to be tested. + * @return TRUE if the text is modifiable. + * + * @see utext_freeze() + * @see utext_replace() + * @see utext_copy() + * @stable ICU 3.4 + * + */ +U_STABLE UBool U_EXPORT2 +utext_isWritable(const UText *ut); + + +/** + * Test whether there is meta data associated with the text. + * @see Replaceable::hasMetaData() + * + * @param ut The UText to be tested + * @return TRUE if the underlying text includes meta data. + * @stable ICU 3.4 + */ +U_STABLE UBool U_EXPORT2 +utext_hasMetaData(const UText *ut); + + +/** + * Replace a range of the original text with a replacement text. + * + * Leaves the current iteration position at the position following the + * newly inserted replacement text. + * + * This function is only available on UText types that support writing, + * that is, ones where utext_isWritable() returns TRUE. + * + * When using this function, there should be only a single UText opened onto the + * underlying native text string. Behavior after a replace operation + * on a UText is undefined for any other additional UTexts that refer to the + * modified string. + * + * @param ut the UText representing the text to be operated on. + * @param nativeStart the native index of the start of the region to be replaced + * @param nativeLimit the native index of the character following the region to be replaced. + * @param replacementText pointer to the replacement text + * @param replacementLength length of the replacement text, or -1 if the text is NUL terminated. + * @param status receives any error status. Possible errors include + * U_NO_WRITE_PERMISSION + * + * @return The signed number of (native) storage units by which + * the length of the text expanded or contracted. + * + * @stable ICU 3.4 + */ +U_STABLE int32_t U_EXPORT2 +utext_replace(UText *ut, + int64_t nativeStart, int64_t nativeLimit, + const UChar *replacementText, int32_t replacementLength, + UErrorCode *status); + + + +/** + * + * Copy or move a substring from one position to another within the text, + * while retaining any metadata associated with the text. + * This function is used to duplicate or reorder substrings. + * The destination index must not overlap the source range. + * + * The text to be copied or moved is inserted at destIndex; + * it does not replace or overwrite any existing text. + * + * The iteration position is left following the newly inserted text + * at the destination position. + * + * This function is only available on UText types that support writing, + * that is, ones where utext_isWritable() returns TRUE. + * + * When using this function, there should be only a single UText opened onto the + * underlying native text string. Behavior after a copy operation + * on a UText is undefined in any other additional UTexts that refer to the + * modified string. + * + * @param ut The UText representing the text to be operated on. + * @param nativeStart The native index of the start of the region to be copied or moved + * @param nativeLimit The native index of the character position following the region + * to be copied. + * @param destIndex The native destination index to which the source substring is + * copied or moved. + * @param move If TRUE, then the substring is moved, not copied/duplicated. + * @param status receives any error status. Possible errors include U_NO_WRITE_PERMISSION + * + * @stable ICU 3.4 + */ +U_STABLE void U_EXPORT2 +utext_copy(UText *ut, + int64_t nativeStart, int64_t nativeLimit, + int64_t destIndex, + UBool move, + UErrorCode *status); + + +/** + *

+ * Freeze a UText. This prevents any modification to the underlying text itself + * by means of functions operating on this UText. + *

+ *

+ * Once frozen, a UText can not be unfrozen. The intent is to ensure + * that a the text underlying a frozen UText wrapper cannot be modified via that UText. + *

+ *

+ * Caution: freezing a UText will disable changes made via the specific + * frozen UText wrapper only; it will not have any effect on the ability to + * directly modify the text by bypassing the UText. Any such backdoor modifications + * are always an error while UText access is occuring because the underlying + * text can get out of sync with UText's buffering. + *

+ * + * @param ut The UText to be frozen. + * @see utext_isWritable() + * @stable ICU 3.6 + */ +U_STABLE void U_EXPORT2 +utext_freeze(UText *ut); + + +/** + * UText provider properties (bit field indexes). + * + * @see UText + * @stable ICU 3.4 + */ +enum { + /** + * It is potentially time consuming for the provider to determine the length of the text. + * @stable ICU 3.4 + */ + UTEXT_PROVIDER_LENGTH_IS_EXPENSIVE = 1, + /** + * Text chunks remain valid and usable until the text object is modified or + * deleted, not just until the next time the access() function is called + * (which is the default). + * @stable ICU 3.4 + */ + UTEXT_PROVIDER_STABLE_CHUNKS = 2, + /** + * The provider supports modifying the text via the replace() and copy() + * functions. + * @see Replaceable + * @stable ICU 3.4 + */ + UTEXT_PROVIDER_WRITABLE = 3, + /** + * There is meta data associated with the text. + * @see Replaceable::hasMetaData() + * @stable ICU 3.4 + */ + UTEXT_PROVIDER_HAS_META_DATA = 4, + /** + * Text provider owns the text storage. + * Generally occurs as the result of a deep clone of the UText. + * When closing the UText, the associated text must + * also be closed/deleted/freed/ whatever is appropriate. + * @stable ICU 3.6 + */ + UTEXT_PROVIDER_OWNS_TEXT = 5 +}; + +/** + * Function type declaration for UText.clone(). + * + * clone a UText. Much like opening a UText where the source text is itself + * another UText. + * + * A deep clone will copy both the UText data structures and the underlying text. + * The original and cloned UText will operate completely independently; modifications + * made to the text in one will not effect the other. Text providers are not + * required to support deep clones. The user of clone() must check the status return + * and be prepared to handle failures. + * + * A shallow clone replicates only the UText data structures; it does not make + * a copy of the underlying text. Shallow clones can be used as an efficient way to + * have multiple iterators active in a single text string that is not being + * modified. + * + * A shallow clone operation must not fail except for truly exceptional conditions such + * as memory allocation failures. + * + * A UText and its clone may be safely concurrently accessed by separate threads. + * This is true for both shallow and deep clones. + * It is the responsibility of the Text Provider to ensure that this thread safety + * constraint is met. + + * + * @param dest A UText struct to be filled in with the result of the clone operation, + * or NULL if the clone function should heap-allocate a new UText struct. + * @param src The UText to be cloned. + * @param deep TRUE to request a deep clone, FALSE for a shallow clone. + * @param status Errors are returned here. For deep clones, U_UNSUPPORTED_ERROR + * should be returned if the text provider is unable to clone the + * original text. + * @return The newly created clone, or NULL if the clone operation failed. + * + * @stable ICU 3.4 + */ +typedef UText * U_CALLCONV +UTextClone(UText *dest, const UText *src, UBool deep, UErrorCode *status); + + +/** + * Function type declaration for UText.nativeLength(). + * + * @param ut the UText to get the length of. + * @return the length, in the native units of the original text string. + * @see UText + * @stable ICU 3.4 + */ +typedef int64_t U_CALLCONV +UTextNativeLength(UText *ut); + +/** + * Function type declaration for UText.access(). Get the description of the text chunk + * containing the text at a requested native index. The UText's iteration + * position will be left at the requested index. If the index is out + * of bounds, the iteration position will be left at the start or end + * of the string, as appropriate. + * + * Chunks must begin and end on code point boundaries. A single code point + * comprised of multiple storage units must never span a chunk boundary. + * + * + * @param ut the UText being accessed. + * @param nativeIndex Requested index of the text to be accessed. + * @param forward If TRUE, then the returned chunk must contain text + * starting from the index, so that start<=index + * The size (number of 16 bit UChars) in the data to be extracted is returned. The + * full amount is returned, even when the specified buffer size is smaller. + *

+ * The extracted string will (if you are a user) / must (if you are a text provider) + * be NUL-terminated if there is sufficient space in the destination buffer. + * + * @param ut the UText from which to extract data. + * @param nativeStart the native index of the first characer to extract. + * @param nativeLimit the native string index of the position following the last + * character to extract. + * @param dest the UChar (UTF-16) buffer into which the extracted text is placed + * @param destCapacity The size, in UChars, of the destination buffer. May be zero + * for precomputing the required size. + * @param status receives any error status. + * If U_BUFFER_OVERFLOW_ERROR: Returns number of UChars for + * preflighting. + * @return Number of UChars in the data. Does not include a trailing NUL. + * + * @stable ICU 3.4 + */ +typedef int32_t U_CALLCONV +UTextExtract(UText *ut, + int64_t nativeStart, int64_t nativeLimit, + UChar *dest, int32_t destCapacity, + UErrorCode *status); + +/** + * Function type declaration for UText.replace(). + * + * Replace a range of the original text with a replacement text. + * + * Leaves the current iteration position at the position following the + * newly inserted replacement text. + * + * This function need only be implemented on UText types that support writing. + * + * When using this function, there should be only a single UText opened onto the + * underlying native text string. The function is responsible for updating the + * text chunk within the UText to reflect the updated iteration position, + * taking into account any changes to the underlying string's structure caused + * by the replace operation. + * + * @param ut the UText representing the text to be operated on. + * @param nativeStart the index of the start of the region to be replaced + * @param nativeLimit the index of the character following the region to be replaced. + * @param replacementText pointer to the replacement text + * @param replacmentLength length of the replacement text in UChars, or -1 if the text is NUL terminated. + * @param status receives any error status. Possible errors include + * U_NO_WRITE_PERMISSION + * + * @return The signed number of (native) storage units by which + * the length of the text expanded or contracted. + * + * @stable ICU 3.4 + */ +typedef int32_t U_CALLCONV +UTextReplace(UText *ut, + int64_t nativeStart, int64_t nativeLimit, + const UChar *replacementText, int32_t replacmentLength, + UErrorCode *status); + +/** + * Function type declaration for UText.copy(). + * + * Copy or move a substring from one position to another within the text, + * while retaining any metadata associated with the text. + * This function is used to duplicate or reorder substrings. + * The destination index must not overlap the source range. + * + * The text to be copied or moved is inserted at destIndex; + * it does not replace or overwrite any existing text. + * + * This function need only be implemented for UText types that support writing. + * + * When using this function, there should be only a single UText opened onto the + * underlying native text string. The function is responsible for updating the + * text chunk within the UText to reflect the updated iteration position, + * taking into account any changes to the underlying string's structure caused + * by the replace operation. + * + * @param ut The UText representing the text to be operated on. + * @param nativeStart The index of the start of the region to be copied or moved + * @param nativeLimit The index of the character following the region to be replaced. + * @param nativeDest The destination index to which the source substring is copied or moved. + * @param move If TRUE, then the substring is moved, not copied/duplicated. + * @param status receives any error status. Possible errors include U_NO_WRITE_PERMISSION + * + * @stable ICU 3.4 + */ +typedef void U_CALLCONV +UTextCopy(UText *ut, + int64_t nativeStart, int64_t nativeLimit, + int64_t nativeDest, + UBool move, + UErrorCode *status); + +/** + * Function type declaration for UText.mapOffsetToNative(). + * Map from the current UChar offset within the current text chunk to + * the corresponding native index in the original source text. + * + * This is required only for text providers that do not use native UTF-16 indexes. + * + * @param ut the UText. + * @return Absolute (native) index corresponding to chunkOffset in the current chunk. + * The returned native index should always be to a code point boundary. + * + * @stable ICU 3.4 + */ +typedef int64_t U_CALLCONV +UTextMapOffsetToNative(const UText *ut); + +/** + * Function type declaration for UText.mapIndexToUTF16(). + * Map from a native index to a UChar offset within a text chunk. + * Behavior is undefined if the native index does not fall within the + * current chunk. + * + * This function is required only for text providers that do not use native UTF-16 indexes. + * + * @param ut The UText containing the text chunk. + * @param nativeIndex Absolute (native) text index, chunk->start<=index<=chunk->limit. + * @return Chunk-relative UTF-16 offset corresponding to the specified native + * index. + * + * @stable ICU 3.4 + */ +typedef int32_t U_CALLCONV +UTextMapNativeIndexToUTF16(const UText *ut, int64_t nativeIndex); + + +/** + * Function type declaration for UText.utextClose(). + * + * A Text Provider close function is only required for provider types that make + * allocations in their open function (or other functions) that must be + * cleaned when the UText is closed. + * + * The allocation of the UText struct itself and any "extra" storage + * associated with the UText is handled by the common UText implementation + * and does not require provider specific cleanup in a close function. + * + * Most UText provider implementations do not need to implement this function. + * + * @param ut A UText object to be closed. + * + * @stable ICU 3.4 + */ +typedef void U_CALLCONV +UTextClose(UText *ut); + + +/** + * (public) Function dispatch table for UText. + * Conceptually very much like a C++ Virtual Function Table. + * This struct defines the organization of the table. + * Each text provider implementation must provide an + * actual table that is initialized with the appropriate functions + * for the type of text being handled. + * @stable ICU 3.6 + */ +struct UTextFuncs { + /** + * (public) Function table size, sizeof(UTextFuncs) + * Intended for use should the table grow to accomodate added + * functions in the future, to allow tests for older format + * function tables that do not contain the extensions. + * + * Fields are placed for optimal alignment on + * 32/64/128-bit-pointer machines, by normally grouping together + * 4 32-bit fields, + * 4 pointers, + * 2 64-bit fields + * in sequence. + * @stable ICU 3.6 + */ + int32_t tableSize; + + /** + * (private) Alignment padding. + * Do not use, reserved for use by the UText framework only. + * @internal + */ + int32_t reserved1, reserved2, reserved3; + + + /** + * (public) Function pointer for UTextClone + * + * @see UTextClone + * @stable ICU 3.6 + */ + UTextClone *clone; + + /** + * (public) function pointer for UTextLength + * May be expensive to compute! + * + * @see UTextLength + * @stable ICU 3.6 + */ + UTextNativeLength *nativeLength; + + /** + * (public) Function pointer for UTextAccess. + * + * @see UTextAccess + * @stable ICU 3.6 + */ + UTextAccess *access; + + /** + * (public) Function pointer for UTextExtract. + * + * @see UTextExtract + * @stable ICU 3.6 + */ + UTextExtract *extract; + + /** + * (public) Function pointer for UTextReplace. + * + * @see UTextReplace + * @stable ICU 3.6 + */ + UTextReplace *replace; + + /** + * (public) Function pointer for UTextCopy. + * + * @see UTextCopy + * @stable ICU 3.6 + */ + UTextCopy *copy; + + /** + * (public) Function pointer for UTextMapOffsetToNative. + * + * @see UTextMapOffsetToNative + * @stable ICU 3.6 + */ + UTextMapOffsetToNative *mapOffsetToNative; + + /** + * (public) Function pointer for UTextMapNativeIndexToUTF16. + * + * @see UTextMapNativeIndexToUTF16 + * @stable ICU 3.6 + */ + UTextMapNativeIndexToUTF16 *mapNativeIndexToUTF16; + + /** + * (public) Function pointer for UTextClose. + * + * @see UTextClose + * @stable ICU 3.6 + */ + UTextClose *close; + + /** + * (private) Spare function pointer + * @internal + */ + + UTextClose *spare1; + /** + * (private) Spare function pointer + * @internal + */ + UTextClose *spare2; + + /** + * (private) Spare function pointer + * @internal + */ + UTextClose *spare3; + +}; +typedef struct UTextFuncs UTextFuncs; + + /** + * UText struct. Provides the interface between the generic UText access code + * and the UText provider code that works on specific kinds of + * text (UTF-8, noncontiguous UTF-16, whatever.) + * + * Applications that are using predefined types of text providers + * to pass text data to ICU services will have no need to view the + * internals of the UText structs that they open. + * + * @stable ICU 3.6 + */ +struct UText { + /** + * (private) Magic. Used to help detect when UText functions are handed + * invalid or unitialized UText structs. + * utext_openXYZ() functions take an initialized, + * but not necessarily open, UText struct as an + * optional fill-in parameter. This magic field + * is used to check for that initialization. + * Text provider close functions must NOT clear + * the magic field because that would prevent + * reuse of the UText struct. + * @internal + */ + uint32_t magic; + + + /** + * (private) Flags for managing the allocation and freeing of + * memory associated with this UText. + * @internal + */ + int32_t flags; + + + /** + * Text provider properties. This set of flags is maintainted by the + * text provider implementation. + * @stable ICU 3.4 + */ + int32_t providerProperties; + + /** + * (public) sizeOfStruct=sizeof(UText) + * Allows possible backward compatible extension. + * + * @stable ICU 3.4 + */ + int32_t sizeOfStruct; + + /* ------ 16 byte alignment boundary ----------- */ + + + /** + * (protected) Native index of the first character position following + * the current chunk. + * @stable ICU 3.6 + */ + int64_t chunkNativeLimit; + + /** + * (protected) Size in bytes of the extra space (pExtra). + * @stable ICU 3.4 + */ + int32_t extraSize; + + /** + * (protected) The highest chunk offset where native indexing and + * chunk (UTF-16) indexing correspond. For UTF-16 sources, value + * will be equal to chunkLength. + * + * @stable ICU 3.6 + */ + int32_t nativeIndexingLimit; + + /* ---- 16 byte alignment boundary------ */ + + /** + * (protected) Native index of the first character in the text chunk. + * @stable ICU 3.6 + */ + int64_t chunkNativeStart; + + /** + * (protected) Current iteration position within the text chunk (UTF-16 buffer). + * This is the index to the character that will be returned by utext_next32(). + * @stable ICU 3.6 + */ + int32_t chunkOffset; + + /** + * (protected) Length the text chunk (UTF-16 buffer), in UChars. + * @stable ICU 3.6 + */ + int32_t chunkLength; + + /* ---- 16 byte alignment boundary-- */ + + + /** + * (protected) pointer to a chunk of text in UTF-16 format. + * May refer either to original storage of the source of the text, or + * if conversion was required, to a buffer owned by the UText. + * @stable ICU 3.6 + */ + const UChar *chunkContents; + + /** + * (public) Pointer to Dispatch table for accessing functions for this UText. + * @stable ICU 3.6 + */ + UTextFuncs *pFuncs; + + /** + * (protected) Pointer to additional space requested by the + * text provider during the utext_open operation. + * @stable ICU 3.4 + */ + void *pExtra; + + /** + * (protected) Pointer to string or text-containin object or similar. + * This is the source of the text that this UText is wrapping, in a format + * that is known to the text provider functions. + * @stable ICU 3.4 + */ + const void *context; + + /* --- 16 byte alignment boundary--- */ + + /** + * (protected) Pointer fields available for use by the text provider. + * Not used by UText common code. + * @stable ICU 3.6 + */ + const void *p; + /** + * (protected) Pointer fields available for use by the text provider. + * Not used by UText common code. + * @stable ICU 3.6 + */ + const void *q; + /** + * (protected) Pointer fields available for use by the text provider. + * Not used by UText common code. + * @stable ICU 3.6 + */ + const void *r; + + /** + * Private field reserved for future use by the UText framework + * itself. This is not to be touched by the text providers. + * @internal ICU 3.4 + */ + void *privP; + + + /* --- 16 byte alignment boundary--- */ + + + /** + * (protected) Integer field reserved for use by the text provider. + * Not used by the UText framework, or by the client (user) of the UText. + * @stable ICU 3.4 + */ + int64_t a; + + /** + * (protected) Integer field reserved for use by the text provider. + * Not used by the UText framework, or by the client (user) of the UText. + * @stable ICU 3.4 + */ + int32_t b; + + /** + * (protected) Integer field reserved for use by the text provider. + * Not used by the UText framework, or by the client (user) of the UText. + * @stable ICU 3.4 + */ + int32_t c; + + /* ---- 16 byte alignment boundary---- */ + + + /** + * Private field reserved for future use by the UText framework + * itself. This is not to be touched by the text providers. + * @internal ICU 3.4 + */ + int64_t privA; + /** + * Private field reserved for future use by the UText framework + * itself. This is not to be touched by the text providers. + * @internal ICU 3.4 + */ + int32_t privB; + /** + * Private field reserved for future use by the UText framework + * itself. This is not to be touched by the text providers. + * @internal ICU 3.4 + */ + int32_t privC; +}; + + +/** + * Common function for use by Text Provider implementations to allocate and/or initialize + * a new UText struct. To be called in the implementation of utext_open() functions. + * If the supplied UText parameter is null, a new UText struct will be allocated on the heap. + * If the supplied UText is already open, the provider's close function will be called + * so that the struct can be reused by the open that is in progress. + * + * @param ut pointer to a UText struct to be re-used, or null if a new UText + * should be allocated. + * @param extraSpace The amount of additional space to be allocated as part + * of this UText, for use by types of providers that require + * additional storage. + * @param status Errors are returned here. + * @return pointer to the UText, allocated if necessary, with extra space set up if requested. + * @stable ICU 3.4 + */ +U_STABLE UText * U_EXPORT2 +utext_setup(UText *ut, int32_t extraSpace, UErrorCode *status); + +/** + * @internal + * Value used to help identify correctly initialized UText structs. + * Note: must be publicly visible so that UTEXT_INITIALIZER can access it. + */ +enum { + UTEXT_MAGIC = 0x345ad82c +}; + +/** + * initializer to be used with local (stack) instances of a UText + * struct. UText structs must be initialized before passing + * them to one of the utext_open functions. + * + * @stable ICU 3.6 + */ +#define UTEXT_INITIALIZER { \ + UTEXT_MAGIC, /* magic */ \ + 0, /* flags */ \ + 0, /* providerProps */ \ + sizeof(UText), /* sizeOfStruct */ \ + 0, /* chunkNativeLimit */ \ + 0, /* extraSize */ \ + 0, /* nativeIndexingLimit */ \ + 0, /* chunkNativeStart */ \ + 0, /* chunkOffset */ \ + 0, /* chunkLength */ \ + NULL, /* chunkContents */ \ + NULL, /* pFuncs */ \ + NULL, /* pExtra */ \ + NULL, /* context */ \ + NULL, NULL, NULL, /* p, q, r */ \ + NULL, /* privP */ \ + 0, 0, 0, /* a, b, c */ \ + 0, 0, 0 /* privA,B,C, */ \ + } + + +U_CDECL_END + + + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/utf.h b/Build/source/libs/icu/icu-xetex/common/unicode/utf.h new file mode 100644 index 00000000000..1682283cc6d --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/utf.h @@ -0,0 +1,227 @@ +/* +******************************************************************************* +* +* Copyright (C) 1999-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: utf.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 1999sep09 +* created by: Markus W. Scherer +*/ + +/** + * \file + * \brief C API: Code point macros + * + * This file defines macros for checking whether a code point is + * a surrogate or a non-character etc. + * + * The UChar and UChar32 data types for Unicode code units and code points + * are defined in umachines.h because they can be machine-dependent. + * + * utf.h is included by utypes.h and itself includes utf8.h and utf16.h after some + * common definitions. Those files define macros for efficiently getting code points + * in and out of UTF-8/16 strings. + * utf16.h macros have "U16_" prefixes. + * utf8.h defines similar macros with "U8_" prefixes for UTF-8 string handling. + * + * ICU processes 16-bit Unicode strings. + * Most of the time, such strings are well-formed UTF-16. + * Single, unpaired surrogates must be handled as well, and are treated in ICU + * like regular code points where possible. + * (Pairs of surrogate code points are indistinguishable from supplementary + * code points encoded as pairs of supplementary code units.) + * + * In fact, almost all Unicode code points in normal text (>99%) + * are on the BMP (<=U+ffff) and even <=U+d7ff. + * ICU functions handle supplementary code points (U+10000..U+10ffff) + * but are optimized for the much more frequently occurring BMP code points. + * + * utf.h defines UChar to be an unsigned 16-bit integer. If this matches wchar_t, then + * UChar is defined to be exactly wchar_t, otherwise uint16_t. + * + * UChar32 is defined to be a signed 32-bit integer (int32_t), large enough for a 21-bit + * Unicode code point (Unicode scalar value, 0..0x10ffff). + * Before ICU 2.4, the definition of UChar32 was similarly platform-dependent as + * the definition of UChar. For details see the documentation for UChar32 itself. + * + * utf.h also defines a small number of C macros for single Unicode code points. + * These are simple checks for surrogates and non-characters. + * For actual Unicode character properties see uchar.h. + * + * By default, string operations must be done with error checking in case + * a string is not well-formed UTF-16. + * The macros will detect if a surrogate code unit is unpaired + * (lead unit without trail unit or vice versa) and just return the unit itself + * as the code point. + * (It is an accidental property of Unicode and UTF-16 that all + * malformed sequences can be expressed unambiguously with a distinct subrange + * of Unicode code points.) + * + * The regular "safe" macros require that the initial, passed-in string index + * is within bounds. They only check the index when they read more than one + * code unit. This is usually done with code similar to the following loop: + *

while(i
+ *
+ * When it is safe to assume that text is well-formed UTF-16
+ * (does not contain single, unpaired surrogates), then one can use
+ * U16_..._UNSAFE macros.
+ * These do not check for proper code unit sequences or truncated text and may
+ * yield wrong results or even cause a crash if they are used with "malformed"
+ * text.
+ * In practice, U16_..._UNSAFE macros will produce slightly less code but
+ * should not be faster because the processing is only different when a
+ * surrogate code unit is detected, which will be rare.
+ *
+ * Similarly for UTF-8, there are "safe" macros without a suffix,
+ * and U8_..._UNSAFE versions.
+ * The performance differences are much larger here because UTF-8 provides so
+ * many opportunities for malformed sequences.
+ * The unsafe UTF-8 macros are entirely implemented inside the macro definitions
+ * and are fast, while the safe UTF-8 macros call functions for all but the
+ * trivial (ASCII) cases.
+ * (ICU 3.6 optimizes U8_NEXT() and U8_APPEND() to handle most other common
+ * characters inline as well.)
+ *
+ * Unlike with UTF-16, malformed sequences cannot be expressed with distinct
+ * code point values (0..U+10ffff). They are indicated with negative values instead.
+ *
+ * For more information see the ICU User Guide Strings chapter
+ * (http://icu-project.org/userguide/strings.html).
+ *
+ * Usage:
+ * ICU coding guidelines for if() statements should be followed when using these macros.
+ * Compound statements (curly braces {}) must be used  for if-else-while... 
+ * bodies and all macro statements should be terminated with semicolon.
+ *
+ * @stable ICU 2.4
+ */
+
+#ifndef __UTF_H__
+#define __UTF_H__
+
+#include "unicode/utypes.h"
+/* include the utfXX.h after the following definitions */
+
+/* single-code point definitions -------------------------------------------- */
+
+/**
+ * This value is intended for sentinel values for APIs that
+ * (take or) return single code points (UChar32).
+ * It is outside of the Unicode code point range 0..0x10ffff.
+ * 
+ * For example, a "done" or "error" value in a new API
+ * could be indicated with U_SENTINEL.
+ *
+ * ICU APIs designed before ICU 2.4 usually define service-specific "done"
+ * values, mostly 0xffff.
+ * Those may need to be distinguished from
+ * actual U+ffff text contents by calling functions like
+ * CharacterIterator::hasNext() or UnicodeString::length().
+ *
+ * @return -1
+ * @see UChar32
+ * @stable ICU 2.4
+ */
+#define U_SENTINEL (-1)
+
+/**
+ * Is this code point a Unicode noncharacter?
+ * @param c 32-bit code point
+ * @return TRUE or FALSE
+ * @stable ICU 2.4
+ */
+#define U_IS_UNICODE_NONCHAR(c) \
+    ((c)>=0xfdd0 && \
+     ((uint32_t)(c)<=0xfdef || ((c)&0xfffe)==0xfffe) && \
+     (uint32_t)(c)<=0x10ffff)
+
+/**
+ * Is c a Unicode code point value (0..U+10ffff)
+ * that can be assigned a character?
+ *
+ * Code points that are not characters include:
+ * - single surrogate code points (U+d800..U+dfff, 2048 code points)
+ * - the last two code points on each plane (U+__fffe and U+__ffff, 34 code points)
+ * - U+fdd0..U+fdef (new with Unicode 3.1, 32 code points)
+ * - the highest Unicode code point value is U+10ffff
+ *
+ * This means that all code points below U+d800 are character code points,
+ * and that boundary is tested first for performance.
+ *
+ * @param c 32-bit code point
+ * @return TRUE or FALSE
+ * @stable ICU 2.4
+ */
+#define U_IS_UNICODE_CHAR(c) \
+    ((uint32_t)(c)<0xd800 || \
+        ((uint32_t)(c)>0xdfff && \
+         (uint32_t)(c)<=0x10ffff && \
+         !U_IS_UNICODE_NONCHAR(c)))
+
+/**
+ * Is this code point a BMP code point (U+0000..U+ffff)?
+ * @param c 32-bit code point
+ * @return TRUE or FALSE
+ * @stable ICU 2.8
+ */
+#define U_IS_BMP(c) ((uint32_t)(c)<=0xffff)
+
+/**
+ * Is this code point a supplementary code point (U+10000..U+10ffff)?
+ * @param c 32-bit code point
+ * @return TRUE or FALSE
+ * @stable ICU 2.8
+ */
+#define U_IS_SUPPLEMENTARY(c) ((uint32_t)((c)-0x10000)<=0xfffff)
+ 
+/**
+ * Is this code point a lead surrogate (U+d800..U+dbff)?
+ * @param c 32-bit code point
+ * @return TRUE or FALSE
+ * @stable ICU 2.4
+ */
+#define U_IS_LEAD(c) (((c)&0xfffffc00)==0xd800)
+
+/**
+ * Is this code point a trail surrogate (U+dc00..U+dfff)?
+ * @param c 32-bit code point
+ * @return TRUE or FALSE
+ * @stable ICU 2.4
+ */
+#define U_IS_TRAIL(c) (((c)&0xfffffc00)==0xdc00)
+
+/**
+ * Is this code point a surrogate (U+d800..U+dfff)?
+ * @param c 32-bit code point
+ * @return TRUE or FALSE
+ * @stable ICU 2.4
+ */
+#define U_IS_SURROGATE(c) (((c)&0xfffff800)==0xd800)
+
+/**
+ * Assuming c is a surrogate code point (U_IS_SURROGATE(c)),
+ * is it a lead surrogate?
+ * @param c 32-bit code point
+ * @return TRUE or FALSE
+ * @stable ICU 2.4
+ */
+#define U_IS_SURROGATE_LEAD(c) (((c)&0x400)==0)
+
+/* include the utfXX.h ------------------------------------------------------ */
+
+#include "unicode/utf8.h"
+#include "unicode/utf16.h"
+
+/* utf_old.h contains deprecated, pre-ICU 2.4 definitions */
+#include "unicode/utf_old.h"
+
+#endif
diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/utf16.h b/Build/source/libs/icu/icu-xetex/common/unicode/utf16.h
new file mode 100644
index 00000000000..719bc043bb2
--- /dev/null
+++ b/Build/source/libs/icu/icu-xetex/common/unicode/utf16.h
@@ -0,0 +1,605 @@
+/*
+*******************************************************************************
+*
+*   Copyright (C) 1999-2007, International Business Machines
+*   Corporation and others.  All Rights Reserved.
+*
+*******************************************************************************
+*   file name:  utf16.h
+*   encoding:   US-ASCII
+*   tab size:   8 (not used)
+*   indentation:4
+*
+*   created on: 1999sep09
+*   created by: Markus W. Scherer
+*/
+
+/**
+ * \file
+ * \brief C API: 16-bit Unicode handling macros
+ * 
+ * This file defines macros to deal with 16-bit Unicode (UTF-16) code units and strings.
+ * utf16.h is included by utf.h after unicode/umachine.h
+ * and some common definitions.
+ *
+ * For more information see utf.h and the ICU User Guide Strings chapter
+ * (http://icu-project.org/userguide/strings.html).
+ *
+ * Usage:
+ * ICU coding guidelines for if() statements should be followed when using these macros.
+ * Compound statements (curly braces {}) must be used  for if-else-while... 
+ * bodies and all macro statements should be terminated with semicolon.
+ */
+
+#ifndef __UTF16_H__
+#define __UTF16_H__
+
+/* utf.h must be included first. */
+#ifndef __UTF_H__
+#   include "unicode/utf.h"
+#endif
+
+/* single-code point definitions -------------------------------------------- */
+
+/**
+ * Does this code unit alone encode a code point (BMP, not a surrogate)?
+ * @param c 16-bit code unit
+ * @return TRUE or FALSE
+ * @stable ICU 2.4
+ */
+#define U16_IS_SINGLE(c) !U_IS_SURROGATE(c)
+
+/**
+ * Is this code unit a lead surrogate (U+d800..U+dbff)?
+ * @param c 16-bit code unit
+ * @return TRUE or FALSE
+ * @stable ICU 2.4
+ */
+#define U16_IS_LEAD(c) (((c)&0xfffffc00)==0xd800)
+
+/**
+ * Is this code unit a trail surrogate (U+dc00..U+dfff)?
+ * @param c 16-bit code unit
+ * @return TRUE or FALSE
+ * @stable ICU 2.4
+ */
+#define U16_IS_TRAIL(c) (((c)&0xfffffc00)==0xdc00)
+
+/**
+ * Is this code unit a surrogate (U+d800..U+dfff)?
+ * @param c 16-bit code unit
+ * @return TRUE or FALSE
+ * @stable ICU 2.4
+ */
+#define U16_IS_SURROGATE(c) U_IS_SURROGATE(c)
+
+/**
+ * Assuming c is a surrogate code point (U16_IS_SURROGATE(c)),
+ * is it a lead surrogate?
+ * @param c 16-bit code unit
+ * @return TRUE or FALSE
+ * @stable ICU 2.4
+ */
+#define U16_IS_SURROGATE_LEAD(c) (((c)&0x400)==0)
+
+/**
+ * Helper constant for U16_GET_SUPPLEMENTARY.
+ * @internal
+ */
+#define U16_SURROGATE_OFFSET ((0xd800<<10UL)+0xdc00-0x10000)
+
+/**
+ * Get a supplementary code point value (U+10000..U+10ffff)
+ * from its lead and trail surrogates.
+ * The result is undefined if the input values are not
+ * lead and trail surrogates.
+ *
+ * @param lead lead surrogate (U+d800..U+dbff)
+ * @param trail trail surrogate (U+dc00..U+dfff)
+ * @return supplementary code point (U+10000..U+10ffff)
+ * @stable ICU 2.4
+ */
+#define U16_GET_SUPPLEMENTARY(lead, trail) \
+    (((UChar32)(lead)<<10UL)+(UChar32)(trail)-U16_SURROGATE_OFFSET)
+
+
+/**
+ * Get the lead surrogate (0xd800..0xdbff) for a
+ * supplementary code point (0x10000..0x10ffff).
+ * @param supplementary 32-bit code point (U+10000..U+10ffff)
+ * @return lead surrogate (U+d800..U+dbff) for supplementary
+ * @stable ICU 2.4
+ */
+#define U16_LEAD(supplementary) (UChar)(((supplementary)>>10)+0xd7c0)
+
+/**
+ * Get the trail surrogate (0xdc00..0xdfff) for a
+ * supplementary code point (0x10000..0x10ffff).
+ * @param supplementary 32-bit code point (U+10000..U+10ffff)
+ * @return trail surrogate (U+dc00..U+dfff) for supplementary
+ * @stable ICU 2.4
+ */
+#define U16_TRAIL(supplementary) (UChar)(((supplementary)&0x3ff)|0xdc00)
+
+/**
+ * How many 16-bit code units are used to encode this Unicode code point? (1 or 2)
+ * The result is not defined if c is not a Unicode code point (U+0000..U+10ffff).
+ * @param c 32-bit code point
+ * @return 1 or 2
+ * @stable ICU 2.4
+ */
+#define U16_LENGTH(c) ((uint32_t)(c)<=0xffff ? 1 : 2)
+
+/**
+ * The maximum number of 16-bit code units per Unicode code point (U+0000..U+10ffff).
+ * @return 2
+ * @stable ICU 2.4
+ */
+#define U16_MAX_LENGTH 2
+
+/**
+ * Get a code point from a string at a random-access offset,
+ * without changing the offset.
+ * "Unsafe" macro, assumes well-formed UTF-16.
+ *
+ * The offset may point to either the lead or trail surrogate unit
+ * for a supplementary code point, in which case the macro will read
+ * the adjacent matching surrogate as well.
+ * The result is undefined if the offset points to a single, unpaired surrogate.
+ * Iteration through a string is more efficient with U16_NEXT_UNSAFE or U16_NEXT.
+ *
+ * @param s const UChar * string
+ * @param i string offset
+ * @param c output UChar32 variable
+ * @see U16_GET
+ * @stable ICU 2.4
+ */
+#define U16_GET_UNSAFE(s, i, c) { \
+    (c)=(s)[i]; \
+    if(U16_IS_SURROGATE(c)) { \
+        if(U16_IS_SURROGATE_LEAD(c)) { \
+            (c)=U16_GET_SUPPLEMENTARY((c), (s)[(i)+1]); \
+        } else { \
+            (c)=U16_GET_SUPPLEMENTARY((s)[(i)-1], (c)); \
+        } \
+    } \
+}
+
+/**
+ * Get a code point from a string at a random-access offset,
+ * without changing the offset.
+ * "Safe" macro, handles unpaired surrogates and checks for string boundaries.
+ *
+ * The offset may point to either the lead or trail surrogate unit
+ * for a supplementary code point, in which case the macro will read
+ * the adjacent matching surrogate as well.
+ * If the offset points to a single, unpaired surrogate, then that itself
+ * will be returned as the code point.
+ * Iteration through a string is more efficient with U16_NEXT_UNSAFE or U16_NEXT.
+ *
+ * @param s const UChar * string
+ * @param start starting string offset (usually 0)
+ * @param i string offset, must be start<=i=(start) && U16_IS_LEAD(__c2=(s)[(i)-1])) { \
+                (c)=U16_GET_SUPPLEMENTARY(__c2, (c)); \
+            } \
+        } \
+    } \
+}
+
+/* definitions with forward iteration --------------------------------------- */
+
+/**
+ * Get a code point from a string at a code point boundary offset,
+ * and advance the offset to the next code point boundary.
+ * (Post-incrementing forward iteration.)
+ * "Unsafe" macro, assumes well-formed UTF-16.
+ *
+ * The offset may point to the lead surrogate unit
+ * for a supplementary code point, in which case the macro will read
+ * the following trail surrogate as well.
+ * If the offset points to a trail surrogate, then that itself
+ * will be returned as the code point.
+ * The result is undefined if the offset points to a single, unpaired lead surrogate.
+ *
+ * @param s const UChar * string
+ * @param i string offset
+ * @param c output UChar32 variable
+ * @see U16_NEXT
+ * @stable ICU 2.4
+ */
+#define U16_NEXT_UNSAFE(s, i, c) { \
+    (c)=(s)[(i)++]; \
+    if(U16_IS_LEAD(c)) { \
+        (c)=U16_GET_SUPPLEMENTARY((c), (s)[(i)++]); \
+    } \
+}
+
+/**
+ * Get a code point from a string at a code point boundary offset,
+ * and advance the offset to the next code point boundary.
+ * (Post-incrementing forward iteration.)
+ * "Safe" macro, handles unpaired surrogates and checks for string boundaries.
+ *
+ * The offset may point to the lead surrogate unit
+ * for a supplementary code point, in which case the macro will read
+ * the following trail surrogate as well.
+ * If the offset points to a trail surrogate or
+ * to a single, unpaired lead surrogate, then that itself
+ * will be returned as the code point.
+ *
+ * @param s const UChar * string
+ * @param i string offset, must be i>10)+0xd7c0); \
+        (s)[(i)++]=(uint16_t)(((c)&0x3ff)|0xdc00); \
+    } \
+}
+
+/**
+ * Append a code point to a string, overwriting 1 or 2 code units.
+ * The offset points to the current end of the string contents
+ * and is advanced (post-increment).
+ * "Safe" macro, checks for a valid code point.
+ * If a surrogate pair is written, checks for sufficient space in the string.
+ * If the code point is not valid or a trail surrogate does not fit,
+ * then isError is set to TRUE.
+ *
+ * @param s const UChar * string buffer
+ * @param i string offset, must be i>10)+0xd7c0); \
+        (s)[(i)++]=(uint16_t)(((c)&0x3ff)|0xdc00); \
+    } else /* c>0x10ffff or not enough space */ { \
+        (isError)=TRUE; \
+    } \
+}
+
+/**
+ * Advance the string offset from one code point boundary to the next.
+ * (Post-incrementing iteration.)
+ * "Unsafe" macro, assumes well-formed UTF-16.
+ *
+ * @param s const UChar * string
+ * @param i string offset
+ * @see U16_FWD_1
+ * @stable ICU 2.4
+ */
+#define U16_FWD_1_UNSAFE(s, i) { \
+    if(U16_IS_LEAD((s)[(i)++])) { \
+        ++(i); \
+    } \
+}
+
+/**
+ * Advance the string offset from one code point boundary to the next.
+ * (Post-incrementing iteration.)
+ * "Safe" macro, handles unpaired surrogates and checks for string boundaries.
+ *
+ * @param s const UChar * string
+ * @param i string offset, must be i0) { \
+        U16_FWD_1_UNSAFE(s, i); \
+        --__N; \
+    } \
+}
+
+/**
+ * Advance the string offset from one code point boundary to the n-th next one,
+ * i.e., move forward by n code points.
+ * (Post-incrementing iteration.)
+ * "Safe" macro, handles unpaired surrogates and checks for string boundaries.
+ *
+ * @param s const UChar * string
+ * @param i string offset, must be i0 && (i)<(length)) { \
+        U16_FWD_1(s, i, length); \
+        --__N; \
+    } \
+}
+
+/**
+ * Adjust a random-access offset to a code point boundary
+ * at the start of a code point.
+ * If the offset points to the trail surrogate of a surrogate pair,
+ * then the offset is decremented.
+ * Otherwise, it is not modified.
+ * "Unsafe" macro, assumes well-formed UTF-16.
+ *
+ * @param s const UChar * string
+ * @param i string offset
+ * @see U16_SET_CP_START
+ * @stable ICU 2.4
+ */
+#define U16_SET_CP_START_UNSAFE(s, i) { \
+    if(U16_IS_TRAIL((s)[i])) { \
+        --(i); \
+    } \
+}
+
+/**
+ * Adjust a random-access offset to a code point boundary
+ * at the start of a code point.
+ * If the offset points to the trail surrogate of a surrogate pair,
+ * then the offset is decremented.
+ * Otherwise, it is not modified.
+ * "Safe" macro, handles unpaired surrogates and checks for string boundaries.
+ *
+ * @param s const UChar * string
+ * @param start starting string offset (usually 0)
+ * @param i string offset, must be start<=i
+ * @see U16_SET_CP_START_UNSAFE
+ * @stable ICU 2.4
+ */
+#define U16_SET_CP_START(s, start, i) { \
+    if(U16_IS_TRAIL((s)[i]) && (i)>(start) && U16_IS_LEAD((s)[(i)-1])) { \
+        --(i); \
+    } \
+}
+
+/* definitions with backward iteration -------------------------------------- */
+
+/**
+ * Move the string offset from one code point boundary to the previous one
+ * and get the code point between them.
+ * (Pre-decrementing backward iteration.)
+ * "Unsafe" macro, assumes well-formed UTF-16.
+ *
+ * The input offset may be the same as the string length.
+ * If the offset is behind a trail surrogate unit
+ * for a supplementary code point, then the macro will read
+ * the preceding lead surrogate as well.
+ * If the offset is behind a lead surrogate, then that itself
+ * will be returned as the code point.
+ * The result is undefined if the offset is behind a single, unpaired trail surrogate.
+ *
+ * @param s const UChar * string
+ * @param i string offset
+ * @param c output UChar32 variable
+ * @see U16_PREV
+ * @stable ICU 2.4
+ */
+#define U16_PREV_UNSAFE(s, i, c) { \
+    (c)=(s)[--(i)]; \
+    if(U16_IS_TRAIL(c)) { \
+        (c)=U16_GET_SUPPLEMENTARY((s)[--(i)], (c)); \
+    } \
+}
+
+/**
+ * Move the string offset from one code point boundary to the previous one
+ * and get the code point between them.
+ * (Pre-decrementing backward iteration.)
+ * "Safe" macro, handles unpaired surrogates and checks for string boundaries.
+ *
+ * The input offset may be the same as the string length.
+ * If the offset is behind a trail surrogate unit
+ * for a supplementary code point, then the macro will read
+ * the preceding lead surrogate as well.
+ * If the offset is behind a lead surrogate or behind a single, unpaired
+ * trail surrogate, then that itself
+ * will be returned as the code point.
+ *
+ * @param s const UChar * string
+ * @param start starting string offset (usually 0)
+ * @param i string offset, must be start(start) && U16_IS_LEAD(__c2=(s)[(i)-1])) { \
+            --(i); \
+            (c)=U16_GET_SUPPLEMENTARY(__c2, (c)); \
+        } \
+    } \
+}
+
+/**
+ * Move the string offset from one code point boundary to the previous one.
+ * (Pre-decrementing backward iteration.)
+ * The input offset may be the same as the string length.
+ * "Unsafe" macro, assumes well-formed UTF-16.
+ *
+ * @param s const UChar * string
+ * @param i string offset
+ * @see U16_BACK_1
+ * @stable ICU 2.4
+ */
+#define U16_BACK_1_UNSAFE(s, i) { \
+    if(U16_IS_TRAIL((s)[--(i)])) { \
+        --(i); \
+    } \
+}
+
+/**
+ * Move the string offset from one code point boundary to the previous one.
+ * (Pre-decrementing backward iteration.)
+ * The input offset may be the same as the string length.
+ * "Safe" macro, handles unpaired surrogates and checks for string boundaries.
+ *
+ * @param s const UChar * string
+ * @param start starting string offset (usually 0)
+ * @param i string offset, must be start(start) && U16_IS_LEAD((s)[(i)-1])) { \
+        --(i); \
+    } \
+}
+
+/**
+ * Move the string offset from one code point boundary to the n-th one before it,
+ * i.e., move backward by n code points.
+ * (Pre-decrementing backward iteration.)
+ * The input offset may be the same as the string length.
+ * "Unsafe" macro, assumes well-formed UTF-16.
+ *
+ * @param s const UChar * string
+ * @param i string offset
+ * @param n number of code points to skip
+ * @see U16_BACK_N
+ * @stable ICU 2.4
+ */
+#define U16_BACK_N_UNSAFE(s, i, n) { \
+    int32_t __N=(n); \
+    while(__N>0) { \
+        U16_BACK_1_UNSAFE(s, i); \
+        --__N; \
+    } \
+}
+
+/**
+ * Move the string offset from one code point boundary to the n-th one before it,
+ * i.e., move backward by n code points.
+ * (Pre-decrementing backward iteration.)
+ * The input offset may be the same as the string length.
+ * "Safe" macro, handles unpaired surrogates and checks for string boundaries.
+ *
+ * @param s const UChar * string
+ * @param start start of string
+ * @param i string offset, must be start0 && (i)>(start)) { \
+        U16_BACK_1(s, start, i); \
+        --__N; \
+    } \
+}
+
+/**
+ * Adjust a random-access offset to a code point boundary after a code point.
+ * If the offset is behind the lead surrogate of a surrogate pair,
+ * then the offset is incremented.
+ * Otherwise, it is not modified.
+ * The input offset may be the same as the string length.
+ * "Unsafe" macro, assumes well-formed UTF-16.
+ *
+ * @param s const UChar * string
+ * @param i string offset
+ * @see U16_SET_CP_LIMIT
+ * @stable ICU 2.4
+ */
+#define U16_SET_CP_LIMIT_UNSAFE(s, i) { \
+    if(U16_IS_LEAD((s)[(i)-1])) { \
+        ++(i); \
+    } \
+}
+
+/**
+ * Adjust a random-access offset to a code point boundary after a code point.
+ * If the offset is behind the lead surrogate of a surrogate pair,
+ * then the offset is incremented.
+ * Otherwise, it is not modified.
+ * The input offset may be the same as the string length.
+ * "Safe" macro, handles unpaired surrogates and checks for string boundaries.
+ *
+ * @param s const UChar * string
+ * @param start starting string offset (usually 0)
+ * @param i string offset, start<=i<=length
+ * @param length string length
+ * @see U16_SET_CP_LIMIT_UNSAFE
+ * @stable ICU 2.4
+ */
+#define U16_SET_CP_LIMIT(s, start, i, length) { \
+    if((start)<(i) && (i)<(length) && U16_IS_LEAD((s)[(i)-1]) && U16_IS_TRAIL((s)[i])) { \
+        ++(i); \
+    } \
+}
+
+#endif
diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/utf32.h b/Build/source/libs/icu/icu-xetex/common/unicode/utf32.h
new file mode 100644
index 00000000000..bf63e69dba0
--- /dev/null
+++ b/Build/source/libs/icu/icu-xetex/common/unicode/utf32.h
@@ -0,0 +1,23 @@
+/*
+*******************************************************************************
+*
+*   Copyright (C) 1999-2001, International Business Machines
+*   Corporation and others.  All Rights Reserved.
+*
+*******************************************************************************
+*   file name:  utf32.h
+*   encoding:   US-ASCII
+*   tab size:   8 (not used)
+*   indentation:4
+*
+*   created on: 1999sep20
+*   created by: Markus W. Scherer
+*/
+/**
+ * \file
+ * \brief C API: UTF-32 macros
+ *
+ * This file is obsolete and its contents moved to utf_old.h.
+ * See utf_old.h and Jitterbug 2150 and its discussion on the ICU mailing list
+ * in September 2002.
+ */
diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/utf8.h b/Build/source/libs/icu/icu-xetex/common/unicode/utf8.h
new file mode 100644
index 00000000000..1142c44df5f
--- /dev/null
+++ b/Build/source/libs/icu/icu-xetex/common/unicode/utf8.h
@@ -0,0 +1,652 @@
+/*
+*******************************************************************************
+*
+*   Copyright (C) 1999-2007, International Business Machines
+*   Corporation and others.  All Rights Reserved.
+*
+*******************************************************************************
+*   file name:  utf8.h
+*   encoding:   US-ASCII
+*   tab size:   8 (not used)
+*   indentation:4
+*
+*   created on: 1999sep13
+*   created by: Markus W. Scherer
+*/
+
+/**
+ * \file
+ * \brief C API: 8-bit Unicode handling macros
+ * 
+ * This file defines macros to deal with 8-bit Unicode (UTF-8) code units (bytes) and strings.
+ * utf8.h is included by utf.h after unicode/umachine.h
+ * and some common definitions.
+ *
+ * For more information see utf.h and the ICU User Guide Strings chapter
+ * (http://icu-project.org/userguide/strings.html).
+ *
+ * Usage:
+ * ICU coding guidelines for if() statements should be followed when using these macros.
+ * Compound statements (curly braces {}) must be used  for if-else-while... 
+ * bodies and all macro statements should be terminated with semicolon.
+ */
+
+#ifndef __UTF8_H__
+#define __UTF8_H__
+
+/* utf.h must be included first. */
+#ifndef __UTF_H__
+#   include "unicode/utf.h"
+#endif
+
+/* internal definitions ----------------------------------------------------- */
+
+/**
+ * \var utf8_countTrailBytes
+ * Internal array with numbers of trail bytes for any given byte used in
+ * lead byte position.
+ * @internal
+ */
+#ifdef U_UTF8_IMPL
+U_EXPORT const uint8_t 
+#elif defined(U_STATIC_IMPLEMENTATION) || defined(U_COMMON_IMPLEMENTATION)
+U_CFUNC const uint8_t
+#else
+U_CFUNC U_IMPORT const uint8_t /* U_IMPORT2? */ /*U_IMPORT*/ 
+#endif
+utf8_countTrailBytes[256];
+
+/**
+ * Count the trail bytes for a UTF-8 lead byte.
+ * @internal
+ */
+#define U8_COUNT_TRAIL_BYTES(leadByte) (utf8_countTrailBytes[(uint8_t)leadByte])
+
+/**
+ * Mask a UTF-8 lead byte, leave only the lower bits that form part of the code point value.
+ * @internal
+ */
+#define U8_MASK_LEAD_BYTE(leadByte, countTrailBytes) ((leadByte)&=(1<<(6-(countTrailBytes)))-1)
+
+/**
+ * Function for handling "next code point" with error-checking.
+ * @internal
+ */
+U_INTERNAL UChar32 U_EXPORT2
+utf8_nextCharSafeBody(const uint8_t *s, int32_t *pi, int32_t length, UChar32 c, UBool strict);
+
+/**
+ * Function for handling "append code point" with error-checking.
+ * @internal
+ */
+U_INTERNAL int32_t U_EXPORT2
+utf8_appendCharSafeBody(uint8_t *s, int32_t i, int32_t length, UChar32 c, UBool *pIsError);
+
+/**
+ * Function for handling "previous code point" with error-checking.
+ * @internal
+ */
+U_INTERNAL UChar32 U_EXPORT2
+utf8_prevCharSafeBody(const uint8_t *s, int32_t start, int32_t *pi, UChar32 c, UBool strict);
+
+/**
+ * Function for handling "skip backward one code point" with error-checking.
+ * @internal
+ */
+U_INTERNAL int32_t U_EXPORT2
+utf8_back1SafeBody(const uint8_t *s, int32_t start, int32_t i);
+
+/* single-code point definitions -------------------------------------------- */
+
+/**
+ * Does this code unit (byte) encode a code point by itself (US-ASCII 0..0x7f)?
+ * @param c 8-bit code unit (byte)
+ * @return TRUE or FALSE
+ * @stable ICU 2.4
+ */
+#define U8_IS_SINGLE(c) (((c)&0x80)==0)
+
+/**
+ * Is this code unit (byte) a UTF-8 lead byte?
+ * @param c 8-bit code unit (byte)
+ * @return TRUE or FALSE
+ * @stable ICU 2.4
+ */
+#define U8_IS_LEAD(c) ((uint8_t)((c)-0xc0)<0x3e)
+
+/**
+ * Is this code unit (byte) a UTF-8 trail byte?
+ * @param c 8-bit code unit (byte)
+ * @return TRUE or FALSE
+ * @stable ICU 2.4
+ */
+#define U8_IS_TRAIL(c) (((c)&0xc0)==0x80)
+
+/**
+ * How many code units (bytes) are used for the UTF-8 encoding
+ * of this Unicode code point?
+ * @param c 32-bit code point
+ * @return 1..4, or 0 if c is a surrogate or not a Unicode code point
+ * @stable ICU 2.4
+ */
+#define U8_LENGTH(c) \
+    ((uint32_t)(c)<=0x7f ? 1 : \
+        ((uint32_t)(c)<=0x7ff ? 2 : \
+            ((uint32_t)(c)<=0xd7ff ? 3 : \
+                ((uint32_t)(c)<=0xdfff || (uint32_t)(c)>0x10ffff ? 0 : \
+                    ((uint32_t)(c)<=0xffff ? 3 : 4)\
+                ) \
+            ) \
+        ) \
+    )
+
+/**
+ * The maximum number of UTF-8 code units (bytes) per Unicode code point (U+0000..U+10ffff).
+ * @return 4
+ * @stable ICU 2.4
+ */
+#define U8_MAX_LENGTH 4
+
+/**
+ * Get a code point from a string at a random-access offset,
+ * without changing the offset.
+ * The offset may point to either the lead byte or one of the trail bytes
+ * for a code point, in which case the macro will read all of the bytes
+ * for the code point.
+ * The result is undefined if the offset points to an illegal UTF-8
+ * byte sequence.
+ * Iteration through a string is more efficient with U8_NEXT_UNSAFE or U8_NEXT.
+ *
+ * @param s const uint8_t * string
+ * @param i string offset
+ * @param c output UChar32 variable
+ * @see U8_GET
+ * @stable ICU 2.4
+ */
+#define U8_GET_UNSAFE(s, i, c) { \
+    int32_t _u8_get_unsafe_index=(int32_t)(i); \
+    U8_SET_CP_START_UNSAFE(s, _u8_get_unsafe_index); \
+    U8_NEXT_UNSAFE(s, _u8_get_unsafe_index, c); \
+}
+
+/**
+ * Get a code point from a string at a random-access offset,
+ * without changing the offset.
+ * The offset may point to either the lead byte or one of the trail bytes
+ * for a code point, in which case the macro will read all of the bytes
+ * for the code point.
+ * If the offset points to an illegal UTF-8 byte sequence, then
+ * c is set to a negative value.
+ * Iteration through a string is more efficient with U8_NEXT_UNSAFE or U8_NEXT.
+ *
+ * @param s const uint8_t * string
+ * @param start starting string offset
+ * @param i string offset, must be start<=i=0x80) { \
+        uint8_t __t1, __t2; \
+        if( /* handle U+1000..U+CFFF inline */ \
+            (0xe0<(c) && (c)<=0xec) && \
+            (((i)+1)<(length)) && \
+            (__t1=(uint8_t)((s)[i]-0x80))<=0x3f && \
+            (__t2=(uint8_t)((s)[(i)+1]-0x80))<= 0x3f \
+        ) { \
+            /* no need for (c&0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ \
+            (c)=(UChar)(((c)<<12)|(__t1<<6)|__t2); \
+            (i)+=2; \
+        } else if( /* handle U+0080..U+07FF inline */ \
+            ((c)<0xe0 && (c)>=0xc2) && \
+            ((i)<(length)) && \
+            (__t1=(uint8_t)((s)[i]-0x80))<=0x3f \
+        ) { \
+            (c)=(UChar)((((c)&0x1f)<<6)|__t1); \
+            ++(i); \
+        } else if(U8_IS_LEAD(c)) { \
+            /* function call for "complicated" and error cases */ \
+            (c)=utf8_nextCharSafeBody((const uint8_t *)s, &(i), (int32_t)(length), c, -1); \
+        } else { \
+            (c)=U_SENTINEL; \
+        } \
+    } \
+}
+
+/**
+ * Append a code point to a string, overwriting 1 to 4 bytes.
+ * The offset points to the current end of the string contents
+ * and is advanced (post-increment).
+ * "Unsafe" macro, assumes a valid code point and sufficient space in the string.
+ * Otherwise, the result is undefined.
+ *
+ * @param s const uint8_t * string buffer
+ * @param i string offset
+ * @param c code point to append
+ * @see U8_APPEND
+ * @stable ICU 2.4
+ */
+#define U8_APPEND_UNSAFE(s, i, c) { \
+    if((uint32_t)(c)<=0x7f) { \
+        (s)[(i)++]=(uint8_t)(c); \
+    } else { \
+        if((uint32_t)(c)<=0x7ff) { \
+            (s)[(i)++]=(uint8_t)(((c)>>6)|0xc0); \
+        } else { \
+            if((uint32_t)(c)<=0xffff) { \
+                (s)[(i)++]=(uint8_t)(((c)>>12)|0xe0); \
+            } else { \
+                (s)[(i)++]=(uint8_t)(((c)>>18)|0xf0); \
+                (s)[(i)++]=(uint8_t)((((c)>>12)&0x3f)|0x80); \
+            } \
+            (s)[(i)++]=(uint8_t)((((c)>>6)&0x3f)|0x80); \
+        } \
+        (s)[(i)++]=(uint8_t)(((c)&0x3f)|0x80); \
+    } \
+}
+
+/**
+ * Append a code point to a string, overwriting 1 to 4 bytes.
+ * The offset points to the current end of the string contents
+ * and is advanced (post-increment).
+ * "Safe" macro, checks for a valid code point.
+ * If a non-ASCII code point is written, checks for sufficient space in the string.
+ * If the code point is not valid or trail bytes do not fit,
+ * then isError is set to TRUE.
+ *
+ * @param s const uint8_t * string buffer
+ * @param i string offset, must be i>6)|0xc0); \
+        (s)[(i)++]=(uint8_t)(((c)&0x3f)|0x80); \
+    } else if((uint32_t)(c)<=0xd7ff && (i)+2<(capacity)) { \
+        (s)[(i)++]=(uint8_t)(((c)>>12)|0xe0); \
+        (s)[(i)++]=(uint8_t)((((c)>>6)&0x3f)|0x80); \
+        (s)[(i)++]=(uint8_t)(((c)&0x3f)|0x80); \
+    } else { \
+        (i)=utf8_appendCharSafeBody(s, (int32_t)(i), (int32_t)(capacity), c, &(isError)); \
+    } \
+}
+
+/**
+ * Advance the string offset from one code point boundary to the next.
+ * (Post-incrementing iteration.)
+ * "Unsafe" macro, assumes well-formed UTF-8.
+ *
+ * @param s const uint8_t * string
+ * @param i string offset
+ * @see U8_FWD_1
+ * @stable ICU 2.4
+ */
+#define U8_FWD_1_UNSAFE(s, i) { \
+    (i)+=1+U8_COUNT_TRAIL_BYTES((s)[i]); \
+}
+
+/**
+ * Advance the string offset from one code point boundary to the next.
+ * (Post-incrementing iteration.)
+ * "Safe" macro, checks for illegal sequences and for string boundaries.
+ *
+ * @param s const uint8_t * string
+ * @param i string offset, must be i(length)) { \
+            __count=(uint8_t)((length)-(i)); \
+        } \
+        while(__count>0 && U8_IS_TRAIL((s)[i])) { \
+            ++(i); \
+            --__count; \
+        } \
+    } \
+}
+
+/**
+ * Advance the string offset from one code point boundary to the n-th next one,
+ * i.e., move forward by n code points.
+ * (Post-incrementing iteration.)
+ * "Unsafe" macro, assumes well-formed UTF-8.
+ *
+ * @param s const uint8_t * string
+ * @param i string offset
+ * @param n number of code points to skip
+ * @see U8_FWD_N
+ * @stable ICU 2.4
+ */
+#define U8_FWD_N_UNSAFE(s, i, n) { \
+    int32_t __N=(n); \
+    while(__N>0) { \
+        U8_FWD_1_UNSAFE(s, i); \
+        --__N; \
+    } \
+}
+
+/**
+ * Advance the string offset from one code point boundary to the n-th next one,
+ * i.e., move forward by n code points.
+ * (Post-incrementing iteration.)
+ * "Safe" macro, checks for illegal sequences and for string boundaries.
+ *
+ * @param s const uint8_t * string
+ * @param i string offset, must be i0 && (i)<(length)) { \
+        U8_FWD_1(s, i, length); \
+        --__N; \
+    } \
+}
+
+/**
+ * Adjust a random-access offset to a code point boundary
+ * at the start of a code point.
+ * If the offset points to a UTF-8 trail byte,
+ * then the offset is moved backward to the corresponding lead byte.
+ * Otherwise, it is not modified.
+ * "Unsafe" macro, assumes well-formed UTF-8.
+ *
+ * @param s const uint8_t * string
+ * @param i string offset
+ * @see U8_SET_CP_START
+ * @stable ICU 2.4
+ */
+#define U8_SET_CP_START_UNSAFE(s, i) { \
+    while(U8_IS_TRAIL((s)[i])) { --(i); } \
+}
+
+/**
+ * Adjust a random-access offset to a code point boundary
+ * at the start of a code point.
+ * If the offset points to a UTF-8 trail byte,
+ * then the offset is moved backward to the corresponding lead byte.
+ * Otherwise, it is not modified.
+ * "Safe" macro, checks for illegal sequences and for string boundaries.
+ *
+ * @param s const uint8_t * string
+ * @param start starting string offset (usually 0)
+ * @param i string offset, must be start<=i
+ * @see U8_SET_CP_START_UNSAFE
+ * @stable ICU 2.4
+ */
+#define U8_SET_CP_START(s, start, i) { \
+    if(U8_IS_TRAIL((s)[(i)])) { \
+        (i)=utf8_back1SafeBody(s, start, (int32_t)(i)); \
+    } \
+}
+
+/* definitions with backward iteration -------------------------------------- */
+
+/**
+ * Move the string offset from one code point boundary to the previous one
+ * and get the code point between them.
+ * (Pre-decrementing backward iteration.)
+ * "Unsafe" macro, assumes well-formed UTF-8.
+ *
+ * The input offset may be the same as the string length.
+ * If the offset is behind a multi-byte sequence, then the macro will read
+ * the whole sequence.
+ * If the offset is behind a lead byte, then that itself
+ * will be returned as the code point.
+ * The result is undefined if the offset is behind an illegal UTF-8 sequence.
+ *
+ * @param s const uint8_t * string
+ * @param i string offset
+ * @param c output UChar32 variable
+ * @see U8_PREV
+ * @stable ICU 2.4
+ */
+#define U8_PREV_UNSAFE(s, i, c) { \
+    (c)=(uint8_t)(s)[--(i)]; \
+    if(U8_IS_TRAIL(c)) { \
+        uint8_t __b, __count=1, __shift=6; \
+\
+        /* c is a trail byte */ \
+        (c)&=0x3f; \
+        for(;;) { \
+            __b=(uint8_t)(s)[--(i)]; \
+            if(__b>=0xc0) { \
+                U8_MASK_LEAD_BYTE(__b, __count); \
+                (c)|=(UChar32)__b<<__shift; \
+                break; \
+            } else { \
+                (c)|=(UChar32)(__b&0x3f)<<__shift; \
+                ++__count; \
+                __shift+=6; \
+            } \
+        } \
+    } \
+}
+
+/**
+ * Move the string offset from one code point boundary to the previous one
+ * and get the code point between them.
+ * (Pre-decrementing backward iteration.)
+ * "Safe" macro, checks for illegal sequences and for string boundaries.
+ *
+ * The input offset may be the same as the string length.
+ * If the offset is behind a multi-byte sequence, then the macro will read
+ * the whole sequence.
+ * If the offset is behind a lead byte, then that itself
+ * will be returned as the code point.
+ * If the offset is behind an illegal UTF-8 sequence, then c is set to a negative value.
+ *
+ * @param s const uint8_t * string
+ * @param start starting string offset (usually 0)
+ * @param i string offset, must be start=0x80) { \
+        if((c)<=0xbf) { \
+            (c)=utf8_prevCharSafeBody((const uint8_t *)s, start, &(i), c, -1); \
+        } else { \
+            (c)=U_SENTINEL; \
+        } \
+    } \
+}
+
+/**
+ * Move the string offset from one code point boundary to the previous one.
+ * (Pre-decrementing backward iteration.)
+ * The input offset may be the same as the string length.
+ * "Unsafe" macro, assumes well-formed UTF-8.
+ *
+ * @param s const uint8_t * string
+ * @param i string offset
+ * @see U8_BACK_1
+ * @stable ICU 2.4
+ */
+#define U8_BACK_1_UNSAFE(s, i) { \
+    while(U8_IS_TRAIL((s)[--(i)])) {} \
+}
+
+/**
+ * Move the string offset from one code point boundary to the previous one.
+ * (Pre-decrementing backward iteration.)
+ * The input offset may be the same as the string length.
+ * "Safe" macro, checks for illegal sequences and for string boundaries.
+ *
+ * @param s const uint8_t * string
+ * @param start starting string offset (usually 0)
+ * @param i string offset, must be start0) { \
+        U8_BACK_1_UNSAFE(s, i); \
+        --__N; \
+    } \
+}
+
+/**
+ * Move the string offset from one code point boundary to the n-th one before it,
+ * i.e., move backward by n code points.
+ * (Pre-decrementing backward iteration.)
+ * The input offset may be the same as the string length.
+ * "Safe" macro, checks for illegal sequences and for string boundaries.
+ *
+ * @param s const uint8_t * string
+ * @param start index of the start of the string
+ * @param i string offset, must be start0 && (i)>(start)) { \
+        U8_BACK_1(s, start, i); \
+        --__N; \
+    } \
+}
+
+/**
+ * Adjust a random-access offset to a code point boundary after a code point.
+ * If the offset is behind a partial multi-byte sequence,
+ * then the offset is incremented to behind the whole sequence.
+ * Otherwise, it is not modified.
+ * The input offset may be the same as the string length.
+ * "Unsafe" macro, assumes well-formed UTF-8.
+ *
+ * @param s const uint8_t * string
+ * @param i string offset
+ * @see U8_SET_CP_LIMIT
+ * @stable ICU 2.4
+ */
+#define U8_SET_CP_LIMIT_UNSAFE(s, i) { \
+    U8_BACK_1_UNSAFE(s, i); \
+    U8_FWD_1_UNSAFE(s, i); \
+}
+
+/**
+ * Adjust a random-access offset to a code point boundary after a code point.
+ * If the offset is behind a partial multi-byte sequence,
+ * then the offset is incremented to behind the whole sequence.
+ * Otherwise, it is not modified.
+ * The input offset may be the same as the string length.
+ * "Safe" macro, checks for illegal sequences and for string boundaries.
+ *
+ * @param s const uint8_t * string
+ * @param start starting string offset (usually 0)
+ * @param i string offset, must be start<=i<=length
+ * @param length string length
+ * @see U8_SET_CP_LIMIT_UNSAFE
+ * @stable ICU 2.4
+ */
+#define U8_SET_CP_LIMIT(s, start, i, length) { \
+    if((start)<(i) && (i)<(length)) { \
+        U8_BACK_1(s, start, i); \
+        U8_FWD_1(s, i, length); \
+    } \
+}
+
+#endif
diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/utf_old.h b/Build/source/libs/icu/icu-xetex/common/unicode/utf_old.h
new file mode 100644
index 00000000000..2397889960e
--- /dev/null
+++ b/Build/source/libs/icu/icu-xetex/common/unicode/utf_old.h
@@ -0,0 +1,1171 @@
+/*
+*******************************************************************************
+*
+*   Copyright (C) 2002-2005, International Business Machines
+*   Corporation and others.  All Rights Reserved.
+*
+*******************************************************************************
+*   file name:  utf.h
+*   encoding:   US-ASCII
+*   tab size:   8 (not used)
+*   indentation:4
+*
+*   created on: 2002sep21
+*   created by: Markus W. Scherer
+*/
+
+/**
+ * \file 
+ * \brief C API: Deprecated macros for Unicode string handling
+ */
+
+/**
+ * 
+ * The macros in utf_old.h are all deprecated and their use discouraged.
+ * Some of the design principles behind the set of UTF macros
+ * have changed or proved impractical.
+ * Almost all of the old "UTF macros" are at least renamed.
+ * If you are looking for a new equivalent to an old macro, please see the
+ * comment at the old one.
+ *
+ * utf_old.h is included by utf.h after unicode/umachine.h
+ * and some common definitions, to not break old code.
+ *
+ * Brief summary of reasons for deprecation:
+ * - Switch on UTF_SIZE (selection of UTF-8/16/32 default string processing)
+ *   was impractical.
+ * - Switch on UTF_SAFE etc. (selection of unsafe/safe/strict default string processing)
+ *   was of little use and impractical.
+ * - Whole classes of macros became obsolete outside of the UTF_SIZE/UTF_SAFE
+ *   selection framework: UTF32_ macros (all trivial)
+ *   and UTF_ default and intermediate macros (all aliases).
+ * - The selection framework also caused many macro aliases.
+ * - Change in Unicode standard: "irregular" sequences (3.0) became illegal (3.2).
+ * - Change of language in Unicode standard:
+ *   Growing distinction between internal x-bit Unicode strings and external UTF-x
+ *   forms, with the former more lenient.
+ *   Suggests renaming of UTF16_ macros to U16_.
+ * - The prefix "UTF_" without a width number confused some users.
+ * - "Safe" append macros needed the addition of an error indicator output.
+ * - "Safe" UTF-8 macros used legitimate (if rarely used) code point values
+ *   to indicate error conditions.
+ * - The use of the "_CHAR" infix for code point operations confused some users.
+ *
+ * More details:
+ *
+ * Until ICU 2.2, utf.h theoretically allowed to choose among UTF-8/16/32
+ * for string processing, and among unsafe/safe/strict default macros for that.
+ *
+ * It proved nearly impossible to write non-trivial, high-performance code
+ * that is UTF-generic.
+ * Unsafe default macros would be dangerous for default string processing,
+ * and the main reason for the "strict" versions disappeared:
+ * Between Unicode 3.0 and 3.2 all "irregular" UTF-8 sequences became illegal.
+ * The only other conditions that "strict" checked for were non-characters,
+ * which are valid during processing. Only during text input/output should they
+ * be checked, and at that time other well-formedness checks may be
+ * necessary or useful as well.
+ * This can still be done by using U16_NEXT and U_IS_UNICODE_NONCHAR
+ * or U_IS_UNICODE_CHAR.
+ *
+ * The old UTF8_..._SAFE macros also used some normal Unicode code points
+ * to indicate malformed sequences.
+ * The new UTF8_ macros without suffix use negative values instead.
+ *
+ * The entire contents of utf32.h was moved here without replacement
+ * because all those macros were trivial and
+ * were meaningful only in the framework of choosing the UTF size.
+ *
+ * See Jitterbug 2150 and its discussion on the ICU mailing list
+ * in September 2002.
+ *
+ * 
+ * + * Obsolete part of pre-ICU 2.4 utf.h file documentation: + * + *

The original concept for these files was for ICU to allow + * in principle to set which UTF (UTF-8/16/32) is used internally + * by defining UTF_SIZE to either 8, 16, or 32. utf.h would then define the UChar type + * accordingly. UTF-16 was the default.

+ * + *

This concept has been abandoned. + * A lot of the ICU source code assumes UChar strings are in UTF-16. + * This is especially true for low-level code like + * conversion, normalization, and collation. + * The utf.h header enforces the default of UTF-16. + * The UTF-8 and UTF-32 macros remain for now for completeness and backward compatibility.

+ * + *

Accordingly, utf.h defines UChar to be an unsigned 16-bit integer. If this matches wchar_t, then + * UChar is defined to be exactly wchar_t, otherwise uint16_t.

+ * + *

UChar32 is defined to be a signed 32-bit integer (int32_t), large enough for a 21-bit + * Unicode code point (Unicode scalar value, 0..0x10ffff). + * Before ICU 2.4, the definition of UChar32 was similarly platform-dependent as + * the definition of UChar. For details see the documentation for UChar32 itself.

+ * + *

utf.h also defines a number of C macros for handling single Unicode code points and + * for using UTF Unicode strings. It includes utf8.h, utf16.h, and utf32.h for the actual + * implementations of those macros and then aliases one set of them (for UTF-16) for general use. + * The UTF-specific macros have the UTF size in the macro name prefixes (UTF16_...), while + * the general alias macros always begin with UTF_...

+ * + *

Many string operations can be done with or without error checking. + * Where such a distinction is useful, there are two versions of the macros, "unsafe" and "safe" + * ones with ..._UNSAFE and ..._SAFE suffixes. The unsafe macros are fast but may cause + * program failures if the strings are not well-formed. The safe macros have an additional, boolean + * parameter "strict". If strict is FALSE, then only illegal sequences are detected. + * Otherwise, irregular sequences and non-characters are detected as well (like single surrogates). + * Safe macros return special error code points for illegal/irregular sequences: + * Typically, U+ffff, or values that would result in a code unit sequence of the same length + * as the erroneous input sequence.
+ * Note that _UNSAFE macros have fewer parameters: They do not have the strictness parameter, and + * they do not have start/length parameters for boundary checking.

+ * + *

Here, the macros are aliased in two steps: + * In the first step, the UTF-specific macros with UTF16_ prefix and _UNSAFE and _SAFE suffixes are + * aliased according to the UTF_SIZE to macros with UTF_ prefix and the same suffixes and signatures. + * Then, in a second step, the default, general alias macros are set to use either the unsafe or + * the safe/not strict (default) or the safe/strict macro; + * these general macros do not have a strictness parameter.

+ * + *

It is possible to change the default choice for the general alias macros to be unsafe, safe/not strict or safe/strict. + * The default is safe/not strict. It is not recommended to select the unsafe macros as the basis for + * Unicode string handling in ICU! To select this, define UTF_SAFE, UTF_STRICT, or UTF_UNSAFE.

+ * + *

For general use, one should use the default, general macros with UTF_ prefix and no _SAFE/_UNSAFE suffix. + * Only in some cases it may be necessary to control the choice of macro directly and use a less generic alias. + * For example, if it can be assumed that a string is well-formed and the index will stay within the bounds, + * then the _UNSAFE version may be used. + * If a UTF-8 string is to be processed, then the macros with UTF8_ prefixes need to be used.

+ * + *
+ * + * @deprecated ICU 2.4. Use the macros in utf.h, utf16.h, utf8.h instead. + */ + +#ifndef __UTF_OLD_H__ +#define __UTF_OLD_H__ + +#ifndef U_HIDE_DEPRECATED_API + +/* utf.h must be included first. */ +#ifndef __UTF_H__ +# include "unicode/utf.h" +#endif + +/* Formerly utf.h, part 1 --------------------------------------------------- */ + +#ifdef U_USE_UTF_DEPRECATES +/** + * Unicode string and array offset and index type. + * ICU always counts Unicode code units (UChars) for + * string offsets, indexes, and lengths, not Unicode code points. + * + * @obsolete ICU 2.6. Use int32_t directly instead since this API will be removed in that release. + */ +typedef int32_t UTextOffset; +#endif + +/** Number of bits in a Unicode string code unit - ICU uses 16-bit Unicode. @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF_SIZE 16 + +/** + * The default choice for general Unicode string macros is to use the ..._SAFE macro implementations + * with strict=FALSE. + * + * @deprecated ICU 2.4. Obsolete, see utf_old.h. + */ +#define UTF_SAFE +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#undef UTF_UNSAFE +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#undef UTF_STRICT + +/** + *

UTF8_ERROR_VALUE_1 and UTF8_ERROR_VALUE_2 are special error values for UTF-8, + * which need 1 or 2 bytes in UTF-8:
+ * U+0015 = NAK = Negative Acknowledge, C0 control character
+ * U+009f = highest C1 control character

+ * + *

These are used by UTF8_..._SAFE macros so that they can return an error value + * that needs the same number of code units (bytes) as were seen by + * a macro. They should be tested with UTF_IS_ERROR() or UTF_IS_VALID().

+ * + * @deprecated ICU 2.4. Obsolete, see utf_old.h. + */ +#define UTF8_ERROR_VALUE_1 0x15 + +/** + * See documentation on UTF8_ERROR_VALUE_1 for details. + * + * @deprecated ICU 2.4. Obsolete, see utf_old.h. + */ +#define UTF8_ERROR_VALUE_2 0x9f + +/** + * Error value for all UTFs. This code point value will be set by macros with error + * checking if an error is detected. + * + * @deprecated ICU 2.4. Obsolete, see utf_old.h. + */ +#define UTF_ERROR_VALUE 0xffff + +/** + * Is a given 32-bit code an error value + * as returned by one of the macros for any UTF? + * + * @deprecated ICU 2.4. Obsolete, see utf_old.h. + */ +#define UTF_IS_ERROR(c) \ + (((c)&0xfffe)==0xfffe || (c)==UTF8_ERROR_VALUE_1 || (c)==UTF8_ERROR_VALUE_2) + +/** + * This is a combined macro: Is c a valid Unicode value _and_ not an error code? + * + * @deprecated ICU 2.4. Obsolete, see utf_old.h. + */ +#define UTF_IS_VALID(c) \ + (UTF_IS_UNICODE_CHAR(c) && \ + (c)!=UTF8_ERROR_VALUE_1 && (c)!=UTF8_ERROR_VALUE_2) + +/** + * Is this code unit or code point a surrogate (U+d800..U+dfff)? + * @deprecated ICU 2.4. Renamed to U_IS_SURROGATE and U16_IS_SURROGATE, see utf_old.h. + */ +#define UTF_IS_SURROGATE(uchar) (((uchar)&0xfffff800)==0xd800) + +/** + * Is a given 32-bit code point a Unicode noncharacter? + * + * @deprecated ICU 2.4. Renamed to U_IS_UNICODE_NONCHAR, see utf_old.h. + */ +#define UTF_IS_UNICODE_NONCHAR(c) \ + ((c)>=0xfdd0 && \ + ((uint32_t)(c)<=0xfdef || ((c)&0xfffe)==0xfffe) && \ + (uint32_t)(c)<=0x10ffff) + +/** + * Is a given 32-bit value a Unicode code point value (0..U+10ffff) + * that can be assigned a character? + * + * Code points that are not characters include: + * - single surrogate code points (U+d800..U+dfff, 2048 code points) + * - the last two code points on each plane (U+__fffe and U+__ffff, 34 code points) + * - U+fdd0..U+fdef (new with Unicode 3.1, 32 code points) + * - the highest Unicode code point value is U+10ffff + * + * This means that all code points below U+d800 are character code points, + * and that boundary is tested first for performance. + * + * @deprecated ICU 2.4. Renamed to U_IS_UNICODE_CHAR, see utf_old.h. + */ +#define UTF_IS_UNICODE_CHAR(c) \ + ((uint32_t)(c)<0xd800 || \ + ((uint32_t)(c)>0xdfff && \ + (uint32_t)(c)<=0x10ffff && \ + !UTF_IS_UNICODE_NONCHAR(c))) + +/* Formerly utf8.h ---------------------------------------------------------- */ + +/** + * Count the trail bytes for a UTF-8 lead byte. + * @deprecated ICU 2.4. Renamed to U8_COUNT_TRAIL_BYTES, see utf_old.h. + */ +#define UTF8_COUNT_TRAIL_BYTES(leadByte) (utf8_countTrailBytes[(uint8_t)leadByte]) + +/** + * Mask a UTF-8 lead byte, leave only the lower bits that form part of the code point value. + * @deprecated ICU 2.4. Renamed to U8_MASK_LEAD_BYTE, see utf_old.h. + */ +#define UTF8_MASK_LEAD_BYTE(leadByte, countTrailBytes) ((leadByte)&=(1<<(6-(countTrailBytes)))-1) + +/** Is this this code point a single code unit (byte)? @deprecated ICU 2.4. Renamed to U8_IS_SINGLE, see utf_old.h. */ +#define UTF8_IS_SINGLE(uchar) (((uchar)&0x80)==0) +/** Is this this code unit the lead code unit (byte) of a code point? @deprecated ICU 2.4. Renamed to U8_IS_LEAD, see utf_old.h. */ +#define UTF8_IS_LEAD(uchar) ((uint8_t)((uchar)-0xc0)<0x3e) +/** Is this this code unit a trailing code unit (byte) of a code point? @deprecated ICU 2.4. Renamed to U8_IS_TRAIL, see utf_old.h. */ +#define UTF8_IS_TRAIL(uchar) (((uchar)&0xc0)==0x80) + +/** Does this scalar Unicode value need multiple code units for storage? @deprecated ICU 2.4. Use U8_LENGTH or test ((uint32_t)(c)>0x7f) instead, see utf_old.h. */ +#define UTF8_NEED_MULTIPLE_UCHAR(c) ((uint32_t)(c)>0x7f) + +/** + * Given the lead character, how many bytes are taken by this code point. + * ICU does not deal with code points >0x10ffff + * unless necessary for advancing in the byte stream. + * + * These length macros take into account that for values >0x10ffff + * the UTF8_APPEND_CHAR_SAFE macros would write the error code point 0xffff + * with 3 bytes. + * Code point comparisons need to be in uint32_t because UChar32 + * may be a signed type, and negative values must be recognized. + * + * @deprecated ICU 2.4. Use U8_LENGTH instead, see utf_old.h. + */ +#if 1 +# define UTF8_CHAR_LENGTH(c) \ + ((uint32_t)(c)<=0x7f ? 1 : \ + ((uint32_t)(c)<=0x7ff ? 2 : \ + ((uint32_t)((c)-0x10000)>0xfffff ? 3 : 4) \ + ) \ + ) +#else +# define UTF8_CHAR_LENGTH(c) \ + ((uint32_t)(c)<=0x7f ? 1 : \ + ((uint32_t)(c)<=0x7ff ? 2 : \ + ((uint32_t)(c)<=0xffff ? 3 : \ + ((uint32_t)(c)<=0x10ffff ? 4 : \ + ((uint32_t)(c)<=0x3ffffff ? 5 : \ + ((uint32_t)(c)<=0x7fffffff ? 6 : 3) \ + ) \ + ) \ + ) \ + ) \ + ) +#endif + +/** The maximum number of bytes per code point. @deprecated ICU 2.4. Renamed to U8_MAX_LENGTH, see utf_old.h. */ +#define UTF8_MAX_CHAR_LENGTH 4 + +/** Average number of code units compared to UTF-16. @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF8_ARRAY_SIZE(size) ((5*(size))/2) + +/** @deprecated ICU 2.4. Renamed to U8_GET_UNSAFE, see utf_old.h. */ +#define UTF8_GET_CHAR_UNSAFE(s, i, c) { \ + int32_t _utf8_get_char_unsafe_index=(int32_t)(i); \ + UTF8_SET_CHAR_START_UNSAFE(s, _utf8_get_char_unsafe_index); \ + UTF8_NEXT_CHAR_UNSAFE(s, _utf8_get_char_unsafe_index, c); \ +} + +/** @deprecated ICU 2.4. Use U8_GET instead, see utf_old.h. */ +#define UTF8_GET_CHAR_SAFE(s, start, i, length, c, strict) { \ + int32_t _utf8_get_char_safe_index=(int32_t)(i); \ + UTF8_SET_CHAR_START_SAFE(s, start, _utf8_get_char_safe_index); \ + UTF8_NEXT_CHAR_SAFE(s, _utf8_get_char_safe_index, length, c, strict); \ +} + +/** @deprecated ICU 2.4. Renamed to U8_NEXT_UNSAFE, see utf_old.h. */ +#define UTF8_NEXT_CHAR_UNSAFE(s, i, c) { \ + (c)=(s)[(i)++]; \ + if((uint8_t)((c)-0xc0)<0x35) { \ + uint8_t __count=UTF8_COUNT_TRAIL_BYTES(c); \ + UTF8_MASK_LEAD_BYTE(c, __count); \ + switch(__count) { \ + /* each following branch falls through to the next one */ \ + case 3: \ + (c)=((c)<<6)|((s)[(i)++]&0x3f); \ + case 2: \ + (c)=((c)<<6)|((s)[(i)++]&0x3f); \ + case 1: \ + (c)=((c)<<6)|((s)[(i)++]&0x3f); \ + /* no other branches to optimize switch() */ \ + break; \ + } \ + } \ +} + +/** @deprecated ICU 2.4. Renamed to U8_APPEND_UNSAFE, see utf_old.h. */ +#define UTF8_APPEND_CHAR_UNSAFE(s, i, c) { \ + if((uint32_t)(c)<=0x7f) { \ + (s)[(i)++]=(uint8_t)(c); \ + } else { \ + if((uint32_t)(c)<=0x7ff) { \ + (s)[(i)++]=(uint8_t)(((c)>>6)|0xc0); \ + } else { \ + if((uint32_t)(c)<=0xffff) { \ + (s)[(i)++]=(uint8_t)(((c)>>12)|0xe0); \ + } else { \ + (s)[(i)++]=(uint8_t)(((c)>>18)|0xf0); \ + (s)[(i)++]=(uint8_t)((((c)>>12)&0x3f)|0x80); \ + } \ + (s)[(i)++]=(uint8_t)((((c)>>6)&0x3f)|0x80); \ + } \ + (s)[(i)++]=(uint8_t)(((c)&0x3f)|0x80); \ + } \ +} + +/** @deprecated ICU 2.4. Renamed to U8_FWD_1_UNSAFE, see utf_old.h. */ +#define UTF8_FWD_1_UNSAFE(s, i) { \ + (i)+=1+UTF8_COUNT_TRAIL_BYTES((s)[i]); \ +} + +/** @deprecated ICU 2.4. Renamed to U8_FWD_N_UNSAFE, see utf_old.h. */ +#define UTF8_FWD_N_UNSAFE(s, i, n) { \ + int32_t __N=(n); \ + while(__N>0) { \ + UTF8_FWD_1_UNSAFE(s, i); \ + --__N; \ + } \ +} + +/** @deprecated ICU 2.4. Renamed to U8_SET_CP_START_UNSAFE, see utf_old.h. */ +#define UTF8_SET_CHAR_START_UNSAFE(s, i) { \ + while(UTF8_IS_TRAIL((s)[i])) { --(i); } \ +} + +/** @deprecated ICU 2.4. Use U8_NEXT instead, see utf_old.h. */ +#define UTF8_NEXT_CHAR_SAFE(s, i, length, c, strict) { \ + (c)=(s)[(i)++]; \ + if((c)>=0x80) { \ + if(UTF8_IS_LEAD(c)) { \ + (c)=utf8_nextCharSafeBody(s, &(i), (int32_t)(length), c, strict); \ + } else { \ + (c)=UTF8_ERROR_VALUE_1; \ + } \ + } \ +} + +/** @deprecated ICU 2.4. Use U8_APPEND instead, see utf_old.h. */ +#define UTF8_APPEND_CHAR_SAFE(s, i, length, c) { \ + if((uint32_t)(c)<=0x7f) { \ + (s)[(i)++]=(uint8_t)(c); \ + } else { \ + (i)=utf8_appendCharSafeBody(s, (int32_t)(i), (int32_t)(length), c, NULL); \ + } \ +} + +/** @deprecated ICU 2.4. Renamed to U8_FWD_1, see utf_old.h. */ +#define UTF8_FWD_1_SAFE(s, i, length) U8_FWD_1(s, i, length) + +/** @deprecated ICU 2.4. Renamed to U8_FWD_N, see utf_old.h. */ +#define UTF8_FWD_N_SAFE(s, i, length, n) U8_FWD_N(s, i, length, n) + +/** @deprecated ICU 2.4. Renamed to U8_SET_CP_START, see utf_old.h. */ +#define UTF8_SET_CHAR_START_SAFE(s, start, i) U8_SET_CP_START(s, start, i) + +/** @deprecated ICU 2.4. Renamed to U8_PREV_UNSAFE, see utf_old.h. */ +#define UTF8_PREV_CHAR_UNSAFE(s, i, c) { \ + (c)=(s)[--(i)]; \ + if(UTF8_IS_TRAIL(c)) { \ + uint8_t __b, __count=1, __shift=6; \ +\ + /* c is a trail byte */ \ + (c)&=0x3f; \ + for(;;) { \ + __b=(s)[--(i)]; \ + if(__b>=0xc0) { \ + UTF8_MASK_LEAD_BYTE(__b, __count); \ + (c)|=(UChar32)__b<<__shift; \ + break; \ + } else { \ + (c)|=(UChar32)(__b&0x3f)<<__shift; \ + ++__count; \ + __shift+=6; \ + } \ + } \ + } \ +} + +/** @deprecated ICU 2.4. Renamed to U8_BACK_1_UNSAFE, see utf_old.h. */ +#define UTF8_BACK_1_UNSAFE(s, i) { \ + while(UTF8_IS_TRAIL((s)[--(i)])) {} \ +} + +/** @deprecated ICU 2.4. Renamed to U8_BACK_N_UNSAFE, see utf_old.h. */ +#define UTF8_BACK_N_UNSAFE(s, i, n) { \ + int32_t __N=(n); \ + while(__N>0) { \ + UTF8_BACK_1_UNSAFE(s, i); \ + --__N; \ + } \ +} + +/** @deprecated ICU 2.4. Renamed to U8_SET_CP_LIMIT_UNSAFE, see utf_old.h. */ +#define UTF8_SET_CHAR_LIMIT_UNSAFE(s, i) { \ + UTF8_BACK_1_UNSAFE(s, i); \ + UTF8_FWD_1_UNSAFE(s, i); \ +} + +/** @deprecated ICU 2.4. Use U8_PREV instead, see utf_old.h. */ +#define UTF8_PREV_CHAR_SAFE(s, start, i, c, strict) { \ + (c)=(s)[--(i)]; \ + if((c)>=0x80) { \ + if((c)<=0xbf) { \ + (c)=utf8_prevCharSafeBody(s, start, &(i), c, strict); \ + } else { \ + (c)=UTF8_ERROR_VALUE_1; \ + } \ + } \ +} + +/** @deprecated ICU 2.4. Renamed to U8_BACK_1, see utf_old.h. */ +#define UTF8_BACK_1_SAFE(s, start, i) U8_BACK_1(s, start, i) + +/** @deprecated ICU 2.4. Renamed to U8_BACK_N, see utf_old.h. */ +#define UTF8_BACK_N_SAFE(s, start, i, n) U8_BACK_N(s, start, i, n) + +/** @deprecated ICU 2.4. Renamed to U8_SET_CP_LIMIT, see utf_old.h. */ +#define UTF8_SET_CHAR_LIMIT_SAFE(s, start, i, length) U8_SET_CP_LIMIT(s, start, i, length) + +/* Formerly utf16.h --------------------------------------------------------- */ + +/** Is uchar a first/lead surrogate? @deprecated ICU 2.4. Renamed to U_IS_LEAD and U16_IS_LEAD, see utf_old.h. */ +#define UTF_IS_FIRST_SURROGATE(uchar) (((uchar)&0xfffffc00)==0xd800) + +/** Is uchar a second/trail surrogate? @deprecated ICU 2.4. Renamed to U_IS_TRAIL and U16_IS_TRAIL, see utf_old.h. */ +#define UTF_IS_SECOND_SURROGATE(uchar) (((uchar)&0xfffffc00)==0xdc00) + +/** Assuming c is a surrogate, is it a first/lead surrogate? @deprecated ICU 2.4. Renamed to U_IS_SURROGATE_LEAD and U16_IS_SURROGATE_LEAD, see utf_old.h. */ +#define UTF_IS_SURROGATE_FIRST(c) (((c)&0x400)==0) + +/** Helper constant for UTF16_GET_PAIR_VALUE. @deprecated ICU 2.4. Renamed to U16_SURROGATE_OFFSET, see utf_old.h. */ +#define UTF_SURROGATE_OFFSET ((0xd800<<10UL)+0xdc00-0x10000) + +/** Get the UTF-32 value from the surrogate code units. @deprecated ICU 2.4. Renamed to U16_GET_SUPPLEMENTARY, see utf_old.h. */ +#define UTF16_GET_PAIR_VALUE(first, second) \ + (((first)<<10UL)+(second)-UTF_SURROGATE_OFFSET) + +/** @deprecated ICU 2.4. Renamed to U16_LEAD, see utf_old.h. */ +#define UTF_FIRST_SURROGATE(supplementary) (UChar)(((supplementary)>>10)+0xd7c0) + +/** @deprecated ICU 2.4. Renamed to U16_TRAIL, see utf_old.h. */ +#define UTF_SECOND_SURROGATE(supplementary) (UChar)(((supplementary)&0x3ff)|0xdc00) + +/** @deprecated ICU 2.4. Renamed to U16_LEAD, see utf_old.h. */ +#define UTF16_LEAD(supplementary) UTF_FIRST_SURROGATE(supplementary) + +/** @deprecated ICU 2.4. Renamed to U16_TRAIL, see utf_old.h. */ +#define UTF16_TRAIL(supplementary) UTF_SECOND_SURROGATE(supplementary) + +/** @deprecated ICU 2.4. Renamed to U16_IS_SINGLE, see utf_old.h. */ +#define UTF16_IS_SINGLE(uchar) !UTF_IS_SURROGATE(uchar) + +/** @deprecated ICU 2.4. Renamed to U16_IS_LEAD, see utf_old.h. */ +#define UTF16_IS_LEAD(uchar) UTF_IS_FIRST_SURROGATE(uchar) + +/** @deprecated ICU 2.4. Renamed to U16_IS_TRAIL, see utf_old.h. */ +#define UTF16_IS_TRAIL(uchar) UTF_IS_SECOND_SURROGATE(uchar) + +/** Does this scalar Unicode value need multiple code units for storage? @deprecated ICU 2.4. Use U16_LENGTH or test ((uint32_t)(c)>0xffff) instead, see utf_old.h. */ +#define UTF16_NEED_MULTIPLE_UCHAR(c) ((uint32_t)(c)>0xffff) + +/** @deprecated ICU 2.4. Renamed to U16_LENGTH, see utf_old.h. */ +#define UTF16_CHAR_LENGTH(c) ((uint32_t)(c)<=0xffff ? 1 : 2) + +/** @deprecated ICU 2.4. Renamed to U16_MAX_LENGTH, see utf_old.h. */ +#define UTF16_MAX_CHAR_LENGTH 2 + +/** Average number of code units compared to UTF-16. @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF16_ARRAY_SIZE(size) (size) + +/** + * Get a single code point from an offset that points to any + * of the code units that belong to that code point. + * Assume 0<=i=(start) && UTF_IS_FIRST_SURROGATE(__c2=(s)[(i)-1])) { \ + (c)=UTF16_GET_PAIR_VALUE(__c2, (c)); \ + /* strict: ((c)&0xfffe)==0xfffe is caught by UTF_IS_ERROR() and UTF_IS_UNICODE_CHAR() */ \ + } else if(strict) {\ + /* unmatched second surrogate */ \ + (c)=UTF_ERROR_VALUE; \ + } \ + } \ + } else if((strict) && !UTF_IS_UNICODE_CHAR(c)) { \ + (c)=UTF_ERROR_VALUE; \ + } \ +} + +/** @deprecated ICU 2.4. Renamed to U16_NEXT_UNSAFE, see utf_old.h. */ +#define UTF16_NEXT_CHAR_UNSAFE(s, i, c) { \ + (c)=(s)[(i)++]; \ + if(UTF_IS_FIRST_SURROGATE(c)) { \ + (c)=UTF16_GET_PAIR_VALUE((c), (s)[(i)++]); \ + } \ +} + +/** @deprecated ICU 2.4. Renamed to U16_APPEND_UNSAFE, see utf_old.h. */ +#define UTF16_APPEND_CHAR_UNSAFE(s, i, c) { \ + if((uint32_t)(c)<=0xffff) { \ + (s)[(i)++]=(uint16_t)(c); \ + } else { \ + (s)[(i)++]=(uint16_t)(((c)>>10)+0xd7c0); \ + (s)[(i)++]=(uint16_t)(((c)&0x3ff)|0xdc00); \ + } \ +} + +/** @deprecated ICU 2.4. Renamed to U16_FWD_1_UNSAFE, see utf_old.h. */ +#define UTF16_FWD_1_UNSAFE(s, i) { \ + if(UTF_IS_FIRST_SURROGATE((s)[(i)++])) { \ + ++(i); \ + } \ +} + +/** @deprecated ICU 2.4. Renamed to U16_FWD_N_UNSAFE, see utf_old.h. */ +#define UTF16_FWD_N_UNSAFE(s, i, n) { \ + int32_t __N=(n); \ + while(__N>0) { \ + UTF16_FWD_1_UNSAFE(s, i); \ + --__N; \ + } \ +} + +/** @deprecated ICU 2.4. Renamed to U16_SET_CP_START_UNSAFE, see utf_old.h. */ +#define UTF16_SET_CHAR_START_UNSAFE(s, i) { \ + if(UTF_IS_SECOND_SURROGATE((s)[i])) { \ + --(i); \ + } \ +} + +/** @deprecated ICU 2.4. Use U16_NEXT instead, see utf_old.h. */ +#define UTF16_NEXT_CHAR_SAFE(s, i, length, c, strict) { \ + (c)=(s)[(i)++]; \ + if(UTF_IS_FIRST_SURROGATE(c)) { \ + uint16_t __c2; \ + if((i)<(length) && UTF_IS_SECOND_SURROGATE(__c2=(s)[(i)])) { \ + ++(i); \ + (c)=UTF16_GET_PAIR_VALUE((c), __c2); \ + /* strict: ((c)&0xfffe)==0xfffe is caught by UTF_IS_ERROR() and UTF_IS_UNICODE_CHAR() */ \ + } else if(strict) {\ + /* unmatched first surrogate */ \ + (c)=UTF_ERROR_VALUE; \ + } \ + } else if((strict) && !UTF_IS_UNICODE_CHAR(c)) { \ + /* unmatched second surrogate or other non-character */ \ + (c)=UTF_ERROR_VALUE; \ + } \ +} + +/** @deprecated ICU 2.4. Use U16_APPEND instead, see utf_old.h. */ +#define UTF16_APPEND_CHAR_SAFE(s, i, length, c) { \ + if((uint32_t)(c)<=0xffff) { \ + (s)[(i)++]=(uint16_t)(c); \ + } else if((uint32_t)(c)<=0x10ffff) { \ + if((i)+1<(length)) { \ + (s)[(i)++]=(uint16_t)(((c)>>10)+0xd7c0); \ + (s)[(i)++]=(uint16_t)(((c)&0x3ff)|0xdc00); \ + } else /* not enough space */ { \ + (s)[(i)++]=UTF_ERROR_VALUE; \ + } \ + } else /* c>0x10ffff, write error value */ { \ + (s)[(i)++]=UTF_ERROR_VALUE; \ + } \ +} + +/** @deprecated ICU 2.4. Renamed to U16_FWD_1, see utf_old.h. */ +#define UTF16_FWD_1_SAFE(s, i, length) U16_FWD_1(s, i, length) + +/** @deprecated ICU 2.4. Renamed to U16_FWD_N, see utf_old.h. */ +#define UTF16_FWD_N_SAFE(s, i, length, n) U16_FWD_N(s, i, length, n) + +/** @deprecated ICU 2.4. Renamed to U16_SET_CP_START, see utf_old.h. */ +#define UTF16_SET_CHAR_START_SAFE(s, start, i) U16_SET_CP_START(s, start, i) + +/** @deprecated ICU 2.4. Renamed to U16_PREV_UNSAFE, see utf_old.h. */ +#define UTF16_PREV_CHAR_UNSAFE(s, i, c) { \ + (c)=(s)[--(i)]; \ + if(UTF_IS_SECOND_SURROGATE(c)) { \ + (c)=UTF16_GET_PAIR_VALUE((s)[--(i)], (c)); \ + } \ +} + +/** @deprecated ICU 2.4. Renamed to U16_BACK_1_UNSAFE, see utf_old.h. */ +#define UTF16_BACK_1_UNSAFE(s, i) { \ + if(UTF_IS_SECOND_SURROGATE((s)[--(i)])) { \ + --(i); \ + } \ +} + +/** @deprecated ICU 2.4. Renamed to U16_BACK_N_UNSAFE, see utf_old.h. */ +#define UTF16_BACK_N_UNSAFE(s, i, n) { \ + int32_t __N=(n); \ + while(__N>0) { \ + UTF16_BACK_1_UNSAFE(s, i); \ + --__N; \ + } \ +} + +/** @deprecated ICU 2.4. Renamed to U16_SET_CP_LIMIT_UNSAFE, see utf_old.h. */ +#define UTF16_SET_CHAR_LIMIT_UNSAFE(s, i) { \ + if(UTF_IS_FIRST_SURROGATE((s)[(i)-1])) { \ + ++(i); \ + } \ +} + +/** @deprecated ICU 2.4. Use U16_PREV instead, see utf_old.h. */ +#define UTF16_PREV_CHAR_SAFE(s, start, i, c, strict) { \ + (c)=(s)[--(i)]; \ + if(UTF_IS_SECOND_SURROGATE(c)) { \ + uint16_t __c2; \ + if((i)>(start) && UTF_IS_FIRST_SURROGATE(__c2=(s)[(i)-1])) { \ + --(i); \ + (c)=UTF16_GET_PAIR_VALUE(__c2, (c)); \ + /* strict: ((c)&0xfffe)==0xfffe is caught by UTF_IS_ERROR() and UTF_IS_UNICODE_CHAR() */ \ + } else if(strict) {\ + /* unmatched second surrogate */ \ + (c)=UTF_ERROR_VALUE; \ + } \ + } else if((strict) && !UTF_IS_UNICODE_CHAR(c)) { \ + /* unmatched first surrogate or other non-character */ \ + (c)=UTF_ERROR_VALUE; \ + } \ +} + +/** @deprecated ICU 2.4. Renamed to U16_BACK_1, see utf_old.h. */ +#define UTF16_BACK_1_SAFE(s, start, i) U16_BACK_1(s, start, i) + +/** @deprecated ICU 2.4. Renamed to U16_BACK_N, see utf_old.h. */ +#define UTF16_BACK_N_SAFE(s, start, i, n) U16_BACK_N(s, start, i, n) + +/** @deprecated ICU 2.4. Renamed to U16_SET_CP_LIMIT, see utf_old.h. */ +#define UTF16_SET_CHAR_LIMIT_SAFE(s, start, i, length) U16_SET_CP_LIMIT(s, start, i, length) + +/* Formerly utf32.h --------------------------------------------------------- */ + +/* +* Old documentation: +* +* This file defines macros to deal with UTF-32 code units and code points. +* Signatures and semantics are the same as for the similarly named macros +* in utf16.h. +* utf32.h is included by utf.h after unicode/umachine.h

+* and some common definitions. +*

Usage: ICU coding guidelines for if() statements should be followed when using these macros. +* Compound statements (curly braces {}) must be used for if-else-while... +* bodies and all macro statements should be terminated with semicolon.

+*/ + +/* internal definitions ----------------------------------------------------- */ + +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_IS_SAFE(c, strict) \ + (!(strict) ? \ + (uint32_t)(c)<=0x10ffff : \ + UTF_IS_UNICODE_CHAR(c)) + +/* + * For the semantics of all of these macros, see utf16.h. + * The UTF-32 versions are trivial because any code point is + * encoded using exactly one code unit. + */ + +/* single-code point definitions -------------------------------------------- */ + +/* classes of code unit values */ + +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_IS_SINGLE(uchar) 1 +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_IS_LEAD(uchar) 0 +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_IS_TRAIL(uchar) 0 + +/* number of code units per code point */ + +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_NEED_MULTIPLE_UCHAR(c) 0 +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_CHAR_LENGTH(c) 1 +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_MAX_CHAR_LENGTH 1 + +/* average number of code units compared to UTF-16 */ + +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_ARRAY_SIZE(size) (size) + +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_GET_CHAR_UNSAFE(s, i, c) { \ + (c)=(s)[i]; \ +} + +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_GET_CHAR_SAFE(s, start, i, length, c, strict) { \ + (c)=(s)[i]; \ + if(!UTF32_IS_SAFE(c, strict)) { \ + (c)=UTF_ERROR_VALUE; \ + } \ +} + +/* definitions with forward iteration --------------------------------------- */ + +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_NEXT_CHAR_UNSAFE(s, i, c) { \ + (c)=(s)[(i)++]; \ +} + +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_APPEND_CHAR_UNSAFE(s, i, c) { \ + (s)[(i)++]=(c); \ +} + +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_FWD_1_UNSAFE(s, i) { \ + ++(i); \ +} + +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_FWD_N_UNSAFE(s, i, n) { \ + (i)+=(n); \ +} + +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_SET_CHAR_START_UNSAFE(s, i) { \ +} + +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_NEXT_CHAR_SAFE(s, i, length, c, strict) { \ + (c)=(s)[(i)++]; \ + if(!UTF32_IS_SAFE(c, strict)) { \ + (c)=UTF_ERROR_VALUE; \ + } \ +} + +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_APPEND_CHAR_SAFE(s, i, length, c) { \ + if((uint32_t)(c)<=0x10ffff) { \ + (s)[(i)++]=(c); \ + } else /* c>0x10ffff, write 0xfffd */ { \ + (s)[(i)++]=0xfffd; \ + } \ +} + +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_FWD_1_SAFE(s, i, length) { \ + ++(i); \ +} + +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_FWD_N_SAFE(s, i, length, n) { \ + if(((i)+=(n))>(length)) { \ + (i)=(length); \ + } \ +} + +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_SET_CHAR_START_SAFE(s, start, i) { \ +} + +/* definitions with backward iteration -------------------------------------- */ + +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_PREV_CHAR_UNSAFE(s, i, c) { \ + (c)=(s)[--(i)]; \ +} + +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_BACK_1_UNSAFE(s, i) { \ + --(i); \ +} + +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_BACK_N_UNSAFE(s, i, n) { \ + (i)-=(n); \ +} + +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_SET_CHAR_LIMIT_UNSAFE(s, i) { \ +} + +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_PREV_CHAR_SAFE(s, start, i, c, strict) { \ + (c)=(s)[--(i)]; \ + if(!UTF32_IS_SAFE(c, strict)) { \ + (c)=UTF_ERROR_VALUE; \ + } \ +} + +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_BACK_1_SAFE(s, start, i) { \ + --(i); \ +} + +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_BACK_N_SAFE(s, start, i, n) { \ + (i)-=(n); \ + if((i)<(start)) { \ + (i)=(start); \ + } \ +} + +/** @deprecated ICU 2.4. Obsolete, see utf_old.h. */ +#define UTF32_SET_CHAR_LIMIT_SAFE(s, i, length) { \ +} + +/* Formerly utf.h, part 2 --------------------------------------------------- */ + +/** + * Estimate the number of code units for a string based on the number of UTF-16 code units. + * + * @deprecated ICU 2.4. Obsolete, see utf_old.h. + */ +#define UTF_ARRAY_SIZE(size) UTF16_ARRAY_SIZE(size) + +/** @deprecated ICU 2.4. Renamed to U16_GET_UNSAFE, see utf_old.h. */ +#define UTF_GET_CHAR_UNSAFE(s, i, c) UTF16_GET_CHAR_UNSAFE(s, i, c) + +/** @deprecated ICU 2.4. Use U16_GET instead, see utf_old.h. */ +#define UTF_GET_CHAR_SAFE(s, start, i, length, c, strict) UTF16_GET_CHAR_SAFE(s, start, i, length, c, strict) + + +/** @deprecated ICU 2.4. Renamed to U16_NEXT_UNSAFE, see utf_old.h. */ +#define UTF_NEXT_CHAR_UNSAFE(s, i, c) UTF16_NEXT_CHAR_UNSAFE(s, i, c) + +/** @deprecated ICU 2.4. Use U16_NEXT instead, see utf_old.h. */ +#define UTF_NEXT_CHAR_SAFE(s, i, length, c, strict) UTF16_NEXT_CHAR_SAFE(s, i, length, c, strict) + + +/** @deprecated ICU 2.4. Renamed to U16_APPEND_UNSAFE, see utf_old.h. */ +#define UTF_APPEND_CHAR_UNSAFE(s, i, c) UTF16_APPEND_CHAR_UNSAFE(s, i, c) + +/** @deprecated ICU 2.4. Use U16_APPEND instead, see utf_old.h. */ +#define UTF_APPEND_CHAR_SAFE(s, i, length, c) UTF16_APPEND_CHAR_SAFE(s, i, length, c) + + +/** @deprecated ICU 2.4. Renamed to U16_FWD_1_UNSAFE, see utf_old.h. */ +#define UTF_FWD_1_UNSAFE(s, i) UTF16_FWD_1_UNSAFE(s, i) + +/** @deprecated ICU 2.4. Renamed to U16_FWD_1, see utf_old.h. */ +#define UTF_FWD_1_SAFE(s, i, length) UTF16_FWD_1_SAFE(s, i, length) + + +/** @deprecated ICU 2.4. Renamed to U16_FWD_N_UNSAFE, see utf_old.h. */ +#define UTF_FWD_N_UNSAFE(s, i, n) UTF16_FWD_N_UNSAFE(s, i, n) + +/** @deprecated ICU 2.4. Renamed to U16_FWD_N, see utf_old.h. */ +#define UTF_FWD_N_SAFE(s, i, length, n) UTF16_FWD_N_SAFE(s, i, length, n) + + +/** @deprecated ICU 2.4. Renamed to U16_SET_CP_START_UNSAFE, see utf_old.h. */ +#define UTF_SET_CHAR_START_UNSAFE(s, i) UTF16_SET_CHAR_START_UNSAFE(s, i) + +/** @deprecated ICU 2.4. Renamed to U16_SET_CP_START, see utf_old.h. */ +#define UTF_SET_CHAR_START_SAFE(s, start, i) UTF16_SET_CHAR_START_SAFE(s, start, i) + + +/** @deprecated ICU 2.4. Renamed to U16_PREV_UNSAFE, see utf_old.h. */ +#define UTF_PREV_CHAR_UNSAFE(s, i, c) UTF16_PREV_CHAR_UNSAFE(s, i, c) + +/** @deprecated ICU 2.4. Use U16_PREV instead, see utf_old.h. */ +#define UTF_PREV_CHAR_SAFE(s, start, i, c, strict) UTF16_PREV_CHAR_SAFE(s, start, i, c, strict) + + +/** @deprecated ICU 2.4. Renamed to U16_BACK_1_UNSAFE, see utf_old.h. */ +#define UTF_BACK_1_UNSAFE(s, i) UTF16_BACK_1_UNSAFE(s, i) + +/** @deprecated ICU 2.4. Renamed to U16_BACK_1, see utf_old.h. */ +#define UTF_BACK_1_SAFE(s, start, i) UTF16_BACK_1_SAFE(s, start, i) + + +/** @deprecated ICU 2.4. Renamed to U16_BACK_N_UNSAFE, see utf_old.h. */ +#define UTF_BACK_N_UNSAFE(s, i, n) UTF16_BACK_N_UNSAFE(s, i, n) + +/** @deprecated ICU 2.4. Renamed to U16_BACK_N, see utf_old.h. */ +#define UTF_BACK_N_SAFE(s, start, i, n) UTF16_BACK_N_SAFE(s, start, i, n) + + +/** @deprecated ICU 2.4. Renamed to U16_SET_CP_LIMIT_UNSAFE, see utf_old.h. */ +#define UTF_SET_CHAR_LIMIT_UNSAFE(s, i) UTF16_SET_CHAR_LIMIT_UNSAFE(s, i) + +/** @deprecated ICU 2.4. Renamed to U16_SET_CP_LIMIT, see utf_old.h. */ +#define UTF_SET_CHAR_LIMIT_SAFE(s, start, i, length) UTF16_SET_CHAR_LIMIT_SAFE(s, start, i, length) + +/* Define default macros (UTF-16 "safe") ------------------------------------ */ + +/** + * Does this code unit alone encode a code point (BMP, not a surrogate)? + * Same as UTF16_IS_SINGLE. + * @deprecated ICU 2.4. Renamed to U_IS_SINGLE and U16_IS_SINGLE, see utf_old.h. + */ +#define UTF_IS_SINGLE(uchar) U16_IS_SINGLE(uchar) + +/** + * Is this code unit the first one of several (a lead surrogate)? + * Same as UTF16_IS_LEAD. + * @deprecated ICU 2.4. Renamed to U_IS_LEAD and U16_IS_LEAD, see utf_old.h. + */ +#define UTF_IS_LEAD(uchar) U16_IS_LEAD(uchar) + +/** + * Is this code unit one of several but not the first one (a trail surrogate)? + * Same as UTF16_IS_TRAIL. + * @deprecated ICU 2.4. Renamed to U_IS_TRAIL and U16_IS_TRAIL, see utf_old.h. + */ +#define UTF_IS_TRAIL(uchar) U16_IS_TRAIL(uchar) + +/** + * Does this code point require multiple code units (is it a supplementary code point)? + * Same as UTF16_NEED_MULTIPLE_UCHAR. + * @deprecated ICU 2.4. Use U16_LENGTH or test ((uint32_t)(c)>0xffff) instead. + */ +#define UTF_NEED_MULTIPLE_UCHAR(c) UTF16_NEED_MULTIPLE_UCHAR(c) + +/** + * How many code units are used to encode this code point (1 or 2)? + * Same as UTF16_CHAR_LENGTH. + * @deprecated ICU 2.4. Renamed to U16_LENGTH, see utf_old.h. + */ +#define UTF_CHAR_LENGTH(c) U16_LENGTH(c) + +/** + * How many code units are used at most for any Unicode code point (2)? + * Same as UTF16_MAX_CHAR_LENGTH. + * @deprecated ICU 2.4. Renamed to U16_MAX_LENGTH, see utf_old.h. + */ +#define UTF_MAX_CHAR_LENGTH U16_MAX_LENGTH + +/** + * Set c to the code point that contains the code unit i. + * i could point to the lead or the trail surrogate for the code point. + * i is not modified. + * Same as UTF16_GET_CHAR. + * \pre 0<=i +#include "unicode/utypes.h" + +/** + * \file + * \brief C API: Definitions for ICU tracing/logging. + * + * This provides API for debugging the internals of ICU without the use of + * a traditional debugger. + * + * By default, tracing is disabled in ICU. If you need to debug ICU with + * tracing, please compile ICU with the --enable-tracing configure option. + */ + +U_CDECL_BEGIN + +/** + * Trace severity levels. Higher levels increase the verbosity of the trace output. + * @see utrace_setLevel + * @stable ICU 2.8 + */ +typedef enum UTraceLevel { + /** Disable all tracing @stable ICU 2.8*/ + UTRACE_OFF=-1, + /** Trace error conditions only @stable ICU 2.8*/ + UTRACE_ERROR=0, + /** Trace errors and warnings @stable ICU 2.8*/ + UTRACE_WARNING=3, + /** Trace opens and closes of ICU services @stable ICU 2.8*/ + UTRACE_OPEN_CLOSE=5, + /** Trace an intermediate number of ICU operations @stable ICU 2.8*/ + UTRACE_INFO=7, + /** Trace the maximum number of ICU operations @stable ICU 2.8*/ + UTRACE_VERBOSE=9 +} UTraceLevel; + +/** + * These are the ICU functions that will be traced when tracing is enabled. + * @stable ICU 2.8 + */ +typedef enum UTraceFunctionNumber { + UTRACE_FUNCTION_START=0, + UTRACE_U_INIT=UTRACE_FUNCTION_START, + UTRACE_U_CLEANUP, + UTRACE_FUNCTION_LIMIT, + + UTRACE_CONVERSION_START=0x1000, + UTRACE_UCNV_OPEN=UTRACE_CONVERSION_START, + UTRACE_UCNV_OPEN_PACKAGE, + UTRACE_UCNV_OPEN_ALGORITHMIC, + UTRACE_UCNV_CLONE, + UTRACE_UCNV_CLOSE, + UTRACE_UCNV_FLUSH_CACHE, + UTRACE_UCNV_LOAD, + UTRACE_UCNV_UNLOAD, + UTRACE_CONVERSION_LIMIT, + + UTRACE_COLLATION_START=0x2000, + UTRACE_UCOL_OPEN=UTRACE_COLLATION_START, + UTRACE_UCOL_CLOSE, + UTRACE_UCOL_STRCOLL, + UTRACE_UCOL_GET_SORTKEY, + UTRACE_UCOL_GETLOCALE, + UTRACE_UCOL_NEXTSORTKEYPART, + UTRACE_UCOL_STRCOLLITER, + UTRACE_UCOL_OPEN_FROM_SHORT_STRING, + UTRACE_COLLATION_LIMIT +} UTraceFunctionNumber; + +/** + * Setter for the trace level. + * @param traceLevel A UTraceLevel value. + * @stable ICU 2.8 + */ +U_STABLE void U_EXPORT2 +utrace_setLevel(int32_t traceLevel); + +/** + * Getter for the trace level. + * @return The UTraceLevel value being used by ICU. + * @stable ICU 2.8 + */ +U_STABLE int32_t U_EXPORT2 +utrace_getLevel(void); + +/* Trace function pointers types ----------------------------- */ + +/** + * Type signature for the trace function to be called when entering a function. + * @param context value supplied at the time the trace functions are set. + * @param fnNumber Enum value indicating the ICU function being entered. + * @stable ICU 2.8 + */ +typedef void U_CALLCONV +UTraceEntry(const void *context, int32_t fnNumber); + +/** + * Type signature for the trace function to be called when exiting from a function. + * @param context value supplied at the time the trace functions are set. + * @param fnNumber Enum value indicating the ICU function being exited. + * @param fmt A formatting string that describes the number and types + * of arguments included with the variable args. The fmt + * string has the same form as the utrace_vformat format + * string. + * @param args A variable arguments list. Contents are described by + * the fmt parameter. + * @see utrace_vformat + * @stable ICU 2.8 + */ +typedef void U_CALLCONV +UTraceExit(const void *context, int32_t fnNumber, + const char *fmt, va_list args); + +/** + * Type signature for the trace function to be called from within an ICU function + * to display data or messages. + * @param context value supplied at the time the trace functions are set. + * @param fnNumber Enum value indicating the ICU function being exited. + * @param level The current tracing level + * @param fmt A format string describing the tracing data that is supplied + * as variable args + * @param args The data being traced, passed as variable args. + * @stable ICU 2.8 + */ +typedef void U_CALLCONV +UTraceData(const void *context, int32_t fnNumber, int32_t level, + const char *fmt, va_list args); + +/** + * Set ICU Tracing functions. Installs application-provided tracing + * functions into ICU. After doing this, subsequent ICU operations + * will call back to the installed functions, providing a trace + * of the use of ICU. Passing a NULL pointer for a tracing function + * is allowed, and inhibits tracing action at points where that function + * would be called. + *

+ * Tracing and Threads: Tracing functions are global to a process, and + * will be called in response to ICU operations performed by any + * thread. If tracing of an individual thread is desired, the + * tracing functions must themselves filter by checking that the + * current thread is the desired thread. + * + * @param context an uninterpretted pointer. Whatever is passed in + * here will in turn be passed to each of the tracing + * functions UTraceEntry, UTraceExit and UTraceData. + * ICU does not use or alter this pointer. + * @param e Callback function to be called on entry to a + * a traced ICU function. + * @param x Callback function to be called on exit from a + * traced ICU function. + * @param d Callback function to be called from within a + * traced ICU function, for the purpose of providing + * data to the trace. + * + * @stable ICU 2.8 + */ +U_STABLE void U_EXPORT2 +utrace_setFunctions(const void *context, + UTraceEntry *e, UTraceExit *x, UTraceData *d); + +/** + * Get the currently installed ICU tracing functions. Note that a null function + * pointer will be returned if no trace function has been set. + * + * @param context The currently installed tracing context. + * @param e The currently installed UTraceEntry function. + * @param x The currently installed UTraceExit function. + * @param d The currently installed UTraceData function. + * @stable ICU 2.8 + */ +U_STABLE void U_EXPORT2 +utrace_getFunctions(const void **context, + UTraceEntry **e, UTraceExit **x, UTraceData **d); + + + +/* + * + * ICU trace format string syntax + * + * Format Strings are passed to UTraceData functions, and define the + * number and types of the trace data being passed on each call. + * + * The UTraceData function, which is supplied by the application, + * not by ICU, can either forward the trace data (passed via + * varargs) and the format string back to ICU for formatting into + * a displayable string, or it can interpret the format itself, + * and do as it wishes with the trace data. + * + * + * Goals for the format string + * - basic data output + * - easy to use for trace programmer + * - sufficient provision for data types for trace output readability + * - well-defined types and binary portable APIs + * + * Non-goals + * - printf compatibility + * - fancy formatting + * - argument reordering and other internationalization features + * + * ICU trace format strings contain plain text with argument inserts, + * much like standard printf format strings. + * Each insert begins with a '%', then optionally contains a 'v', + * then exactly one type character. + * Two '%' in a row represent a '%' instead of an insert. + * The trace format strings need not have \n at the end. + * + * + * Types + * ----- + * + * Type characters: + * - c A char character in the default codepage. + * - s A NUL-terminated char * string in the default codepage. + * - S A UChar * string. Requires two params, (ptr, length). Length=-1 for nul term. + * - b A byte (8-bit integer). + * - h A 16-bit integer. Also a 16 bit Unicode code unit. + * - d A 32-bit integer. Also a 20 bit Unicode code point value. + * - l A 64-bit integer. + * - p A data pointer. + * + * Vectors + * ------- + * + * If the 'v' is not specified, then one item of the specified type + * is passed in. + * If the 'v' (for "vector") is specified, then a vector of items of the + * specified type is passed in, via a pointer to the first item + * and an int32_t value for the length of the vector. + * Length==-1 means zero or NUL termination. Works for vectors of all types. + * + * Note: %vS is a vector of (UChar *) strings. The strings must + * be nul terminated as there is no way to provide a + * separate length parameter for each string. The length + * parameter (required for all vectors) is the number of + * strings, not the length of the strings. + * + * Examples + * -------- + * + * These examples show the parameters that will be passed to an application's + * UTraceData() function for various formats. + * + * - the precise formatting is up to the application! + * - the examples use type casts for arguments only to _show_ the types of + * arguments without needing variable declarations in the examples; + * the type casts will not be necessary in actual code + * + * UTraceDataFunc(context, fnNumber, level, + * "There is a character %c in the string %s.", // Format String + * (char)c, (const char *)s); // varargs parameters + * -> There is a character 0x42 'B' in the string "Bravo". + * + * UTraceDataFunc(context, fnNumber, level, + * "Vector of bytes %vb vector of chars %vc", + * (const uint8_t *)bytes, (int32_t)bytesLength, + * (const char *)chars, (int32_t)charsLength); + * -> Vector of bytes + * 42 63 64 3f [4] + * vector of chars + * "Bcd?"[4] + * + * UTraceDataFunc(context, fnNumber, level, + * "An int32_t %d and a whole bunch of them %vd", + * (int32_t)-5, (const int32_t *)ints, (int32_t)intsLength); + * -> An int32_t 0xfffffffb and a whole bunch of them + * fffffffb 00000005 0000010a [3] + * + */ + + + +/** + * Trace output Formatter. An application's UTraceData tracing functions may call + * back to this function to format the trace output in a + * human readable form. Note that a UTraceData function may choose + * to not format the data; it could, for example, save it in + * in the raw form it was received (more compact), leaving + * formatting for a later trace analyis tool. + * @param outBuf pointer to a buffer to receive the formatted output. Output + * will be nul terminated if there is space in the buffer - + * if the length of the requested output < the output buffer size. + * @param capacity Length of the output buffer. + * @param indent Number of spaces to indent the output. Intended to allow + * data displayed from nested functions to be indented for readability. + * @param fmt Format specification for the data to output + * @param args Data to be formatted. + * @return Length of formatted output, including the terminating NUL. + * If buffer capacity is insufficient, the required capacity is returned. + * @stable ICU 2.8 + */ +U_STABLE int32_t U_EXPORT2 +utrace_vformat(char *outBuf, int32_t capacity, + int32_t indent, const char *fmt, va_list args); + +/** + * Trace output Formatter. An application's UTraceData tracing functions may call + * this function to format any additional trace data, beyond that + * provided by default, in human readable form with the same + * formatting conventions used by utrace_vformat(). + * @param outBuf pointer to a buffer to receive the formatted output. Output + * will be nul terminated if there is space in the buffer - + * if the length of the requested output < the output buffer size. + * @param capacity Length of the output buffer. + * @param indent Number of spaces to indent the output. Intended to allow + * data displayed from nested functions to be indented for readability. + * @param fmt Format specification for the data to output + * @param ... Data to be formatted. + * @return Length of formatted output, including the terminating NUL. + * If buffer capacity is insufficient, the required capacity is returned. + * @stable ICU 2.8 + */ +U_STABLE int32_t U_EXPORT2 +utrace_format(char *outBuf, int32_t capacity, + int32_t indent, const char *fmt, ...); + + + +/* Trace function numbers --------------------------------------------------- */ + +/** + * Get the name of a function from its trace function number. + * + * @param fnNumber The trace number for an ICU function. + * @return The name string for the function. + * + * @see UTraceFunctionNumber + * @stable ICU 2.8 + */ +U_STABLE const char * U_EXPORT2 +utrace_functionName(int32_t fnNumber); + +U_CDECL_END + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/utypes.h b/Build/source/libs/icu/icu-xetex/common/unicode/utypes.h new file mode 100644 index 00000000000..902db881338 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/utypes.h @@ -0,0 +1,787 @@ +/* +********************************************************************** +* Copyright (C) 1996-2007, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* +* FILE NAME : UTYPES.H (formerly ptypes.h) +* +* Date Name Description +* 12/11/96 helena Creation. +* 02/27/97 aliu Added typedefs for UClassID, int8, int16, int32, +* uint8, uint16, and uint32. +* 04/01/97 aliu Added XP_CPLUSPLUS and modified to work under C as +* well as C++. +* Modified to use memcpy() for uprv_arrayCopy() fns. +* 04/14/97 aliu Added TPlatformUtilities. +* 05/07/97 aliu Added import/export specifiers (replacing the old +* broken EXT_CLASS). Added version number for our +* code. Cleaned up header. +* 6/20/97 helena Java class name change. +* 08/11/98 stephen UErrorCode changed from typedef to enum +* 08/12/98 erm Changed T_ANALYTIC_PACKAGE_VERSION to 3 +* 08/14/98 stephen Added uprv_arrayCopy() for int8_t, int16_t, int32_t +* 12/09/98 jfitz Added BUFFER_OVERFLOW_ERROR (bug 1100066) +* 04/20/99 stephen Cleaned up & reworked for autoconf. +* Renamed to utypes.h. +* 05/05/99 stephen Changed to use +* 12/07/99 helena Moved copyright notice string from ucnv_bld.h here. +******************************************************************************* +*/ + +#ifndef UTYPES_H +#define UTYPES_H + + +#include "unicode/umachine.h" +#include "unicode/utf.h" +#include "unicode/uversion.h" +#include "unicode/uconfig.h" + +#if !U_DEFAULT_SHOW_DRAFT && !defined(U_SHOW_DRAFT_API) +#define U_HIDE_DRAFT_API 1 +#endif + +#ifdef U_HIDE_DRAFT_API +#include "unicode/udraft.h" +#endif + +#ifdef U_HIDE_DEPRECATED_API +#include "unicode/udeprctd.h" +#endif + +#ifdef U_HIDE_DEPRECATED_API +#include "unicode/uobslete.h" +#endif + +#ifdef U_HIDE_INTERNAL_API +#include "unicode/uintrnal.h" +#endif + +#ifdef U_HIDE_SYSTEM_API +#include "unicode/usystem.h" +#endif + +/*! + * \file + * \brief Basic definitions for ICU, for both C and C++ APIs + * + * This file defines basic types, constants, and enumerations directly or + * indirectly by including other header files, especially utf.h for the + * basic character and string definitions and umachine.h for consistent + * integer and other types. + */ + +/*===========================================================================*/ +/* char Character set family */ +/*===========================================================================*/ + +/** + * U_CHARSET_FAMILY is equal to this value when the platform is an ASCII based platform. + * @stable ICU 2.0 + */ +#define U_ASCII_FAMILY 0 + +/** + * U_CHARSET_FAMILY is equal to this value when the platform is an EBCDIC based platform. + * @stable ICU 2.0 + */ +#define U_EBCDIC_FAMILY 1 + +/** + * \def U_CHARSET_FAMILY + * + *

These definitions allow to specify the encoding of text + * in the char data type as defined by the platform and the compiler. + * It is enough to determine the code point values of "invariant characters", + * which are the ones shared by all encodings that are in use + * on a given platform.

+ * + *

Those "invariant characters" should be all the uppercase and lowercase + * latin letters, the digits, the space, and "basic punctuation". + * Also, '\\n', '\\r', '\\t' should be available.

+ * + *

The list of "invariant characters" is:
+ * \code + * A-Z a-z 0-9 SPACE " % & ' ( ) * + , - . / : ; < = > ? _ + * \endcode + *
+ * (52 letters + 10 numbers + 20 punc/sym/space = 82 total)

+ * + *

This matches the IBM Syntactic Character Set (CS 640).

+ * + *

In other words, all the graphic characters in 7-bit ASCII should + * be safely accessible except the following:

+ * + * \code + * '\' + * '[' + * ']' + * '{' + * '}' + * '^' + * '~' + * '!' + * '#' + * '|' + * '$' + * '@' + * '`' + * \endcode + * @stable ICU 2.0 + */ + +#ifndef U_CHARSET_FAMILY +# define U_CHARSET_FAMILY 0 +#endif + +/*===========================================================================*/ +/* ICUDATA naming scheme */ +/*===========================================================================*/ + +/** + * \def U_ICUDATA_TYPE_LETTER + * + * This is a platform-dependent string containing one letter: + * - b for big-endian, ASCII-family platforms + * - l for little-endian, ASCII-family platforms + * - e for big-endian, EBCDIC-family platforms + * This letter is part of the common data file name. + * @stable ICU 2.0 + */ + +/** + * \def U_ICUDATA_TYPE_LITLETTER + * The non-string form of U_ICUDATA_TYPE_LETTER + * @stable ICU 2.0 + */ +#if U_CHARSET_FAMILY +# if U_IS_BIG_ENDIAN + /* EBCDIC - should always be BE */ +# define U_ICUDATA_TYPE_LETTER "e" +# define U_ICUDATA_TYPE_LITLETTER e +# else +# error "Don't know what to do with little endian EBCDIC!" +# define U_ICUDATA_TYPE_LETTER "x" +# define U_ICUDATA_TYPE_LITLETTER x +# endif +#else +# if U_IS_BIG_ENDIAN + /* Big-endian ASCII */ +# define U_ICUDATA_TYPE_LETTER "b" +# define U_ICUDATA_TYPE_LITLETTER b +# else + /* Little-endian ASCII */ +# define U_ICUDATA_TYPE_LETTER "l" +# define U_ICUDATA_TYPE_LITLETTER l +# endif +#endif + +/** + * A single string literal containing the icudata stub name. i.e. 'icudt18e' for + * ICU 1.8.x on EBCDIC, etc.. + * @stable ICU 2.0 + */ +#define U_ICUDATA_NAME "icudt" U_ICU_VERSION_SHORT U_ICUDATA_TYPE_LETTER + + +/** + * U_ICU_ENTRY_POINT is the name of the DLL entry point to the ICU data library. + * Defined as a literal, not a string. + * Tricky Preprocessor use - ## operator replaces macro paramters with the literal string + * from the corresponding macro invocation, _before_ other macro substitutions. + * Need a nested \#defines to get the actual version numbers rather than + * the literal text U_ICU_VERSION_MAJOR_NUM into the name. + * The net result will be something of the form + * \#define U_ICU_ENTRY_POINT icudt19_dat + * @stable ICU 2.4 + */ +#define U_ICUDATA_ENTRY_POINT U_DEF2_ICUDATA_ENTRY_POINT(U_ICU_VERSION_MAJOR_NUM, U_ICU_VERSION_MINOR_NUM) + +/** + * Do not use. + * @internal + */ +#define U_DEF2_ICUDATA_ENTRY_POINT(major, minor) U_DEF_ICUDATA_ENTRY_POINT(major, minor) +/** + * Do not use. + * @internal + */ +#define U_DEF_ICUDATA_ENTRY_POINT(major, minor) icudt##major##minor##_dat + +/** + * \def U_CALLCONV + * Similar to U_CDECL_BEGIN/U_CDECL_END, this qualifier is necessary + * in callback function typedefs to make sure that the calling convention + * is compatible. + * + * This is only used for non-ICU-API functions. + * When a function is a public ICU API, + * you must use the U_CAPI and U_EXPORT2 qualifiers. + * @stable ICU 2.0 + */ +#if defined(OS390) && (__COMPILER_VER__ < 0x41020000) && defined(XP_CPLUSPLUS) +# define U_CALLCONV __cdecl +#else +# define U_CALLCONV U_EXPORT2 +#endif + +/** + * \def NULL + * Define NULL if necessary, to 0 for C++ and to ((void *)0) for C. + * @stable ICU 2.0 + */ +#ifndef NULL +#ifdef XP_CPLUSPLUS +#define NULL 0 +#else +#define NULL ((void *)0) +#endif +#endif + +/*===========================================================================*/ +/* Calendar/TimeZone data types */ +/*===========================================================================*/ + +/** + * Date and Time data type. + * This is a primitive data type that holds the date and time + * as the number of milliseconds since 1970-jan-01, 00:00 UTC. + * UTC leap seconds are ignored. + * @stable ICU 2.0 + */ +typedef double UDate; + +/** The number of milliseconds per second @stable ICU 2.0 */ +#define U_MILLIS_PER_SECOND (1000) +/** The number of milliseconds per minute @stable ICU 2.0 */ +#define U_MILLIS_PER_MINUTE (60000) +/** The number of milliseconds per hour @stable ICU 2.0 */ +#define U_MILLIS_PER_HOUR (3600000) +/** The number of milliseconds per day @stable ICU 2.0 */ +#define U_MILLIS_PER_DAY (86400000) + + +/*===========================================================================*/ +/* UClassID-based RTTI */ +/*===========================================================================*/ + +/** + * UClassID is used to identify classes without using RTTI, since RTTI + * is not yet supported by all C++ compilers. Each class hierarchy which needs + * to implement polymorphic clone() or operator==() defines two methods, + * described in detail below. UClassID values can be compared using + * operator==(). Nothing else should be done with them. + * + * \par + * getDynamicClassID() is declared in the base class of the hierarchy as + * a pure virtual. Each concrete subclass implements it in the same way: + * + * \code + * class Base { + * public: + * virtual UClassID getDynamicClassID() const = 0; + * } + * + * class Derived { + * public: + * virtual UClassID getDynamicClassID() const + * { return Derived::getStaticClassID(); } + * } + * \endcode + * + * Each concrete class implements getStaticClassID() as well, which allows + * clients to test for a specific type. + * + * \code + * class Derived { + * public: + * static UClassID U_EXPORT2 getStaticClassID(); + * private: + * static char fgClassID; + * } + * + * // In Derived.cpp: + * UClassID Derived::getStaticClassID() + * { return (UClassID)&Derived::fgClassID; } + * char Derived::fgClassID = 0; // Value is irrelevant + * \endcode + * @stable ICU 2.0 + */ +typedef void* UClassID; + +/*===========================================================================*/ +/* Shared library/DLL import-export API control */ +/*===========================================================================*/ + +/* + * Control of symbol import/export. + * ICU is separated into three libraries. + */ + +/* + * \def U_COMBINED_IMPLEMENTATION + * Set to export library symbols from inside the ICU library + * when all of ICU is in a single library. + * This can be set as a compiler option while building ICU, and it + * needs to be the first one tested to override U_COMMON_API, U_I18N_API, etc. + * @stable ICU 2.0 + */ + +/** + * \def U_DATA_API + * Set to export library symbols from inside the stubdata library, + * and to import them from outside. + * @stable ICU 3.0 + */ + +/** + * \def U_COMMON_API + * Set to export library symbols from inside the common library, + * and to import them from outside. + * @stable ICU 2.0 + */ + +/** + * \def U_I18N_API + * Set to export library symbols from inside the i18n library, + * and to import them from outside. + * @stable ICU 2.0 + */ + +/** + * \def U_LAYOUT_API + * Set to export library symbols from inside the layout engine library, + * and to import them from outside. + * @stable ICU 2.0 + */ + +/** + * \def U_LAYOUTEX_API + * Set to export library symbols from inside the layout extensions library, + * and to import them from outside. + * @stable ICU 2.6 + */ + +/** + * \def U_IO_API + * Set to export library symbols from inside the ustdio library, + * and to import them from outside. + * @stable ICU 2.0 + */ + +/** + * \def U_TOOLUTIL_API + * Set to export library symbols from inside the toolutil library, + * and to import them from outside. + * @stable ICU 3.4 + */ + +#if defined(U_COMBINED_IMPLEMENTATION) +#define U_DATA_API U_EXPORT +#define U_COMMON_API U_EXPORT +#define U_I18N_API U_EXPORT +#define U_LAYOUT_API U_EXPORT +#define U_LAYOUTEX_API U_EXPORT +#define U_IO_API U_EXPORT +#define U_TOOLUTIL_API U_EXPORT +#elif defined(U_STATIC_IMPLEMENTATION) +#define U_DATA_API +#define U_COMMON_API +#define U_I18N_API +#define U_LAYOUT_API +#define U_LAYOUTEX_API +#define U_IO_API +#define U_TOOLUTIL_API +#elif defined(U_COMMON_IMPLEMENTATION) +#define U_DATA_API U_IMPORT +#define U_COMMON_API U_EXPORT +#define U_I18N_API U_IMPORT +#define U_LAYOUT_API U_IMPORT +#define U_LAYOUTEX_API U_IMPORT +#define U_IO_API U_IMPORT +#define U_TOOLUTIL_API U_IMPORT +#elif defined(U_I18N_IMPLEMENTATION) +#define U_DATA_API U_IMPORT +#define U_COMMON_API U_IMPORT +#define U_I18N_API U_EXPORT +#define U_LAYOUT_API U_IMPORT +#define U_LAYOUTEX_API U_IMPORT +#define U_IO_API U_IMPORT +#define U_TOOLUTIL_API U_IMPORT +#elif defined(U_LAYOUT_IMPLEMENTATION) +#define U_DATA_API U_IMPORT +#define U_COMMON_API U_IMPORT +#define U_I18N_API U_IMPORT +#define U_LAYOUT_API U_EXPORT +#define U_LAYOUTEX_API U_IMPORT +#define U_IO_API U_IMPORT +#define U_TOOLUTIL_API U_IMPORT +#elif defined(U_LAYOUTEX_IMPLEMENTATION) +#define U_DATA_API U_IMPORT +#define U_COMMON_API U_IMPORT +#define U_I18N_API U_IMPORT +#define U_LAYOUT_API U_IMPORT +#define U_LAYOUTEX_API U_EXPORT +#define U_IO_API U_IMPORT +#define U_TOOLUTIL_API U_IMPORT +#elif defined(U_IO_IMPLEMENTATION) +#define U_DATA_API U_IMPORT +#define U_COMMON_API U_IMPORT +#define U_I18N_API U_IMPORT +#define U_LAYOUT_API U_IMPORT +#define U_LAYOUTEX_API U_IMPORT +#define U_IO_API U_EXPORT +#define U_TOOLUTIL_API U_IMPORT +#elif defined(U_TOOLUTIL_IMPLEMENTATION) +#define U_DATA_API U_IMPORT +#define U_COMMON_API U_IMPORT +#define U_I18N_API U_IMPORT +#define U_LAYOUT_API U_IMPORT +#define U_LAYOUTEX_API U_IMPORT +#define U_IO_API U_IMPORT +#define U_TOOLUTIL_API U_EXPORT +#else +#define U_DATA_API U_IMPORT +#define U_COMMON_API U_IMPORT +#define U_I18N_API U_IMPORT +#define U_LAYOUT_API U_IMPORT +#define U_LAYOUTEX_API U_IMPORT +#define U_IO_API U_IMPORT +#define U_TOOLUTIL_API U_IMPORT +#endif + +/** + * \def U_STANDARD_CPP_NAMESPACE + * Control of C++ Namespace + * @stable ICU 2.0 + */ +#ifdef __cplusplus +#define U_STANDARD_CPP_NAMESPACE :: +#else +#define U_STANDARD_CPP_NAMESPACE +#endif + + +/*===========================================================================*/ +/* Global delete operator */ +/*===========================================================================*/ + +/* + * The ICU4C library must not use the global new and delete operators. + * These operators here are defined to enable testing for this. + * See Jitterbug 2581 for details of why this is necessary. + * + * Verification that ICU4C's memory usage is correct, i.e., + * that global new/delete are not used: + * + * a) Check for imports of global new/delete (see uobject.cpp for details) + * b) Verify that new is never imported. + * c) Verify that delete is only imported from object code for interface/mixin classes. + * d) Add global delete and delete[] only for the ICU4C library itself + * and define them in a way that crashes or otherwise easily shows a problem. + * + * The following implements d). + * The operator implementations crash; this is intentional and used for library debugging. + * + * Note: This is currently only done on Windows because + * some Linux/Unix compilers have problems with defining global new/delete. + * On Windows, U_WINDOWS is defined, and it is _MSC_VER>=1200 for MSVC 6.0 and higher. + */ +#if defined(XP_CPLUSPLUS) && defined(U_WINDOWS) && U_DEBUG && U_OVERRIDE_CXX_ALLOCATION && (_MSC_VER>=1200) && !defined(U_STATIC_IMPLEMENTATION) && (defined(U_COMMON_IMPLEMENTATION) || defined(U_I18N_IMPLEMENTATION) || defined(U_IO_IMPLEMENTATION) || defined(U_LAYOUT_IMPLEMENTATION) || defined(U_LAYOUTEX_IMPLEMENTATION)) + +#ifndef U_HIDE_INTERNAL_API +/** + * Global operator new, defined only inside ICU4C, must not be used. + * Crashes intentionally. + * @internal + */ +inline void * +operator new(size_t /*size*/) { + char *q=NULL; + *q=5; /* break it */ + return q; +} + +/** + * Global operator new[], defined only inside ICU4C, must not be used. + * Crashes intentionally. + * @internal + */ +inline void * +operator new[](size_t /*size*/) { + char *q=NULL; + *q=5; /* break it */ + return q; +} + +/** + * Global operator delete, defined only inside ICU4C, must not be used. + * Crashes intentionally. + * @internal + */ +inline void +operator delete(void * /*p*/) { + char *q=NULL; + *q=5; /* break it */ +} + +/** + * Global operator delete[], defined only inside ICU4C, must not be used. + * Crashes intentionally. + * @internal + */ +inline void +operator delete[](void * /*p*/) { + char *q=NULL; + *q=5; /* break it */ +} + +#endif /* U_HIDE_INTERNAL_API */ +#endif + +/*===========================================================================*/ +/* UErrorCode */ +/*===========================================================================*/ + +/** + * Error code to replace exception handling, so that the code is compatible with all C++ compilers, + * and to use the same mechanism for C and C++. + * + * \par + * ICU functions that take a reference (C++) or a pointer (C) to a UErrorCode + * first test if(U_FAILURE(errorCode)) { return immediately; } + * so that in a chain of such functions the first one that sets an error code + * causes the following ones to not perform any operations. + * + * \par + * Error codes should be tested using U_FAILURE() and U_SUCCESS(). + * @stable ICU 2.0 + */ +typedef enum UErrorCode { + /* The ordering of U_ERROR_INFO_START Vs U_USING_FALLBACK_WARNING looks weird + * and is that way because VC++ debugger displays first encountered constant, + * which is not the what the code is used for + */ + + U_USING_FALLBACK_WARNING = -128, /**< A resource bundle lookup returned a fallback result (not an error) */ + + U_ERROR_WARNING_START = -128, /**< Start of information results (semantically successful) */ + + U_USING_DEFAULT_WARNING = -127, /**< A resource bundle lookup returned a result from the root locale (not an error) */ + + U_SAFECLONE_ALLOCATED_WARNING = -126, /**< A SafeClone operation required allocating memory (informational only) */ + + U_STATE_OLD_WARNING = -125, /**< ICU has to use compatibility layer to construct the service. Expect performance/memory usage degradation. Consider upgrading */ + + U_STRING_NOT_TERMINATED_WARNING = -124,/**< An output string could not be NUL-terminated because output length==destCapacity. */ + + U_SORT_KEY_TOO_SHORT_WARNING = -123, /**< Number of levels requested in getBound is higher than the number of levels in the sort key */ + + U_AMBIGUOUS_ALIAS_WARNING = -122, /**< This converter alias can go to different converter implementations */ + + U_DIFFERENT_UCA_VERSION = -121, /**< ucol_open encountered a mismatch between UCA version and collator image version, so the collator was constructed from rules. No impact to further function */ + + U_ERROR_WARNING_LIMIT, /**< This must always be the last warning value to indicate the limit for UErrorCode warnings (last warning code +1) */ + + + U_ZERO_ERROR = 0, /**< No error, no warning. */ + + U_ILLEGAL_ARGUMENT_ERROR = 1, /**< Start of codes indicating failure */ + U_MISSING_RESOURCE_ERROR = 2, /**< The requested resource cannot be found */ + U_INVALID_FORMAT_ERROR = 3, /**< Data format is not what is expected */ + U_FILE_ACCESS_ERROR = 4, /**< The requested file cannot be found */ + U_INTERNAL_PROGRAM_ERROR = 5, /**< Indicates a bug in the library code */ + U_MESSAGE_PARSE_ERROR = 6, /**< Unable to parse a message (message format) */ + U_MEMORY_ALLOCATION_ERROR = 7, /**< Memory allocation error */ + U_INDEX_OUTOFBOUNDS_ERROR = 8, /**< Trying to access the index that is out of bounds */ + U_PARSE_ERROR = 9, /**< Equivalent to Java ParseException */ + U_INVALID_CHAR_FOUND = 10, /**< Character conversion: Unmappable input sequence. In other APIs: Invalid character. */ + U_TRUNCATED_CHAR_FOUND = 11, /**< Character conversion: Incomplete input sequence. */ + U_ILLEGAL_CHAR_FOUND = 12, /**< Character conversion: Illegal input sequence/combination of input units. */ + U_INVALID_TABLE_FORMAT = 13, /**< Conversion table file found, but corrupted */ + U_INVALID_TABLE_FILE = 14, /**< Conversion table file not found */ + U_BUFFER_OVERFLOW_ERROR = 15, /**< A result would not fit in the supplied buffer */ + U_UNSUPPORTED_ERROR = 16, /**< Requested operation not supported in current context */ + U_RESOURCE_TYPE_MISMATCH = 17, /**< an operation is requested over a resource that does not support it */ + U_ILLEGAL_ESCAPE_SEQUENCE = 18, /**< ISO-2022 illlegal escape sequence */ + U_UNSUPPORTED_ESCAPE_SEQUENCE = 19, /**< ISO-2022 unsupported escape sequence */ + U_NO_SPACE_AVAILABLE = 20, /**< No space available for in-buffer expansion for Arabic shaping */ + U_CE_NOT_FOUND_ERROR = 21, /**< Currently used only while setting variable top, but can be used generally */ + U_PRIMARY_TOO_LONG_ERROR = 22, /**< User tried to set variable top to a primary that is longer than two bytes */ + U_STATE_TOO_OLD_ERROR = 23, /**< ICU cannot construct a service from this state, as it is no longer supported */ + U_TOO_MANY_ALIASES_ERROR = 24, /**< There are too many aliases in the path to the requested resource. + It is very possible that a circular alias definition has occured */ + U_ENUM_OUT_OF_SYNC_ERROR = 25, /**< UEnumeration out of sync with underlying collection */ + U_INVARIANT_CONVERSION_ERROR = 26, /**< Unable to convert a UChar* string to char* with the invariant converter. */ + U_INVALID_STATE_ERROR = 27, /**< Requested operation can not be completed with ICU in its current state */ + U_COLLATOR_VERSION_MISMATCH = 28, /**< Collator version is not compatible with the base version */ + U_USELESS_COLLATOR_ERROR = 29, /**< Collator is options only and no base is specified */ + U_NO_WRITE_PERMISSION = 30, /**< Attempt to modify read-only or constant data. */ + + U_STANDARD_ERROR_LIMIT, /**< This must always be the last value to indicate the limit for standard errors */ + /* + * the error code range 0x10000 0x10100 are reserved for Transliterator + */ + U_BAD_VARIABLE_DEFINITION=0x10000,/**< Missing '$' or duplicate variable name */ + U_PARSE_ERROR_START = 0x10000, /**< Start of Transliterator errors */ + U_MALFORMED_RULE, /**< Elements of a rule are misplaced */ + U_MALFORMED_SET, /**< A UnicodeSet pattern is invalid*/ + U_MALFORMED_SYMBOL_REFERENCE, /**< UNUSED as of ICU 2.4 */ + U_MALFORMED_UNICODE_ESCAPE, /**< A Unicode escape pattern is invalid*/ + U_MALFORMED_VARIABLE_DEFINITION, /**< A variable definition is invalid */ + U_MALFORMED_VARIABLE_REFERENCE, /**< A variable reference is invalid */ + U_MISMATCHED_SEGMENT_DELIMITERS, /**< UNUSED as of ICU 2.4 */ + U_MISPLACED_ANCHOR_START, /**< A start anchor appears at an illegal position */ + U_MISPLACED_CURSOR_OFFSET, /**< A cursor offset occurs at an illegal position */ + U_MISPLACED_QUANTIFIER, /**< A quantifier appears after a segment close delimiter */ + U_MISSING_OPERATOR, /**< A rule contains no operator */ + U_MISSING_SEGMENT_CLOSE, /**< UNUSED as of ICU 2.4 */ + U_MULTIPLE_ANTE_CONTEXTS, /**< More than one ante context */ + U_MULTIPLE_CURSORS, /**< More than one cursor */ + U_MULTIPLE_POST_CONTEXTS, /**< More than one post context */ + U_TRAILING_BACKSLASH, /**< A dangling backslash */ + U_UNDEFINED_SEGMENT_REFERENCE, /**< A segment reference does not correspond to a defined segment */ + U_UNDEFINED_VARIABLE, /**< A variable reference does not correspond to a defined variable */ + U_UNQUOTED_SPECIAL, /**< A special character was not quoted or escaped */ + U_UNTERMINATED_QUOTE, /**< A closing single quote is missing */ + U_RULE_MASK_ERROR, /**< A rule is hidden by an earlier more general rule */ + U_MISPLACED_COMPOUND_FILTER, /**< A compound filter is in an invalid location */ + U_MULTIPLE_COMPOUND_FILTERS, /**< More than one compound filter */ + U_INVALID_RBT_SYNTAX, /**< A "::id" rule was passed to the RuleBasedTransliterator parser */ + U_INVALID_PROPERTY_PATTERN, /**< UNUSED as of ICU 2.4 */ + U_MALFORMED_PRAGMA, /**< A 'use' pragma is invlalid */ + U_UNCLOSED_SEGMENT, /**< A closing ')' is missing */ + U_ILLEGAL_CHAR_IN_SEGMENT, /**< UNUSED as of ICU 2.4 */ + U_VARIABLE_RANGE_EXHAUSTED, /**< Too many stand-ins generated for the given variable range */ + U_VARIABLE_RANGE_OVERLAP, /**< The variable range overlaps characters used in rules */ + U_ILLEGAL_CHARACTER, /**< A special character is outside its allowed context */ + U_INTERNAL_TRANSLITERATOR_ERROR, /**< Internal transliterator system error */ + U_INVALID_ID, /**< A "::id" rule specifies an unknown transliterator */ + U_INVALID_FUNCTION, /**< A "&fn()" rule specifies an unknown transliterator */ + U_PARSE_ERROR_LIMIT, /**< The limit for Transliterator errors */ + + /* + * the error code range 0x10100 0x10200 are reserved for formatting API parsing error + */ + U_UNEXPECTED_TOKEN=0x10100, /**< Syntax error in format pattern */ + U_FMT_PARSE_ERROR_START=0x10100, /**< Start of format library errors */ + U_MULTIPLE_DECIMAL_SEPARATORS, /**< More than one decimal separator in number pattern */ + U_MULTIPLE_DECIMAL_SEPERATORS = U_MULTIPLE_DECIMAL_SEPARATORS, /**< Typo: kept for backward compatibility. Use U_MULTIPLE_DECIMAL_SEPARATORS */ + U_MULTIPLE_EXPONENTIAL_SYMBOLS, /**< More than one exponent symbol in number pattern */ + U_MALFORMED_EXPONENTIAL_PATTERN, /**< Grouping symbol in exponent pattern */ + U_MULTIPLE_PERCENT_SYMBOLS, /**< More than one percent symbol in number pattern */ + U_MULTIPLE_PERMILL_SYMBOLS, /**< More than one permill symbol in number pattern */ + U_MULTIPLE_PAD_SPECIFIERS, /**< More than one pad symbol in number pattern */ + U_PATTERN_SYNTAX_ERROR, /**< Syntax error in format pattern */ + U_ILLEGAL_PAD_POSITION, /**< Pad symbol misplaced in number pattern */ + U_UNMATCHED_BRACES, /**< Braces do not match in message pattern */ + U_UNSUPPORTED_PROPERTY, /**< UNUSED as of ICU 2.4 */ + U_UNSUPPORTED_ATTRIBUTE, /**< UNUSED as of ICU 2.4 */ + U_FMT_PARSE_ERROR_LIMIT, /**< The limit for format library errors */ + + /* + * the error code range 0x10200 0x102ff are reserved for Break Iterator related error + */ + U_BRK_INTERNAL_ERROR=0x10200, /**< An internal error (bug) was detected. */ + U_BRK_ERROR_START=0x10200, /**< Start of codes indicating Break Iterator failures */ + U_BRK_HEX_DIGITS_EXPECTED, /**< Hex digits expected as part of a escaped char in a rule. */ + U_BRK_SEMICOLON_EXPECTED, /**< Missing ';' at the end of a RBBI rule. */ + U_BRK_RULE_SYNTAX, /**< Syntax error in RBBI rule. */ + U_BRK_UNCLOSED_SET, /**< UnicodeSet witing an RBBI rule missing a closing ']'. */ + U_BRK_ASSIGN_ERROR, /**< Syntax error in RBBI rule assignment statement. */ + U_BRK_VARIABLE_REDFINITION, /**< RBBI rule $Variable redefined. */ + U_BRK_MISMATCHED_PAREN, /**< Mis-matched parentheses in an RBBI rule. */ + U_BRK_NEW_LINE_IN_QUOTED_STRING, /**< Missing closing quote in an RBBI rule. */ + U_BRK_UNDEFINED_VARIABLE, /**< Use of an undefined $Variable in an RBBI rule. */ + U_BRK_INIT_ERROR, /**< Initialization failure. Probable missing ICU Data. */ + U_BRK_RULE_EMPTY_SET, /**< Rule contains an empty Unicode Set. */ + U_BRK_UNRECOGNIZED_OPTION, /**< !!option in RBBI rules not recognized. */ + U_BRK_MALFORMED_RULE_TAG, /**< The {nnn} tag on a rule is mal formed */ + U_BRK_ERROR_LIMIT, /**< This must always be the last value to indicate the limit for Break Iterator failures */ + + /* + * The error codes in the range 0x10300-0x103ff are reserved for regular expression related errrs + */ + U_REGEX_INTERNAL_ERROR=0x10300, /**< An internal error (bug) was detected. */ + U_REGEX_ERROR_START=0x10300, /**< Start of codes indicating Regexp failures */ + U_REGEX_RULE_SYNTAX, /**< Syntax error in regexp pattern. */ + U_REGEX_INVALID_STATE, /**< RegexMatcher in invalid state for requested operation */ + U_REGEX_BAD_ESCAPE_SEQUENCE, /**< Unrecognized backslash escape sequence in pattern */ + U_REGEX_PROPERTY_SYNTAX, /**< Incorrect Unicode property */ + U_REGEX_UNIMPLEMENTED, /**< Use of regexp feature that is not yet implemented. */ + U_REGEX_MISMATCHED_PAREN, /**< Incorrectly nested parentheses in regexp pattern. */ + U_REGEX_NUMBER_TOO_BIG, /**< Decimal number is too large. */ + U_REGEX_BAD_INTERVAL, /**< Error in {min,max} interval */ + U_REGEX_MAX_LT_MIN, /**< In {min,max}, max is less than min. */ + U_REGEX_INVALID_BACK_REF, /**< Back-reference to a non-existent capture group. */ + U_REGEX_INVALID_FLAG, /**< Invalid value for match mode flags. */ + U_REGEX_LOOK_BEHIND_LIMIT, /**< Look-Behind pattern matches must have a bounded maximum length. */ + U_REGEX_SET_CONTAINS_STRING, /**< Regexps cannot have UnicodeSets containing strings.*/ + U_REGEX_ERROR_LIMIT, /**< This must always be the last value to indicate the limit for regexp errors */ + + /* + * The error code in the range 0x10400-0x104ff are reserved for IDNA related error codes + */ + U_IDNA_PROHIBITED_ERROR=0x10400, + U_IDNA_ERROR_START=0x10400, + U_IDNA_UNASSIGNED_ERROR, + U_IDNA_CHECK_BIDI_ERROR, + U_IDNA_STD3_ASCII_RULES_ERROR, + U_IDNA_ACE_PREFIX_ERROR, + U_IDNA_VERIFICATION_ERROR, + U_IDNA_LABEL_TOO_LONG_ERROR, + U_IDNA_ZERO_LENGTH_LABEL_ERROR, + U_IDNA_DOMAIN_NAME_TOO_LONG_ERROR, + U_IDNA_ERROR_LIMIT, + /* + * Aliases for StringPrep + */ + U_STRINGPREP_PROHIBITED_ERROR = U_IDNA_PROHIBITED_ERROR, + U_STRINGPREP_UNASSIGNED_ERROR = U_IDNA_UNASSIGNED_ERROR, + U_STRINGPREP_CHECK_BIDI_ERROR = U_IDNA_CHECK_BIDI_ERROR, + + + U_ERROR_LIMIT=U_IDNA_ERROR_LIMIT /**< This must always be the last value to indicate the limit for UErrorCode (last error code +1) */ +} UErrorCode; + +/* Use the following to determine if an UErrorCode represents */ +/* operational success or failure. */ + +#ifdef XP_CPLUSPLUS + /** + * Does the error code indicate success? + * @stable ICU 2.0 + */ + static + inline UBool U_SUCCESS(UErrorCode code) { return (UBool)(code<=U_ZERO_ERROR); } + /** + * Does the error code indicate a failure? + * @stable ICU 2.0 + */ + static + inline UBool U_FAILURE(UErrorCode code) { return (UBool)(code>U_ZERO_ERROR); } +#else + /** + * Does the error code indicate success? + * @stable ICU 2.0 + */ +# define U_SUCCESS(x) ((x)<=U_ZERO_ERROR) + /** + * Does the error code indicate a failure? + * @stable ICU 2.0 + */ +# define U_FAILURE(x) ((x)>U_ZERO_ERROR) +#endif + +/** + * Return a string for a UErrorCode value. + * The string will be the same as the name of the error code constant + * in the UErrorCode enum above. + * @stable ICU 2.0 + */ +U_STABLE const char * U_EXPORT2 +u_errorName(UErrorCode code); + + +#endif /* _UTYPES */ diff --git a/Build/source/libs/icu/icu-xetex/common/unicode/uversion.h b/Build/source/libs/icu/icu-xetex/common/unicode/uversion.h new file mode 100644 index 00000000000..f026aed8ac6 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unicode/uversion.h @@ -0,0 +1,266 @@ +/* +******************************************************************************* +* Copyright (C) 2000-2007, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* +* file name: uversion.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* Created by: Vladimir Weinstein +* +* Contains all the important version numbers for ICU. +* Gets included by utypes.h and Windows .rc files +*/ + +/** + * \file + * \brief C API: Contains all the important version numbers for ICU. + */ +/*===========================================================================*/ +/* Main ICU version information */ +/*===========================================================================*/ + +#ifndef UVERSION_H +#define UVERSION_H + +/** + * IMPORTANT: When updating version, the following things need to be done: + * source/common/unicode/uversion.h - this file: update major, minor, + * patchlevel, suffix, version, short version constants, namespace, + * and copyright + * source/common/common.vcproj - update 'Output file name' on the link tab so + * that it contains the new major/minor combination + * source/i18n/i18n.vcproj - same as for the common.vcproj + * source/layout/layout.vcproj - same as for the common.vcproj + * source/stubdata/stubdata.vcproj - same as for the common.vcproj + * source/io/io.vcproj - same as for the common.vcproj + * source/data/makedata.mak - change U_ICUDATA_NAME so that it contains + * the new major/minor combination + * source/tools/genren/genren.pl - use this script according to the README + * in that folder + */ + +#include "unicode/umachine.h" + +/** The standard copyright notice that gets compiled into each library. + * This value will change in the subsequent releases of ICU + * @stable ICU 2.4 + */ +#define U_COPYRIGHT_STRING \ + " Copyright (C) 2007, International Business Machines Corporation and others. All Rights Reserved. " + +/** Maximum length of the copyright string. + * @stable ICU 2.4 + */ +#define U_COPYRIGHT_STRING_LENGTH 128 + +/** The current ICU major version as an integer. + * This value will change in the subsequent releases of ICU + * @stable ICU 2.4 + */ +#define U_ICU_VERSION_MAJOR_NUM 3 + +/** The current ICU minor version as an integer. + * This value will change in the subsequent releases of ICU + * @stable ICU 2.6 + */ +#define U_ICU_VERSION_MINOR_NUM 8 + +/** The current ICU patchlevel version as an integer. + * This value will change in the subsequent releases of ICU + * @stable ICU 2.4 + */ +#define U_ICU_VERSION_PATCHLEVEL_NUM 1 + +/** Glued version suffix for renamers + * This value will change in the subsequent releases of ICU + * @stable ICU 2.6 + */ +#define U_ICU_VERSION_SUFFIX _3_8 + +/** The current ICU library version as a dotted-decimal string. The patchlevel + * only appears in this string if it non-zero. + * This value will change in the subsequent releases of ICU + * @stable ICU 2.4 + */ +#define U_ICU_VERSION "3.8.1" + +/** The current ICU library major/minor version as a string without dots, for library name suffixes. + * This value will change in the subsequent releases of ICU + * @stable ICU 2.6 + */ +#define U_ICU_VERSION_SHORT "38" + +/** An ICU version consists of up to 4 numbers from 0..255. + * @stable ICU 2.4 + */ +#define U_MAX_VERSION_LENGTH 4 + +/** In a string, ICU version fields are delimited by dots. + * @stable ICU 2.4 + */ +#define U_VERSION_DELIMITER '.' + +/** The maximum length of an ICU version string. + * @stable ICU 2.4 + */ +#define U_MAX_VERSION_STRING_LENGTH 20 + +/** The binary form of a version on ICU APIs is an array of 4 uint8_t. + * @stable ICU 2.4 + */ +typedef uint8_t UVersionInfo[U_MAX_VERSION_LENGTH]; + +/*===========================================================================*/ +/* C++ namespace if supported. Versioned unless versioning is disabled. */ +/*===========================================================================*/ + +/** + * \def U_NAMESPACE_BEGIN + * This is used to begin a declaration of a public ICU C++ API. + * If the compiler doesn't support namespaces, this does nothing. + * @stable ICU 2.4 + */ + +/** + * \def U_NAMESPACE_END + * This is used to end a declaration of a public ICU C++ API + * If the compiler doesn't support namespaces, this does nothing. + * @stable ICU 2.4 + */ + +/** + * \def U_NAMESPACE_USE + * This is used to specify that the rest of the code uses the + * public ICU C++ API namespace. + * If the compiler doesn't support namespaces, this does nothing. + * @stable ICU 2.4 + */ + +/** + * \def U_NAMESPACE_QUALIFIER + * This is used to qualify that a function or class is part of + * the public ICU C++ API namespace. + * If the compiler doesn't support namespaces, this does nothing. + * @stable ICU 2.4 + */ + +/* Define namespace symbols if the compiler supports it. */ +#if U_HAVE_NAMESPACE && defined(XP_CPLUSPLUS) +# if U_DISABLE_RENAMING +# define U_ICU_NAMESPACE icu + namespace U_ICU_NAMESPACE { } +# else +# define U_ICU_NAMESPACE icu_3_8 + namespace U_ICU_NAMESPACE { } + namespace icu = U_ICU_NAMESPACE; +# endif + +# define U_NAMESPACE_BEGIN namespace U_ICU_NAMESPACE { +# define U_NAMESPACE_END } +# define U_NAMESPACE_USE using namespace U_ICU_NAMESPACE; +# define U_NAMESPACE_QUALIFIER U_ICU_NAMESPACE:: + +# ifndef U_USING_ICU_NAMESPACE +# define U_USING_ICU_NAMESPACE 1 +# endif +# if U_USING_ICU_NAMESPACE + U_NAMESPACE_USE +# endif +#else +# define U_NAMESPACE_BEGIN +# define U_NAMESPACE_END +# define U_NAMESPACE_USE +# define U_NAMESPACE_QUALIFIER +#endif + + +/*===========================================================================*/ +/* General version helper functions. Definitions in putil.c */ +/*===========================================================================*/ + +/** + * Parse a string with dotted-decimal version information and + * fill in a UVersionInfo structure with the result. + * Definition of this function lives in putil.c + * + * @param versionArray The destination structure for the version information. + * @param versionString A string with dotted-decimal version information, + * with up to four non-negative number fields with + * values of up to 255 each. + * @stable ICU 2.4 + */ +U_STABLE void U_EXPORT2 +u_versionFromString(UVersionInfo versionArray, const char *versionString); + +/** + * Write a string with dotted-decimal version information according + * to the input UVersionInfo. + * Definition of this function lives in putil.c + * + * @param versionArray The version information to be written as a string. + * @param versionString A string buffer that will be filled in with + * a string corresponding to the numeric version + * information in versionArray. + * The buffer size must be at least U_MAX_VERSION_STRING_LENGTH. + * @stable ICU 2.4 + */ +U_STABLE void U_EXPORT2 +u_versionToString(UVersionInfo versionArray, char *versionString); + +/** + * Gets the ICU release version. The version array stores the version information + * for ICU. For example, release "1.3.31.2" is then represented as 0x01031F02. + * Definition of this function lives in putil.c + * + * @param versionArray the version # information, the result will be filled in + * @stable ICU 2.0 + */ +U_STABLE void U_EXPORT2 +u_getVersion(UVersionInfo versionArray); + + +/*=========================================================================== + * ICU collation framework version information + * Version info that can be obtained from a collator is affected by these + * numbers in a secret and magic way. Please use collator version as whole + *=========================================================================== + */ + +/** Collation runtime version (sort key generator, strcoll). + * If the version is different, sortkeys for the same string could be different + * version 2 was in ICU 1.8.1. changed is: compression intervals, French secondary + * compression, generating quad level always when strength is quad or more + * version 4 - ICU 2.2 - tracking UCA changes, ignore completely ignorables + * in contractions, ignore primary ignorables after shifted + * version 5 - ICU 2.8 - changed implicit generation code + * version 6 - ICU 3.4 - with the UCA 4.1, Thai tag is no longer generated or used + * This value may change in the subsequent releases of ICU + * @stable ICU 2.4 + */ +#define UCOL_RUNTIME_VERSION 6 + +/** Builder code version. When this is different, same tailoring might result + * in assigning different collation elements to code points + * version 2 was in ICU 1.8.1. added support for prefixes, tweaked canonical + * closure. However, the tailorings should probably get same CEs assigned + * version 5 - ICU 2.2 - fixed some bugs, renamed some indirect values. + * version 6 - ICU 2.8 - fixed bug in builder that allowed 0xFF in primary values + * version 7 - ICU 3.4 - with the UCA 4.1 Thai tag is no longer processed, complete ignorables + * now break contractions + * Backward compatible with the old rules. + * This value may change in the subsequent releases of ICU + * @stable ICU 2.4 + */ +#define UCOL_BUILDER_VERSION 7 + +/** This is the version of the tailorings + * This value may change in the subsequent releases of ICU + * @stable ICU 2.4 + */ +#define UCOL_TAILORINGS_VERSION 1 + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unifilt.cpp b/Build/source/libs/icu/icu-xetex/common/unifilt.cpp new file mode 100644 index 00000000000..69ac3070c79 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unifilt.cpp @@ -0,0 +1,68 @@ +/* +********************************************************************** +* Copyright (c) 2001-2004, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 07/18/01 aliu Creation. +********************************************************************** +*/ + +#include "unicode/unifilt.h" +#include "unicode/rep.h" + +U_NAMESPACE_BEGIN +UOBJECT_DEFINE_ABSTRACT_RTTI_IMPLEMENTATION(UnicodeFilter) + + +/* Define this here due to the lack of another file. + It can't be defined in the header */ +UnicodeMatcher::~UnicodeMatcher() {} + +UnicodeFilter::~UnicodeFilter() {} + +/** + * UnicodeFunctor API. Cast 'this' to a UnicodeMatcher* pointer + * and return the pointer. + */ +UnicodeMatcher* UnicodeFilter::toMatcher() const { + return (UnicodeMatcher*) this; +} + +void UnicodeFilter::setData(const TransliterationRuleData*) {} + +/** + * Default implementation of UnicodeMatcher::matches() for Unicode + * filters. Matches a single code point at offset (either one or + * two 16-bit code units). + */ +UMatchDegree UnicodeFilter::matches(const Replaceable& text, + int32_t& offset, + int32_t limit, + UBool incremental) { + UChar32 c; + if (offset < limit && + contains(c = text.char32At(offset))) { + offset += UTF_CHAR_LENGTH(c); + return U_MATCH; + } + if (offset > limit && + contains(c = text.char32At(offset))) { + // Backup offset by 1, unless the preceding character is a + // surrogate pair -- then backup by 2 (keep offset pointing at + // the lead surrogate). + --offset; + if (offset >= 0) { + offset -= UTF_CHAR_LENGTH(text.char32At(offset)) - 1; + } + return U_MATCH; + } + if (incremental && offset == limit) { + return U_PARTIAL_MATCH; + } + return U_MISMATCH; +} + +U_NAMESPACE_END + +//eof diff --git a/Build/source/libs/icu/icu-xetex/common/unifunct.cpp b/Build/source/libs/icu/icu-xetex/common/unifunct.cpp new file mode 100644 index 00000000000..b834eeed535 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unifunct.cpp @@ -0,0 +1,26 @@ +/* +********************************************************************** +* Copyright (c) 2002-2004, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +*/ + +#include "unicode/unifunct.h" + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_ABSTRACT_RTTI_IMPLEMENTATION(UnicodeFunctor) + +UnicodeFunctor::~UnicodeFunctor() {} + +UnicodeMatcher* UnicodeFunctor::toMatcher() const { + return 0; +} + +UnicodeReplacer* UnicodeFunctor::toReplacer() const { + return 0; +} + +U_NAMESPACE_END + +//eof diff --git a/Build/source/libs/icu/icu-xetex/common/uniset.cpp b/Build/source/libs/icu/icu-xetex/common/uniset.cpp new file mode 100644 index 00000000000..82dcce6cf5c --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uniset.cpp @@ -0,0 +1,2145 @@ +/* +********************************************************************** +* Copyright (C) 1999-2007, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 10/20/99 alan Creation. +********************************************************************** +*/ + +#include "unicode/utypes.h" +#include "unicode/uniset.h" +#include "unicode/parsepos.h" +#include "unicode/symtable.h" +#include "ruleiter.h" +#include "cmemory.h" +#include "cstring.h" +#include "uhash.h" +#include "util.h" +#include "uvector.h" +#include "charstr.h" +#include "ustrfmt.h" +#include "uassert.h" +#include "hash.h" +#include "bmpset.h" +#include "unisetspan.h" + +// Define UChar constants using hex for EBCDIC compatibility +// Used #define to reduce private static exports and memory access time. +#define SET_OPEN ((UChar)0x005B) /*[*/ +#define SET_CLOSE ((UChar)0x005D) /*]*/ +#define HYPHEN ((UChar)0x002D) /*-*/ +#define COMPLEMENT ((UChar)0x005E) /*^*/ +#define COLON ((UChar)0x003A) /*:*/ +#define BACKSLASH ((UChar)0x005C) /*\*/ +#define INTERSECTION ((UChar)0x0026) /*&*/ +#define UPPER_U ((UChar)0x0055) /*U*/ +#define LOWER_U ((UChar)0x0075) /*u*/ +#define OPEN_BRACE ((UChar)123) /*{*/ +#define CLOSE_BRACE ((UChar)125) /*}*/ +#define UPPER_P ((UChar)0x0050) /*P*/ +#define LOWER_P ((UChar)0x0070) /*p*/ +#define UPPER_N ((UChar)78) /*N*/ +#define EQUALS ((UChar)0x003D) /*=*/ + +// HIGH_VALUE > all valid values. 110000 for codepoints +#define UNICODESET_HIGH 0x0110000 + +// LOW <= all valid values. ZERO for codepoints +#define UNICODESET_LOW 0x000000 + +// initial storage. Must be >= 0 +#define START_EXTRA 16 + +// extra amount for growth. Must be >= 0 +#define GROW_EXTRA START_EXTRA + +U_NAMESPACE_BEGIN + +SymbolTable::~SymbolTable() {} + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(UnicodeSet) + +/** + * Modify the given UChar32 variable so that it is in range, by + * pinning values < UNICODESET_LOW to UNICODESET_LOW, and + * pinning values > UNICODESET_HIGH-1 to UNICODESET_HIGH-1. + * It modifies its argument in-place and also returns it. + */ +static inline UChar32 pinCodePoint(UChar32& c) { + if (c < UNICODESET_LOW) { + c = UNICODESET_LOW; + } else if (c > (UNICODESET_HIGH-1)) { + c = (UNICODESET_HIGH-1); + } + return c; +} + +//---------------------------------------------------------------- +// Debugging +//---------------------------------------------------------------- + +// DO NOT DELETE THIS CODE. This code is used to debug memory leaks. +// To enable the debugging, define the symbol DEBUG_MEM in the line +// below. This will result in text being sent to stdout that looks +// like this: +// DEBUG UnicodeSet: ct 0x00A39B20; 397 [\u0A81-\u0A83\u0A85- +// DEBUG UnicodeSet: dt 0x00A39B20; 396 [\u0A81-\u0A83\u0A85- +// Each line lists a construction (ct) or destruction (dt) event, the +// object address, the number of outstanding objects after the event, +// and the pattern of the object in question. + +// #define DEBUG_MEM + +#ifdef DEBUG_MEM +#include +static int32_t _dbgCount = 0; + +static inline void _dbgct(UnicodeSet* set) { + UnicodeString str; + set->toPattern(str, TRUE); + char buf[40]; + str.extract(0, 39, buf, ""); + printf("DEBUG UnicodeSet: ct 0x%08X; %d %s\n", set, ++_dbgCount, buf); +} + +static inline void _dbgdt(UnicodeSet* set) { + UnicodeString str; + set->toPattern(str, TRUE); + char buf[40]; + str.extract(0, 39, buf, ""); + printf("DEBUG UnicodeSet: dt 0x%08X; %d %s\n", set, --_dbgCount, buf); +} + +#else + +#define _dbgct(set) +#define _dbgdt(set) + +#endif + +//---------------------------------------------------------------- +// UnicodeString in UVector support +//---------------------------------------------------------------- + +static void U_CALLCONV cloneUnicodeString(UHashTok *dst, UHashTok *src) { + dst->pointer = new UnicodeString(*(UnicodeString*)src->pointer); +} + +static int8_t U_CALLCONV compareUnicodeString(UHashTok t1, UHashTok t2) { + const UnicodeString &a = *(const UnicodeString*)t1.pointer; + const UnicodeString &b = *(const UnicodeString*)t2.pointer; + return a.compare(b); +} + +//---------------------------------------------------------------- +// Constructors &c +//---------------------------------------------------------------- + +/** + * Constructs an empty set. + */ +UnicodeSet::UnicodeSet() : + len(1), capacity(1 + START_EXTRA), list(0), bmpSet(0), buffer(0), + bufferCapacity(0), patLen(0), pat(NULL), strings(NULL), stringSpan(NULL) +{ + list = (UChar32*) uprv_malloc(sizeof(UChar32) * capacity); + if(list!=NULL){ + list[0] = UNICODESET_HIGH; + } + UErrorCode status = U_ZERO_ERROR; + allocateStrings(status); + _dbgct(this); +} + +/** + * Constructs a set containing the given range. If end > + * start then an empty set is created. + * + * @param start first character, inclusive, of range + * @param end last character, inclusive, of range + */ +UnicodeSet::UnicodeSet(UChar32 start, UChar32 end) : + len(1), capacity(1 + START_EXTRA), list(0), bmpSet(0), buffer(0), + bufferCapacity(0), patLen(0), pat(NULL), strings(NULL), stringSpan(NULL) +{ + list = (UChar32*) uprv_malloc(sizeof(UChar32) * capacity); + if(list!=NULL){ + UErrorCode status = U_ZERO_ERROR; + + list[0] = UNICODESET_HIGH; + allocateStrings(status); + complement(start, end); + } + _dbgct(this); +} + +/** + * Constructs a set that is identical to the given UnicodeSet. + */ +UnicodeSet::UnicodeSet(const UnicodeSet& o) : + UnicodeFilter(o), + len(0), capacity(o.isFrozen() ? o.len : o.len + GROW_EXTRA), list(0), + bmpSet(0), + buffer(0), bufferCapacity(0), + patLen(0), pat(NULL), strings(NULL), stringSpan(NULL) +{ + list = (UChar32*) uprv_malloc(sizeof(UChar32) * capacity); + if(list!=NULL){ + UErrorCode status = U_ZERO_ERROR; + allocateStrings(status); + *this = o; + } + _dbgct(this); +} + +// Copy-construct as thawed. +UnicodeSet::UnicodeSet(const UnicodeSet& o, UBool /* asThawed */) : + UnicodeFilter(o), + len(0), capacity(o.len + GROW_EXTRA), list(0), + bmpSet(0), + buffer(0), bufferCapacity(0), + patLen(0), pat(NULL), strings(NULL), stringSpan(NULL) +{ + list = (UChar32*) uprv_malloc(sizeof(UChar32) * capacity); + if(list!=NULL){ + UErrorCode status = U_ZERO_ERROR; + allocateStrings(status); + // *this = o except for bmpSet and stringSpan + len = o.len; + uprv_memcpy(list, o.list, len*sizeof(UChar32)); + strings->assign(*o.strings, cloneUnicodeString, status); + if (o.pat) { + setPattern(UnicodeString(o.pat, o.patLen)); + } + } + _dbgct(this); +} + +/** + * Destructs the set. + */ +UnicodeSet::~UnicodeSet() { + _dbgdt(this); // first! + uprv_free(list); + delete bmpSet; + if (buffer) { + uprv_free(buffer); + } + delete strings; + delete stringSpan; + releasePattern(); +} + +/** + * Assigns this object to be a copy of another. + */ +UnicodeSet& UnicodeSet::operator=(const UnicodeSet& o) { + if (this == &o) { + return *this; + } + if (isFrozen()) { + return *this; + } + ensureCapacity(o.len); + len = o.len; + uprv_memcpy(list, o.list, len*sizeof(UChar32)); + if (o.bmpSet == NULL) { + bmpSet = NULL; + } else { + bmpSet = new BMPSet(*o.bmpSet, list, len); + } + UErrorCode ec = U_ZERO_ERROR; + strings->assign(*o.strings, cloneUnicodeString, ec); + if (o.stringSpan == NULL) { + stringSpan = NULL; + } else { + stringSpan = new UnicodeSetStringSpan(*o.stringSpan, *strings); + } + releasePattern(); + if (o.pat) { + setPattern(UnicodeString(o.pat, o.patLen)); + } + return *this; +} + +/** + * Returns a copy of this object. All UnicodeMatcher objects have + * to support cloning in order to allow classes using + * UnicodeMatchers, such as Transliterator, to implement cloning. + */ +UnicodeFunctor* UnicodeSet::clone() const { + return new UnicodeSet(*this); +} + +UnicodeFunctor *UnicodeSet::cloneAsThawed() const { + return new UnicodeSet(*this, TRUE); +} + +/** + * Compares the specified object with this set for equality. Returns + * true if the two sets + * have the same size, and every member of the specified set is + * contained in this set (or equivalently, every member of this set is + * contained in the specified set). + * + * @param o set to be compared for equality with this set. + * @return true if the specified set is equal to this set. + */ +UBool UnicodeSet::operator==(const UnicodeSet& o) const { + if (len != o.len) return FALSE; + for (int32_t i = 0; i < len; ++i) { + if (list[i] != o.list[i]) return FALSE; + } + if (*strings != *o.strings) return FALSE; + return TRUE; +} + +/** + * Returns the hash code value for this set. + * + * @return the hash code value for this set. + * @see Object#hashCode() + */ +int32_t UnicodeSet::hashCode(void) const { + int32_t result = len; + for (int32_t i = 0; i < len; ++i) { + result *= 1000003; + result += list[i]; + } + return result; +} + +//---------------------------------------------------------------- +// Public API +//---------------------------------------------------------------- + +/** + * Returns the number of elements in this set (its cardinality), + * Note than the elements of a set may include both individual + * codepoints and strings. + * + * @return the number of elements in this set (its cardinality). + */ +int32_t UnicodeSet::size(void) const { + int32_t n = 0; + int32_t count = getRangeCount(); + for (int32_t i = 0; i < count; ++i) { + n += getRangeEnd(i) - getRangeStart(i) + 1; + } + return n + strings->size(); +} + +/** + * Returns true if this set contains no elements. + * + * @return true if this set contains no elements. + */ +UBool UnicodeSet::isEmpty(void) const { + return len == 1 && strings->size() == 0; +} + +/** + * Returns true if this set contains the given character. + * @param c character to be checked for containment + * @return true if the test condition is met + */ +UBool UnicodeSet::contains(UChar32 c) const { + // Set i to the index of the start item greater than ch + // We know we will terminate without length test! + // LATER: for large sets, add binary search + //int32_t i = -1; + //for (;;) { + // if (c < list[++i]) break; + //} + if (bmpSet != NULL) { + return bmpSet->contains(c); + } + if (stringSpan != NULL) { + return stringSpan->contains(c); + } + if (c >= UNICODESET_HIGH) { // Don't need to check LOW bound + return FALSE; + } + int32_t i = findCodePoint(c); + return (UBool)(i & 1); // return true if odd +} + +/** + * Returns the smallest value i such that c < list[i]. Caller + * must ensure that c is a legal value or this method will enter + * an infinite loop. This method performs a binary search. + * @param c a character in the range MIN_VALUE..MAX_VALUE + * inclusive + * @return the smallest integer i in the range 0..len-1, + * inclusive, such that c < list[i] + */ +int32_t UnicodeSet::findCodePoint(UChar32 c) const { + /* Examples: + findCodePoint(c) + set list[] c=0 1 3 4 7 8 + === ============== =========== + [] [110000] 0 0 0 0 0 0 + [\u0000-\u0003] [0, 4, 110000] 1 1 1 2 2 2 + [\u0004-\u0007] [4, 8, 110000] 0 0 0 1 1 2 + [:Any:] [0, 110000] 1 1 1 1 1 1 + */ + + // Return the smallest i such that c < list[i]. Assume + // list[len - 1] == HIGH and that c is legal (0..HIGH-1). + if (c < list[0]) + return 0; + // High runner test. c is often after the last range, so an + // initial check for this condition pays off. + int32_t lo = 0; + int32_t hi = len - 1; + if (lo >= hi || c >= list[hi-1]) + return hi; + // invariant: c >= list[lo] + // invariant: c < list[hi] + for (;;) { + int32_t i = (lo + hi) >> 1; + if (i == lo) { + break; // Found! + } else if (c < list[i]) { + hi = i; + } else { + lo = i; + } + } + return hi; +} + +/** + * Returns true if this set contains every character + * of the given range. + * @param start first character, inclusive, of the range + * @param end last character, inclusive, of the range + * @return true if the test condition is met + */ +UBool UnicodeSet::contains(UChar32 start, UChar32 end) const { + //int32_t i = -1; + //for (;;) { + // if (start < list[++i]) break; + //} + int32_t i = findCodePoint(start); + return ((i & 1) != 0 && end < list[i]); +} + +/** + * Returns true if this set contains the given + * multicharacter string. + * @param s string to be checked for containment + * @return true if this set contains the specified string + */ +UBool UnicodeSet::contains(const UnicodeString& s) const { + if (s.length() == 0) return FALSE; + int32_t cp = getSingleCP(s); + if (cp < 0) { + return strings->contains((void*) &s); + } else { + return contains((UChar32) cp); + } +} + +/** + * Returns true if this set contains all the characters and strings + * of the given set. + * @param c set to be checked for containment + * @return true if the test condition is met + */ +UBool UnicodeSet::containsAll(const UnicodeSet& c) const { + // The specified set is a subset if all of its pairs are contained in + // this set. It's possible to code this more efficiently in terms of + // direct manipulation of the inversion lists if the need arises. + int32_t n = c.getRangeCount(); + for (int i=0; icontainsAll(*c.strings)) return FALSE; + return TRUE; +} + +/** + * Returns true if this set contains all the characters + * of the given string. + * @param s string containing characters to be checked for containment + * @return true if the test condition is met + */ +UBool UnicodeSet::containsAll(const UnicodeString& s) const { + return (UBool)(span(s.getBuffer(), s.length(), USET_SPAN_CONTAINED) == + s.length()); +} + +/** + * Returns true if this set contains none of the characters + * of the given range. + * @param start first character, inclusive, of the range + * @param end last character, inclusive, of the range + * @return true if the test condition is met + */ +UBool UnicodeSet::containsNone(UChar32 start, UChar32 end) const { + //int32_t i = -1; + //for (;;) { + // if (start < list[++i]) break; + //} + int32_t i = findCodePoint(start); + return ((i & 1) == 0 && end < list[i]); +} + +/** + * Returns true if this set contains none of the characters and strings + * of the given set. + * @param c set to be checked for containment + * @return true if the test condition is met + */ +UBool UnicodeSet::containsNone(const UnicodeSet& c) const { + // The specified set is a subset if all of its pairs are contained in + // this set. It's possible to code this more efficiently in terms of + // direct manipulation of the inversion lists if the need arises. + int32_t n = c.getRangeCount(); + for (int32_t i=0; icontainsNone(*c.strings)) return FALSE; + return TRUE; +} + +/** + * Returns true if this set contains none of the characters + * of the given string. + * @param s string containing characters to be checked for containment + * @return true if the test condition is met + */ +UBool UnicodeSet::containsNone(const UnicodeString& s) const { + return (UBool)(span(s.getBuffer(), s.length(), USET_SPAN_NOT_CONTAINED) == + s.length()); +} + +/** + * Returns true if this set contains any character whose low byte + * is the given value. This is used by RuleBasedTransliterator for + * indexing. + */ +UBool UnicodeSet::matchesIndexValue(uint8_t v) const { + /* The index value v, in the range [0,255], is contained in this set if + * it is contained in any pair of this set. Pairs either have the high + * bytes equal, or unequal. If the high bytes are equal, then we have + * aaxx..aayy, where aa is the high byte. Then v is contained if xx <= + * v <= yy. If the high bytes are unequal we have aaxx..bbyy, bb>aa. + * Then v is contained if xx <= v || v <= yy. (This is identical to the + * time zone month containment logic.) + */ + int32_t i; + int32_t rangeCount=getRangeCount(); + for (i=0; isize() != 0) { + for (i=0; isize(); ++i) { + const UnicodeString& s = *(const UnicodeString*)strings->elementAt(i); + //if (s.length() == 0) { + // // Empty strings match everything + // return TRUE; + //} + // assert(s.length() != 0); // We enforce this elsewhere + UChar32 c = s.char32At(0); + if ((c & 0xFF) == v) { + return TRUE; + } + } + } + return FALSE; +} + +/** + * Implementation of UnicodeMatcher::matches(). Always matches the + * longest possible multichar string. + */ +UMatchDegree UnicodeSet::matches(const Replaceable& text, + int32_t& offset, + int32_t limit, + UBool incremental) { + if (offset == limit) { + // Strings, if any, have length != 0, so we don't worry + // about them here. If we ever allow zero-length strings + // we much check for them here. + if (contains(U_ETHER)) { + return incremental ? U_PARTIAL_MATCH : U_MATCH; + } else { + return U_MISMATCH; + } + } else { + if (strings->size() != 0) { // try strings first + + // might separate forward and backward loops later + // for now they are combined + + // TODO Improve efficiency of this, at least in the forward + // direction, if not in both. In the forward direction we + // can assume the strings are sorted. + + int32_t i; + UBool forward = offset < limit; + + // firstChar is the leftmost char to match in the + // forward direction or the rightmost char to match in + // the reverse direction. + UChar firstChar = text.charAt(offset); + + // If there are multiple strings that can match we + // return the longest match. + int32_t highWaterLength = 0; + + for (i=0; isize(); ++i) { + const UnicodeString& trial = *(const UnicodeString*)strings->elementAt(i); + + //if (trial.length() == 0) { + // return U_MATCH; // null-string always matches + //} + // assert(trial.length() != 0); // We ensure this elsewhere + + UChar c = trial.charAt(forward ? 0 : trial.length() - 1); + + // Strings are sorted, so we can optimize in the + // forward direction. + if (forward && c > firstChar) break; + if (c != firstChar) continue; + + int32_t matchLen = matchRest(text, offset, limit, trial); + + if (incremental) { + int32_t maxLen = forward ? limit-offset : offset-limit; + if (matchLen == maxLen) { + // We have successfully matched but only up to limit. + return U_PARTIAL_MATCH; + } + } + + if (matchLen == trial.length()) { + // We have successfully matched the whole string. + if (matchLen > highWaterLength) { + highWaterLength = matchLen; + } + // In the forward direction we know strings + // are sorted so we can bail early. + if (forward && matchLen < highWaterLength) { + break; + } + continue; + } + } + + // We've checked all strings without a partial match. + // If we have full matches, return the longest one. + if (highWaterLength != 0) { + offset += forward ? highWaterLength : -highWaterLength; + return U_MATCH; + } + } + return UnicodeFilter::matches(text, offset, limit, incremental); + } +} + +/** + * Returns the longest match for s in text at the given position. + * If limit > start then match forward from start+1 to limit + * matching all characters except s.charAt(0). If limit < start, + * go backward starting from start-1 matching all characters + * except s.charAt(s.length()-1). This method assumes that the + * first character, text.charAt(start), matches s, so it does not + * check it. + * @param text the text to match + * @param start the first character to match. In the forward + * direction, text.charAt(start) is matched against s.charAt(0). + * In the reverse direction, it is matched against + * s.charAt(s.length()-1). + * @param limit the limit offset for matching, either last+1 in + * the forward direction, or last-1 in the reverse direction, + * where last is the index of the last character to match. + * @return If part of s matches up to the limit, return |limit - + * start|. If all of s matches before reaching the limit, return + * s.length(). If there is a mismatch between s and text, return + * 0 + */ +int32_t UnicodeSet::matchRest(const Replaceable& text, + int32_t start, int32_t limit, + const UnicodeString& s) { + int32_t i; + int32_t maxLen; + int32_t slen = s.length(); + if (start < limit) { + maxLen = limit - start; + if (maxLen > slen) maxLen = slen; + for (i = 1; i < maxLen; ++i) { + if (text.charAt(start + i) != s.charAt(i)) return 0; + } + } else { + maxLen = start - limit; + if (maxLen > slen) maxLen = slen; + --slen; // <=> slen = s.length() - 1; + for (i = 1; i < maxLen; ++i) { + if (text.charAt(start - i) != s.charAt(slen - i)) return 0; + } + } + return maxLen; +} + +/** + * Implement of UnicodeMatcher + */ +void UnicodeSet::addMatchSetTo(UnicodeSet& toUnionTo) const { + toUnionTo.addAll(*this); +} + +/** + * Returns the index of the given character within this set, where + * the set is ordered by ascending code point. If the character + * is not in this set, return -1. The inverse of this method is + * charAt(). + * @return an index from 0..size()-1, or -1 + */ +int32_t UnicodeSet::indexOf(UChar32 c) const { + if (c < MIN_VALUE || c > MAX_VALUE) { + return -1; + } + int32_t i = 0; + int32_t n = 0; + for (;;) { + UChar32 start = list[i++]; + if (c < start) { + return -1; + } + UChar32 limit = list[i++]; + if (c < limit) { + return n + c - start; + } + n += limit - start; + } +} + +/** + * Returns the character at the given index within this set, where + * the set is ordered by ascending code point. If the index is + * out of range, return (UChar32)-1. The inverse of this method is + * indexOf(). + * @param index an index from 0..size()-1 + * @return the character at the given index, or (UChar32)-1. + */ +UChar32 UnicodeSet::charAt(int32_t index) const { + if (index >= 0) { + // len2 is the largest even integer <= len, that is, it is len + // for even values and len-1 for odd values. With odd values + // the last entry is UNICODESET_HIGH. + int32_t len2 = len & ~1; + for (int32_t i=0; i < len2;) { + UChar32 start = list[i++]; + int32_t count = list[i++] - start; + if (index < count) { + return (UChar32)(start + index); + } + index -= count; + } + } + return (UChar32)-1; +} + +/** + * Make this object represent the range start - end. + * If end > start then this object is set to an + * an empty range. + * + * @param start first character in the set, inclusive + * @rparam end last character in the set, inclusive + */ +UnicodeSet& UnicodeSet::set(UChar32 start, UChar32 end) { + clear(); + complement(start, end); + return *this; +} + +/** + * Adds the specified range to this set if it is not already + * present. If this set already contains the specified range, + * the call leaves this set unchanged. If end > start + * then an empty range is added, leaving the set unchanged. + * + * @param start first character, inclusive, of range to be added + * to this set. + * @param end last character, inclusive, of range to be added + * to this set. + */ +UnicodeSet& UnicodeSet::add(UChar32 start, UChar32 end) { + if (pinCodePoint(start) < pinCodePoint(end)) { + UChar32 range[3] = { start, end+1, UNICODESET_HIGH }; + add(range, 2, 0); + } else if (start == end) { + add(start); + } + return *this; +} + +// #define DEBUG_US_ADD + +#ifdef DEBUG_US_ADD +#include +void dump(UChar32 c) { + if (c <= 0xFF) { + printf("%c", (char)c); + } else { + printf("U+%04X", c); + } +} +void dump(const UChar32* list, int32_t len) { + printf("["); + for (int32_t i=0; i "); +#endif + + if (c == list[i]-1) { + // c is before start of next range + list[i] = c; + // if we touched the HIGH mark, then add a new one + if (c == (UNICODESET_HIGH - 1)) { + ensureCapacity(len+1); + list[len++] = UNICODESET_HIGH; + } + if (i > 0 && c == list[i-1]) { + // collapse adjacent ranges + + // [..., start_k-1, c, c, limit_k, ..., HIGH] + // ^ + // list[i] + + //for (int32_t k=i-1; k 0 && c == list[i-1]) { + // c is after end of prior range + list[i-1]++; + // no need to check for collapse here + } + + else { + // At this point we know the new char is not adjacent to + // any existing ranges, and it is not 10FFFF. + + + // [..., start_k-1, limit_k-1, start_k, limit_k, ..., HIGH] + // ^ + // list[i] + + // [..., start_k-1, limit_k-1, c, c+1, start_k, limit_k, ..., HIGH] + // ^ + // list[i] + + ensureCapacity(len+2); + + //for (int32_t k=len-1; k>=i; --k) { + // list[k+2] = list[k]; + //} + UChar32* src = list + len; + UChar32* dst = src + 2; + UChar32* srclimit = list + i; + while (src > srclimit) *(--dst) = *(--src); + + list[i] = c; + list[i+1] = c+1; + len += 2; + } + +#ifdef DEBUG_US_ADD + dump(list, len); + printf("\n"); + + for (i=1; i {"ch"} + *
Warning: you cannot add an empty string ("") to a UnicodeSet. + * @param s the source string + * @return the modified set, for chaining + */ +UnicodeSet& UnicodeSet::add(const UnicodeString& s) { + if (s.length() == 0 || isFrozen()) return *this; + int32_t cp = getSingleCP(s); + if (cp < 0) { + if (!strings->contains((void*) &s)) { + _add(s); + releasePattern(); + } + } else { + add((UChar32)cp); + } + return *this; +} + +/** + * Adds the given string, in order, to 'strings'. The given string + * must have been checked by the caller to not be empty and to not + * already be in 'strings'. + */ +void UnicodeSet::_add(const UnicodeString& s) { + if (isFrozen()) { + return; + } + UnicodeString* t = new UnicodeString(s); + UErrorCode ec = U_ZERO_ERROR; + strings->sortedInsert(t, compareUnicodeString, ec); +} + +/** + * @return a code point IF the string consists of a single one. + * otherwise returns -1. + * @param string to test + */ +int32_t UnicodeSet::getSingleCP(const UnicodeString& s) { + //if (s.length() < 1) { + // throw new IllegalArgumentException("Can't use zero-length strings in UnicodeSet"); + //} + if (s.length() > 2) return -1; + if (s.length() == 1) return s.charAt(0); + + // at this point, len = 2 + UChar32 cp = s.char32At(0); + if (cp > 0xFFFF) { // is surrogate pair + return cp; + } + return -1; +} + +/** + * Adds each of the characters in this string to the set. Thus "ch" => {"c", "h"} + * If this set already any particular character, it has no effect on that character. + * @param the source string + * @return the modified set, for chaining + */ +UnicodeSet& UnicodeSet::addAll(const UnicodeString& s) { + UChar32 cp; + for (int32_t i = 0; i < s.length(); i += UTF_CHAR_LENGTH(cp)) { + cp = s.char32At(i); + add(cp); + } + return *this; +} + +/** + * Retains EACH of the characters in this string. Note: "ch" == {"c", "h"} + * If this set already any particular character, it has no effect on that character. + * @param the source string + * @return the modified set, for chaining + */ +UnicodeSet& UnicodeSet::retainAll(const UnicodeString& s) { + UnicodeSet set; + set.addAll(s); + retainAll(set); + return *this; +} + +/** + * Complement EACH of the characters in this string. Note: "ch" == {"c", "h"} + * If this set already any particular character, it has no effect on that character. + * @param the source string + * @return the modified set, for chaining + */ +UnicodeSet& UnicodeSet::complementAll(const UnicodeString& s) { + UnicodeSet set; + set.addAll(s); + complementAll(set); + return *this; +} + +/** + * Remove EACH of the characters in this string. Note: "ch" == {"c", "h"} + * If this set already any particular character, it has no effect on that character. + * @param the source string + * @return the modified set, for chaining + */ +UnicodeSet& UnicodeSet::removeAll(const UnicodeString& s) { + UnicodeSet set; + set.addAll(s); + removeAll(set); + return *this; +} + +/** + * Makes a set from a multicharacter string. Thus "ch" => {"ch"} + *
Warning: you cannot add an empty string ("") to a UnicodeSet. + * @param the source string + * @return a newly created set containing the given string + */ +UnicodeSet* U_EXPORT2 UnicodeSet::createFrom(const UnicodeString& s) { + UnicodeSet *set = new UnicodeSet(); + set->add(s); + return set; +} + + +/** + * Makes a set from each of the characters in the string. Thus "ch" => {"c", "h"} + * @param the source string + * @return a newly created set containing the given characters + */ +UnicodeSet* U_EXPORT2 UnicodeSet::createFromAll(const UnicodeString& s) { + UnicodeSet *set = new UnicodeSet(); + set->addAll(s); + return set; +} + +/** + * Retain only the elements in this set that are contained in the + * specified range. If end > start then an empty range is + * retained, leaving the set empty. + * + * @param start first character, inclusive, of range to be retained + * to this set. + * @param end last character, inclusive, of range to be retained + * to this set. + */ +UnicodeSet& UnicodeSet::retain(UChar32 start, UChar32 end) { + if (pinCodePoint(start) <= pinCodePoint(end)) { + UChar32 range[3] = { start, end+1, UNICODESET_HIGH }; + retain(range, 2, 0); + } else { + clear(); + } + return *this; +} + +UnicodeSet& UnicodeSet::retain(UChar32 c) { + return retain(c, c); +} + +/** + * Removes the specified range from this set if it is present. + * The set will not contain the specified range once the call + * returns. If end > start then an empty range is + * removed, leaving the set unchanged. + * + * @param start first character, inclusive, of range to be removed + * from this set. + * @param end last character, inclusive, of range to be removed + * from this set. + */ +UnicodeSet& UnicodeSet::remove(UChar32 start, UChar32 end) { + if (pinCodePoint(start) <= pinCodePoint(end)) { + UChar32 range[3] = { start, end+1, UNICODESET_HIGH }; + retain(range, 2, 2); + } + return *this; +} + +/** + * Removes the specified character from this set if it is present. + * The set will not contain the specified range once the call + * returns. + */ +UnicodeSet& UnicodeSet::remove(UChar32 c) { + return remove(c, c); +} + +/** + * Removes the specified string from this set if it is present. + * The set will not contain the specified character once the call + * returns. + * @param the source string + * @return the modified set, for chaining + */ +UnicodeSet& UnicodeSet::remove(const UnicodeString& s) { + if (s.length() == 0 || isFrozen()) return *this; + int32_t cp = getSingleCP(s); + if (cp < 0) { + strings->removeElement((void*) &s); + releasePattern(); + } else { + remove((UChar32)cp, (UChar32)cp); + } + return *this; +} + +/** + * Complements the specified range in this set. Any character in + * the range will be removed if it is in this set, or will be + * added if it is not in this set. If end > start + * then an empty range is xor'ed, leaving the set unchanged. + * + * @param start first character, inclusive, of range to be removed + * from this set. + * @param end last character, inclusive, of range to be removed + * from this set. + */ +UnicodeSet& UnicodeSet::complement(UChar32 start, UChar32 end) { + if (isFrozen()) { + return *this; + } + if (pinCodePoint(start) <= pinCodePoint(end)) { + UChar32 range[3] = { start, end+1, UNICODESET_HIGH }; + exclusiveOr(range, 2, 0); + } + releasePattern(); + return *this; +} + +UnicodeSet& UnicodeSet::complement(UChar32 c) { + return complement(c, c); +} + +/** + * This is equivalent to + * complement(MIN_VALUE, MAX_VALUE). + */ +UnicodeSet& UnicodeSet::complement(void) { + if (isFrozen()) { + return *this; + } + if (list[0] == UNICODESET_LOW) { + ensureBufferCapacity(len-1); + uprv_memcpy(buffer, list + 1, (len-1)*sizeof(UChar32)); + --len; + } else { + ensureBufferCapacity(len+1); + uprv_memcpy(buffer + 1, list, len*sizeof(UChar32)); + buffer[0] = UNICODESET_LOW; + ++len; + } + swapBuffers(); + releasePattern(); + return *this; +} + +/** + * Complement the specified string in this set. + * The set will not contain the specified string once the call + * returns. + *
Warning: you cannot add an empty string ("") to a UnicodeSet. + * @param s the string to complement + * @return this object, for chaining + */ +UnicodeSet& UnicodeSet::complement(const UnicodeString& s) { + if (s.length() == 0 || isFrozen()) return *this; + int32_t cp = getSingleCP(s); + if (cp < 0) { + if (strings->contains((void*) &s)) { + strings->removeElement((void*) &s); + } else { + _add(s); + } + releasePattern(); + } else { + complement((UChar32)cp, (UChar32)cp); + } + return *this; +} + +/** + * Adds all of the elements in the specified set to this set if + * they're not already present. This operation effectively + * modifies this set so that its value is the union of the two + * sets. The behavior of this operation is unspecified if the specified + * collection is modified while the operation is in progress. + * + * @param c set whose elements are to be added to this set. + * @see #add(char, char) + */ +UnicodeSet& UnicodeSet::addAll(const UnicodeSet& c) { + add(c.list, c.len, 0); + + // Add strings in order + for (int32_t i=0; isize(); ++i) { + const UnicodeString* s = (const UnicodeString*)c.strings->elementAt(i); + if (!strings->contains((void*) s)) { + _add(*s); + } + } + return *this; +} + +/** + * Retains only the elements in this set that are contained in the + * specified set. In other words, removes from this set all of + * its elements that are not contained in the specified set. This + * operation effectively modifies this set so that its value is + * the intersection of the two sets. + * + * @param c set that defines which elements this set will retain. + */ +UnicodeSet& UnicodeSet::retainAll(const UnicodeSet& c) { + if (isFrozen()) { + return *this; + } + retain(c.list, c.len, 0); + strings->retainAll(*c.strings); + return *this; +} + +/** + * Removes from this set all of its elements that are contained in the + * specified set. This operation effectively modifies this + * set so that its value is the asymmetric set difference of + * the two sets. + * + * @param c set that defines which elements will be removed from + * this set. + */ +UnicodeSet& UnicodeSet::removeAll(const UnicodeSet& c) { + if (isFrozen()) { + return *this; + } + retain(c.list, c.len, 2); + strings->removeAll(*c.strings); + return *this; +} + +/** + * Complements in this set all elements contained in the specified + * set. Any character in the other set will be removed if it is + * in this set, or will be added if it is not in this set. + * + * @param c set that defines which elements will be xor'ed from + * this set. + */ +UnicodeSet& UnicodeSet::complementAll(const UnicodeSet& c) { + if (isFrozen()) { + return *this; + } + exclusiveOr(c.list, c.len, 0); + + for (int32_t i=0; isize(); ++i) { + void* e = c.strings->elementAt(i); + if (!strings->removeElement(e)) { + _add(*(const UnicodeString*)e); + } + } + return *this; +} + +/** + * Removes all of the elements from this set. This set will be + * empty after this call returns. + */ +UnicodeSet& UnicodeSet::clear(void) { + if (isFrozen()) { + return *this; + } + list[0] = UNICODESET_HIGH; + len = 1; + releasePattern(); + strings->removeAllElements(); + return *this; +} + +/** + * Iteration method that returns the number of ranges contained in + * this set. + * @see #getRangeStart + * @see #getRangeEnd + */ +int32_t UnicodeSet::getRangeCount() const { + return len/2; +} + +/** + * Iteration method that returns the first character in the + * specified range of this set. + * @see #getRangeCount + * @see #getRangeEnd + */ +UChar32 UnicodeSet::getRangeStart(int32_t index) const { + return list[index*2]; +} + +/** + * Iteration method that returns the last character in the + * specified range of this set. + * @see #getRangeStart + * @see #getRangeEnd + */ +UChar32 UnicodeSet::getRangeEnd(int32_t index) const { + return list[index*2 + 1] - 1; +} + +int32_t UnicodeSet::getStringCount() const { + return strings->size(); +} + +const UnicodeString* UnicodeSet::getString(int32_t index) const { + return (const UnicodeString*) strings->elementAt(index); +} + +/** + * Reallocate this objects internal structures to take up the least + * possible space, without changing this object's value. + */ +UnicodeSet& UnicodeSet::compact() { + if (isFrozen()) { + return *this; + } + // Delete buffer first to defragment memory less. + if (buffer != NULL) { + uprv_free(buffer); + buffer = NULL; + } + if (len < capacity) { + // Make the capacity equal to len or 1. + // We don't want to realloc of 0 size. + capacity = len + (len == 0); + list = (UChar32*) uprv_realloc(list, sizeof(UChar32) * capacity); + } + return *this; +} + +int32_t UnicodeSet::serialize(uint16_t *dest, int32_t destCapacity, UErrorCode& ec) const { + int32_t bmpLength, length, destLength; + + if (U_FAILURE(ec)) { + return 0; + } + + if (destCapacity<0 || (destCapacity>0 && dest==NULL)) { + ec=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* count necessary 16-bit units */ + length=this->len-1; // Subtract 1 to ignore final UNICODESET_HIGH + // assert(length>=0); + if (length==0) { + /* empty set */ + if (destCapacity>0) { + *dest=0; + } else { + ec=U_BUFFER_OVERFLOW_ERROR; + } + return 1; + } + /* now length>0 */ + + if (this->list[length-1]<=0xffff) { + /* all BMP */ + bmpLength=length; + } else if (this->list[0]>=0x10000) { + /* all supplementary */ + bmpLength=0; + length*=2; + } else { + /* some BMP, some supplementary */ + for (bmpLength=0; bmpLengthlist[bmpLength]<=0xffff; ++bmpLength) {} + length=bmpLength+2*(length-bmpLength); + } + + /* length: number of 16-bit array units */ + if (length>0x7fff) { + /* there are only 15 bits for the length in the first serialized word */ + ec=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + + /* + * total serialized length: + * number of 16-bit array units (length) + + * 1 length unit (always) + + * 1 bmpLength unit (if there are supplementary values) + */ + destLength=length+((length>bmpLength)?2:1); + if (destLength<=destCapacity) { + const UChar32 *p; + int32_t i; + + *dest=(uint16_t)length; + if (length>bmpLength) { + *dest|=0x8000; + *++dest=(uint16_t)bmpLength; + } + ++dest; + + /* write the BMP part of the array */ + p=this->list; + for (i=0; i>16); + *dest++=(uint16_t)*p++; + } + } else { + ec=U_BUFFER_OVERFLOW_ERROR; + } + return destLength; +} + +//---------------------------------------------------------------- +// Implementation: Utility methods +//---------------------------------------------------------------- + +/** + * Allocate our strings vector and return TRUE if successful. + */ +UBool UnicodeSet::allocateStrings(UErrorCode &status) { + if (U_FAILURE(status)) { + return FALSE; + } + strings = new UVector(uhash_deleteUnicodeString, + uhash_compareUnicodeString, 1, status); + if (U_FAILURE(status)) { + delete strings; + strings = NULL; + return FALSE; + } + return TRUE; +} + +void UnicodeSet::ensureCapacity(int32_t newLen) { + if (newLen <= capacity) + return; + capacity = newLen + GROW_EXTRA; + UChar32* temp = (UChar32*) uprv_malloc(sizeof(UChar32) * capacity); + uprv_memcpy(temp, list, len*sizeof(UChar32)); + uprv_free(list); + list = temp; +} + +void UnicodeSet::ensureBufferCapacity(int32_t newLen) { + if (buffer != NULL && newLen <= bufferCapacity) + return; + if (buffer) { + uprv_free(buffer); + } + bufferCapacity = newLen + GROW_EXTRA; + buffer = (UChar32*) uprv_malloc(sizeof(UChar32) * bufferCapacity); +} + +/** + * Swap list and buffer. + */ +void UnicodeSet::swapBuffers(void) { + // swap list and buffer + UChar32* temp = list; + list = buffer; + buffer = temp; + + int32_t c = capacity; + capacity = bufferCapacity; + bufferCapacity = c; +} + +//---------------------------------------------------------------- +// Implementation: Fundamental operators +//---------------------------------------------------------------- + +static inline UChar32 max(UChar32 a, UChar32 b) { + return (a > b) ? a : b; +} + +// polarity = 0, 3 is normal: x xor y +// polarity = 1, 2: x xor ~y == x === y + +void UnicodeSet::exclusiveOr(const UChar32* other, int32_t otherLen, int8_t polarity) { + if (isFrozen()) { + return; + } + ensureBufferCapacity(len + otherLen); + int32_t i = 0, j = 0, k = 0; + UChar32 a = list[i++]; + UChar32 b; + if (polarity == 1 || polarity == 2) { + b = UNICODESET_LOW; + if (other[j] == UNICODESET_LOW) { // skip base if already LOW + ++j; + b = other[j]; + } + } else { + b = other[j++]; + } + // simplest of all the routines + // sort the values, discarding identicals! + for (;;) { + if (a < b) { + buffer[k++] = a; + a = list[i++]; + } else if (b < a) { + buffer[k++] = b; + b = other[j++]; + } else if (a != UNICODESET_HIGH) { // at this point, a == b + // discard both values! + a = list[i++]; + b = other[j++]; + } else { // DONE! + buffer[k++] = UNICODESET_HIGH; + len = k; + break; + } + } + swapBuffers(); + releasePattern(); +} + +// polarity = 0 is normal: x union y +// polarity = 2: x union ~y +// polarity = 1: ~x union y +// polarity = 3: ~x union ~y + +void UnicodeSet::add(const UChar32* other, int32_t otherLen, int8_t polarity) { + if (isFrozen()) { + return; + } + ensureBufferCapacity(len + otherLen); + int32_t i = 0, j = 0, k = 0; + UChar32 a = list[i++]; + UChar32 b = other[j++]; + // change from xor is that we have to check overlapping pairs + // polarity bit 1 means a is second, bit 2 means b is. + for (;;) { + switch (polarity) { + case 0: // both first; take lower if unequal + if (a < b) { // take a + // Back up over overlapping ranges in buffer[] + if (k > 0 && a <= buffer[k-1]) { + // Pick latter end value in buffer[] vs. list[] + a = max(list[i], buffer[--k]); + } else { + // No overlap + buffer[k++] = a; + a = list[i]; + } + i++; // Common if/else code factored out + polarity ^= 1; + } else if (b < a) { // take b + if (k > 0 && b <= buffer[k-1]) { + b = max(other[j], buffer[--k]); + } else { + buffer[k++] = b; + b = other[j]; + } + j++; + polarity ^= 2; + } else { // a == b, take a, drop b + if (a == UNICODESET_HIGH) goto loop_end; + // This is symmetrical; it doesn't matter if + // we backtrack with a or b. - liu + if (k > 0 && a <= buffer[k-1]) { + a = max(list[i], buffer[--k]); + } else { + // No overlap + buffer[k++] = a; + a = list[i]; + } + i++; + polarity ^= 1; + b = other[j++]; + polarity ^= 2; + } + break; + case 3: // both second; take higher if unequal, and drop other + if (b <= a) { // take a + if (a == UNICODESET_HIGH) goto loop_end; + buffer[k++] = a; + } else { // take b + if (b == UNICODESET_HIGH) goto loop_end; + buffer[k++] = b; + } + a = list[i++]; + polarity ^= 1; // factored common code + b = other[j++]; + polarity ^= 2; + break; + case 1: // a second, b first; if b < a, overlap + if (a < b) { // no overlap, take a + buffer[k++] = a; a = list[i++]; polarity ^= 1; + } else if (b < a) { // OVERLAP, drop b + b = other[j++]; + polarity ^= 2; + } else { // a == b, drop both! + if (a == UNICODESET_HIGH) goto loop_end; + a = list[i++]; + polarity ^= 1; + b = other[j++]; + polarity ^= 2; + } + break; + case 2: // a first, b second; if a < b, overlap + if (b < a) { // no overlap, take b + buffer[k++] = b; + b = other[j++]; + polarity ^= 2; + } else if (a < b) { // OVERLAP, drop a + a = list[i++]; + polarity ^= 1; + } else { // a == b, drop both! + if (a == UNICODESET_HIGH) goto loop_end; + a = list[i++]; + polarity ^= 1; + b = other[j++]; + polarity ^= 2; + } + break; + } + } + loop_end: + buffer[k++] = UNICODESET_HIGH; // terminate + len = k; + swapBuffers(); + releasePattern(); +} + +// polarity = 0 is normal: x intersect y +// polarity = 2: x intersect ~y == set-minus +// polarity = 1: ~x intersect y +// polarity = 3: ~x intersect ~y + +void UnicodeSet::retain(const UChar32* other, int32_t otherLen, int8_t polarity) { + if (isFrozen()) { + return; + } + ensureBufferCapacity(len + otherLen); + int32_t i = 0, j = 0, k = 0; + UChar32 a = list[i++]; + UChar32 b = other[j++]; + // change from xor is that we have to check overlapping pairs + // polarity bit 1 means a is second, bit 2 means b is. + for (;;) { + switch (polarity) { + case 0: // both first; drop the smaller + if (a < b) { // drop a + a = list[i++]; + polarity ^= 1; + } else if (b < a) { // drop b + b = other[j++]; + polarity ^= 2; + } else { // a == b, take one, drop other + if (a == UNICODESET_HIGH) goto loop_end; + buffer[k++] = a; + a = list[i++]; + polarity ^= 1; + b = other[j++]; + polarity ^= 2; + } + break; + case 3: // both second; take lower if unequal + if (a < b) { // take a + buffer[k++] = a; + a = list[i++]; + polarity ^= 1; + } else if (b < a) { // take b + buffer[k++] = b; + b = other[j++]; + polarity ^= 2; + } else { // a == b, take one, drop other + if (a == UNICODESET_HIGH) goto loop_end; + buffer[k++] = a; + a = list[i++]; + polarity ^= 1; + b = other[j++]; + polarity ^= 2; + } + break; + case 1: // a second, b first; + if (a < b) { // NO OVERLAP, drop a + a = list[i++]; + polarity ^= 1; + } else if (b < a) { // OVERLAP, take b + buffer[k++] = b; + b = other[j++]; + polarity ^= 2; + } else { // a == b, drop both! + if (a == UNICODESET_HIGH) goto loop_end; + a = list[i++]; + polarity ^= 1; + b = other[j++]; + polarity ^= 2; + } + break; + case 2: // a first, b second; if a < b, overlap + if (b < a) { // no overlap, drop b + b = other[j++]; + polarity ^= 2; + } else if (a < b) { // OVERLAP, take a + buffer[k++] = a; + a = list[i++]; + polarity ^= 1; + } else { // a == b, drop both! + if (a == UNICODESET_HIGH) goto loop_end; + a = list[i++]; + polarity ^= 1; + b = other[j++]; + polarity ^= 2; + } + break; + } + } + loop_end: + buffer[k++] = UNICODESET_HIGH; // terminate + len = k; + swapBuffers(); + releasePattern(); +} + +/** + * Append the toPattern() representation of a + * string to the given StringBuffer. + */ +void UnicodeSet::_appendToPat(UnicodeString& buf, const UnicodeString& s, UBool +escapeUnprintable) { + UChar32 cp; + for (int32_t i = 0; i < s.length(); i += UTF_CHAR_LENGTH(cp)) { + _appendToPat(buf, cp = s.char32At(i), escapeUnprintable); + } +} + +/** + * Append the toPattern() representation of a + * character to the given StringBuffer. + */ +void UnicodeSet::_appendToPat(UnicodeString& buf, UChar32 c, UBool +escapeUnprintable) { + if (escapeUnprintable && ICU_Utility::isUnprintable(c)) { + // Use hex escape notation (\uxxxx or \Uxxxxxxxx) for anything + // unprintable + if (ICU_Utility::escapeUnprintable(buf, c)) { + return; + } + } + // Okay to let ':' pass through + switch (c) { + case SET_OPEN: + case SET_CLOSE: + case HYPHEN: + case COMPLEMENT: + case INTERSECTION: + case BACKSLASH: + case OPEN_BRACE: + case CLOSE_BRACE: + case COLON: + case SymbolTable::SYMBOL_REF: + buf.append(BACKSLASH); + break; + default: + // Escape whitespace + if (uprv_isRuleWhiteSpace(c)) { + buf.append(BACKSLASH); + } + break; + } + buf.append(c); +} + +/** + * Append a string representation of this set to result. This will be + * a cleaned version of the string passed to applyPattern(), if there + * is one. Otherwise it will be generated. + */ +UnicodeString& UnicodeSet::_toPattern(UnicodeString& result, + UBool escapeUnprintable) const +{ + if (pat != NULL) { + int32_t i; + int32_t backslashCount = 0; + for (i=0; i 1 && + getRangeStart(0) == MIN_VALUE && + getRangeEnd(count-1) == MAX_VALUE) { + + // Emit the inverse + result.append(COMPLEMENT); + + for (int32_t i = 1; i < count; ++i) { + UChar32 start = getRangeEnd(i-1)+1; + UChar32 end = getRangeStart(i)-1; + _appendToPat(result, start, escapeUnprintable); + if (start != end) { + if ((start+1) != end) { + result.append(HYPHEN); + } + _appendToPat(result, end, escapeUnprintable); + } + } + } + + // Default; emit the ranges as pairs + else { + for (int32_t i = 0; i < count; ++i) { + UChar32 start = getRangeStart(i); + UChar32 end = getRangeEnd(i); + _appendToPat(result, start, escapeUnprintable); + if (start != end) { + if ((start+1) != end) { + result.append(HYPHEN); + } + _appendToPat(result, end, escapeUnprintable); + } + } + } + + for (int32_t i = 0; isize(); ++i) { + result.append(OPEN_BRACE); + _appendToPat(result, + *(const UnicodeString*) strings->elementAt(i), + escapeUnprintable); + result.append(CLOSE_BRACE); + } + return result.append(SET_CLOSE); +} + +/** +* Release existing cached pattern +*/ +void UnicodeSet::releasePattern() { + if (pat) { + uprv_free(pat); + pat = NULL; + patLen = 0; + } +} + +/** +* Set the new pattern to cache. +*/ +void UnicodeSet::setPattern(const UnicodeString& newPat) { + releasePattern(); + int32_t newPatLen = newPat.length(); + pat = (UChar *)uprv_malloc((newPatLen + 1) * sizeof(UChar)); + if (pat) { + patLen = newPatLen; + newPat.extractBetween(0, patLen, pat); + pat[patLen] = 0; + } + // else we don't care if malloc failed. This was just a nice cache. + // We can regenerate an equivalent pattern later when requested. +} + +UnicodeFunctor *UnicodeSet::freeze() { + if(!isFrozen()) { + // Do most of what compact() does before freezing because + // compact() will not work when the set is frozen. + // Small modification: Don't shrink if the savings would be tiny (<=GROW_EXTRA). + + // Delete buffer first to defragment memory less. + if (buffer != NULL) { + uprv_free(buffer); + buffer = NULL; + } + if (capacity > (len + GROW_EXTRA)) { + // Make the capacity equal to len or 1. + // We don't want to realloc of 0 size. + capacity = len + (len == 0); + list = (UChar32*) uprv_realloc(list, sizeof(UChar32) * capacity); + } + + // Optimize contains() and span() and similar functions. + if (!strings->isEmpty()) { + stringSpan = new UnicodeSetStringSpan(*this, *strings, UnicodeSetStringSpan::ALL); + if (stringSpan != NULL && !stringSpan->needsStringSpanUTF16()) { + // All strings are irrelevant for span() etc. because + // all of each string's code points are contained in this set. + // Do not check needsStringSpanUTF8() because UTF-8 has at most as + // many relevant strings as UTF-16. + // (Thus needsStringSpanUTF8() implies needsStringSpanUTF16().) + delete stringSpan; + stringSpan = NULL; + } + } + if (stringSpan == NULL) { + // No span-relevant strings: Optimize for code point spans. + bmpSet=new BMPSet(list, len); + } + } + return this; +} + +int32_t UnicodeSet::span(const UChar *s, int32_t length, USetSpanCondition spanCondition) const { + if(length>0 && bmpSet!=NULL) { + return (int32_t)(bmpSet->span(s, s+length, spanCondition)-s); + } + if(length<0) { + length=u_strlen(s); + } + if(length==0) { + return 0; + } + if(stringSpan!=NULL) { + return stringSpan->span(s, length, spanCondition); + } else if(!strings->isEmpty()) { + uint32_t which= spanCondition==USET_SPAN_NOT_CONTAINED ? + UnicodeSetStringSpan::FWD_UTF16_NOT_CONTAINED : + UnicodeSetStringSpan::FWD_UTF16_CONTAINED; + UnicodeSetStringSpan strSpan(*this, *strings, which); + if(strSpan.needsStringSpanUTF16()) { + return strSpan.span(s, length, spanCondition); + } + } + + if(spanCondition!=USET_SPAN_NOT_CONTAINED) { + spanCondition=USET_SPAN_CONTAINED; // Pin to 0/1 values. + } + + UChar32 c; + int32_t start=0, prev=0; + do { + U16_NEXT(s, start, length, c); + if(spanCondition!=contains(c)) { + break; + } + } while((prev=start)0 && bmpSet!=NULL) { + return (int32_t)(bmpSet->spanBack(s, s+length, spanCondition)-s); + } + if(length<0) { + length=u_strlen(s); + } + if(length==0) { + return 0; + } + if(stringSpan!=NULL) { + return stringSpan->spanBack(s, length, spanCondition); + } else if(!strings->isEmpty()) { + uint32_t which= spanCondition==USET_SPAN_NOT_CONTAINED ? + UnicodeSetStringSpan::BACK_UTF16_NOT_CONTAINED : + UnicodeSetStringSpan::BACK_UTF16_CONTAINED; + UnicodeSetStringSpan strSpan(*this, *strings, which); + if(strSpan.needsStringSpanUTF16()) { + return strSpan.spanBack(s, length, spanCondition); + } + } + + if(spanCondition!=USET_SPAN_NOT_CONTAINED) { + spanCondition=USET_SPAN_CONTAINED; // Pin to 0/1 values. + } + + UChar32 c; + int32_t prev=length; + do { + U16_PREV(s, 0, length, c); + if(spanCondition!=contains(c)) { + break; + } + } while((prev=length)>0); + return prev; +} + +int32_t UnicodeSet::spanUTF8(const char *s, int32_t length, USetSpanCondition spanCondition) const { + if(length>0 && bmpSet!=NULL) { + const uint8_t *s0=(const uint8_t *)s; + return (int32_t)(bmpSet->spanUTF8(s0, length, spanCondition)-s0); + } + if(length<0) { + length=uprv_strlen(s); + } + if(length==0) { + return 0; + } + if(stringSpan!=NULL) { + return stringSpan->spanUTF8((const uint8_t *)s, length, spanCondition); + } else if(!strings->isEmpty()) { + uint32_t which= spanCondition==USET_SPAN_NOT_CONTAINED ? + UnicodeSetStringSpan::FWD_UTF8_NOT_CONTAINED : + UnicodeSetStringSpan::FWD_UTF8_CONTAINED; + UnicodeSetStringSpan strSpan(*this, *strings, which); + if(strSpan.needsStringSpanUTF8()) { + return strSpan.spanUTF8((const uint8_t *)s, length, spanCondition); + } + } + + if(spanCondition!=USET_SPAN_NOT_CONTAINED) { + spanCondition=USET_SPAN_CONTAINED; // Pin to 0/1 values. + } + + UChar32 c; + int32_t start=0, prev=0; + do { + U8_NEXT(s, start, length, c); + if(c<0) { + c=0xfffd; + } + if(spanCondition!=contains(c)) { + break; + } + } while((prev=start)0 && bmpSet!=NULL) { + const uint8_t *s0=(const uint8_t *)s; + return bmpSet->spanBackUTF8(s0, length, spanCondition); + } + if(length<0) { + length=uprv_strlen(s); + } + if(length==0) { + return 0; + } + if(stringSpan!=NULL) { + return stringSpan->spanBackUTF8((const uint8_t *)s, length, spanCondition); + } else if(!strings->isEmpty()) { + uint32_t which= spanCondition==USET_SPAN_NOT_CONTAINED ? + UnicodeSetStringSpan::BACK_UTF8_NOT_CONTAINED : + UnicodeSetStringSpan::BACK_UTF8_CONTAINED; + UnicodeSetStringSpan strSpan(*this, *strings, which); + if(strSpan.needsStringSpanUTF8()) { + return strSpan.spanBackUTF8((const uint8_t *)s, length, spanCondition); + } + } + + if(spanCondition!=USET_SPAN_NOT_CONTAINED) { + spanCondition=USET_SPAN_CONTAINED; // Pin to 0/1 values. + } + + UChar32 c; + int32_t prev=length; + do { + U8_PREV(s, 0, length, c); + if(c<0) { + c=0xfffd; + } + if(spanCondition!=contains(c)) { + break; + } + } while((prev=length)>0); + return prev; +} + +U_NAMESPACE_END diff --git a/Build/source/libs/icu/icu-xetex/common/uniset_props.cpp b/Build/source/libs/icu/icu-xetex/common/uniset_props.cpp new file mode 100644 index 00000000000..71dfce9c18a --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uniset_props.cpp @@ -0,0 +1,1394 @@ +/* +******************************************************************************* +* +* Copyright (C) 1999-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: uniset_props.cpp +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2004aug25 +* created by: Markus W. Scherer +* +* Character property dependent functions moved here from uniset.cpp +*/ + +#include "unicode/utypes.h" +#include "unicode/uniset.h" +#include "unicode/parsepos.h" +#include "unicode/uchar.h" +#include "unicode/uscript.h" +#include "unicode/symtable.h" +#include "unicode/uset.h" +#include "unicode/locid.h" +#include "unicode/brkiter.h" +#include "uset_imp.h" +#include "ruleiter.h" +#include "cmemory.h" +#include "ucln_cmn.h" +#include "util.h" +#include "uvector.h" +#include "uprops.h" +#include "propname.h" +#include "unormimp.h" +#include "ucase.h" +#include "ubidi_props.h" +#include "uinvchar.h" +#include "charstr.h" +#include "cstring.h" +#include "umutex.h" +#include "uassert.h" +#include "hash.h" + +U_NAMESPACE_USE + +#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) + +// initial storage. Must be >= 0 +// *** same as in uniset.cpp ! *** +#define START_EXTRA 16 + +// Define UChar constants using hex for EBCDIC compatibility +// Used #define to reduce private static exports and memory access time. +#define SET_OPEN ((UChar)0x005B) /*[*/ +#define SET_CLOSE ((UChar)0x005D) /*]*/ +#define HYPHEN ((UChar)0x002D) /*-*/ +#define COMPLEMENT ((UChar)0x005E) /*^*/ +#define COLON ((UChar)0x003A) /*:*/ +#define BACKSLASH ((UChar)0x005C) /*\*/ +#define INTERSECTION ((UChar)0x0026) /*&*/ +#define UPPER_U ((UChar)0x0055) /*U*/ +#define LOWER_U ((UChar)0x0075) /*u*/ +#define OPEN_BRACE ((UChar)123) /*{*/ +#define CLOSE_BRACE ((UChar)125) /*}*/ +#define UPPER_P ((UChar)0x0050) /*P*/ +#define LOWER_P ((UChar)0x0070) /*p*/ +#define UPPER_N ((UChar)78) /*N*/ +#define EQUALS ((UChar)0x003D) /*=*/ + +//static const UChar POSIX_OPEN[] = { SET_OPEN,COLON,0 }; // "[:" +static const UChar POSIX_CLOSE[] = { COLON,SET_CLOSE,0 }; // ":]" +//static const UChar PERL_OPEN[] = { BACKSLASH,LOWER_P,0 }; // "\\p" +static const UChar PERL_CLOSE[] = { CLOSE_BRACE,0 }; // "}" +//static const UChar NAME_OPEN[] = { BACKSLASH,UPPER_N,0 }; // "\\N" +static const UChar HYPHEN_RIGHT_BRACE[] = {HYPHEN,SET_CLOSE,0}; /*-]*/ + +// Special property set IDs +static const char ANY[] = "ANY"; // [\u0000-\U0010FFFF] +static const char ASCII[] = "ASCII"; // [\u0000-\u007F] +static const char ASSIGNED[] = "Assigned"; // [:^Cn:] + +// Unicode name property alias +#define NAME_PROP "na" +#define NAME_PROP_LENGTH 2 + +/** + * Delimiter string used in patterns to close a category reference: + * ":]". Example: "[:Lu:]". + */ +//static const UChar CATEGORY_CLOSE[] = {COLON, SET_CLOSE, 0x0000}; /* ":]" */ + +U_CDECL_BEGIN + +static UnicodeSet *INCLUSIONS[UPROPS_SRC_COUNT] = { NULL }; // cached getInclusions() + +//---------------------------------------------------------------- +// Inclusions list +//---------------------------------------------------------------- + +// USetAdder implementation +// Does not use uset.h to reduce code dependencies +static void U_CALLCONV +_set_add(USet *set, UChar32 c) { + ((UnicodeSet *)set)->add(c); +} + +static void U_CALLCONV +_set_addRange(USet *set, UChar32 start, UChar32 end) { + ((UnicodeSet *)set)->add(start, end); +} + +static void U_CALLCONV +_set_addString(USet *set, const UChar *str, int32_t length) { + ((UnicodeSet *)set)->add(UnicodeString((UBool)(length<0), str, length)); +} + +/** + * Cleanup function for UnicodeSet + */ +static UBool U_CALLCONV uset_cleanup(void) { + int32_t i; + + for(i = UPROPS_SRC_NONE; i < UPROPS_SRC_COUNT; ++i) { + if (INCLUSIONS[i] != NULL) { + delete INCLUSIONS[i]; + INCLUSIONS[i] = NULL; + } + } + + return TRUE; +} + +U_CDECL_END + +U_NAMESPACE_BEGIN + +static const UnicodeSet* getInclusions(int32_t src, UErrorCode &status) { + UBool needInit; + UMTX_CHECK(NULL, (INCLUSIONS[src] == NULL), needInit); + if (needInit) { + UnicodeSet* incl = new UnicodeSet(); + USetAdder sa = { + (USet *)incl, + _set_add, + _set_addRange, + _set_addString, + NULL // don't need remove() + }; + + if (incl != NULL) { + switch(src) { + case UPROPS_SRC_CHAR: + uchar_addPropertyStarts(&sa, &status); + break; + case UPROPS_SRC_PROPSVEC: + upropsvec_addPropertyStarts(&sa, &status); + break; + case UPROPS_SRC_CHAR_AND_PROPSVEC: + uchar_addPropertyStarts(&sa, &status); + upropsvec_addPropertyStarts(&sa, &status); + break; + case UPROPS_SRC_HST: + uhst_addPropertyStarts(&sa, &status); + break; +#if !UCONFIG_NO_NORMALIZATION + case UPROPS_SRC_NORM: + unorm_addPropertyStarts(&sa, &status); + break; +#endif + case UPROPS_SRC_CASE: + ucase_addPropertyStarts(ucase_getSingleton(&status), &sa, &status); + break; + case UPROPS_SRC_BIDI: + ubidi_addPropertyStarts(ubidi_getSingleton(&status), &sa, &status); + break; + default: + status = U_INTERNAL_PROGRAM_ERROR; + break; + } + if (U_SUCCESS(status)) { + // Compact for caching + incl->compact(); + umtx_lock(NULL); + if (INCLUSIONS[src] == NULL) { + INCLUSIONS[src] = incl; + incl = NULL; + ucln_common_registerCleanup(UCLN_COMMON_USET, uset_cleanup); + } + umtx_unlock(NULL); + } + delete incl; + } else { + status = U_MEMORY_ALLOCATION_ERROR; + } + } + return INCLUSIONS[src]; +} + +// helper functions for matching of pattern syntax pieces ------------------ *** +// these functions are parallel to the PERL_OPEN etc. strings above + +// using these functions is not only faster than UnicodeString::compare() and +// caseCompare(), but they also make UnicodeSet work for simple patterns when +// no Unicode properties data is available - when caseCompare() fails + +static inline UBool +isPerlOpen(const UnicodeString &pattern, int32_t pos) { + UChar c; + return pattern.charAt(pos)==BACKSLASH && ((c=pattern.charAt(pos+1))==LOWER_P || c==UPPER_P); +} + +/*static inline UBool +isPerlClose(const UnicodeString &pattern, int32_t pos) { + return pattern.charAt(pos)==CLOSE_BRACE; +}*/ + +static inline UBool +isNameOpen(const UnicodeString &pattern, int32_t pos) { + return pattern.charAt(pos)==BACKSLASH && pattern.charAt(pos+1)==UPPER_N; +} + +static inline UBool +isPOSIXOpen(const UnicodeString &pattern, int32_t pos) { + return pattern.charAt(pos)==SET_OPEN && pattern.charAt(pos+1)==COLON; +} + +/*static inline UBool +isPOSIXClose(const UnicodeString &pattern, int32_t pos) { + return pattern.charAt(pos)==COLON && pattern.charAt(pos+1)==SET_CLOSE; +}*/ + +// TODO memory debugging provided inside uniset.cpp +// could be made available here but probably obsolete with use of modern +// memory leak checker tools +#define _dbgct(me) + +//---------------------------------------------------------------- +// Constructors &c +//---------------------------------------------------------------- + +/** + * Constructs a set from the given pattern, optionally ignoring + * white space. See the class description for the syntax of the + * pattern language. + * @param pattern a string specifying what characters are in the set + */ +UnicodeSet::UnicodeSet(const UnicodeString& pattern, + UErrorCode& status) : + len(0), capacity(START_EXTRA), list(0), bmpSet(0), buffer(0), + bufferCapacity(0), patLen(0), pat(NULL), strings(NULL), stringSpan(NULL) +{ + if(U_SUCCESS(status)){ + list = (UChar32*) uprv_malloc(sizeof(UChar32) * capacity); + /* test for NULL */ + if(list == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + }else{ + allocateStrings(status); + applyPattern(pattern, USET_IGNORE_SPACE, NULL, status); + } + } + _dbgct(this); +} + +/** + * Constructs a set from the given pattern, optionally ignoring + * white space. See the class description for the syntax of the + * pattern language. + * @param pattern a string specifying what characters are in the set + * @param options bitmask for options to apply to the pattern. + * Valid options are USET_IGNORE_SPACE and USET_CASE_INSENSITIVE. + */ +UnicodeSet::UnicodeSet(const UnicodeString& pattern, + uint32_t options, + const SymbolTable* symbols, + UErrorCode& status) : + len(0), capacity(START_EXTRA), list(0), bmpSet(0), buffer(0), + bufferCapacity(0), patLen(0), pat(NULL), strings(NULL), stringSpan(NULL) +{ + if(U_SUCCESS(status)){ + list = (UChar32*) uprv_malloc(sizeof(UChar32) * capacity); + /* test for NULL */ + if(list == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + }else{ + allocateStrings(status); + applyPattern(pattern, options, symbols, status); + } + } + _dbgct(this); +} + +UnicodeSet::UnicodeSet(const UnicodeString& pattern, ParsePosition& pos, + uint32_t options, + const SymbolTable* symbols, + UErrorCode& status) : + len(0), capacity(START_EXTRA), list(0), bmpSet(0), buffer(0), + bufferCapacity(0), patLen(0), pat(NULL), strings(NULL), stringSpan(NULL) +{ + if(U_SUCCESS(status)){ + list = (UChar32*) uprv_malloc(sizeof(UChar32) * capacity); + /* test for NULL */ + if(list == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + }else{ + allocateStrings(status); + applyPattern(pattern, pos, options, symbols, status); + } + } + _dbgct(this); +} + +//---------------------------------------------------------------- +// Public API +//---------------------------------------------------------------- + +/** + * Modifies this set to represent the set specified by the given + * pattern, optionally ignoring white space. See the class + * description for the syntax of the pattern language. + * @param pattern a string specifying what characters are in the set + * @param ignoreSpaces if true, all spaces in the + * pattern are ignored. Spaces are those characters for which + * uprv_isRuleWhiteSpace() is true. + * Characters preceded by '\\' are escaped, losing any special + * meaning they otherwise have. Spaces may be included by + * escaping them. + * @exception IllegalArgumentException if the pattern + * contains a syntax error. + */ +UnicodeSet& UnicodeSet::applyPattern(const UnicodeString& pattern, + UErrorCode& status) { + return applyPattern(pattern, USET_IGNORE_SPACE, NULL, status); +} + + +/** + * Modifies this set to represent the set specified by the given + * pattern, optionally ignoring white space. See the class + * description for the syntax of the pattern language. + * @param pattern a string specifying what characters are in the set + * @param options bitmask for options to apply to the pattern. + * Valid options are USET_IGNORE_SPACE and USET_CASE_INSENSITIVE. + */ +UnicodeSet& UnicodeSet::applyPattern(const UnicodeString& pattern, + uint32_t options, + const SymbolTable* symbols, + UErrorCode& status) { + if (U_FAILURE(status) || isFrozen()) { + return *this; + } + + ParsePosition pos(0); + applyPattern(pattern, pos, options, symbols, status); + if (U_FAILURE(status)) return *this; + + int32_t i = pos.getIndex(); + + if (options & USET_IGNORE_SPACE) { + // Skip over trailing whitespace + ICU_Utility::skipWhitespace(pattern, i, TRUE); + } + + if (i != pattern.length()) { + status = U_ILLEGAL_ARGUMENT_ERROR; + } + return *this; +} + +UnicodeSet& UnicodeSet::applyPattern(const UnicodeString& pattern, + ParsePosition& pos, + uint32_t options, + const SymbolTable* symbols, + UErrorCode& status) { + if (U_FAILURE(status) || isFrozen()) { + return *this; + } + // Need to build the pattern in a temporary string because + // _applyPattern calls add() etc., which set pat to empty. + UnicodeString rebuiltPat; + RuleCharacterIterator chars(pattern, symbols, pos); + applyPattern(chars, symbols, rebuiltPat, options, status); + if (U_FAILURE(status)) return *this; + if (chars.inVariable()) { + // syntaxError(chars, "Extra chars in variable value"); + status = U_MALFORMED_SET; + return *this; + } + setPattern(rebuiltPat); + return *this; +} + +/** + * Return true if the given position, in the given pattern, appears + * to be the start of a UnicodeSet pattern. + */ +UBool UnicodeSet::resemblesPattern(const UnicodeString& pattern, int32_t pos) { + return ((pos+1) < pattern.length() && + pattern.charAt(pos) == (UChar)91/*[*/) || + resemblesPropertyPattern(pattern, pos); +} + +//---------------------------------------------------------------- +// Implementation: Pattern parsing +//---------------------------------------------------------------- + +/** + * A small all-inline class to manage a UnicodeSet pointer. Add + * operator->() etc. as needed. + */ +class UnicodeSetPointer { + UnicodeSet* p; +public: + inline UnicodeSetPointer() : p(0) {} + inline ~UnicodeSetPointer() { delete p; } + inline UnicodeSet* pointer() { return p; } + inline UBool allocate() { + if (p == 0) { + p = new UnicodeSet(); + } + return p != 0; + } +}; + +/** + * Parse the pattern from the given RuleCharacterIterator. The + * iterator is advanced over the parsed pattern. + * @param chars iterator over the pattern characters. Upon return + * it will be advanced to the first character after the parsed + * pattern, or the end of the iteration if all characters are + * parsed. + * @param symbols symbol table to use to parse and dereference + * variables, or null if none. + * @param rebuiltPat the pattern that was parsed, rebuilt or + * copied from the input pattern, as appropriate. + * @param options a bit mask of zero or more of the following: + * IGNORE_SPACE, CASE. + */ +void UnicodeSet::applyPattern(RuleCharacterIterator& chars, + const SymbolTable* symbols, + UnicodeString& rebuiltPat, + uint32_t options, + UErrorCode& ec) { + if (U_FAILURE(ec)) return; + + // Syntax characters: [ ] ^ - & { } + + // Recognized special forms for chars, sets: c-c s-s s&s + + int32_t opts = RuleCharacterIterator::PARSE_VARIABLES | + RuleCharacterIterator::PARSE_ESCAPES; + if ((options & USET_IGNORE_SPACE) != 0) { + opts |= RuleCharacterIterator::SKIP_WHITESPACE; + } + + UnicodeString patLocal, buf; + UBool usePat = FALSE; + UnicodeSetPointer scratch; + RuleCharacterIterator::Pos backup; + + // mode: 0=before [, 1=between [...], 2=after ] + // lastItem: 0=none, 1=char, 2=set + int8_t lastItem = 0, mode = 0; + UChar32 lastChar = 0; + UChar op = 0; + + UBool invert = FALSE; + + clear(); + + while (mode != 2 && !chars.atEnd()) { + U_ASSERT((lastItem == 0 && op == 0) || + (lastItem == 1 && (op == 0 || op == HYPHEN /*'-'*/)) || + (lastItem == 2 && (op == 0 || op == HYPHEN /*'-'*/ || + op == INTERSECTION /*'&'*/))); + + UChar32 c = 0; + UBool literal = FALSE; + UnicodeSet* nested = 0; // alias - do not delete + + // -------- Check for property pattern + + // setMode: 0=none, 1=unicodeset, 2=propertypat, 3=preparsed + int8_t setMode = 0; + if (resemblesPropertyPattern(chars, opts)) { + setMode = 2; + } + + // -------- Parse '[' of opening delimiter OR nested set. + // If there is a nested set, use `setMode' to define how + // the set should be parsed. If the '[' is part of the + // opening delimiter for this pattern, parse special + // strings "[", "[^", "[-", and "[^-". Check for stand-in + // characters representing a nested set in the symbol + // table. + + else { + // Prepare to backup if necessary + chars.getPos(backup); + c = chars.next(opts, literal, ec); + if (U_FAILURE(ec)) return; + + if (c == 0x5B /*'['*/ && !literal) { + if (mode == 1) { + chars.setPos(backup); // backup + setMode = 1; + } else { + // Handle opening '[' delimiter + mode = 1; + patLocal.append((UChar) 0x5B /*'['*/); + chars.getPos(backup); // prepare to backup + c = chars.next(opts, literal, ec); + if (U_FAILURE(ec)) return; + if (c == 0x5E /*'^'*/ && !literal) { + invert = TRUE; + patLocal.append((UChar) 0x5E /*'^'*/); + chars.getPos(backup); // prepare to backup + c = chars.next(opts, literal, ec); + if (U_FAILURE(ec)) return; + } + // Fall through to handle special leading '-'; + // otherwise restart loop for nested [], \p{}, etc. + if (c == HYPHEN /*'-'*/) { + literal = TRUE; + // Fall through to handle literal '-' below + } else { + chars.setPos(backup); // backup + continue; + } + } + } else if (symbols != 0) { + const UnicodeFunctor *m = symbols->lookupMatcher(c); + if (m != 0) { + if (m->getDynamicClassID() != UnicodeSet::getStaticClassID()) { + ec = U_MALFORMED_SET; + return; + } + // casting away const, but `nested' won't be modified + // (important not to modify stored set) + nested = (UnicodeSet*) m; + setMode = 3; + } + } + } + + // -------- Handle a nested set. This either is inline in + // the pattern or represented by a stand-in that has + // previously been parsed and was looked up in the symbol + // table. + + if (setMode != 0) { + if (lastItem == 1) { + if (op != 0) { + // syntaxError(chars, "Char expected after operator"); + ec = U_MALFORMED_SET; + return; + } + add(lastChar, lastChar); + _appendToPat(patLocal, lastChar, FALSE); + lastItem = 0; + op = 0; + } + + if (op == HYPHEN /*'-'*/ || op == INTERSECTION /*'&'*/) { + patLocal.append(op); + } + + if (nested == 0) { + // lazy allocation + if (!scratch.allocate()) { + ec = U_MEMORY_ALLOCATION_ERROR; + return; + } + nested = scratch.pointer(); + } + switch (setMode) { + case 1: + nested->applyPattern(chars, symbols, patLocal, options, ec); + break; + case 2: + chars.skipIgnored(opts); + nested->applyPropertyPattern(chars, patLocal, ec); + if (U_FAILURE(ec)) return; + break; + case 3: // `nested' already parsed + nested->_toPattern(patLocal, FALSE); + break; + } + + usePat = TRUE; + + if (mode == 0) { + // Entire pattern is a category; leave parse loop + *this = *nested; + mode = 2; + break; + } + + switch (op) { + case HYPHEN: /*'-'*/ + removeAll(*nested); + break; + case INTERSECTION: /*'&'*/ + retainAll(*nested); + break; + case 0: + addAll(*nested); + break; + } + + op = 0; + lastItem = 2; + + continue; + } + + if (mode == 0) { + // syntaxError(chars, "Missing '['"); + ec = U_MALFORMED_SET; + return; + } + + // -------- Parse special (syntax) characters. If the + // current character is not special, or if it is escaped, + // then fall through and handle it below. + + if (!literal) { + switch (c) { + case 0x5D /*']'*/: + if (lastItem == 1) { + add(lastChar, lastChar); + _appendToPat(patLocal, lastChar, FALSE); + } + // Treat final trailing '-' as a literal + if (op == HYPHEN /*'-'*/) { + add(op, op); + patLocal.append(op); + } else if (op == INTERSECTION /*'&'*/) { + // syntaxError(chars, "Trailing '&'"); + ec = U_MALFORMED_SET; + return; + } + patLocal.append((UChar) 0x5D /*']'*/); + mode = 2; + continue; + case HYPHEN /*'-'*/: + if (op == 0) { + if (lastItem != 0) { + op = (UChar) c; + continue; + } else { + // Treat final trailing '-' as a literal + add(c, c); + c = chars.next(opts, literal, ec); + if (U_FAILURE(ec)) return; + if (c == 0x5D /*']'*/ && !literal) { + patLocal.append(HYPHEN_RIGHT_BRACE); + mode = 2; + continue; + } + } + } + // syntaxError(chars, "'-' not after char or set"); + ec = U_MALFORMED_SET; + return; + case INTERSECTION /*'&'*/: + if (lastItem == 2 && op == 0) { + op = (UChar) c; + continue; + } + // syntaxError(chars, "'&' not after set"); + ec = U_MALFORMED_SET; + return; + case 0x5E /*'^'*/: + // syntaxError(chars, "'^' not after '['"); + ec = U_MALFORMED_SET; + return; + case 0x7B /*'{'*/: + if (op != 0) { + // syntaxError(chars, "Missing operand after operator"); + ec = U_MALFORMED_SET; + return; + } + if (lastItem == 1) { + add(lastChar, lastChar); + _appendToPat(patLocal, lastChar, FALSE); + } + lastItem = 0; + buf.truncate(0); + { + UBool ok = FALSE; + while (!chars.atEnd()) { + c = chars.next(opts, literal, ec); + if (U_FAILURE(ec)) return; + if (c == 0x7D /*'}'*/ && !literal) { + ok = TRUE; + break; + } + buf.append(c); + } + if (buf.length() < 1 || !ok) { + // syntaxError(chars, "Invalid multicharacter string"); + ec = U_MALFORMED_SET; + return; + } + } + // We have new string. Add it to set and continue; + // we don't need to drop through to the further + // processing + add(buf); + patLocal.append((UChar) 0x7B /*'{'*/); + _appendToPat(patLocal, buf, FALSE); + patLocal.append((UChar) 0x7D /*'}'*/); + continue; + case SymbolTable::SYMBOL_REF: + // symbols nosymbols + // [a-$] error error (ambiguous) + // [a$] anchor anchor + // [a-$x] var "x"* literal '$' + // [a-$.] error literal '$' + // *We won't get here in the case of var "x" + { + chars.getPos(backup); + c = chars.next(opts, literal, ec); + if (U_FAILURE(ec)) return; + UBool anchor = (c == 0x5D /*']'*/ && !literal); + if (symbols == 0 && !anchor) { + c = SymbolTable::SYMBOL_REF; + chars.setPos(backup); + break; // literal '$' + } + if (anchor && op == 0) { + if (lastItem == 1) { + add(lastChar, lastChar); + _appendToPat(patLocal, lastChar, FALSE); + } + add(U_ETHER); + usePat = TRUE; + patLocal.append((UChar) SymbolTable::SYMBOL_REF); + patLocal.append((UChar) 0x5D /*']'*/); + mode = 2; + continue; + } + // syntaxError(chars, "Unquoted '$'"); + ec = U_MALFORMED_SET; + return; + } + default: + break; + } + } + + // -------- Parse literal characters. This includes both + // escaped chars ("\u4E01") and non-syntax characters + // ("a"). + + switch (lastItem) { + case 0: + lastItem = 1; + lastChar = c; + break; + case 1: + if (op == HYPHEN /*'-'*/) { + if (lastChar >= c) { + // Don't allow redundant (a-a) or empty (b-a) ranges; + // these are most likely typos. + // syntaxError(chars, "Invalid range"); + ec = U_MALFORMED_SET; + return; + } + add(lastChar, c); + _appendToPat(patLocal, lastChar, FALSE); + patLocal.append(op); + _appendToPat(patLocal, c, FALSE); + lastItem = 0; + op = 0; + } else { + add(lastChar, lastChar); + _appendToPat(patLocal, lastChar, FALSE); + lastChar = c; + } + break; + case 2: + if (op != 0) { + // syntaxError(chars, "Set expected after operator"); + ec = U_MALFORMED_SET; + return; + } + lastChar = c; + lastItem = 1; + break; + } + } + + if (mode != 2) { + // syntaxError(chars, "Missing ']'"); + ec = U_MALFORMED_SET; + return; + } + + chars.skipIgnored(opts); + + /** + * Handle global flags (invert, case insensitivity). If this + * pattern should be compiled case-insensitive, then we need + * to close over case BEFORE COMPLEMENTING. This makes + * patterns like /[^abc]/i work. + */ + if ((options & USET_CASE_INSENSITIVE) != 0) { + closeOver(USET_CASE_INSENSITIVE); + } + else if ((options & USET_ADD_CASE_MAPPINGS) != 0) { + closeOver(USET_ADD_CASE_MAPPINGS); + } + if (invert) { + complement(); + } + + // Use the rebuilt pattern (patLocal) only if necessary. Prefer the + // generated pattern. + if (usePat) { + rebuiltPat.append(patLocal); + } else { + _generatePattern(rebuiltPat, FALSE); + } +} + +//---------------------------------------------------------------- +// Property set implementation +//---------------------------------------------------------------- + +static UBool numericValueFilter(UChar32 ch, void* context) { + return u_getNumericValue(ch) == *(double*)context; +} + +static UBool generalCategoryMaskFilter(UChar32 ch, void* context) { + int32_t value = *(int32_t*)context; + return (U_GET_GC_MASK((UChar32) ch) & value) != 0; +} + +static UBool versionFilter(UChar32 ch, void* context) { + UVersionInfo v, none = { 0, 0, 0, 0}; + UVersionInfo* version = (UVersionInfo*)context; + u_charAge(ch, v); + return uprv_memcmp(&v, &none, sizeof(v)) > 0 && uprv_memcmp(&v, version, sizeof(v)) <= 0; +} + +typedef struct { + UProperty prop; + int32_t value; +} IntPropertyContext; + +static UBool intPropertyFilter(UChar32 ch, void* context) { + IntPropertyContext* c = (IntPropertyContext*)context; + return u_getIntPropertyValue((UChar32) ch, c->prop) == c->value; +} + + +/** + * Generic filter-based scanning code for UCD property UnicodeSets. + */ +void UnicodeSet::applyFilter(UnicodeSet::Filter filter, + void* context, + int32_t src, + UErrorCode &status) { + // Walk through all Unicode characters, noting the start + // and end of each range for which filter.contain(c) is + // true. Add each range to a set. + // + // To improve performance, use the INCLUSIONS set, which + // encodes information about character ranges that are known + // to have identical properties. INCLUSIONS contains + // only the first characters of such ranges. + // + // TODO Where possible, instead of scanning over code points, + // use internal property data to initialize UnicodeSets for + // those properties. Scanning code points is slow. + if (U_FAILURE(status)) return; + + const UnicodeSet* inclusions = getInclusions(src, status); + if (U_FAILURE(status)) { + return; + } + + clear(); + + UChar32 startHasProperty = -1; + int limitRange = inclusions->getRangeCount(); + + for (int j=0; jgetRangeStart(j); + UChar32 end = inclusions->getRangeEnd(j); + + // for all the code points in the range, process + for (UChar32 ch = start; ch <= end; ++ch) { + // only add to this UnicodeSet on inflection points -- + // where the hasProperty value changes to false + if ((*filter)(ch, context)) { + if (startHasProperty < 0) { + startHasProperty = ch; + } + } else if (startHasProperty >= 0) { + add(startHasProperty, ch-1); + startHasProperty = -1; + } + } + } + if (startHasProperty >= 0) { + add((UChar32)startHasProperty, (UChar32)0x10FFFF); + } +} + +static UBool mungeCharName(char* dst, const char* src, int32_t dstCapacity) { + /* Note: we use ' ' in compiler code page */ + int32_t j = 0; + char ch; + --dstCapacity; /* make room for term. zero */ + while ((ch = *src++) != 0) { + if (ch == ' ' && (j==0 || (j>0 && dst[j-1]==' '))) { + continue; + } + if (j >= dstCapacity) return FALSE; + dst[j++] = ch; + } + if (j > 0 && dst[j-1] == ' ') --j; + dst[j] = 0; + return TRUE; +} + +//---------------------------------------------------------------- +// Property set API +//---------------------------------------------------------------- + +#define FAIL(ec) {ec=U_ILLEGAL_ARGUMENT_ERROR; return *this;} + +UnicodeSet& +UnicodeSet::applyIntPropertyValue(UProperty prop, int32_t value, UErrorCode& ec) { + if (U_FAILURE(ec) || isFrozen()) return *this; + + if (prop == UCHAR_GENERAL_CATEGORY_MASK) { + applyFilter(generalCategoryMaskFilter, &value, UPROPS_SRC_CHAR, ec); + } else { + IntPropertyContext c = {prop, value}; + applyFilter(intPropertyFilter, &c, uprops_getSource(prop), ec); + } + return *this; +} + +UnicodeSet& +UnicodeSet::applyPropertyAlias(const UnicodeString& prop, + const UnicodeString& value, + UErrorCode& ec) { + if (U_FAILURE(ec) || isFrozen()) return *this; + + // prop and value used to be converted to char * using the default + // converter instead of the invariant conversion. + // This should not be necessary because all Unicode property and value + // names use only invariant characters. + // If there are any variant characters, then we won't find them anyway. + // Checking first avoids assertion failures in the conversion. + if( !uprv_isInvariantUString(prop.getBuffer(), prop.length()) || + !uprv_isInvariantUString(value.getBuffer(), value.length()) + ) { + FAIL(ec); + } + CharString pname(prop); + CharString vname(value); + + UProperty p; + int32_t v; + UBool mustNotBeEmpty = FALSE, invert = FALSE; + + if (value.length() > 0) { + p = u_getPropertyEnum(pname); + if (p == UCHAR_INVALID_CODE) FAIL(ec); + + // Treat gc as gcm + if (p == UCHAR_GENERAL_CATEGORY) { + p = UCHAR_GENERAL_CATEGORY_MASK; + } + + if ((p >= UCHAR_BINARY_START && p < UCHAR_BINARY_LIMIT) || + (p >= UCHAR_INT_START && p < UCHAR_INT_LIMIT) || + (p >= UCHAR_MASK_START && p < UCHAR_MASK_LIMIT)) { + v = u_getPropertyValueEnum(p, vname); + if (v == UCHAR_INVALID_CODE) { + // Handle numeric CCC + if (p == UCHAR_CANONICAL_COMBINING_CLASS || + p == UCHAR_TRAIL_CANONICAL_COMBINING_CLASS || + p == UCHAR_LEAD_CANONICAL_COMBINING_CLASS) { + char* end; + double value = uprv_strtod(vname, &end); + v = (int32_t) value; + if (v != value || v < 0 || *end != 0) { + // non-integral or negative value, or trailing junk + FAIL(ec); + } + // If the resultant set is empty then the numeric value + // was invalid. + mustNotBeEmpty = TRUE; + } else { + FAIL(ec); + } + } + } + + else { + + switch (p) { + case UCHAR_NUMERIC_VALUE: + { + char* end; + double value = uprv_strtod(vname, &end); + if (*end != 0) { + FAIL(ec); + } + applyFilter(numericValueFilter, &value, UPROPS_SRC_CHAR, ec); + return *this; + } + break; + case UCHAR_NAME: + case UCHAR_UNICODE_1_NAME: + { + // Must munge name, since u_charFromName() does not do + // 'loose' matching. + char buf[128]; // it suffices that this be > uprv_getMaxCharNameLength + if (!mungeCharName(buf, vname, sizeof(buf))) FAIL(ec); + UCharNameChoice choice = (p == UCHAR_NAME) ? + U_EXTENDED_CHAR_NAME : U_UNICODE_10_CHAR_NAME; + UChar32 ch = u_charFromName(choice, buf, &ec); + if (U_SUCCESS(ec)) { + clear(); + add(ch); + return *this; + } else { + FAIL(ec); + } + } + break; + case UCHAR_AGE: + { + // Must munge name, since u_versionFromString() does not do + // 'loose' matching. + char buf[128]; + if (!mungeCharName(buf, vname, sizeof(buf))) FAIL(ec); + UVersionInfo version; + u_versionFromString(version, buf); + applyFilter(versionFilter, &version, UPROPS_SRC_PROPSVEC, ec); + return *this; + } + break; + default: + // p is a non-binary, non-enumerated property that we + // don't support (yet). + FAIL(ec); + } + } + } + + else { + // value is empty. Interpret as General Category, Script, or + // Binary property. + p = UCHAR_GENERAL_CATEGORY_MASK; + v = u_getPropertyValueEnum(p, pname); + if (v == UCHAR_INVALID_CODE) { + p = UCHAR_SCRIPT; + v = u_getPropertyValueEnum(p, pname); + if (v == UCHAR_INVALID_CODE) { + p = u_getPropertyEnum(pname); + if (p >= UCHAR_BINARY_START && p < UCHAR_BINARY_LIMIT) { + v = 1; + } else if (0 == uprv_comparePropertyNames(ANY, pname)) { + set(MIN_VALUE, MAX_VALUE); + return *this; + } else if (0 == uprv_comparePropertyNames(ASCII, pname)) { + set(0, 0x7F); + return *this; + } else if (0 == uprv_comparePropertyNames(ASSIGNED, pname)) { + // [:Assigned:]=[:^Cn:] + p = UCHAR_GENERAL_CATEGORY_MASK; + v = U_GC_CN_MASK; + invert = TRUE; + } else { + FAIL(ec); + } + } + } + } + + applyIntPropertyValue(p, v, ec); + if(invert) { + complement(); + } + + if (U_SUCCESS(ec) && (mustNotBeEmpty && isEmpty())) { + // mustNotBeEmpty is set to true if an empty set indicates + // invalid input. + ec = U_ILLEGAL_ARGUMENT_ERROR; + } + + return *this; +} + +//---------------------------------------------------------------- +// Property set patterns +//---------------------------------------------------------------- + +/** + * Return true if the given position, in the given pattern, appears + * to be the start of a property set pattern. + */ +UBool UnicodeSet::resemblesPropertyPattern(const UnicodeString& pattern, + int32_t pos) { + // Patterns are at least 5 characters long + if ((pos+5) > pattern.length()) { + return FALSE; + } + + // Look for an opening [:, [:^, \p, or \P + return isPOSIXOpen(pattern, pos) || isPerlOpen(pattern, pos) || isNameOpen(pattern, pos); +} + +/** + * Return true if the given iterator appears to point at a + * property pattern. Regardless of the result, return with the + * iterator unchanged. + * @param chars iterator over the pattern characters. Upon return + * it will be unchanged. + * @param iterOpts RuleCharacterIterator options + */ +UBool UnicodeSet::resemblesPropertyPattern(RuleCharacterIterator& chars, + int32_t iterOpts) { + // NOTE: literal will always be FALSE, because we don't parse escapes. + UBool result = FALSE, literal; + UErrorCode ec = U_ZERO_ERROR; + iterOpts &= ~RuleCharacterIterator::PARSE_ESCAPES; + RuleCharacterIterator::Pos pos; + chars.getPos(pos); + UChar32 c = chars.next(iterOpts, literal, ec); + if (c == 0x5B /*'['*/ || c == 0x5C /*'\\'*/) { + UChar32 d = chars.next(iterOpts & ~RuleCharacterIterator::SKIP_WHITESPACE, + literal, ec); + result = (c == 0x5B /*'['*/) ? (d == 0x3A /*':'*/) : + (d == 0x4E /*'N'*/ || d == 0x70 /*'p'*/ || d == 0x50 /*'P'*/); + } + chars.setPos(pos); + return result && U_SUCCESS(ec); +} + +/** + * Parse the given property pattern at the given parse position. + */ +UnicodeSet& UnicodeSet::applyPropertyPattern(const UnicodeString& pattern, + ParsePosition& ppos, + UErrorCode &ec) { + int32_t pos = ppos.getIndex(); + + UBool posix = FALSE; // true for [:pat:], false for \p{pat} \P{pat} \N{pat} + UBool isName = FALSE; // true for \N{pat}, o/w false + UBool invert = FALSE; + + if (U_FAILURE(ec)) return *this; + + // Minimum length is 5 characters, e.g. \p{L} + if ((pos+5) > pattern.length()) { + FAIL(ec); + } + + // On entry, ppos should point to one of the following locations: + // Look for an opening [:, [:^, \p, or \P + if (isPOSIXOpen(pattern, pos)) { + posix = TRUE; + pos += 2; + pos = ICU_Utility::skipWhitespace(pattern, pos); + if (pos < pattern.length() && pattern.charAt(pos) == COMPLEMENT) { + ++pos; + invert = TRUE; + } + } else if (isPerlOpen(pattern, pos) || isNameOpen(pattern, pos)) { + UChar c = pattern.charAt(pos+1); + invert = (c == UPPER_P); + isName = (c == UPPER_N); + pos += 2; + pos = ICU_Utility::skipWhitespace(pattern, pos); + if (pos == pattern.length() || pattern.charAt(pos++) != OPEN_BRACE) { + // Syntax error; "\p" or "\P" not followed by "{" + FAIL(ec); + } + } else { + // Open delimiter not seen + FAIL(ec); + } + + // Look for the matching close delimiter, either :] or } + int32_t close = pattern.indexOf(posix ? POSIX_CLOSE : PERL_CLOSE, pos); + if (close < 0) { + // Syntax error; close delimiter missing + FAIL(ec); + } + + // Look for an '=' sign. If this is present, we will parse a + // medium \p{gc=Cf} or long \p{GeneralCategory=Format} + // pattern. + int32_t equals = pattern.indexOf(EQUALS, pos); + UnicodeString propName, valueName; + if (equals >= 0 && equals < close && !isName) { + // Equals seen; parse medium/long pattern + pattern.extractBetween(pos, equals, propName); + pattern.extractBetween(equals+1, close, valueName); + } + + else { + // Handle case where no '=' is seen, and \N{} + pattern.extractBetween(pos, close, propName); + + // Handle \N{name} + if (isName) { + // This is a little inefficient since it means we have to + // parse NAME_PROP back to UCHAR_NAME even though we already + // know it's UCHAR_NAME. If we refactor the API to + // support args of (UProperty, char*) then we can remove + // NAME_PROP and make this a little more efficient. + valueName = propName; + propName = UnicodeString(NAME_PROP, NAME_PROP_LENGTH, US_INV); + } + } + + applyPropertyAlias(propName, valueName, ec); + + if (U_SUCCESS(ec)) { + if (invert) { + complement(); + } + + // Move to the limit position after the close delimiter if the + // parse succeeded. + ppos.setIndex(close + (posix ? 2 : 1)); + } + + return *this; +} + +/** + * Parse a property pattern. + * @param chars iterator over the pattern characters. Upon return + * it will be advanced to the first character after the parsed + * pattern, or the end of the iteration if all characters are + * parsed. + * @param rebuiltPat the pattern that was parsed, rebuilt or + * copied from the input pattern, as appropriate. + */ +void UnicodeSet::applyPropertyPattern(RuleCharacterIterator& chars, + UnicodeString& rebuiltPat, + UErrorCode& ec) { + if (U_FAILURE(ec)) return; + UnicodeString pattern; + chars.lookahead(pattern); + ParsePosition pos(0); + applyPropertyPattern(pattern, pos, ec); + if (U_FAILURE(ec)) return; + if (pos.getIndex() == 0) { + // syntaxError(chars, "Invalid property pattern"); + ec = U_MALFORMED_SET; + return; + } + chars.jumpahead(pos.getIndex()); + rebuiltPat.append(pattern, 0, pos.getIndex()); +} + +//---------------------------------------------------------------- +// Case folding API +//---------------------------------------------------------------- + +// add the result of a full case mapping to the set +// use str as a temporary string to avoid constructing one +static inline void +addCaseMapping(UnicodeSet &set, int32_t result, const UChar *full, UnicodeString &str) { + if(result >= 0) { + if(result > UCASE_MAX_STRING_LENGTH) { + // add a single-code point case mapping + set.add(result); + } else { + // add a string case mapping from full with length result + str.setTo((UBool)FALSE, full, result); + set.add(str); + } + } + // result < 0: the code point mapped to itself, no need to add it + // see ucase.h +} + +UnicodeSet& UnicodeSet::closeOver(int32_t attribute) { + if (isFrozen()) { + return *this; + } + if (attribute & (USET_CASE_INSENSITIVE | USET_ADD_CASE_MAPPINGS)) { + UErrorCode status = U_ZERO_ERROR; + const UCaseProps *csp = ucase_getSingleton(&status); + if (U_SUCCESS(status)) { + UnicodeSet foldSet(*this); + UnicodeString str; + USetAdder sa = { + (USet *)&foldSet, + _set_add, + _set_addRange, + _set_addString, + NULL // don't need remove() + }; + + // start with input set to guarantee inclusion + // USET_CASE: remove strings because the strings will actually be reduced (folded); + // therefore, start with no strings and add only those needed + if (attribute & USET_CASE_INSENSITIVE) { + foldSet.strings->removeAllElements(); + } + + int32_t n = getRangeCount(); + UChar32 result; + const UChar *full; + int32_t locCache = 0; + + for (int32_t i=0; isize() > 0) { + if (attribute & USET_CASE_INSENSITIVE) { + for (int32_t j=0; jsize(); ++j) { + str = *(const UnicodeString *) strings->elementAt(j); + str.foldCase(); + if(!ucase_addStringCaseClosure(csp, str.getBuffer(), str.length(), &sa)) { + foldSet.add(str); // does not map to code points: add the folded string itself + } + } + } else { + Locale root(""); +#if !UCONFIG_NO_BREAK_ITERATION + BreakIterator *bi = BreakIterator::createWordInstance(root, status); +#endif + if (U_SUCCESS(status)) { + const UnicodeString *pStr; + + for (int32_t j=0; jsize(); ++j) { + pStr = (const UnicodeString *) strings->elementAt(j); + (str = *pStr).toLower(root); + foldSet.add(str); +#if !UCONFIG_NO_BREAK_ITERATION + (str = *pStr).toTitle(bi, root); + foldSet.add(str); +#endif + (str = *pStr).toUpper(root); + foldSet.add(str); + (str = *pStr).foldCase(); + foldSet.add(str); + } + } +#if !UCONFIG_NO_BREAK_ITERATION + delete bi; +#endif + } + } + *this = foldSet; + } + } + return *this; +} + +U_NAMESPACE_END diff --git a/Build/source/libs/icu/icu-xetex/common/unisetspan.cpp b/Build/source/libs/icu/icu-xetex/common/unisetspan.cpp new file mode 100644 index 00000000000..0e43dfc3f75 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unisetspan.cpp @@ -0,0 +1,1508 @@ +/* +****************************************************************************** +* +* Copyright (C) 2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: unisetspan.cpp +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2007mar01 +* created by: Markus W. Scherer +*/ + +#include "unicode/utypes.h" +#include "unicode/uniset.h" +#include "unicode/ustring.h" +#include "cmemory.h" +#include "uvector.h" +#include "unisetspan.h" + +U_NAMESPACE_BEGIN + +/* + * List of offsets from the current position from where to try matching + * a code point or a string. + * Store offsets rather than indexes to simplify the code and use the same list + * for both increments (in span()) and decrements (in spanBack()). + * + * Assumption: The maximum offset is limited, and the offsets that are stored + * at any one time are relatively dense, that is, there are normally no gaps of + * hundreds or thousands of offset values. + * + * The implementation uses a circular buffer of byte flags, + * each indicating whether the corresponding offset is in the list. + * This avoids inserting into a sorted list of offsets (or absolute indexes) and + * physically moving part of the list. + * + * Note: In principle, the caller should setMaxLength() to the maximum of the + * max string length and U16_LENGTH/U8_LENGTH to account for + * "long" single code points. + * However, this implementation uses at least a staticList with more than + * U8_LENGTH entries anyway. + * + * Note: If maxLength were guaranteed to be no more than 32 or 64, + * the list could be stored as bit flags in a single integer. + * Rather than handling a circular buffer with a start list index, + * the integer would simply be shifted when lower offsets are removed. + * UnicodeSet does not have a limit on the lengths of strings. + */ +class OffsetList { // Only ever stack-allocated, does not need to inherit UMemory. +public: + OffsetList() : list(staticList), capacity(0), length(0), start(0) {} + + ~OffsetList() { + if(list!=staticList) { + uprv_free(list); + } + } + + // Call exactly once if the list is to be used. + void setMaxLength(int32_t maxLength) { + if(maxLength<=(int32_t)sizeof(staticList)) { + capacity=(int32_t)sizeof(staticList); + } else { + UBool *l=(UBool *)uprv_malloc(maxLength); + if(l!=NULL) { + list=l; + capacity=maxLength; + } + } + uprv_memset(list, 0, capacity); + } + + void clear() { + uprv_memset(list, 0, capacity); + start=length=0; + } + + UBool isEmpty() const { + return (UBool)(length==0); + } + + // Reduce all stored offsets by delta, used when the current position + // moves by delta. + // There must not be any offsets lower than delta. + // If there is an offset equal to delta, it is removed. + // delta=[1..maxLength] + void shift(int32_t delta) { + int32_t i=start+delta; + if(i>=capacity) { + i-=capacity; + } + if(list[i]) { + list[i]=FALSE; + --length; + } + start=i; + } + + // Add an offset. The list must not contain it yet. + // offset=[1..maxLength] + void addOffset(int32_t offset) { + int32_t i=start+offset; + if(i>=capacity) { + i-=capacity; + } + list[i]=TRUE; + ++length; + } + + // offset=[1..maxLength] + UBool containsOffset(int32_t offset) const { + int32_t i=start+offset; + if(i>=capacity) { + i-=capacity; + } + return list[i]; + } + + // Find the lowest stored offset from a non-empty list, remove it, + // and reduce all other offsets by this minimum. + // Returns [1..maxLength]. + int32_t popMinimum() { + // Look for the next offset in list[start+1..capacity-1]. + int32_t i=start, result; + while(++imaxLength16) { + maxLength16=length16; + } + if((which&UTF8) && (thisRelevant || (which&CONTAINED))) { + int32_t length8=getUTF8Length(s16, length16); + utf8Length+=length8; + if(length8>maxLength8) { + maxLength8=length8; + } + } + } + if(!someRelevant) { + maxLength16=maxLength8=0; + return; + } + + // Freeze after checking for the need to use strings at all because freezing + // a set takes some time and memory which are wasted if there are no relevant strings. + if(all) { + spanSet.freeze(); + } + + uint8_t *spanBackLengths; + uint8_t *spanUTF8Lengths; + uint8_t *spanBackUTF8Lengths; + + // Allocate a block of meta data. + int32_t allocSize; + if(all) { + // UTF-8 lengths, 4 sets of span lengths, UTF-8 strings. + allocSize=stringsLength*(4+1+1+1+1)+utf8Length; + } else { + allocSize=stringsLength; // One set of span lengths. + if(which&UTF8) { + // UTF-8 lengths and UTF-8 strings. + allocSize+=stringsLength*4+utf8Length; + } + } + if(allocSize<=(int32_t)sizeof(staticLengths)) { + utf8Lengths=staticLengths; + } else { + utf8Lengths=(int32_t *)uprv_malloc(allocSize); + if(utf8Lengths==NULL) { + maxLength16=maxLength8=0; // Prevent usage by making needsStringSpanUTF16/8() return FALSE. + return; // Out of memory. + } + } + + if(all) { + // Store span lengths for all span() variants. + spanLengths=(uint8_t *)(utf8Lengths+stringsLength); + spanBackLengths=spanLengths+stringsLength; + spanUTF8Lengths=spanBackLengths+stringsLength; + spanBackUTF8Lengths=spanUTF8Lengths+stringsLength; + utf8=spanBackUTF8Lengths+stringsLength; + } else { + // Store span lengths for only one span() variant. + if(which&UTF8) { + spanLengths=(uint8_t *)(utf8Lengths+stringsLength); + utf8=spanLengths+stringsLength; + } else { + spanLengths=(uint8_t *)utf8Lengths; + } + spanBackLengths=spanUTF8Lengths=spanBackUTF8Lengths=spanLengths; + } + + // Set the meta data and pSpanNotSet and write the UTF-8 strings. + int32_t utf8Count=0; // Count UTF-8 bytes written so far. + + for(i=0; ifreeze(); + } +} + +// Copy constructor. Assumes which==ALL for a frozen set. +UnicodeSetStringSpan::UnicodeSetStringSpan(const UnicodeSetStringSpan &otherStringSpan, + const UVector &newParentSetStrings) + : spanSet(otherStringSpan.spanSet), pSpanNotSet(NULL), strings(newParentSetStrings), + utf8Lengths(NULL), spanLengths(NULL), utf8(NULL), + utf8Length(otherStringSpan.utf8Length), + maxLength16(otherStringSpan.maxLength16), maxLength8(otherStringSpan.maxLength8), + all(TRUE) { + if(otherStringSpan.pSpanNotSet==&otherStringSpan.spanSet) { + pSpanNotSet=&spanSet; + } else { + pSpanNotSet=(UnicodeSet *)otherStringSpan.pSpanNotSet->clone(); + } + + // Allocate a block of meta data. + // UTF-8 lengths, 4 sets of span lengths, UTF-8 strings. + int32_t stringsLength=strings.size(); + int32_t allocSize=stringsLength*(4+1+1+1+1)+utf8Length; + if(allocSize<=(int32_t)sizeof(staticLengths)) { + utf8Lengths=staticLengths; + } else { + utf8Lengths=(int32_t *)uprv_malloc(allocSize); + if(utf8Lengths==NULL) { + maxLength16=maxLength8=0; // Prevent usage by making needsStringSpanUTF16/8() return FALSE. + return; // Out of memory. + } + } + + spanLengths=(uint8_t *)(utf8Lengths+stringsLength); + utf8=spanLengths+stringsLength*4; + uprv_memcpy(utf8Lengths, otherStringSpan.utf8Lengths, allocSize); +} + +UnicodeSetStringSpan::~UnicodeSetStringSpan() { + if(pSpanNotSet!=NULL && pSpanNotSet!=&spanSet) { + delete pSpanNotSet; + } + if(utf8Lengths!=NULL && utf8Lengths!=staticLengths) { + uprv_free(utf8Lengths); + } +} + +void UnicodeSetStringSpan::addToSpanNotSet(UChar32 c) { + if(pSpanNotSet==NULL || pSpanNotSet==&spanSet) { + if(spanSet.contains(c)) { + return; // Nothing to do. + } + UnicodeSet *newSet=(UnicodeSet *)spanSet.cloneAsThawed(); + if(newSet==NULL) { + return; // Out of memory. + } else { + pSpanNotSet=newSet; + } + } + pSpanNotSet->add(c); +} + +// Compare strings without any argument checks. Requires length>0. +static inline UBool +matches16(const UChar *s, const UChar *t, int32_t length) { + do { + if(*s++!=*t++) { + return FALSE; + } + } while(--length>0); + return TRUE; +} + +static inline UBool +matches8(const uint8_t *s, const uint8_t *t, int32_t length) { + do { + if(*s++!=*t++) { + return FALSE; + } + } while(--length>0); + return TRUE; +} + +// Compare 16-bit Unicode strings (which may be malformed UTF-16) +// at code point boundaries. +// That is, each edge of a match must not be in the middle of a surrogate pair. +static inline UBool +matches16CPB(const UChar *s, int32_t start, int32_t limit, const UChar *t, int32_t length) { + s+=start; + limit-=start; + return matches16(s, t, length) && + !(0=0xd800 && c<=0xdbff && length>=2 && U16_IS_TRAIL(c2=s[1])) { + return set.contains(U16_GET_SUPPLEMENTARY(c, c2)) ? 2 : -2; + } + return set.contains(c) ? 1 : -1; +} + +static inline int32_t +spanOneBack(const UnicodeSet &set, const UChar *s, int32_t length) { + UChar c=s[length-1], c2; + if(c>=0xdc00 && c<=0xdfff && length>=2 && U16_IS_LEAD(c2=s[length-2])) { + return set.contains(U16_GET_SUPPLEMENTARY(c2, c)) ? 2 : -2; + } + return set.contains(c) ? 1 : -1; +} + +static inline int32_t +spanOneUTF8(const UnicodeSet &set, const uint8_t *s, int32_t length) { + UChar32 c=*s; + if((int8_t)c>=0) { + return set.contains(c) ? 1 : -1; + } + // Take advantage of non-ASCII fastpaths in U8_NEXT(). + int32_t i=0; + U8_NEXT(s, i, length, c); + return set.contains(c) ? i : -i; +} + +static inline int32_t +spanOneBackUTF8(const UnicodeSet &set, const uint8_t *s, int32_t length) { + UChar32 c=s[length-1]; + if((int8_t)c>=0) { + return set.contains(c) ? 1 : -1; + } + int32_t i=length-1; + c=utf8_prevCharSafeBody(s, 0, &i, c, -1); + length-=i; + return set.contains(c) ? length : -length; +} + +/* + * Note: In span() when spanLength==0 (after a string match, or at the beginning + * after an empty code point span) and in spanNot() and spanNotUTF8(), + * string matching could use a binary search + * because all string matches are done from the same start index. + * + * For UTF-8, this would require a comparison function that returns UTF-16 order. + * + * This optimization should not be necessary for normal UnicodeSets because + * most sets have no strings, and most sets with strings have + * very few very short strings. + * For cases with many strings, it might be better to use a different API + * and implementation with a DFA (state machine). + */ + +/* + * Algorithm for span(USET_SPAN_CONTAINED) + * + * Theoretical algorithm: + * - Iterate through the string, and at each code point boundary: + * + If the code point there is in the set, then remember to continue after it. + * + If a set string matches at the current position, then remember to continue after it. + * + Either recursively span for each code point or string match, + * or recursively span for all but the shortest one and + * iteratively continue the span with the shortest local match. + * + Remember the longest recursive span (the farthest end point). + * + If there is no match at the current position, neither for the code point there + * nor for any set string, then stop and return the longest recursive span length. + * + * Optimized implementation: + * + * (We assume that most sets will have very few very short strings. + * A span using a string-less set is extremely fast.) + * + * Create and cache a spanSet which contains all of the single code points + * of the original set but none of its strings. + * + * - Start with spanLength=spanSet.span(USET_SPAN_CONTAINED). + * - Loop: + * + Try to match each set string at the end of the spanLength. + * ~ Set strings that start with set-contained code points must be matched + * with a partial overlap because the recursive algorithm would have tried + * to match them at every position. + * ~ Set strings that entirely consist of set-contained code points + * are irrelevant for span(USET_SPAN_CONTAINED) because the + * recursive algorithm would continue after them anyway + * and find the longest recursive match from their end. + * ~ Rather than recursing, note each end point of a set string match. + * + If no set string matched after spanSet.span(), then return + * with where the spanSet.span() ended. + * + If at least one set string matched after spanSet.span(), then + * pop the shortest string match end point and continue + * the loop, trying to match all set strings from there. + * + If at least one more set string matched after a previous string match, + * then test if the code point after the previous string match is also + * contained in the set. + * Continue the loop with the shortest end point of either this code point + * or a matching set string. + * + If no more set string matched after a previous string match, + * then try another spanLength=spanSet.span(USET_SPAN_CONTAINED). + * Stop if spanLength==0, otherwise continue the loop. + * + * By noting each end point of a set string match, + * the function visits each string position at most once and finishes + * in linear time. + * + * The recursive algorithm may visit the same string position many times + * if multiple paths lead to it and finishes in exponential time. + */ + +/* + * Algorithm for span(USET_SPAN_SIMPLE) + * + * Theoretical algorithm: + * - Iterate through the string, and at each code point boundary: + * + If the code point there is in the set, then remember to continue after it. + * + If a set string matches at the current position, then remember to continue after it. + * + Continue from the farthest match position and ignore all others. + * + If there is no match at the current position, + * then stop and return the current position. + * + * Optimized implementation: + * + * (Same assumption and spanSet as above.) + * + * - Start with spanLength=spanSet.span(USET_SPAN_CONTAINED). + * - Loop: + * + Try to match each set string at the end of the spanLength. + * ~ Set strings that start with set-contained code points must be matched + * with a partial overlap because the standard algorithm would have tried + * to match them earlier. + * ~ Set strings that entirely consist of set-contained code points + * must be matched with a full overlap because the longest-match algorithm + * would hide set string matches that end earlier. + * Such set strings need not be matched earlier inside the code point span + * because the standard algorithm would then have continued after + * the set string match anyway. + * ~ Remember the longest set string match (farthest end point) from the earliest + * starting point. + * + If no set string matched after spanSet.span(), then return + * with where the spanSet.span() ended. + * + If at least one set string matched, then continue the loop after the + * longest match from the earliest position. + * + If no more set string matched after a previous string match, + * then try another spanLength=spanSet.span(USET_SPAN_CONTAINED). + * Stop if spanLength==0, otherwise continue the loop. + */ + +int32_t UnicodeSetStringSpan::span(const UChar *s, int32_t length, USetSpanCondition spanCondition) const { + if(spanCondition==USET_SPAN_NOT_CONTAINED) { + return spanNot(s, length); + } + int32_t spanLength=spanSet.span(s, length, USET_SPAN_CONTAINED); + if(spanLength==length) { + return length; + } + + // Consider strings; they may overlap with the span. + OffsetList offsets; + if(spanCondition==USET_SPAN_CONTAINED) { + // Use offset list to try all possibilities. + offsets.setMaxLength(maxLength16); + } + int32_t pos=spanLength, rest=length-pos; + int32_t i, stringsLength=strings.size(); + for(;;) { + if(spanCondition==USET_SPAN_CONTAINED) { + for(i=0; i=LONG_SPAN) { + overlap=length16; + // While contained: No point matching fully inside the code point span. + U16_BACK_1(s16, 0, overlap); // Length of the string minus the last code point. + } + if(overlap>spanLength) { + overlap=spanLength; + } + int32_t inc=length16-overlap; // Keep overlap+inc==length16. + for(;;) { + if(inc>rest) { + break; + } + // Try to match if the increment is not listed already. + if(!offsets.containsOffset(inc) && matches16CPB(s, pos-overlap, length, s16, length16)) { + if(inc==rest) { + return length; // Reached the end of the string. + } + offsets.addOffset(inc); + } + if(overlap==0) { + break; + } + --overlap; + ++inc; + } + } + } else /* USET_SPAN_SIMPLE */ { + int32_t maxInc=0, maxOverlap=0; + for(i=0; i=LONG_SPAN) { + overlap=length16; + // Longest match: Need to match fully inside the code point span + // to find the match from the earliest start. + } + if(overlap>spanLength) { + overlap=spanLength; + } + int32_t inc=length16-overlap; // Keep overlap+inc==length16. + for(;;) { + if(inc>rest || overlapmaxOverlap || /* redundant overlap==maxOverlap && */ inc>maxInc) && + matches16CPB(s, pos-overlap, length, s16, length16) + ) { + maxInc=inc; // Longest match from earliest start. + maxOverlap=overlap; + break; + } + --overlap; + ++inc; + } + } + + if(maxInc!=0 || maxOverlap!=0) { + // Longest-match algorithm, and there was a string match. + // Simply continue after it. + pos+=maxInc; + rest-=maxInc; + if(rest==0) { + return length; // Reached the end of the string. + } + spanLength=0; // Match strings from after a string match. + continue; + } + } + // Finished trying to match all strings at pos. + + if(spanLength!=0 || pos==0) { + // The position is after an unlimited code point span (spanLength!=0), + // not after a string match. + // The only position where spanLength==0 after a span is pos==0. + // Otherwise, an unlimited code point span is only tried again when no + // strings match, and if such a non-initial span fails we stop. + if(offsets.isEmpty()) { + return pos; // No strings matched after a span. + } + // Match strings from after the next string match. + } else { + // The position is after a string match (or a single code point). + if(offsets.isEmpty()) { + // No more strings matched after a previous string match. + // Try another code point span from after the last string match. + spanLength=spanSet.span(s+pos, rest, USET_SPAN_CONTAINED); + if( spanLength==rest || // Reached the end of the string, or + spanLength==0 // neither strings nor span progressed. + ) { + return pos+spanLength; + } + pos+=spanLength; + rest-=spanLength; + continue; // spanLength>0: Match strings from after a span. + } else { + // Try to match only one code point from after a string match if some + // string matched beyond it, so that we try all possible positions + // and don't overshoot. + spanLength=spanOne(spanSet, s+pos, rest); + if(spanLength>0) { + if(spanLength==rest) { + return length; // Reached the end of the string. + } + // Match strings after this code point. + // There cannot be any increments below it because UnicodeSet strings + // contain multiple code points. + pos+=spanLength; + rest-=spanLength; + offsets.shift(spanLength); + spanLength=0; + continue; // Match strings from after a single code point. + } + // Match strings from after the next string match. + } + } + int32_t minOffset=offsets.popMinimum(); + pos+=minOffset; + rest-=minOffset; + spanLength=0; // Match strings from after a string match. + } +} + +int32_t UnicodeSetStringSpan::spanBack(const UChar *s, int32_t length, USetSpanCondition spanCondition) const { + if(spanCondition==USET_SPAN_NOT_CONTAINED) { + return spanNotBack(s, length); + } + int32_t pos=spanSet.spanBack(s, length, USET_SPAN_CONTAINED); + if(pos==0) { + return 0; + } + int32_t spanLength=length-pos; + + // Consider strings; they may overlap with the span. + OffsetList offsets; + if(spanCondition==USET_SPAN_CONTAINED) { + // Use offset list to try all possibilities. + offsets.setMaxLength(maxLength16); + } + int32_t i, stringsLength=strings.size(); + uint8_t *spanBackLengths=spanLengths; + if(all) { + spanBackLengths+=stringsLength; + } + for(;;) { + if(spanCondition==USET_SPAN_CONTAINED) { + for(i=0; i=LONG_SPAN) { + overlap=length16; + // While contained: No point matching fully inside the code point span. + int32_t len1=0; + U16_FWD_1(s16, len1, overlap); + overlap-=len1; // Length of the string minus the first code point. + } + if(overlap>spanLength) { + overlap=spanLength; + } + int32_t dec=length16-overlap; // Keep dec+overlap==length16. + for(;;) { + if(dec>pos) { + break; + } + // Try to match if the decrement is not listed already. + if(!offsets.containsOffset(dec) && matches16CPB(s, pos-dec, length, s16, length16)) { + if(dec==pos) { + return 0; // Reached the start of the string. + } + offsets.addOffset(dec); + } + if(overlap==0) { + break; + } + --overlap; + ++dec; + } + } + } else /* USET_SPAN_SIMPLE */ { + int32_t maxDec=0, maxOverlap=0; + for(i=0; i=LONG_SPAN) { + overlap=length16; + // Longest match: Need to match fully inside the code point span + // to find the match from the latest end. + } + if(overlap>spanLength) { + overlap=spanLength; + } + int32_t dec=length16-overlap; // Keep dec+overlap==length16. + for(;;) { + if(dec>pos || overlapmaxOverlap || /* redundant overlap==maxOverlap && */ dec>maxDec) && + matches16CPB(s, pos-dec, length, s16, length16) + ) { + maxDec=dec; // Longest match from latest end. + maxOverlap=overlap; + break; + } + --overlap; + ++dec; + } + } + + if(maxDec!=0 || maxOverlap!=0) { + // Longest-match algorithm, and there was a string match. + // Simply continue before it. + pos-=maxDec; + if(pos==0) { + return 0; // Reached the start of the string. + } + spanLength=0; // Match strings from before a string match. + continue; + } + } + // Finished trying to match all strings at pos. + + if(spanLength!=0 || pos==length) { + // The position is before an unlimited code point span (spanLength!=0), + // not before a string match. + // The only position where spanLength==0 before a span is pos==length. + // Otherwise, an unlimited code point span is only tried again when no + // strings match, and if such a non-initial span fails we stop. + if(offsets.isEmpty()) { + return pos; // No strings matched before a span. + } + // Match strings from before the next string match. + } else { + // The position is before a string match (or a single code point). + if(offsets.isEmpty()) { + // No more strings matched before a previous string match. + // Try another code point span from before the last string match. + int32_t oldPos=pos; + pos=spanSet.spanBack(s, oldPos, USET_SPAN_CONTAINED); + spanLength=oldPos-pos; + if( pos==0 || // Reached the start of the string, or + spanLength==0 // neither strings nor span progressed. + ) { + return pos; + } + continue; // spanLength>0: Match strings from before a span. + } else { + // Try to match only one code point from before a string match if some + // string matched beyond it, so that we try all possible positions + // and don't overshoot. + spanLength=spanOneBack(spanSet, s, pos); + if(spanLength>0) { + if(spanLength==pos) { + return 0; // Reached the start of the string. + } + // Match strings before this code point. + // There cannot be any decrements below it because UnicodeSet strings + // contain multiple code points. + pos-=spanLength; + offsets.shift(spanLength); + spanLength=0; + continue; // Match strings from before a single code point. + } + // Match strings from before the next string match. + } + } + pos-=offsets.popMinimum(); + spanLength=0; // Match strings from before a string match. + } +} + +int32_t UnicodeSetStringSpan::spanUTF8(const uint8_t *s, int32_t length, USetSpanCondition spanCondition) const { + if(spanCondition==USET_SPAN_NOT_CONTAINED) { + return spanNotUTF8(s, length); + } + int32_t spanLength=spanSet.spanUTF8((const char *)s, length, USET_SPAN_CONTAINED); + if(spanLength==length) { + return length; + } + + // Consider strings; they may overlap with the span. + OffsetList offsets; + if(spanCondition==USET_SPAN_CONTAINED) { + // Use offset list to try all possibilities. + offsets.setMaxLength(maxLength8); + } + int32_t pos=spanLength, rest=length-pos; + int32_t i, stringsLength=strings.size(); + uint8_t *spanUTF8Lengths=spanLengths; + if(all) { + spanUTF8Lengths+=2*stringsLength; + } + for(;;) { + const uint8_t *s8=utf8; + int32_t length8; + if(spanCondition==USET_SPAN_CONTAINED) { + for(i=0; i=LONG_SPAN) { + overlap=length8; + // While contained: No point matching fully inside the code point span. + U8_BACK_1(s8, 0, overlap); // Length of the string minus the last code point. + } + if(overlap>spanLength) { + overlap=spanLength; + } + int32_t inc=length8-overlap; // Keep overlap+inc==length8. + for(;;) { + if(inc>rest) { + break; + } + // Try to match if the increment is not listed already. + // Match at code point boundaries. (The UTF-8 strings were converted + // from UTF-16 and are guaranteed to be well-formed.) + if( !U8_IS_TRAIL(s[pos-overlap]) && + !offsets.containsOffset(inc) && + matches8(s+pos-overlap, s8, length8) + + ) { + if(inc==rest) { + return length; // Reached the end of the string. + } + offsets.addOffset(inc); + } + if(overlap==0) { + break; + } + --overlap; + ++inc; + } + s8+=length8; + } + } else /* USET_SPAN_SIMPLE */ { + int32_t maxInc=0, maxOverlap=0; + for(i=0; i=LONG_SPAN) { + overlap=length8; + // Longest match: Need to match fully inside the code point span + // to find the match from the earliest start. + } + if(overlap>spanLength) { + overlap=spanLength; + } + int32_t inc=length8-overlap; // Keep overlap+inc==length8. + for(;;) { + if(inc>rest || overlapmaxOverlap || /* redundant overlap==maxOverlap && */ inc>maxInc) && + matches8(s+pos-overlap, s8, length8) + + ) { + maxInc=inc; // Longest match from earliest start. + maxOverlap=overlap; + break; + } + --overlap; + ++inc; + } + s8+=length8; + } + + if(maxInc!=0 || maxOverlap!=0) { + // Longest-match algorithm, and there was a string match. + // Simply continue after it. + pos+=maxInc; + rest-=maxInc; + if(rest==0) { + return length; // Reached the end of the string. + } + spanLength=0; // Match strings from after a string match. + continue; + } + } + // Finished trying to match all strings at pos. + + if(spanLength!=0 || pos==0) { + // The position is after an unlimited code point span (spanLength!=0), + // not after a string match. + // The only position where spanLength==0 after a span is pos==0. + // Otherwise, an unlimited code point span is only tried again when no + // strings match, and if such a non-initial span fails we stop. + if(offsets.isEmpty()) { + return pos; // No strings matched after a span. + } + // Match strings from after the next string match. + } else { + // The position is after a string match (or a single code point). + if(offsets.isEmpty()) { + // No more strings matched after a previous string match. + // Try another code point span from after the last string match. + spanLength=spanSet.spanUTF8((const char *)s+pos, rest, USET_SPAN_CONTAINED); + if( spanLength==rest || // Reached the end of the string, or + spanLength==0 // neither strings nor span progressed. + ) { + return pos+spanLength; + } + pos+=spanLength; + rest-=spanLength; + continue; // spanLength>0: Match strings from after a span. + } else { + // Try to match only one code point from after a string match if some + // string matched beyond it, so that we try all possible positions + // and don't overshoot. + spanLength=spanOneUTF8(spanSet, s+pos, rest); + if(spanLength>0) { + if(spanLength==rest) { + return length; // Reached the end of the string. + } + // Match strings after this code point. + // There cannot be any increments below it because UnicodeSet strings + // contain multiple code points. + pos+=spanLength; + rest-=spanLength; + offsets.shift(spanLength); + spanLength=0; + continue; // Match strings from after a single code point. + } + // Match strings from after the next string match. + } + } + int32_t minOffset=offsets.popMinimum(); + pos+=minOffset; + rest-=minOffset; + spanLength=0; // Match strings from after a string match. + } +} + +int32_t UnicodeSetStringSpan::spanBackUTF8(const uint8_t *s, int32_t length, USetSpanCondition spanCondition) const { + if(spanCondition==USET_SPAN_NOT_CONTAINED) { + return spanNotBackUTF8(s, length); + } + int32_t pos=spanSet.spanBackUTF8((const char *)s, length, USET_SPAN_CONTAINED); + if(pos==0) { + return 0; + } + int32_t spanLength=length-pos; + + // Consider strings; they may overlap with the span. + OffsetList offsets; + if(spanCondition==USET_SPAN_CONTAINED) { + // Use offset list to try all possibilities. + offsets.setMaxLength(maxLength8); + } + int32_t i, stringsLength=strings.size(); + uint8_t *spanBackUTF8Lengths=spanLengths; + if(all) { + spanBackUTF8Lengths+=3*stringsLength; + } + for(;;) { + const uint8_t *s8=utf8; + int32_t length8; + if(spanCondition==USET_SPAN_CONTAINED) { + for(i=0; i=LONG_SPAN) { + overlap=length8; + // While contained: No point matching fully inside the code point span. + int32_t len1=0; + U8_FWD_1(s8, len1, overlap); + overlap-=len1; // Length of the string minus the first code point. + } + if(overlap>spanLength) { + overlap=spanLength; + } + int32_t dec=length8-overlap; // Keep dec+overlap==length8. + for(;;) { + if(dec>pos) { + break; + } + // Try to match if the decrement is not listed already. + // Match at code point boundaries. (The UTF-8 strings were converted + // from UTF-16 and are guaranteed to be well-formed.) + if( !U8_IS_TRAIL(s[pos-dec]) && + !offsets.containsOffset(dec) && + matches8(s+pos-dec, s8, length8) + ) { + if(dec==pos) { + return 0; // Reached the start of the string. + } + offsets.addOffset(dec); + } + if(overlap==0) { + break; + } + --overlap; + ++dec; + } + s8+=length8; + } + } else /* USET_SPAN_SIMPLE */ { + int32_t maxDec=0, maxOverlap=0; + for(i=0; i=LONG_SPAN) { + overlap=length8; + // Longest match: Need to match fully inside the code point span + // to find the match from the latest end. + } + if(overlap>spanLength) { + overlap=spanLength; + } + int32_t dec=length8-overlap; // Keep dec+overlap==length8. + for(;;) { + if(dec>pos || overlapmaxOverlap || /* redundant overlap==maxOverlap && */ dec>maxDec) && + matches8(s+pos-dec, s8, length8) + ) { + maxDec=dec; // Longest match from latest end. + maxOverlap=overlap; + break; + } + --overlap; + ++dec; + } + s8+=length8; + } + + if(maxDec!=0 || maxOverlap!=0) { + // Longest-match algorithm, and there was a string match. + // Simply continue before it. + pos-=maxDec; + if(pos==0) { + return 0; // Reached the start of the string. + } + spanLength=0; // Match strings from before a string match. + continue; + } + } + // Finished trying to match all strings at pos. + + if(spanLength!=0 || pos==length) { + // The position is before an unlimited code point span (spanLength!=0), + // not before a string match. + // The only position where spanLength==0 before a span is pos==length. + // Otherwise, an unlimited code point span is only tried again when no + // strings match, and if such a non-initial span fails we stop. + if(offsets.isEmpty()) { + return pos; // No strings matched before a span. + } + // Match strings from before the next string match. + } else { + // The position is before a string match (or a single code point). + if(offsets.isEmpty()) { + // No more strings matched before a previous string match. + // Try another code point span from before the last string match. + int32_t oldPos=pos; + pos=spanSet.spanBackUTF8((const char *)s, oldPos, USET_SPAN_CONTAINED); + spanLength=oldPos-pos; + if( pos==0 || // Reached the start of the string, or + spanLength==0 // neither strings nor span progressed. + ) { + return pos; + } + continue; // spanLength>0: Match strings from before a span. + } else { + // Try to match only one code point from before a string match if some + // string matched beyond it, so that we try all possible positions + // and don't overshoot. + spanLength=spanOneBackUTF8(spanSet, s, pos); + if(spanLength>0) { + if(spanLength==pos) { + return 0; // Reached the start of the string. + } + // Match strings before this code point. + // There cannot be any decrements below it because UnicodeSet strings + // contain multiple code points. + pos-=spanLength; + offsets.shift(spanLength); + spanLength=0; + continue; // Match strings from before a single code point. + } + // Match strings from before the next string match. + } + } + pos-=offsets.popMinimum(); + spanLength=0; // Match strings from before a string match. + } +} + +/* + * Algorithm for spanNot()==span(USET_SPAN_NOT_CONTAINED) + * + * Theoretical algorithm: + * - Iterate through the string, and at each code point boundary: + * + If the code point there is in the set, then return with the current position. + * + If a set string matches at the current position, then return with the current position. + * + * Optimized implementation: + * + * (Same assumption as for span() above.) + * + * Create and cache a spanNotSet which contains all of the single code points + * of the original set but none of its strings. + * For each set string add its initial code point to the spanNotSet. + * (Also add its final code point for spanNotBack().) + * + * - Loop: + * + Do spanLength=spanNotSet.span(USET_SPAN_NOT_CONTAINED). + * + If the current code point is in the original set, then + * return the current position. + * + If any set string matches at the current position, then + * return the current position. + * + If there is no match at the current position, neither for the code point there + * nor for any set string, then skip this code point and continue the loop. + * This happens for set-string-initial code points that were added to spanNotSet + * when there is not actually a match for such a set string. + */ + +int32_t UnicodeSetStringSpan::spanNot(const UChar *s, int32_t length) const { + int32_t pos=0, rest=length; + int32_t i, stringsLength=strings.size(); + do { + // Span until we find a code point from the set, + // or a code point that starts or ends some string. + i=pSpanNotSet->span(s+pos, rest, USET_SPAN_NOT_CONTAINED); + if(i==rest) { + return length; // Reached the end of the string. + } + pos+=i; + rest-=i; + + // Check whether the current code point is in the original set, + // without the string starts and ends. + int32_t cpLength=spanOne(spanSet, s+pos, rest); + if(cpLength>0) { + return pos; // There is a set element at pos. + } + + // Try to match the strings at pos. + for(i=0; ispanBack(s, pos, USET_SPAN_NOT_CONTAINED); + if(pos==0) { + return 0; // Reached the start of the string. + } + + // Check whether the current code point is in the original set, + // without the string starts and ends. + int32_t cpLength=spanOneBack(spanSet, s, pos); + if(cpLength>0) { + return pos; // There is a set element at pos. + } + + // Try to match the strings at pos. + for(i=0; ispanUTF8((const char *)s+pos, rest, USET_SPAN_NOT_CONTAINED); + if(i==rest) { + return length; // Reached the end of the string. + } + pos+=i; + rest-=i; + + // Check whether the current code point is in the original set, + // without the string starts and ends. + int32_t cpLength=spanOneUTF8(spanSet, s+pos, rest); + if(cpLength>0) { + return pos; // There is a set element at pos. + } + + // Try to match the strings at pos. + const uint8_t *s8=utf8; + int32_t length8; + for(i=0; ispanBackUTF8((const char *)s, pos, USET_SPAN_NOT_CONTAINED); + if(pos==0) { + return 0; // Reached the start of the string. + } + + // Check whether the current code point is in the original set, + // without the string starts and ends. + int32_t cpLength=spanOneBackUTF8(spanSet, s, pos); + if(cpLength>0) { + return pos; // There is a set element at pos. + } + + // Try to match the strings at pos. + const uint8_t *s8=utf8; + int32_t length8; + for(i=0; i=0xfe. + LONG_SPAN=0xfe, + // All code points in the string are contained in the parent set. + ALL_CP_CONTAINED=0xff + }; + + // Add a starting or ending string character to the spanNotSet + // so that a character span ends before any string. + void addToSpanNotSet(UChar32 c); + + int32_t spanNot(const UChar *s, int32_t length) const; + int32_t spanNotBack(const UChar *s, int32_t length) const; + int32_t spanNotUTF8(const uint8_t *s, int32_t length) const; + int32_t spanNotBackUTF8(const uint8_t *s, int32_t length) const; + + // Set for span(). Same as parent but without strings. + UnicodeSet spanSet; + + // Set for span(not contained). + // Same as spanSet, plus characters that start or end strings. + UnicodeSet *pSpanNotSet; + + // The strings of the parent set. + const UVector &strings; + + // Pointer to the UTF-8 string lengths. + // Also pointer to further allocated storage for meta data and + // UTF-8 string contents as necessary. + int32_t *utf8Lengths; + + // Pointer to the part of the (utf8Lengths) memory block that stores + // the lengths of span(), spanBack() etc. for each string. + uint8_t *spanLengths; + + // Pointer to the part of the (utf8Lengths) memory block that stores + // the UTF-8 versions of the parent set's strings. + uint8_t *utf8; + + // Number of bytes for all UTF-8 versions of strings together. + int32_t utf8Length; + + // Maximum lengths of relevant strings. + int32_t maxLength16; + int32_t maxLength8; + + // Set up for all variants of span()? + UBool all; + + // Memory for small numbers and lengths of strings. + // For example, for 8 strings: + // 8 UTF-8 lengths, 8*4 bytes span lengths, 8*2 3-byte UTF-8 characters + // = 112 bytes = int32_t[28]. + int32_t staticLengths[32]; +}; + +UBool UnicodeSetStringSpan::needsStringSpanUTF16() { + return (UBool)(maxLength16!=0); +} + +UBool UnicodeSetStringSpan::needsStringSpanUTF8() { + return (UBool)(maxLength8!=0); +} + +UBool UnicodeSetStringSpan::contains(UChar32 c) const { + return spanSet.contains(c); +} + +U_NAMESPACE_END + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unistr.cpp b/Build/source/libs/icu/icu-xetex/common/unistr.cpp new file mode 100644 index 00000000000..d8bea383267 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unistr.cpp @@ -0,0 +1,1385 @@ +/* +****************************************************************************** +* Copyright (C) 1999-2007, International Business Machines Corporation and * +* others. All Rights Reserved. * +****************************************************************************** +* +* File unistr.cpp +* +* Modification History: +* +* Date Name Description +* 09/25/98 stephen Creation. +* 04/20/99 stephen Overhauled per 4/16 code review. +* 07/09/99 stephen Renamed {hi,lo},{byte,word} to icu_X for HP/UX +* 11/18/99 aliu Added handleReplaceBetween() to make inherit from +* Replaceable. +* 06/25/01 grhoten Removed the dependency on iostream +****************************************************************************** +*/ + +#include "unicode/utypes.h" +#include "unicode/putil.h" +#include "cstring.h" +#include "cmemory.h" +#include "unicode/ustring.h" +#include "unicode/unistr.h" +#include "uhash.h" +#include "ustr_imp.h" +#include "umutex.h" + +#if 0 + +#if U_IOSTREAM_SOURCE >= 199711 +#include +using namespace std; +#elif U_IOSTREAM_SOURCE >= 198506 +#include +#endif + +//DEBUGGING +void +print(const UnicodeString& s, + const char *name) +{ + UChar c; + cout << name << ":|"; + for(int i = 0; i < s.length(); ++i) { + c = s[i]; + if(c>= 0x007E || c < 0x0020) + cout << "[0x" << hex << s[i] << "]"; + else + cout << (char) s[i]; + } + cout << '|' << endl; +} + +void +print(const UChar *s, + int32_t len, + const char *name) +{ + UChar c; + cout << name << ":|"; + for(int i = 0; i < len; ++i) { + c = s[i]; + if(c>= 0x007E || c < 0x0020) + cout << "[0x" << hex << s[i] << "]"; + else + cout << (char) s[i]; + } + cout << '|' << endl; +} +// END DEBUGGING +#endif + +// Local function definitions for now + +// need to copy areas that may overlap +static +inline void +us_arrayCopy(const UChar *src, int32_t srcStart, + UChar *dst, int32_t dstStart, int32_t count) +{ + if(count>0) { + uprv_memmove(dst+dstStart, src+srcStart, (size_t)(count*sizeof(*src))); + } +} + +// u_unescapeAt() callback to get a UChar from a UnicodeString +U_CDECL_BEGIN +static UChar U_CALLCONV +UnicodeString_charAt(int32_t offset, void *context) { + return ((U_NAMESPACE_QUALIFIER UnicodeString*) context)->charAt(offset); +} +U_CDECL_END + +U_NAMESPACE_BEGIN + +/* The Replaceable virtual destructor can't be defined in the header + due to how AIX works with multiple definitions of virtual functions. +*/ +Replaceable::~Replaceable() {} +Replaceable::Replaceable() {} +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(UnicodeString) + +UnicodeString U_EXPORT2 +operator+ (const UnicodeString &s1, const UnicodeString &s2) { + return + UnicodeString(s1.length()+s2.length()+1, (UChar32)0, 0). + append(s1). + append(s2); +} + +//======================================== +// Reference Counting functions, put at top of file so that optimizing compilers +// have a chance to automatically inline. +//======================================== + +void +UnicodeString::addRef() +{ umtx_atomic_inc((int32_t *)fArray - 1);} + +int32_t +UnicodeString::removeRef() +{ return umtx_atomic_dec((int32_t *)fArray - 1);} + +int32_t +UnicodeString::refCount() const +{ + umtx_lock(NULL); + // Note: without the lock to force a memory barrier, we might see a very + // stale value on some multi-processor systems. + int32_t count = *((int32_t *)fArray - 1); + umtx_unlock(NULL); + return count; + } + +void +UnicodeString::releaseArray() { + if((fFlags & kRefCounted) && removeRef() == 0) { + uprv_free((int32_t *)fArray - 1); + } +} + + + +//======================================== +// Constructors +//======================================== +UnicodeString::UnicodeString() + : fLength(0), + fCapacity(US_STACKBUF_SIZE), + fArray(fStackBuffer), + fFlags(kShortString) +{} + +UnicodeString::UnicodeString(int32_t capacity, UChar32 c, int32_t count) + : fLength(0), + fCapacity(US_STACKBUF_SIZE), + fArray(0), + fFlags(0) +{ + if(count <= 0 || (uint32_t)c > 0x10ffff) { + // just allocate and do not do anything else + allocate(capacity); + } else { + // count > 0, allocate and fill the new string with count c's + int32_t unitCount = UTF_CHAR_LENGTH(c), length = count * unitCount; + if(capacity < length) { + capacity = length; + } + if(allocate(capacity)) { + int32_t i = 0; + + // fill the new string with c + if(unitCount == 1) { + // fill with length UChars + while(i < length) { + fArray[i++] = (UChar)c; + } + } else { + // get the code units for c + UChar units[UTF_MAX_CHAR_LENGTH]; + UTF_APPEND_CHAR_UNSAFE(units, i, c); + + // now it must be i==unitCount + i = 0; + + // for Unicode, unitCount can only be 1, 2, 3, or 4 + // 1 is handled above + while(i < length) { + int32_t unitIdx = 0; + while(unitIdx < unitCount) { + fArray[i++]=units[unitIdx++]; + } + } + } + } + fLength = length; + } +} + +UnicodeString::UnicodeString(UChar ch) + : fLength(1), + fCapacity(US_STACKBUF_SIZE), + fArray(fStackBuffer), + fFlags(kShortString) +{ + fStackBuffer[0] = ch; +} + +UnicodeString::UnicodeString(UChar32 ch) + : fLength(1), + fCapacity(US_STACKBUF_SIZE), + fArray(fStackBuffer), + fFlags(kShortString) +{ + int32_t i = 0; + UBool isError = FALSE; + U16_APPEND(fStackBuffer, i, US_STACKBUF_SIZE, ch, isError); + fLength = i; +} + +UnicodeString::UnicodeString(const UChar *text) + : fLength(0), + fCapacity(US_STACKBUF_SIZE), + fArray(fStackBuffer), + fFlags(kShortString) +{ + doReplace(0, 0, text, 0, -1); +} + +UnicodeString::UnicodeString(const UChar *text, + int32_t textLength) + : fLength(0), + fCapacity(US_STACKBUF_SIZE), + fArray(fStackBuffer), + fFlags(kShortString) +{ + doReplace(0, 0, text, 0, textLength); +} + +UnicodeString::UnicodeString(UBool isTerminated, + const UChar *text, + int32_t textLength) + : fLength(textLength), + fCapacity(isTerminated ? textLength + 1 : textLength), + fArray((UChar *)text), + fFlags(kReadonlyAlias) +{ + if(text == NULL) { + // treat as an empty string, do not alias + fLength = 0; + fCapacity = US_STACKBUF_SIZE; + fArray = fStackBuffer; + fFlags = kShortString; + } else if(textLength < -1 || + (textLength == -1 && !isTerminated) || + (textLength >= 0 && isTerminated && text[textLength] != 0) + ) { + setToBogus(); + } else if(textLength == -1) { + // text is terminated, or else it would have failed the above test + fLength = u_strlen(text); + fCapacity = fLength + 1; + } +} + +UnicodeString::UnicodeString(UChar *buff, + int32_t buffLength, + int32_t buffCapacity) + : fLength(buffLength), + fCapacity(buffCapacity), + fArray(buff), + fFlags(kWritableAlias) +{ + if(buff == NULL) { + // treat as an empty string, do not alias + fLength = 0; + fCapacity = US_STACKBUF_SIZE; + fArray = fStackBuffer; + fFlags = kShortString; + } else if(buffLength < -1 || buffCapacity < 0 || buffLength > buffCapacity) { + setToBogus(); + } else if(buffLength == -1) { + // fLength = u_strlen(buff); but do not look beyond buffCapacity + const UChar *p = buff, *limit = buff + buffCapacity; + while(p != limit && *p != 0) { + ++p; + } + fLength = (int32_t)(p - buff); + } +} + +UnicodeString::UnicodeString(const char *src, int32_t length, EInvariant) + : fLength(0), + fCapacity(US_STACKBUF_SIZE), + fArray(fStackBuffer), + fFlags(kShortString) +{ + if(src==NULL) { + // treat as an empty string + } else { + if(length<0) { + length=(int32_t)uprv_strlen(src); + } + if(cloneArrayIfNeeded(length, length, FALSE)) { + u_charsToUChars(src, getArrayStart(), length); + fLength = length; + } else { + setToBogus(); + } + } +} + +UnicodeString::UnicodeString(const UnicodeString& that) + : Replaceable(), + fLength(0), + fCapacity(US_STACKBUF_SIZE), + fArray(fStackBuffer), + fFlags(kShortString) +{ + copyFrom(that); +} + +UnicodeString::UnicodeString(const UnicodeString& that, + int32_t srcStart) + : Replaceable(), + fLength(0), + fCapacity(US_STACKBUF_SIZE), + fArray(fStackBuffer), + fFlags(kShortString) +{ + setTo(that, srcStart); +} + +UnicodeString::UnicodeString(const UnicodeString& that, + int32_t srcStart, + int32_t srcLength) + : Replaceable(), + fLength(0), + fCapacity(US_STACKBUF_SIZE), + fArray(fStackBuffer), + fFlags(kShortString) +{ + setTo(that, srcStart, srcLength); +} + +// Replaceable base class clone() default implementation, does not clone +Replaceable * +Replaceable::clone() const { + return NULL; +} + +// UnicodeString overrides clone() with a real implementation +Replaceable * +UnicodeString::clone() const { + return new UnicodeString(*this); +} + +//======================================== +// array allocation +//======================================== + +UBool +UnicodeString::allocate(int32_t capacity) { + if(capacity <= US_STACKBUF_SIZE) { + fArray = fStackBuffer; + fCapacity = US_STACKBUF_SIZE; + fFlags = kShortString; + } else { + // count bytes for the refCounter and the string capacity, and + // round up to a multiple of 16; then divide by 4 and allocate int32_t's + // to be safely aligned for the refCount + int32_t words = (int32_t)(((sizeof(int32_t) + capacity * U_SIZEOF_UCHAR + 15) & ~15) >> 2); + int32_t *array = (int32_t*) uprv_malloc( sizeof(int32_t) * words ); + if(array != 0) { + // set initial refCount and point behind the refCount + *array++ = 1; + + // have fArray point to the first UChar + fArray = (UChar *)array; + fCapacity = (int32_t)((words - 1) * (sizeof(int32_t) / U_SIZEOF_UCHAR)); + fFlags = kLongString; + } else { + fLength = 0; + fCapacity = 0; + fFlags = kIsBogus; + return FALSE; + } + } + return TRUE; +} + +//======================================== +// Destructor +//======================================== +UnicodeString::~UnicodeString() +{ + releaseArray(); +} + + +//======================================== +// Assignment +//======================================== + +UnicodeString & +UnicodeString::operator=(const UnicodeString &src) { + return copyFrom(src); +} + +UnicodeString & +UnicodeString::fastCopyFrom(const UnicodeString &src) { + return copyFrom(src, TRUE); +} + +UnicodeString & +UnicodeString::copyFrom(const UnicodeString &src, UBool fastCopy) { + // if assigning to ourselves, do nothing + if(this == 0 || this == &src) { + return *this; + } + + // is the right side bogus? + if(&src == 0 || src.isBogus()) { + setToBogus(); + return *this; + } + + // delete the current contents + releaseArray(); + + // we always copy the length + fLength = src.fLength; + if(fLength == 0) { + // empty string - use the stack buffer + fArray = fStackBuffer; + fCapacity = US_STACKBUF_SIZE; + fFlags = kShortString; + return *this; + } + + // fLength>0 and not an "open" src.getBuffer(minCapacity) + switch(src.fFlags) { + case kShortString: + // short string using the stack buffer, do the same + fArray = fStackBuffer; + fCapacity = US_STACKBUF_SIZE; + fFlags = kShortString; + uprv_memcpy(fStackBuffer, src.fArray, fLength * U_SIZEOF_UCHAR); + break; + case kLongString: + // src uses a refCounted string buffer, use that buffer with refCount + // src is const, use a cast - we don't really change it + ((UnicodeString &)src).addRef(); + // copy all fields, share the reference-counted buffer + fArray = src.fArray; + fCapacity = src.fCapacity; + fFlags = src.fFlags; + break; + case kReadonlyAlias: + if(fastCopy) { + // src is a readonly alias, do the same + // -> maintain the readonly alias as such + fArray = src.fArray; + fCapacity = src.fCapacity; + fFlags = src.fFlags; + break; + } + // else if(!fastCopy) fall through to case kWritableAlias + // -> allocate a new buffer and copy the contents + case kWritableAlias: + // src is a writable alias; we make a copy of that instead + if(allocate(fLength)) { + uprv_memcpy(fArray, src.fArray, fLength * U_SIZEOF_UCHAR); + break; + } + // if there is not enough memory, then fall through to setting to bogus + default: + // if src is bogus, set ourselves to bogus + // do not call setToBogus() here because fArray and fFlags are not consistent here + fArray = 0; + fLength = 0; + fCapacity = 0; + fFlags = kIsBogus; + break; + } + + return *this; +} + +//======================================== +// Miscellaneous operations +//======================================== + +UnicodeString UnicodeString::unescape() const { + UnicodeString result; + for (int32_t i=0; i 0 && chars != srcChars) { + int32_t result; + +# if U_IS_BIG_ENDIAN + // big-endian: byte comparison works + result = uprv_memcmp(chars, srcChars, minLength * sizeof(UChar)); + if(result != 0) { + return (int8_t)(result >> 15 | 1); + } +# else + // little-endian: compare UChar units + do { + result = ((int32_t)*(chars++) - (int32_t)*(srcChars++)); + if(result != 0) { + return (int8_t)(result >> 15 | 1); + } + } while(--minLength > 0); +# endif + } + return lengthResult; +} + +/* String compare in code point order - doCompare() compares in code unit order. */ +int8_t +UnicodeString::doCompareCodePointOrder(int32_t start, + int32_t length, + const UChar *srcChars, + int32_t srcStart, + int32_t srcLength) const +{ + // compare illegal string values + // treat const UChar *srcChars==NULL as an empty string + if(isBogus()) { + return -1; + } + + // pin indices to legal values + pinIndices(start, length); + + if(srcChars == NULL) { + srcStart = srcLength = 0; + } + + int32_t diff = uprv_strCompare(fArray + start, length, srcChars + srcStart, srcLength, FALSE, TRUE); + /* translate the 32-bit result into an 8-bit one */ + if(diff!=0) { + return (int8_t)(diff >> 15 | 1); + } else { + return 0; + } +} + +int32_t +UnicodeString::getLength() const { + return length(); +} + +UChar +UnicodeString::getCharAt(int32_t offset) const { + return charAt(offset); +} + +UChar32 +UnicodeString::getChar32At(int32_t offset) const { + return char32At(offset); +} + +int32_t +UnicodeString::countChar32(int32_t start, int32_t length) const { + pinIndices(start, length); + // if(isBogus()) then fArray==0 and start==0 - u_countChar32() checks for NULL + return u_countChar32(fArray+start, length); +} + +UBool +UnicodeString::hasMoreChar32Than(int32_t start, int32_t length, int32_t number) const { + pinIndices(start, length); + // if(isBogus()) then fArray==0 and start==0 - u_strHasMoreChar32Than() checks for NULL + return u_strHasMoreChar32Than(fArray+start, length, number); +} + +int32_t +UnicodeString::moveIndex32(int32_t index, int32_t delta) const { + // pin index + if(index<0) { + index=0; + } else if(index>fLength) { + index=fLength; + } + + if(delta>0) { + UTF_FWD_N(fArray, index, fLength, delta); + } else { + UTF_BACK_N(fArray, 0, index, -delta); + } + + return index; +} + +void +UnicodeString::doExtract(int32_t start, + int32_t length, + UChar *dst, + int32_t dstStart) const +{ + // pin indices to legal values + pinIndices(start, length); + + // do not copy anything if we alias dst itself + if(fArray + start != dst + dstStart) { + us_arrayCopy(getArrayStart(), start, dst, dstStart, length); + } +} + +int32_t +UnicodeString::extract(UChar *dest, int32_t destCapacity, + UErrorCode &errorCode) const { + if(U_SUCCESS(errorCode)) { + if(isBogus() || destCapacity<0 || (destCapacity>0 && dest==0)) { + errorCode=U_ILLEGAL_ARGUMENT_ERROR; + } else { + if(fLength>0 && fLength<=destCapacity && fArray!=dest) { + uprv_memcpy(dest, fArray, fLength*U_SIZEOF_UCHAR); + } + return u_terminateUChars(dest, destCapacity, fLength, &errorCode); + } + } + + return fLength; +} + +int32_t +UnicodeString::extract(int32_t start, + int32_t length, + char *target, + int32_t targetCapacity, + enum EInvariant) const +{ + // if the arguments are illegal, then do nothing + if(targetCapacity < 0 || (targetCapacity > 0 && target == NULL)) { + return 0; + } + + // pin the indices to legal values + pinIndices(start, length); + + if(length <= targetCapacity) { + u_UCharsToChars(getArrayStart() + start, target, length); + } + UErrorCode status = U_ZERO_ERROR; + return u_terminateChars(target, targetCapacity, length, &status); +} + +void +UnicodeString::extractBetween(int32_t start, + int32_t limit, + UnicodeString& target) const { + pinIndex(start); + pinIndex(limit); + doExtract(start, limit - start, target); +} + +int32_t +UnicodeString::indexOf(const UChar *srcChars, + int32_t srcStart, + int32_t srcLength, + int32_t start, + int32_t length) const +{ + if(isBogus() || srcChars == 0 || srcStart < 0 || srcLength == 0) { + return -1; + } + + // UnicodeString does not find empty substrings + if(srcLength < 0 && srcChars[srcStart] == 0) { + return -1; + } + + // get the indices within bounds + pinIndices(start, length); + + // find the first occurrence of the substring + const UChar *match = u_strFindFirst(fArray + start, length, srcChars + srcStart, srcLength); + if(match == NULL) { + return -1; + } else { + return (int32_t)(match - fArray); + } +} + +int32_t +UnicodeString::doIndexOf(UChar c, + int32_t start, + int32_t length) const +{ + // pin indices + pinIndices(start, length); + + // find the first occurrence of c + const UChar *match = u_memchr(fArray + start, c, length); + if(match == NULL) { + return -1; + } else { + return (int32_t)(match - fArray); + } +} + +int32_t +UnicodeString::doIndexOf(UChar32 c, + int32_t start, + int32_t length) const { + // pin indices + pinIndices(start, length); + + // find the first occurrence of c + const UChar *match = u_memchr32(fArray + start, c, length); + if(match == NULL) { + return -1; + } else { + return (int32_t)(match - fArray); + } +} + +int32_t +UnicodeString::lastIndexOf(const UChar *srcChars, + int32_t srcStart, + int32_t srcLength, + int32_t start, + int32_t length) const +{ + if(isBogus() || srcChars == 0 || srcStart < 0 || srcLength == 0) { + return -1; + } + + // UnicodeString does not find empty substrings + if(srcLength < 0 && srcChars[srcStart] == 0) { + return -1; + } + + // get the indices within bounds + pinIndices(start, length); + + // find the last occurrence of the substring + const UChar *match = u_strFindLast(fArray + start, length, srcChars + srcStart, srcLength); + if(match == NULL) { + return -1; + } else { + return (int32_t)(match - fArray); + } +} + +int32_t +UnicodeString::doLastIndexOf(UChar c, + int32_t start, + int32_t length) const +{ + if(isBogus()) { + return -1; + } + + // pin indices + pinIndices(start, length); + + // find the last occurrence of c + const UChar *match = u_memrchr(fArray + start, c, length); + if(match == NULL) { + return -1; + } else { + return (int32_t)(match - fArray); + } +} + +int32_t +UnicodeString::doLastIndexOf(UChar32 c, + int32_t start, + int32_t length) const { + // pin indices + pinIndices(start, length); + + // find the last occurrence of c + const UChar *match = u_memrchr32(fArray + start, c, length); + if(match == NULL) { + return -1; + } else { + return (int32_t)(match - fArray); + } +} + +//======================================== +// Write implementation +//======================================== + +UnicodeString& +UnicodeString::findAndReplace(int32_t start, + int32_t length, + const UnicodeString& oldText, + int32_t oldStart, + int32_t oldLength, + const UnicodeString& newText, + int32_t newStart, + int32_t newLength) +{ + if(isBogus() || oldText.isBogus() || newText.isBogus()) { + return *this; + } + + pinIndices(start, length); + oldText.pinIndices(oldStart, oldLength); + newText.pinIndices(newStart, newLength); + + if(oldLength == 0) { + return *this; + } + + while(length > 0 && length >= oldLength) { + int32_t pos = indexOf(oldText, oldStart, oldLength, start, length); + if(pos < 0) { + // no more oldText's here: done + break; + } else { + // we found oldText, replace it by newText and go beyond it + replace(pos, oldLength, newText, newStart, newLength); + length -= pos + oldLength - start; + start = pos + newLength; + } + } + + return *this; +} + + +void +UnicodeString::setToBogus() +{ + releaseArray(); + + fArray = 0; + fCapacity = fLength = 0; + fFlags = kIsBogus; +} + +// turn a bogus string into an empty one +void +UnicodeString::unBogus() { + if(fFlags & kIsBogus) { + fArray = fStackBuffer; + fLength = 0; + fCapacity = US_STACKBUF_SIZE; + fFlags = kShortString; + } +} + +// setTo() analogous to the readonly-aliasing constructor with the same signature +UnicodeString & +UnicodeString::setTo(UBool isTerminated, + const UChar *text, + int32_t textLength) +{ + if(fFlags & kOpenGetBuffer) { + // do not modify a string that has an "open" getBuffer(minCapacity) + return *this; + } + + if(text == NULL) { + // treat as an empty string, do not alias + releaseArray(); + fLength = 0; + fCapacity = US_STACKBUF_SIZE; + fArray = fStackBuffer; + fFlags = kShortString; + return *this; + } + + if( textLength < -1 || + (textLength == -1 && !isTerminated) || + (textLength >= 0 && isTerminated && text[textLength] != 0) + ) { + setToBogus(); + return *this; + } + + releaseArray(); + + fArray = (UChar *)text; + if(textLength != -1) { + fLength = textLength; + fCapacity = isTerminated ? fLength + 1 : fLength; + } else { + // text is terminated, or else it would have failed the above test + fLength = u_strlen(text); + fCapacity = fLength + 1; + } + + fFlags = kReadonlyAlias; + return *this; +} + +// setTo() analogous to the writable-aliasing constructor with the same signature +UnicodeString & +UnicodeString::setTo(UChar *buffer, + int32_t buffLength, + int32_t buffCapacity) { + if(fFlags & kOpenGetBuffer) { + // do not modify a string that has an "open" getBuffer(minCapacity) + return *this; + } + + if(buffer == NULL) { + // treat as an empty string, do not alias + releaseArray(); + fLength = 0; + fCapacity = US_STACKBUF_SIZE; + fArray = fStackBuffer; + fFlags = kShortString; + return *this; + } + + if(buffLength < -1 || buffCapacity < 0 || buffLength > buffCapacity) { + setToBogus(); + return *this; + } else if(buffLength == -1) { + // buffLength = u_strlen(buff); but do not look beyond buffCapacity + const UChar *p = buffer, *limit = buffer + buffCapacity; + while(p != limit && *p != 0) { + ++p; + } + buffLength = (int32_t)(p - buffer); + } + + releaseArray(); + + fArray = buffer; + fLength = buffLength; + fCapacity = buffCapacity; + fFlags = kWritableAlias; + return *this; +} + +UnicodeString& +UnicodeString::setCharAt(int32_t offset, + UChar c) +{ + if(cloneArrayIfNeeded() && fLength > 0) { + if(offset < 0) { + offset = 0; + } else if(offset >= fLength) { + offset = fLength - 1; + } + + fArray[offset] = c; + } + return *this; +} + +UnicodeString& +UnicodeString::doReplace( int32_t start, + int32_t length, + const UnicodeString& src, + int32_t srcStart, + int32_t srcLength) +{ + if(!src.isBogus()) { + // pin the indices to legal values + src.pinIndices(srcStart, srcLength); + + // get the characters from src + // and replace the range in ourselves with them + return doReplace(start, length, src.getArrayStart(), srcStart, srcLength); + } else { + // remove the range + return doReplace(start, length, 0, 0, 0); + } +} + +UnicodeString& +UnicodeString::doReplace(int32_t start, + int32_t length, + const UChar *srcChars, + int32_t srcStart, + int32_t srcLength) +{ + if(isBogus()) { + return *this; + } + + if(srcChars == 0) { + srcStart = srcLength = 0; + } else if(srcLength < 0) { + // get the srcLength if necessary + srcLength = u_strlen(srcChars + srcStart); + } + + int32_t *bufferToDelete = 0; + + // the following may change fArray but will not copy the current contents; + // therefore we need to keep the current fArray + UChar *oldArray = fArray; + int32_t oldLength = fLength; + + // pin the indices to legal values + pinIndices(start, length); + + // calculate the size of the string after the replace + int32_t newSize = oldLength - length + srcLength; + + // clone our array and allocate a bigger array if needed + if(!cloneArrayIfNeeded(newSize, newSize + (newSize >> 2) + kGrowSize, + FALSE, &bufferToDelete) + ) { + return *this; + } + + // now do the replace + + if(fArray != oldArray) { + // if fArray changed, then we need to copy everything except what will change + us_arrayCopy(oldArray, 0, fArray, 0, start); + us_arrayCopy(oldArray, start + length, + fArray, start + srcLength, + oldLength - (start + length)); + } else if(length != srcLength) { + // fArray did not change; copy only the portion that isn't changing, leaving a hole + us_arrayCopy(oldArray, start + length, + fArray, start + srcLength, + oldLength - (start + length)); + } + + // now fill in the hole with the new string + us_arrayCopy(srcChars, srcStart, getArrayStart(), start, srcLength); + + fLength = newSize; + + // delayed delete in case srcChars == fArray when we started, and + // to keep oldArray alive for the above operations + if (bufferToDelete) { + uprv_free(bufferToDelete); + } + + return *this; +} + +/** + * Replaceable API + */ +void +UnicodeString::handleReplaceBetween(int32_t start, + int32_t limit, + const UnicodeString& text) { + replaceBetween(start, limit, text); +} + +/** + * Replaceable API + */ +void +UnicodeString::copy(int32_t start, int32_t limit, int32_t dest) { + if (limit <= start) { + return; // Nothing to do; avoid bogus malloc call + } + UChar* text = (UChar*) uprv_malloc( sizeof(UChar) * (limit - start) ); + extractBetween(start, limit, text, 0); + insert(dest, text, 0, limit - start); + uprv_free(text); +} + +/** + * Replaceable API + * + * NOTE: This is for the Replaceable class. There is no rep.cpp, + * so we implement this function here. + */ +UBool Replaceable::hasMetaData() const { + return TRUE; +} + +/** + * Replaceable API + */ +UBool UnicodeString::hasMetaData() const { + return FALSE; +} + +UnicodeString& +UnicodeString::doReverse(int32_t start, + int32_t length) +{ + if(fLength <= 1 || !cloneArrayIfNeeded()) { + return *this; + } + + // pin the indices to legal values + pinIndices(start, length); + + UChar *left = getArrayStart() + start; + UChar *right = getArrayStart() + start + length; + UChar swap; + UBool hasSupplementary = FALSE; + + while(left < --right) { + hasSupplementary |= (UBool)UTF_IS_LEAD(swap = *left); + hasSupplementary |= (UBool)UTF_IS_LEAD(*left++ = *right); + *right = swap; + } + + /* if there are supplementary code points in the reversed range, then re-swap their surrogates */ + if(hasSupplementary) { + UChar swap2; + + left = getArrayStart() + start; + right = getArrayStart() + start + length - 1; // -1 so that we can look at *(left+1) if left= targetLength || !cloneArrayIfNeeded(targetLength)) { + return FALSE; + } else { + // move contents up by padding width + int32_t start = targetLength - fLength; + us_arrayCopy(fArray, 0, fArray, start, fLength); + + // fill in padding character + while(--start >= 0) { + fArray[start] = padChar; + } + fLength = targetLength; + return TRUE; + } +} + +UBool +UnicodeString::padTrailing(int32_t targetLength, + UChar padChar) +{ + if(fLength >= targetLength || !cloneArrayIfNeeded(targetLength)) { + return FALSE; + } else { + // fill in padding character + int32_t length = targetLength; + while(--length >= fLength) { + fArray[length] = padChar; + } + fLength = targetLength; + return TRUE; + } +} + +//======================================== +// Hashing +//======================================== +int32_t +UnicodeString::doHashCode() const +{ + /* Delegate hash computation to uhash. This makes UnicodeString + * hashing consistent with UChar* hashing. */ + int32_t hashCode = uhash_hashUCharsN(getArrayStart(), fLength); + if (hashCode == kInvalidHashCode) { + hashCode = kEmptyHashCode; + } + return hashCode; +} + +//======================================== +// External Buffer +//======================================== + +UChar * +UnicodeString::getBuffer(int32_t minCapacity) { + if(minCapacity>=-1 && cloneArrayIfNeeded(minCapacity)) { + fFlags|=kOpenGetBuffer; + fLength=0; + return fArray; + } else { + return 0; + } +} + +void +UnicodeString::releaseBuffer(int32_t newLength) { + if(fFlags&kOpenGetBuffer && newLength>=-1) { + // set the new fLength + if(newLength==-1) { + // the new length is the string length, capped by fCapacity + const UChar *p=fArray, *limit=fArray+fCapacity; + while(p1, or + * the buffer is too small. + * Return FALSE if memory could not be allocated. + */ + if(forceClone || + fFlags & kBufferIsReadonly || + fFlags & kRefCounted && refCount() > 1 || + newCapacity > fCapacity + ) { + // save old values + UChar *array = fArray; + uint16_t flags = fFlags; + + // check growCapacity for default value and use of the stack buffer + if(growCapacity == -1) { + growCapacity = newCapacity; + } else if(newCapacity <= US_STACKBUF_SIZE && growCapacity > US_STACKBUF_SIZE) { + growCapacity = US_STACKBUF_SIZE; + } + + // allocate a new array + if(allocate(growCapacity) || + newCapacity < growCapacity && allocate(newCapacity) + ) { + if(doCopyArray) { + // copy the contents + // do not copy more than what fits - it may be smaller than before + if(fCapacity < fLength) { + fLength = fCapacity; + } + us_arrayCopy(array, 0, fArray, 0, fLength); + } else { + fLength = 0; + } + + // release the old array + if(flags & kRefCounted) { + // the array is refCounted; decrement and release if 0 + int32_t *pRefCount = ((int32_t *)array - 1); + if(umtx_atomic_dec(pRefCount) == 0) { + if(pBufferToDelete == 0) { + uprv_free(pRefCount); + } else { + // the caller requested to delete it himself + *pBufferToDelete = pRefCount; + } + } + } + } else { + // not enough memory for growCapacity and not even for the smaller newCapacity + // reset the old values for setToBogus() to release the array + fArray = array; + fFlags = flags; + setToBogus(); + return FALSE; + } + } + return TRUE; +} +U_NAMESPACE_END + +#ifdef U_STATIC_IMPLEMENTATION +/* +This should never be called. It is defined here to make sure that the +virtual vector deleting destructor is defined within unistr.cpp. +The vector deleting destructor is already a part of UObject, +but defining it here makes sure that it is included with this object file. +This makes sure that static library dependencies are kept to a minimum. +*/ +static void uprv_UnicodeStringDummy(void) { + U_NAMESPACE_USE + delete [] (new UnicodeString[2]); +} +#endif + diff --git a/Build/source/libs/icu/icu-xetex/common/unistr_case.cpp b/Build/source/libs/icu/icu-xetex/common/unistr_case.cpp new file mode 100644 index 00000000000..e557b4da7bd --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unistr_case.cpp @@ -0,0 +1,246 @@ +/* +******************************************************************************* +* +* Copyright (C) 1999-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: unistr_case.cpp +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:2 +* +* created on: 2004aug19 +* created by: Markus W. Scherer +* +* Case-mapping functions moved here from unistr.cpp +*/ + +#include "unicode/utypes.h" +#include "unicode/putil.h" +#include "unicode/locid.h" +#include "cstring.h" +#include "cmemory.h" +#include "unicode/ustring.h" +#include "unicode/unistr.h" +#include "unicode/uchar.h" +#include "unicode/ubrk.h" +#include "ustr_imp.h" +#include "unormimp.h" +#include "uhash.h" + +U_NAMESPACE_BEGIN + +//======================================== +// Read-only implementation +//======================================== + +int8_t +UnicodeString::doCaseCompare(int32_t start, + int32_t length, + const UChar *srcChars, + int32_t srcStart, + int32_t srcLength, + uint32_t options) const +{ + // compare illegal string values + // treat const UChar *srcChars==NULL as an empty string + if(isBogus()) { + return -1; + } + + // pin indices to legal values + pinIndices(start, length); + + if(srcChars == NULL) { + srcStart = srcLength = 0; + } + + // get the correct pointer + const UChar *chars = getArrayStart(); + + chars += start; + srcChars += srcStart; + + if(chars != srcChars) { + UErrorCode errorCode=U_ZERO_ERROR; + int32_t result=u_strcmpFold(chars, length, srcChars, srcLength, + options|U_COMPARE_IGNORE_CASE, &errorCode); + if(result!=0) { + return (int8_t)(result >> 24 | 1); + } + } else { + // get the srcLength if necessary + if(srcLength < 0) { + srcLength = u_strlen(srcChars + srcStart); + } + if(length != srcLength) { + return (int8_t)((length - srcLength) >> 24 | 1); + } + } + return 0; +} + +//======================================== +// Write implementation +//======================================== + +/* + * Implement argument checking and buffer handling + * for string case mapping as a common function. + */ + +UnicodeString & +UnicodeString::caseMap(BreakIterator *titleIter, + const char *locale, + uint32_t options, + int32_t toWhichCase) { + if(fLength <= 0) { + // nothing to do + return *this; + } + + UErrorCode errorCode; + + errorCode = U_ZERO_ERROR; + const UCaseProps *csp=ucase_getSingleton(&errorCode); + if(U_FAILURE(errorCode)) { + setToBogus(); + return *this; + } + + // We need to allocate a new buffer for the internal string case mapping function. + // This is very similar to how doReplace() below keeps the old array pointer + // and deletes the old array itself after it is done. + // In addition, we are forcing cloneArrayIfNeeded() to always allocate a new array. + UChar *oldArray = fArray; + int32_t oldLength = fLength; + int32_t *bufferToDelete = 0; + + // Make sure that if the string is in fStackBuffer we do not overwrite it! + int32_t capacity; + if(fLength <= US_STACKBUF_SIZE) { + if(fArray == fStackBuffer) { + capacity = 2 * US_STACKBUF_SIZE; // make sure that cloneArrayIfNeeded() allocates a new buffer + } else { + capacity = US_STACKBUF_SIZE; + } + } else { + capacity = fLength + 20; + } + if(!cloneArrayIfNeeded(capacity, capacity, FALSE, &bufferToDelete, TRUE)) { + return *this; + } + + // Case-map, and if the result is too long, then reallocate and repeat. + do { + errorCode = U_ZERO_ERROR; + if(toWhichCase==TO_LOWER) { + fLength = ustr_toLower(csp, fArray, fCapacity, + oldArray, oldLength, + locale, &errorCode); + } else if(toWhichCase==TO_UPPER) { + fLength = ustr_toUpper(csp, fArray, fCapacity, + oldArray, oldLength, + locale, &errorCode); + } else if(toWhichCase==TO_TITLE) { +#if UCONFIG_NO_BREAK_ITERATION + errorCode=U_UNSUPPORTED_ERROR; +#else + fLength = ustr_toTitle(csp, fArray, fCapacity, + oldArray, oldLength, + (UBreakIterator *)titleIter, locale, options, &errorCode); +#endif + } else { + fLength = ustr_foldCase(csp, fArray, fCapacity, + oldArray, oldLength, + options, + &errorCode); + } + } while(errorCode==U_BUFFER_OVERFLOW_ERROR && cloneArrayIfNeeded(fLength, fLength, FALSE)); + + if (bufferToDelete) { + uprv_free(bufferToDelete); + } + if(U_FAILURE(errorCode)) { + setToBogus(); + } + return *this; +} + +UnicodeString & +UnicodeString::toLower() { + return caseMap(0, Locale::getDefault().getName(), 0, TO_LOWER); +} + +UnicodeString & +UnicodeString::toLower(const Locale &locale) { + return caseMap(0, locale.getName(), 0, TO_LOWER); +} + +UnicodeString & +UnicodeString::toUpper() { + return caseMap(0, Locale::getDefault().getName(), 0, TO_UPPER); +} + +UnicodeString & +UnicodeString::toUpper(const Locale &locale) { + return caseMap(0, locale.getName(), 0, TO_UPPER); +} + +#if !UCONFIG_NO_BREAK_ITERATION + +UnicodeString & +UnicodeString::toTitle(BreakIterator *titleIter) { + return caseMap(titleIter, Locale::getDefault().getName(), 0, TO_TITLE); +} + +UnicodeString & +UnicodeString::toTitle(BreakIterator *titleIter, const Locale &locale) { + return caseMap(titleIter, locale.getName(), 0, TO_TITLE); +} + +UnicodeString & +UnicodeString::toTitle(BreakIterator *titleIter, const Locale &locale, uint32_t options) { + return caseMap(titleIter, locale.getName(), options, TO_TITLE); +} + +#endif + +UnicodeString & +UnicodeString::foldCase(uint32_t options) { + /* The Locale parameter isn't used. Use "" instead. */ + return caseMap(0, "", options, FOLD_CASE); +} + +U_NAMESPACE_END + +// Defined here to reduce dependencies on break iterator +U_CAPI int32_t U_EXPORT2 +uhash_hashCaselessUnicodeString(const UHashTok key) { + U_NAMESPACE_USE + const UnicodeString *str = (const UnicodeString*) key.pointer; + if (str == NULL) { + return 0; + } + // Inefficient; a better way would be to have a hash function in + // UnicodeString that does case folding on the fly. + UnicodeString copy(*str); + return copy.foldCase().hashCode(); +} + +// Defined here to reduce dependencies on break iterator +U_CAPI UBool U_EXPORT2 +uhash_compareCaselessUnicodeString(const UHashTok key1, const UHashTok key2) { + U_NAMESPACE_USE + const UnicodeString *str1 = (const UnicodeString*) key1.pointer; + const UnicodeString *str2 = (const UnicodeString*) key2.pointer; + if (str1 == str2) { + return TRUE; + } + if (str1 == NULL || str2 == NULL) { + return FALSE; + } + return str1->caseCompare(*str2, U_FOLD_CASE_DEFAULT) == 0; +} + diff --git a/Build/source/libs/icu/icu-xetex/common/unistr_cnv.cpp b/Build/source/libs/icu/icu-xetex/common/unistr_cnv.cpp new file mode 100644 index 00000000000..a5166e09d8b --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unistr_cnv.cpp @@ -0,0 +1,372 @@ +/* +******************************************************************************* +* +* Copyright (C) 1999-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: unistr_cnv.cpp +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:2 +* +* created on: 2004aug19 +* created by: Markus W. Scherer +* +* Character conversion functions moved here from unistr.cpp +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "unicode/putil.h" +#include "cstring.h" +#include "cmemory.h" +#include "unicode/ustring.h" +#include "unicode/unistr.h" +#include "unicode/ucnv.h" +#include "putilimp.h" +#include "ustr_cnv.h" +#include "ustr_imp.h" + +U_NAMESPACE_BEGIN + +//======================================== +// Constructors +//======================================== + +UnicodeString::UnicodeString(const char *codepageData, + const char *codepage) + : fLength(0), + fCapacity(US_STACKBUF_SIZE), + fArray(fStackBuffer), + fFlags(kShortString) +{ + if(codepageData != 0) { + doCodepageCreate(codepageData, (int32_t)uprv_strlen(codepageData), codepage); + } +} + + +UnicodeString::UnicodeString(const char *codepageData, + int32_t dataLength, + const char *codepage) + : fLength(0), + fCapacity(US_STACKBUF_SIZE), + fArray(fStackBuffer), + fFlags(kShortString) +{ + if(codepageData != 0) { + doCodepageCreate(codepageData, dataLength, codepage); + } +} + +UnicodeString::UnicodeString(const char *src, int32_t srcLength, + UConverter *cnv, + UErrorCode &errorCode) + : fLength(0), + fCapacity(US_STACKBUF_SIZE), + fArray(fStackBuffer), + fFlags(kShortString) +{ + if(U_SUCCESS(errorCode)) { + // check arguments + if(src==NULL) { + // treat as an empty string, do nothing more + } else if(srcLength<-1) { + errorCode=U_ILLEGAL_ARGUMENT_ERROR; + } else { + // get input length + if(srcLength==-1) { + srcLength=(int32_t)uprv_strlen(src); + } + if(srcLength>0) { + if(cnv!=0) { + // use the provided converter + ucnv_resetToUnicode(cnv); + doCodepageCreate(src, srcLength, cnv, errorCode); + } else { + // use the default converter + cnv=u_getDefaultConverter(&errorCode); + doCodepageCreate(src, srcLength, cnv, errorCode); + u_releaseDefaultConverter(cnv); + } + } + } + + if(U_FAILURE(errorCode)) { + setToBogus(); + } + } +} + +//======================================== +// Codeset conversion +//======================================== +int32_t +UnicodeString::extract(int32_t start, + int32_t length, + char *target, + uint32_t dstSize, + const char *codepage) const +{ + // if the arguments are illegal, then do nothing + if(/*dstSize < 0 || */(dstSize > 0 && target == 0)) { + return 0; + } + + // pin the indices to legal values + pinIndices(start, length); + + // create the converter + UConverter *converter; + UErrorCode status = U_ZERO_ERROR; + + // just write the NUL if the string length is 0 + if(length == 0) { + if(dstSize >= 0x80000000) { + // careful: dstSize is unsigned! (0xffffffff means "unlimited") + // make sure that the NUL-termination works (takes int32_t) + dstSize=0x7fffffff; + } + return u_terminateChars(target, dstSize, 0, &status); + } + + // if the codepage is the default, use our cache + // if it is an empty string, then use the "invariant character" conversion + if (codepage == 0) { + converter = u_getDefaultConverter(&status); + } else if (*codepage == 0) { + // use the "invariant characters" conversion + int32_t destLength; + // careful: dstSize is unsigned! (0xffffffff means "unlimited") + if(dstSize >= 0x80000000) { + destLength = length; + // make sure that the NUL-termination works (takes int32_t) + dstSize=0x7fffffff; + } else if(length <= (int32_t)dstSize) { + destLength = length; + } else { + destLength = (int32_t)dstSize; + } + u_UCharsToChars(getArrayStart() + start, target, destLength); + return u_terminateChars(target, (int32_t)dstSize, length, &status); + } else { + converter = ucnv_open(codepage, &status); + } + + length = doExtract(start, length, target, (int32_t)dstSize, converter, status); + + // close the converter + if (codepage == 0) { + u_releaseDefaultConverter(converter); + } else { + ucnv_close(converter); + } + + return length; +} + +int32_t +UnicodeString::extract(char *dest, int32_t destCapacity, + UConverter *cnv, + UErrorCode &errorCode) const +{ + if(U_FAILURE(errorCode)) { + return 0; + } + + if(isBogus() || destCapacity<0 || (destCapacity>0 && dest==0)) { + errorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + // nothing to do? + if(fLength<=0) { + return u_terminateChars(dest, destCapacity, 0, &errorCode); + } + + // get the converter + UBool isDefaultConverter; + if(cnv==0) { + isDefaultConverter=TRUE; + cnv=u_getDefaultConverter(&errorCode); + if(U_FAILURE(errorCode)) { + return 0; + } + } else { + isDefaultConverter=FALSE; + ucnv_resetFromUnicode(cnv); + } + + // convert + int32_t length=doExtract(0, fLength, dest, destCapacity, cnv, errorCode); + + // release the converter + if(isDefaultConverter) { + u_releaseDefaultConverter(cnv); + } + + return length; +} + +int32_t +UnicodeString::doExtract(int32_t start, int32_t length, + char *dest, int32_t destCapacity, + UConverter *cnv, + UErrorCode &errorCode) const +{ + if(U_FAILURE(errorCode)) { + if(destCapacity!=0) { + *dest=0; + } + return 0; + } + + const UChar *src=fArray+start, *srcLimit=src+length; + char *originalDest=dest; + const char *destLimit; + + if(destCapacity==0) { + destLimit=dest=0; + } else if(destCapacity==-1) { + // Pin the limit to U_MAX_PTR if the "magic" destCapacity is used. + destLimit=(char*)U_MAX_PTR(dest); + // for NUL-termination, translate into highest int32_t + destCapacity=0x7fffffff; + } else { + destLimit=dest+destCapacity; + } + + // perform the conversion + ucnv_fromUnicode(cnv, &dest, destLimit, &src, srcLimit, 0, TRUE, &errorCode); + length=(int32_t)(dest-originalDest); + + // if an overflow occurs, then get the preflighting length + if(errorCode==U_BUFFER_OVERFLOW_ERROR) { + char buffer[1024]; + + destLimit=buffer+sizeof(buffer); + do { + dest=buffer; + errorCode=U_ZERO_ERROR; + ucnv_fromUnicode(cnv, &dest, destLimit, &src, srcLimit, 0, TRUE, &errorCode); + length+=(int32_t)(dest-buffer); + } while(errorCode==U_BUFFER_OVERFLOW_ERROR); + } + + return u_terminateChars(originalDest, destCapacity, length, &errorCode); +} + +void +UnicodeString::doCodepageCreate(const char *codepageData, + int32_t dataLength, + const char *codepage) +{ + // if there's nothing to convert, do nothing + if(codepageData == 0 || dataLength == 0 || dataLength < -1) { + return; + } + if(dataLength == -1) { + dataLength = (int32_t)uprv_strlen(codepageData); + } + + UErrorCode status = U_ZERO_ERROR; + + // create the converter + // if the codepage is the default, use our cache + // if it is an empty string, then use the "invariant character" conversion + UConverter *converter = (codepage == 0 ? + u_getDefaultConverter(&status) : + *codepage == 0 ? + 0 : + ucnv_open(codepage, &status)); + + // if we failed, set the appropriate flags and return + if(U_FAILURE(status)) { + setToBogus(); + return; + } + + // perform the conversion + if(converter == 0) { + // use the "invariant characters" conversion + if(cloneArrayIfNeeded(dataLength, dataLength, FALSE)) { + u_charsToUChars(codepageData, getArrayStart(), dataLength); + fLength = dataLength; + } else { + setToBogus(); + } + return; + } + + // convert using the real converter + doCodepageCreate(codepageData, dataLength, converter, status); + if(U_FAILURE(status)) { + setToBogus(); + } + + // close the converter + if(codepage == 0) { + u_releaseDefaultConverter(converter); + } else { + ucnv_close(converter); + } +} + +void +UnicodeString::doCodepageCreate(const char *codepageData, + int32_t dataLength, + UConverter *converter, + UErrorCode &status) +{ + if(U_FAILURE(status)) { + return; + } + + // set up the conversion parameters + const char *mySource = codepageData; + const char *mySourceEnd = mySource + dataLength; + UChar *myTarget; + + // estimate the size needed: + // 1.25 UChar's per source byte should cover most cases + int32_t arraySize = dataLength + (dataLength >> 2); + + // we do not care about the current contents + UBool doCopyArray = FALSE; + for(;;) { + if(!cloneArrayIfNeeded(arraySize, arraySize, doCopyArray)) { + setToBogus(); + break; + } + + // perform the conversion + myTarget = fArray + fLength; + ucnv_toUnicode(converter, &myTarget, fArray + fCapacity, + &mySource, mySourceEnd, 0, TRUE, &status); + + // update the conversion parameters + fLength = (int32_t)(myTarget - fArray); + + // allocate more space and copy data, if needed + if(status == U_BUFFER_OVERFLOW_ERROR) { + // reset the error code + status = U_ZERO_ERROR; + + // keep the previous conversion results + doCopyArray = TRUE; + + // estimate the new size needed, larger than before + // try 2 UChar's per remaining source byte + arraySize = (int32_t)(fLength + 2 * (mySourceEnd - mySource)); + } else { + break; + } + } +} + +U_NAMESPACE_END + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unistr_props.cpp b/Build/source/libs/icu/icu-xetex/common/unistr_props.cpp new file mode 100644 index 00000000000..a82408a558a --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unistr_props.cpp @@ -0,0 +1,72 @@ +/* +******************************************************************************* +* +* Copyright (C) 1999-2006, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: unistr_props.cpp +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:2 +* +* created on: 2004aug25 +* created by: Markus W. Scherer +* +* Character property dependent functions moved here from unistr.cpp +*/ + +#include "unicode/utypes.h" +#include "unicode/uchar.h" +#include "unicode/unistr.h" + +U_NAMESPACE_BEGIN + +UnicodeString& +UnicodeString::trim() +{ + if(isBogus()) { + return *this; + } + + UChar32 c; + int32_t i = fLength, length; + + // first cut off trailing white space + for(;;) { + length = i; + if(i <= 0) { + break; + } + UTF_PREV_CHAR(fArray, 0, i, c); + if(!(c == 0x20 || u_isWhitespace(c))) { + break; + } + } + if(length < fLength) { + fLength = length; + } + + // find leading white space + int32_t start; + i = 0; + for(;;) { + start = i; + if(i >= length) { + break; + } + UTF_NEXT_CHAR(fArray, i, length, c); + if(!(c == 0x20 || u_isWhitespace(c))) { + break; + } + } + + // move string forward over leading white space + if(start > 0) { + doReplace(0, start, 0, 0, 0); + } + + return *this; +} + +U_NAMESPACE_END diff --git a/Build/source/libs/icu/icu-xetex/common/unorm.cpp b/Build/source/libs/icu/icu-xetex/common/unorm.cpp new file mode 100644 index 00000000000..00ee9ec3e7d --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unorm.cpp @@ -0,0 +1,4099 @@ +/* +****************************************************************************** +* Copyright (c) 1996-2007, International Business Machines +* Corporation and others. All Rights Reserved. +****************************************************************************** +* File unorm.cpp +* +* Created by: Vladimir Weinstein 12052000 +* +* Modification history : +* +* Date Name Description +* 02/01/01 synwee Added normalization quickcheck enum and method. +* 02/12/01 synwee Commented out quickcheck util api has been approved +* Added private method for doing FCD checks +* 02/23/01 synwee Modified quickcheck and checkFCE to run through +* string for codepoints < 0x300 for the normalization +* mode NFC. +* 05/25/01+ Markus Scherer total rewrite, implement all normalization here +* instead of just wrappers around normlzr.cpp, +* load unorm.dat, support Unicode 3.1 with +* supplementary code points, etc. +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_NORMALIZATION + +#include "unicode/udata.h" +#include "unicode/uchar.h" +#include "unicode/ustring.h" +#include "unicode/uiter.h" +#include "unicode/uniset.h" +#include "unicode/usetiter.h" +#include "unicode/unorm.h" +#include "ucln_cmn.h" +#include "unormimp.h" +#include "ucase.h" +#include "cmemory.h" +#include "umutex.h" +#include "utrie.h" +#include "unicode/uset.h" +#include "udataswp.h" +#include "putilimp.h" + +/* + * Status of tailored normalization + * + * This was done initially for investigation on Unicode public review issue 7 + * (http://www.unicode.org/review/). See Jitterbug 2481. + * While the UTC at meeting #94 (2003mar) did not take up the issue, this is + * a permanent feature in ICU 2.6 in support of IDNA which requires true + * Unicode 3.2 normalization. + * (NormalizationCorrections are rolled into IDNA mapping tables.) + * + * Tailored normalization as implemented here allows to "normalize less" + * than full Unicode normalization would. + * Based internally on a UnicodeSet of code points that are + * "excluded from normalization", the normalization functions leave those + * code points alone ("inert"). This means that tailored normalization + * still transforms text into a canonically equivalent form. + * It does not add decompositions to code points that do not have any or + * change decomposition results. + * + * Any function that searches for a safe boundary has not been touched, + * which means that these functions will be over-pessimistic when + * exclusions are applied. + * This should not matter because subsequent checks and normalizations + * do apply the exclusions; only a little more of the text may be processed + * than necessary under exclusions. + * + * Normalization exclusions have the following effect on excluded code points c: + * - c is not decomposed + * - c is not a composition target + * - c does not combine forward or backward for composition + * except that this is not implemented for Jamo + * - c is treated as having a combining class of 0 + */ +#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) + +U_NAMESPACE_USE + +/* + * This new implementation of the normalization code loads its data from + * unorm.dat, which is generated with the gennorm tool. + * The format of that file is described in unormimp.h . + */ + +/* -------------------------------------------------------------------------- */ + +enum { + _STACK_BUFFER_CAPACITY=100 +}; + +/* + * Constants for the bit fields in the options bit set parameter. + * These need not be public. + * A user only needs to know the currently assigned values. + * The number and positions of reserved bits per field can remain private + * and may change in future implementations. + */ +enum { + _NORM_OPTIONS_NX_MASK=0x1f, + _NORM_OPTIONS_UNICODE_MASK=0x60, + _NORM_OPTIONS_SETS_MASK=0x7f, + + _NORM_OPTIONS_UNICODE_SHIFT=5, + + /* + * The following options are used only in some composition functions. + * They use bits 12 and up to preserve lower bits for the available options + * space in unorm_compare() - + * see documentation for UNORM_COMPARE_NORM_OPTIONS_SHIFT. + */ + + /** Options bit 12, for compatibility vs. canonical decomposition. */ + _NORM_OPTIONS_COMPAT=0x1000, + /** Options bit 13, no discontiguous composition (FCC vs. NFC). */ + _NORM_OPTIONS_COMPOSE_CONTIGUOUS=0x2000 +}; + +U_CDECL_BEGIN +static inline UBool +isHangulWithoutJamoT(UChar c) { + c-=HANGUL_BASE; + return c=_NORM_MIN_HANGUL; +} + +/* + * Given isNorm32HangulOrJamo(), + * is this a Hangul syllable or a Jamo? + */ +/*static inline UBool +isHangulJamoNorm32HangulOrJamoL(uint32_t norm32) { + return norm32<_NORM_MIN_JAMO_V; +}*/ + +/* + * Given norm32 for Jamo V or T, + * is this a Jamo V? + */ +static inline UBool +isJamoVTNorm32JamoV(uint32_t norm32) { + return norm32<_NORM_JAMO_V_TOP; +} + +/* load unorm.dat ----------------------------------------------------------- */ + +/* normTrie: 32-bit trie result may contain a special extraData index with the folding offset */ +static int32_t U_CALLCONV +getFoldingNormOffset(uint32_t norm32) { + if(isNorm32LeadSurrogate(norm32)) { + return + UTRIE_BMP_INDEX_LENGTH+ + (((int32_t)norm32>>(_NORM_EXTRA_SHIFT-UTRIE_SURROGATE_BLOCK_BITS))& + (0x3ff<size>=20 && + pInfo->isBigEndian==U_IS_BIG_ENDIAN && + pInfo->charsetFamily==U_CHARSET_FAMILY && + pInfo->dataFormat[0]==0x4e && /* dataFormat="Norm" */ + pInfo->dataFormat[1]==0x6f && + pInfo->dataFormat[2]==0x72 && + pInfo->dataFormat[3]==0x6d && + pInfo->formatVersion[0]==2 && + pInfo->formatVersion[2]==UTRIE_SHIFT && + pInfo->formatVersion[3]==UTRIE_INDEX_SHIFT + ) { + uprv_memcpy(formatVersion, pInfo->formatVersion, 4); + uprv_memcpy(dataVersion, pInfo->dataVersion, 4); + return TRUE; + } else { + return FALSE; + } +} + +#endif + +static UBool U_CALLCONV +_enumPropertyStartsRange(const void *context, UChar32 start, UChar32 /*limit*/, uint32_t /*value*/) { + /* add the start code point to the USet */ + const USetAdder *sa=(const USetAdder *)context; + sa->add(sa->set, start); + return TRUE; +} + +U_CDECL_END + +#if !UNORM_HARDCODE_DATA + +static int8_t +loadNormData(UErrorCode &errorCode) { + /* load Unicode normalization data from file */ + + /* + * This lazy intialization with double-checked locking (without mutex protection for + * haveNormData==0) is transiently unsafe under certain circumstances. + * Check the readme and use u_init() if necessary. + * + * While u_init() initializes the main normalization data via this functions, + * it does not do so for exclusion sets (which are fully mutexed). + * This is because + * - there can be many exclusion sets + * - they are rarely used + * - they are not usually used in execution paths that are + * as performance-sensitive as others + * (e.g., IDNA takes more time than unorm_quickCheck() anyway) + */ + if(haveNormData==0) { + UTrie _normTrie={ 0,0,0,0,0,0,0 }, _fcdTrie={ 0,0,0,0,0,0,0 }, _auxTrie={ 0,0,0,0,0,0,0 }; + UDataMemory *data; + + const int32_t *p=NULL; + const uint8_t *pb; + + if(&errorCode==NULL || U_FAILURE(errorCode)) { + return 0; + } + + /* open the data outside the mutex block */ + data=udata_openChoice(NULL, DATA_TYPE, DATA_NAME, isAcceptable, NULL, &errorCode); + dataErrorCode=errorCode; + if(U_FAILURE(errorCode)) { + return haveNormData=-1; + } + + p=(const int32_t *)udata_getMemory(data); + pb=(const uint8_t *)(p+_NORM_INDEX_TOP); + utrie_unserialize(&_normTrie, pb, p[_NORM_INDEX_TRIE_SIZE], &errorCode); + _normTrie.getFoldingOffset=getFoldingNormOffset; + + pb+=p[_NORM_INDEX_TRIE_SIZE]+p[_NORM_INDEX_UCHAR_COUNT]*2+p[_NORM_INDEX_COMBINE_DATA_COUNT]*2; + if(p[_NORM_INDEX_FCD_TRIE_SIZE]!=0) { + utrie_unserialize(&_fcdTrie, pb, p[_NORM_INDEX_FCD_TRIE_SIZE], &errorCode); + } + pb+=p[_NORM_INDEX_FCD_TRIE_SIZE]; + + if(p[_NORM_INDEX_AUX_TRIE_SIZE]!=0) { + utrie_unserialize(&_auxTrie, pb, p[_NORM_INDEX_AUX_TRIE_SIZE], &errorCode); + _auxTrie.getFoldingOffset=getFoldingAuxOffset; + } + + if(U_FAILURE(errorCode)) { + dataErrorCode=errorCode; + udata_close(data); + return haveNormData=-1; + } + + /* in the mutex block, set the data for this process */ + umtx_lock(NULL); + if(normData==NULL) { + normData=data; + data=NULL; + + uprv_memcpy(&indexes, p, sizeof(indexes)); + uprv_memcpy(&normTrie, &_normTrie, sizeof(UTrie)); + uprv_memcpy(&fcdTrie, &_fcdTrie, sizeof(UTrie)); + uprv_memcpy(&auxTrie, &_auxTrie, sizeof(UTrie)); + } else { + p=(const int32_t *)udata_getMemory(normData); + } + + /* initialize some variables */ + extraData=(uint16_t *)((uint8_t *)(p+_NORM_INDEX_TOP)+indexes[_NORM_INDEX_TRIE_SIZE]); + combiningTable=extraData+indexes[_NORM_INDEX_UCHAR_COUNT]; + formatVersion_2_1=formatVersion[0]>2 || (formatVersion[0]==2 && formatVersion[1]>=1); + formatVersion_2_2=formatVersion[0]>2 || (formatVersion[0]==2 && formatVersion[1]>=2); + if(formatVersion_2_1) { + canonStartSets=combiningTable+ + indexes[_NORM_INDEX_COMBINE_DATA_COUNT]+ + (indexes[_NORM_INDEX_FCD_TRIE_SIZE]+indexes[_NORM_INDEX_AUX_TRIE_SIZE])/2; + } + haveNormData=1; + ucln_common_registerCleanup(UCLN_COMMON_UNORM, unorm_cleanup); + umtx_unlock(NULL); + + /* if a different thread set it first, then close the extra data */ + if(data!=NULL) { + udata_close(data); /* NULL if it was set correctly */ + } + } + + return haveNormData; +} + +#endif + +static inline UBool +_haveData(UErrorCode &errorCode) { +#if UNORM_HARDCODE_DATA + return U_SUCCESS(errorCode); +#else + if(U_FAILURE(errorCode)) { + return FALSE; + } else if(haveNormData>0) { + return TRUE; + } else if(haveNormData<0) { + errorCode=dataErrorCode; + return FALSE; + } else /* haveNormData==0 */ { + return (UBool)(loadNormData(errorCode)>0); + } +#endif +} + +U_CAPI UBool U_EXPORT2 +unorm_haveData(UErrorCode *pErrorCode) { + return _haveData(*pErrorCode); +} + +U_CAPI const uint16_t * U_EXPORT2 +unorm_getFCDTrie(UErrorCode *pErrorCode) { + if(_haveData(*pErrorCode)) { + return fcdTrie.index; + } else { + return NULL; + } +} + +/* data access primitives --------------------------------------------------- */ + +static inline uint32_t +_getNorm32(UChar c) { + return UTRIE_GET32_FROM_LEAD(&normTrie, c); +} + +static inline uint32_t +_getNorm32FromSurrogatePair(uint32_t norm32, UChar c2) { + /* + * the surrogate index in norm32 stores only the number of the surrogate index block + * see gennorm/store.c/getFoldedNormValue() + */ + norm32= + UTRIE_BMP_INDEX_LENGTH+ + ((norm32>>(_NORM_EXTRA_SHIFT-UTRIE_SURROGATE_BLOCK_BITS))& + (0x3ff<>_NORM_EXTRA_SHIFT); +} + +#if 0 +/* + * It is possible to get the FCD data from the main trie if unorm.icu + * was built without the FCD trie, although it is slower. + * This is not implemented because it is hard to test, and because it seems + * unusual to want to use FCD and not build the data file for it. + * + * Untested sample code: + */ +static inline uint16_t +_getFCD16FromNormData(UChar32 c) { + uint32_t norm32, fcd; + + norm32=_getNorm32(c); + if((norm32&_NORM_QC_NFD) && isNorm32Regular(norm32)) { + /* get the lead/trail cc from the decomposition data */ + const uint16_t *nfd=_getExtraData(norm32); + if(*nfd&_NORM_DECOMP_FLAG_LENGTH_HAS_CC) { + fcd=nfd[1]; + } + } else { + fcd=norm32&_NORM_CC_MASK; + if(fcd!=0) { + /* use the code point cc value for both lead and trail cc's */ + fcd|=fcd>>_NORM_CC_SHIFT; /* assume that the cc is in bits 15..8 */ + } + } + + return (uint16_t)fcd; +} +#endif + +/* normalization exclusion sets --------------------------------------------- */ + +/* + * Normalization exclusion UnicodeSets are used for tailored normalization; + * see the comment near the beginning of this file. + * + * By specifying one or several sets of code points, + * those code points become inert for normalization. + */ + +static const UnicodeSet * +internalGetNXHangul(UErrorCode &errorCode) { + /* internal function, does not check for incoming U_FAILURE */ + UBool isCached; + + UMTX_CHECK(NULL, (UBool)(nxCache[UNORM_NX_HANGUL]!=NULL), isCached); + + if(!isCached) { + UnicodeSet *set=new UnicodeSet(0xac00, 0xd7a3); + if(set==NULL) { + errorCode=U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + // Compact the set for caching. + set->compact(); + + umtx_lock(NULL); + if(nxCache[UNORM_NX_HANGUL]==NULL) { + nxCache[UNORM_NX_HANGUL]=set; + set=NULL; + ucln_common_registerCleanup(UCLN_COMMON_UNORM, unorm_cleanup); + } + umtx_unlock(NULL); + + delete set; + } + + return nxCache[UNORM_NX_HANGUL]; +} + +/* unorm.cpp 1.116 had and used +static const UnicodeSet * +internalGetNXFromPattern(int32_t options, const char *pattern, UErrorCode &errorCode) { + ... +} +*/ + +/* get and set an exclusion set from a serialized UnicodeSet */ +static const UnicodeSet * +internalGetSerializedNX(int32_t options, int32_t nxIndex, UErrorCode &errorCode) { + /* internal function, does not check for incoming U_FAILURE */ + UBool isCached; + + UMTX_CHECK(NULL, (UBool)(nxCache[options]!=NULL), isCached); + + if( !isCached && + canonStartSets!=NULL && + canonStartSets[nxIndex]!=0 && canonStartSets[nxIndex+1]>canonStartSets[nxIndex] + ) { + USerializedSet sset; + UnicodeSet *set; + UChar32 start, end; + int32_t i; + + if( !uset_getSerializedSet( + &sset, + canonStartSets+canonStartSets[nxIndex], + canonStartSets[nxIndex+1]-canonStartSets[nxIndex]) + ) { + errorCode=U_INVALID_FORMAT_ERROR; + return NULL; + } + + /* turn the serialized set into a UnicodeSet */ + set=new UnicodeSet(); + if(set==NULL) { + errorCode=U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + for(i=0; uset_getSerializedRange(&sset, i, &start, &end); ++i) { + set->add(start, end); + } + // Compact the set for caching. + set->compact(); + + umtx_lock(NULL); + if(nxCache[options]==NULL) { + nxCache[options]=set; + set=NULL; + ucln_common_registerCleanup(UCLN_COMMON_UNORM, unorm_cleanup); + } + umtx_unlock(NULL); + + delete set; + } + + return nxCache[options]; +} + +static const UnicodeSet * +internalGetNXCJKCompat(UErrorCode &errorCode) { + /* build a set from [[:Ideographic:]&[:NFD_QC=No:]]=[CJK Ideographs]&[has canonical decomposition] */ + return internalGetSerializedNX( + UNORM_NX_CJK_COMPAT, + _NORM_SET_INDEX_NX_CJK_COMPAT_OFFSET, + errorCode); +} + +static const UnicodeSet * +internalGetNXUnicode(uint32_t options, UErrorCode &errorCode) { + /* internal function, does not check for incoming U_FAILURE */ + int32_t nxIndex; + + options&=_NORM_OPTIONS_UNICODE_MASK; + switch(options) { + case 0: + return NULL; + case UNORM_UNICODE_3_2: + /* [:^Age=3.2:] */ + nxIndex=_NORM_SET_INDEX_NX_UNICODE32_OFFSET; + break; + default: + errorCode=U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + + /* build a set with all code points that were not designated by the specified Unicode version */ + return internalGetSerializedNX(options, nxIndex, errorCode); +} + +/* Get a decomposition exclusion set. The data must be loaded. */ +static const UnicodeSet * +internalGetNX(int32_t options, UErrorCode &errorCode) { + options&=_NORM_OPTIONS_SETS_MASK; + + UBool isCached; + + UMTX_CHECK(NULL, (UBool)(nxCache[options]!=NULL), isCached); + + if(!isCached) { + /* return basic sets */ + if(options==UNORM_NX_HANGUL) { + return internalGetNXHangul(errorCode); + } + if(options==UNORM_NX_CJK_COMPAT) { + return internalGetNXCJKCompat(errorCode); + } + if((options&_NORM_OPTIONS_UNICODE_MASK)!=0 && (options&_NORM_OPTIONS_NX_MASK)==0) { + return internalGetNXUnicode(options, errorCode); + } + + /* build a set from multiple subsets */ + UnicodeSet *set; + const UnicodeSet *other; + + set=new UnicodeSet(); + if(set==NULL) { + errorCode=U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + + if((options&UNORM_NX_HANGUL)!=0 && NULL!=(other=internalGetNXHangul(errorCode))) { + set->addAll(*other); + } + if((options&UNORM_NX_CJK_COMPAT)!=0 && NULL!=(other=internalGetNXCJKCompat(errorCode))) { + set->addAll(*other); + } + if((options&_NORM_OPTIONS_UNICODE_MASK)!=0 && NULL!=(other=internalGetNXUnicode(options, errorCode))) { + set->addAll(*other); + } + + if(U_FAILURE(errorCode)) { + delete set; + return NULL; + } + // Compact the set for caching. + set->compact(); + + umtx_lock(NULL); + if(nxCache[options]==NULL) { + nxCache[options]=set; + set=NULL; + ucln_common_registerCleanup(UCLN_COMMON_UNORM, unorm_cleanup); + } + umtx_unlock(NULL); + + delete set; + } + + return nxCache[options]; +} + +static inline const UnicodeSet * +getNX(int32_t options, UErrorCode &errorCode) { + if(U_FAILURE(errorCode) || (options&=_NORM_OPTIONS_SETS_MASK)==0) { + /* incoming failure, or no decomposition exclusions requested */ + return NULL; + } else { + return internalGetNX(options, errorCode); + } +} + +U_CFUNC const UnicodeSet * +unorm_getNX(int32_t options, UErrorCode *pErrorCode) { + return getNX(options, *pErrorCode); +} + +static inline UBool +nx_contains(const UnicodeSet *nx, UChar32 c) { + return nx!=NULL && nx->contains(c); +} + +static inline UBool +nx_contains(const UnicodeSet *nx, UChar c, UChar c2) { + return nx!=NULL && nx->contains(c2==0 ? c : U16_GET_SUPPLEMENTARY(c, c2)); +} + +/* other normalization primitives ------------------------------------------- */ + +/* get the canonical or compatibility decomposition for one character */ +static inline const UChar * +_decompose(uint32_t norm32, uint32_t qcMask, int32_t &length, + uint8_t &cc, uint8_t &trailCC) { + const UChar *p=(const UChar *)_getExtraData(norm32); + length=*p++; + + if((norm32&qcMask&_NORM_QC_NFKD)!=0 && length>=0x100) { + /* use compatibility decomposition, skip canonical data */ + p+=((length>>7)&1)+(length&_NORM_DECOMP_LENGTH_MASK); + length>>=8; + } + + if(length&_NORM_DECOMP_FLAG_LENGTH_HAS_CC) { + /* get the lead and trail cc's */ + UChar bothCCs=*p++; + cc=(uint8_t)(bothCCs>>8); + trailCC=(uint8_t)bothCCs; + } else { + /* lead and trail cc's are both 0 */ + cc=trailCC=0; + } + + length&=_NORM_DECOMP_LENGTH_MASK; + return p; +} + +/* get the canonical decomposition for one character */ +static inline const UChar * +_decompose(uint32_t norm32, int32_t &length, + uint8_t &cc, uint8_t &trailCC) { + const UChar *p=(const UChar *)_getExtraData(norm32); + length=*p++; + + if(length&_NORM_DECOMP_FLAG_LENGTH_HAS_CC) { + /* get the lead and trail cc's */ + UChar bothCCs=*p++; + cc=(uint8_t)(bothCCs>>8); + trailCC=(uint8_t)bothCCs; + } else { + /* lead and trail cc's are both 0 */ + cc=trailCC=0; + } + + length&=_NORM_DECOMP_LENGTH_MASK; + return p; +} + +/** + * Get the canonical decomposition for one code point. + * @param c code point + * @param buffer out-only buffer for algorithmic decompositions of Hangul + * @param length out-only, takes the length of the decomposition, if any + * @return pointer to decomposition, or 0 if none + * @internal + */ +U_CFUNC const UChar * +unorm_getCanonicalDecomposition(UChar32 c, UChar buffer[4], int32_t *pLength) { + uint32_t norm32; + + if(c0) { + buffer[2]=(UChar)(JAMO_T_BASE+c2); + *pLength=3; + } else { + *pLength=2; + } + + buffer[1]=(UChar)(JAMO_V_BASE+c%JAMO_V_COUNT); + buffer[0]=(UChar)(JAMO_L_BASE+c/JAMO_V_COUNT); + return buffer; + } else { + /* normal decomposition */ + uint8_t cc, trailCC; + return _decompose(norm32, *pLength, cc, trailCC); + } + } else { + return 0; + } +} + +/* + * get the combining class of (c, c2)=*p++ + * before: p>_NORM_CC_SHIFT); + } +} + +/* + * read backwards and get norm32 + * return 0 if the character is >_NORM_CC_SHIFT); +} + +/* + * is this a safe boundary character for NF*D? + * (lead cc==0) + */ +static inline UBool +_isNFDSafe(uint32_t norm32, uint32_t ccOrQCMask, uint32_t decompQCMask) { + if((norm32&ccOrQCMask)==0) { + return TRUE; /* cc==0 and no decomposition: this is NF*D safe */ + } + + /* inspect its decomposition - maybe a Hangul but not a surrogate here */ + if(isNorm32Regular(norm32) && (norm32&decompQCMask)!=0) { + int32_t length; + uint8_t cc, trailCC; + + /* decomposes, get everything from the variable-length extra data */ + _decompose(norm32, decompQCMask, length, cc, trailCC); + return cc==0; + } else { + /* no decomposition (or Hangul), test the cc directly */ + return (norm32&_NORM_CC_MASK)==0; + } +} + +/* + * is this (or does its decomposition begin with) a "true starter"? + * (cc==0 and NF*C_YES) + */ +static inline UBool +_isTrueStarter(uint32_t norm32, uint32_t ccOrQCMask, uint32_t decompQCMask) { + if((norm32&ccOrQCMask)==0) { + return TRUE; /* this is a true starter (could be Hangul or Jamo L) */ + } + + /* inspect its decomposition - not a Hangul or a surrogate here */ + if((norm32&decompQCMask)!=0) { + const UChar *p; + int32_t length; + uint8_t cc, trailCC; + + /* decomposes, get everything from the variable-length extra data */ + p=_decompose(norm32, decompQCMask, length, cc, trailCC); + if(cc==0) { + uint32_t qcMask=ccOrQCMask&_NORM_QC_MASK; + + /* does it begin with NFC_YES? */ + if((_getNorm32(p, qcMask)&qcMask)==0) { + /* yes, the decomposition begins with a true starter */ + return TRUE; + } + } + } + return FALSE; +} + +/* uchar.h */ +U_CAPI uint8_t U_EXPORT2 +u_getCombiningClass(UChar32 c) { +#if !UNORM_HARDCODE_DATA + UErrorCode errorCode=U_ZERO_ERROR; + if(_haveData(errorCode)) { +#endif + uint32_t norm32; + + UTRIE_GET32(&normTrie, c, norm32); + return (uint8_t)(norm32>>_NORM_CC_SHIFT); +#if !UNORM_HARDCODE_DATA + } else { + return 0; + } +#endif +} + +U_CFUNC UBool U_EXPORT2 +unorm_internalIsFullCompositionExclusion(UChar32 c) { +#if UNORM_HARDCODE_DATA + if(auxTrie.index!=NULL) { +#else + UErrorCode errorCode=U_ZERO_ERROR; + if(_haveData(errorCode) && auxTrie.index!=NULL) { +#endif + uint16_t aux; + + UTRIE_GET16(&auxTrie, c, aux); + return (UBool)((aux&_NORM_AUX_COMP_EX_MASK)!=0); + } else { + return FALSE; + } +} + +U_CFUNC UBool U_EXPORT2 +unorm_isCanonSafeStart(UChar32 c) { +#if UNORM_HARDCODE_DATA + if(auxTrie.index!=NULL) { +#else + UErrorCode errorCode=U_ZERO_ERROR; + if(_haveData(errorCode) && auxTrie.index!=NULL) { +#endif + uint16_t aux; + + UTRIE_GET16(&auxTrie, c, aux); + return (UBool)((aux&_NORM_AUX_UNSAFE_MASK)==0); + } else { + return FALSE; + } +} + +U_CAPI void U_EXPORT2 +unorm_getUnicodeVersion(UVersionInfo *versionInfo, UErrorCode *pErrorCode){ + if(unorm_haveData(pErrorCode)){ + uprv_memcpy(*versionInfo, dataVersion, 4); + } +} + + +U_CAPI UBool U_EXPORT2 +unorm_getCanonStartSet(UChar32 c, USerializedSet *fillSet) { +#if !UNORM_HARDCODE_DATA + UErrorCode errorCode=U_ZERO_ERROR; +#endif + if( fillSet!=NULL && (uint32_t)c<=0x10ffff && +#if !UNORM_HARDCODE_DATA + _haveData(errorCode) && +#endif + canonStartSets!=NULL + ) { + const uint16_t *table; + int32_t i, start, limit; + + /* + * binary search for c + * + * There are two search tables, + * one for BMP code points and one for supplementary ones. + * See unormimp.h for details. + */ + if(c<=0xffff) { + table=canonStartSets+canonStartSets[_NORM_SET_INDEX_CANON_SETS_LENGTH]; + start=0; + limit=canonStartSets[_NORM_SET_INDEX_CANON_BMP_TABLE_LENGTH]; + + /* each entry is a pair { c, result } */ + while(start>16); + low=(uint16_t)c; + + /* each entry is a triplet { high(c), low(c), result } */ + while(start0)) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + if(!_haveData(*pErrorCode) || auxTrie.index==NULL) { + return 0; + } + + UTRIE_GET16(&auxTrie, c, aux); + aux&=_NORM_AUX_FNC_MASK; + if(aux!=0) { + const UChar *s; + int32_t length; + + s=(const UChar *)(extraData+aux); + if(*s<0xff00) { + /* s points to the single-unit string */ + length=1; + } else { + length=*s&0xff; + ++s; + } + if(0-skippable code point? See unormimp.h. */ +U_CAPI UBool U_EXPORT2 +unorm_isNFSkippable(UChar32 c, UNormalizationMode mode) { + uint32_t norm32, mask; + uint16_t aux, fcd; + +#if !UNORM_HARDCODE_DATA + UErrorCode errorCode=U_ZERO_ERROR; + if(!_haveData(errorCode)) { + return FALSE; + } +#endif + + /* handle trivial cases; set the comparison mask for the normal ones */ + switch(mode) { + case UNORM_NONE: + return TRUE; + case UNORM_NFD: + mask=_NORM_CC_MASK|_NORM_QC_NFD; + break; + case UNORM_NFKD: + mask=_NORM_CC_MASK|_NORM_QC_NFKD; + break; + case UNORM_NFC: + /* case UNORM_FCC: */ + mask=_NORM_CC_MASK|_NORM_COMBINES_ANY|(_NORM_QC_NFC&_NORM_QC_ANY_NO); + break; + case UNORM_NFKC: + mask=_NORM_CC_MASK|_NORM_COMBINES_ANY|(_NORM_QC_NFKC&_NORM_QC_ANY_NO); + break; + case UNORM_FCD: + /* FCD: skippable if lead cc==0 and trail cc<=1 */ + if(fcdTrie.index!=NULL) { + UTRIE_GET16(&fcdTrie, c, fcd); + return fcd<=1; + } else { + return FALSE; + } + default: + return FALSE; + } + + /* check conditions (a)..(e), see unormimp.h */ + UTRIE_GET32(&normTrie, c, norm32); + if((norm32&mask)!=0) { + return FALSE; /* fails (a)..(e), not skippable */ + } + + if(modeadd(sa->set, c); + sa->add(sa->set, c+1); + } + sa->add(sa->set, HANGUL_BASE+HANGUL_COUNT); /* add Hangul+1 to continue with other properties */ +} + +U_CFUNC UNormalizationCheckResult U_EXPORT2 +unorm_getQuickCheck(UChar32 c, UNormalizationMode mode) { + static const uint32_t qcMask[UNORM_MODE_COUNT]={ + 0, 0, _NORM_QC_NFD, _NORM_QC_NFKD, _NORM_QC_NFC, _NORM_QC_NFKC + }; + + uint32_t norm32; + +#if !UNORM_HARDCODE_DATA + UErrorCode errorCode=U_ZERO_ERROR; + if(!_haveData(errorCode)) { + return UNORM_YES; + } +#endif + + UTRIE_GET32(&normTrie, c, norm32); + norm32&=qcMask[mode]; + + if(norm32==0) { + return UNORM_YES; + } else if(norm32&_NORM_QC_ANY_NO) { + return UNORM_NO; + } else /* _NORM_QC_ANY_MAYBE */ { + return UNORM_MAYBE; + } +} + +U_CFUNC uint16_t U_EXPORT2 +unorm_getFCD16FromCodePoint(UChar32 c) { + uint16_t fcd; +#if !UNORM_HARDCODE_DATA + UErrorCode errorCode; + errorCode=U_ZERO_ERROR; +#endif + + if( +#if !UNORM_HARDCODE_DATA + !_haveData(errorCode) || +#endif + fcdTrie.index==NULL + ) { + return 0; + } + + UTRIE_GET16(&fcdTrie, c, fcd); + return fcd; +} + +/* reorder UTF-16 in-place -------------------------------------------------- */ + +/* + * simpler, single-character version of _mergeOrdered() - + * bubble-insert one single code point into the preceding string + * which is already canonically ordered + * (c, c2) may or may not yet have been inserted at [current..p[ + * + * it must be p=current+lengthof(c, c2) i.e. p=current+(c2==0 ? 1 : 2) + * + * before: [start..current[ is already ordered, and + * [current..p[ may or may not hold (c, c2) but + * must be exactly the same length as (c, c2) + * after: [start..p[ is ordered + * + * returns the trailing combining class + */ +static uint8_t +_insertOrdered(const UChar *start, UChar *current, UChar *p, + UChar c, UChar c2, uint8_t cc) { + const UChar *pBack, *pPreBack; + UChar *r; + uint8_t prevCC, trailCC=cc; + + if(start=prevCC */ + pPreBack=pBack=current; + prevCC=_getPrevCC(start, pPreBack); + if(cc=prevCC) { + break; + } + pBack=pPreBack; + } + + /* + * this is where we are right now with all these pointers: + * [start..pPreBack[ 0..? code points that we can ignore + * [pPreBack..pBack[ 0..1 code points with prevCC<=cc + * [pBack..current[ 0..n code points with >cc, move up to insert (c, c2) + * [current..p[ 1 code point (c, c2) with cc + */ + + /* move the code units in between up */ + r=p; + do { + *--r=*--current; + } while(pBack!=current); + } + } + + /* insert (c, c2) */ + *current=c; + if(c2!=0) { + *(current+1)=c2; + } + + /* we know the cc of the last code point */ + return trailCC; +} + +/* + * merge two UTF-16 string parts together + * to canonically order (order by combining classes) their concatenation + * + * the two strings may already be adjacent, so that the merging is done in-place + * if the two strings are not adjacent, then the buffer holding the first one + * must be large enough + * the second string may or may not be ordered in itself + * + * before: [start..current[ is already ordered, and + * [next..limit[ may be ordered in itself, but + * is not in relation to [start..current[ + * after: [start..current+(limit-next)[ is ordered + * + * the algorithm is a simple bubble-sort that takes the characters from *next++ + * and inserts them in correct combining class order into the preceding part + * of the string + * + * since this function is called much less often than the single-code point + * _insertOrdered(), it just uses that for easier maintenance + * (see file version from before 2001aug31 for a more optimized version) + * + * returns the trailing combining class + */ +static uint8_t +_mergeOrdered(UChar *start, UChar *current, + const UChar *next, const UChar *limit, UBool isOrdered=TRUE) { + UChar *r; + UChar c, c2; + uint8_t cc, trailCC=0; + UBool adjacent; + + adjacent= current==next; + + if(start!=current || !isOrdered) { + while(next0) || destCapacity==0) + ) { + uint32_t norm32, qcMask; + UChar32 minNoMaybe; + int32_t length; + + /* initialize */ + if(!compat) { + minNoMaybe=(UChar32)indexes[_NORM_INDEX_MIN_NFD_NO_MAYBE]; + qcMask=_NORM_QC_NFD; + } else { + minNoMaybe=(UChar32)indexes[_NORM_INDEX_MIN_NFKD_NO_MAYBE]; + qcMask=_NORM_QC_NFKD; + } + + if(c0) { + dest[0]=(UChar)c; + } + return -1; + } + + /* data lookup */ + UTRIE_GET32(&normTrie, c, norm32); + if((norm32&qcMask)==0) { + /* simple case: no decomposition */ + if(c<=0xffff) { + if(destCapacity>0) { + dest[0]=(UChar)c; + } + return -1; + } else { + if(destCapacity>=2) { + dest[0]=UTF16_LEAD(c); + dest[1]=UTF16_TRAIL(c); + } + return -2; + } + } else if(isNorm32HangulOrJamo(norm32)) { + /* Hangul syllable: decompose algorithmically */ + UChar c2; + + c-=HANGUL_BASE; + + c2=(UChar)(c%JAMO_T_COUNT); + c/=JAMO_T_COUNT; + if(c2>0) { + if(destCapacity>=3) { + dest[2]=(UChar)(JAMO_T_BASE+c2); + } + length=3; + } else { + length=2; + } + + if(destCapacity>=2) { + dest[1]=(UChar)(JAMO_V_BASE+c%JAMO_V_COUNT); + dest[0]=(UChar)(JAMO_L_BASE+c/JAMO_V_COUNT); + } + return length; + } else { + /* c decomposes, get everything from the variable-length extra data */ + const UChar *p, *limit; + uint8_t cc, trailCC; + + p=_decompose(norm32, qcMask, length, cc, trailCC); + if(length<=destCapacity) { + limit=p+length; + do { + *dest++=*p++; + } while(p=0) { + /* string with length */ + limit=src+srcLength; + } else /* srcLength==-1 */ { + /* zero-terminated string */ + limit=NULL; + } + + U_ALIGN_CODE(16); + + for(;;) { + /* count code units below the minimum or with irrelevant data for the quick check */ + prevSrc=src; + if(limit==NULL) { + while((c=*src)0) { + buffer[2]=(UChar)(JAMO_T_BASE+c2); + length=3; + } else { + length=2; + } + + buffer[1]=(UChar)(JAMO_V_BASE+c%JAMO_V_COUNT); + buffer[0]=(UChar)(JAMO_L_BASE+c/JAMO_V_COUNT); + } + } else { + if(isNorm32Regular(norm32)) { + c2=0; + length=1; + } else { + /* c is a lead surrogate, get the real norm32 */ + if(src!=limit && UTF_IS_SECOND_SURROGATE(c2=*src)) { + ++src; + length=2; + norm32=_getNorm32FromSurrogatePair(norm32, c2); + } else { + c2=0; + length=1; + norm32=0; + } + } + + /* get the decomposition and the lead and trail cc's */ + if(nx_contains(nx, c, c2)) { + /* excluded: norm32==0 */ + cc=trailCC=0; + p=NULL; + } else if((norm32&qcMask)==0) { + /* c does not decompose */ + cc=trailCC=(uint8_t)(norm32>>_NORM_CC_SHIFT); + p=NULL; + } else { + /* c decomposes, get everything from the variable-length extra data */ + p=_decompose(norm32, qcMask, length, cc, trailCC); + if(length==1) { + /* fastpath a single code unit from decomposition */ + c=*p; + c2=0; + p=NULL; + } + } + } + + /* append the decomposition to the destination buffer, assume length>0 */ + if((destIndex+length)<=destCapacity) { + UChar *reorderSplit=dest+destIndex; + if(p==NULL) { + /* fastpath: single code point */ + if(cc!=0 && cc0); + } + } + } else { + /* buffer overflow */ + /* keep incrementing the destIndex for preflighting */ + destIndex+=length; + } + + prevCC=trailCC; + if(prevCC==0) { + reorderStartIndex=destIndex; + } + } + + outTrailCC=prevCC; + return destIndex; +} + +U_CAPI int32_t U_EXPORT2 +unorm_decompose(UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + UBool compat, int32_t options, + UErrorCode *pErrorCode) { + const UnicodeSet *nx; + int32_t destIndex; + uint8_t trailCC; + + if(!_haveData(*pErrorCode)) { + return 0; + } + + nx=getNX(options, *pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return 0; + } + + destIndex=_decompose(dest, destCapacity, + src, srcLength, + compat, nx, + trailCC); + + return u_terminateUChars(dest, destCapacity, destIndex, pErrorCode); +} + +/* make NFC & NFKC ---------------------------------------------------------- */ + +/* get the composition properties of the next character */ +static inline uint32_t +_getNextCombining(UChar *&p, const UChar *limit, + UChar &c, UChar &c2, + uint16_t &combiningIndex, uint8_t &cc, + const UnicodeSet *nx) { + uint32_t norm32, combineFlags; + + /* get properties */ + c=*p++; + norm32=_getNorm32(c); + + /* preset output values for most characters */ + c2=0; + combiningIndex=0; + cc=0; + + if((norm32&(_NORM_CC_MASK|_NORM_COMBINES_ANY))==0) { + return 0; + } else { + if(isNorm32Regular(norm32)) { + /* set cc etc. below */ + } else if(isNorm32HangulOrJamo(norm32)) { + /* a compatibility decomposition contained Jamos */ + combiningIndex=(uint16_t)(0xfff0|(norm32>>_NORM_EXTRA_SHIFT)); + return norm32&_NORM_COMBINES_ANY; + } else { + /* c is a lead surrogate, get the real norm32 */ + if(p!=limit && UTF_IS_SECOND_SURROGATE(c2=*p)) { + ++p; + norm32=_getNorm32FromSurrogatePair(norm32, c2); + } else { + c2=0; + return 0; + } + } + + if(nx_contains(nx, c, c2)) { + return 0; /* excluded: norm32==0 */ + } + + cc=(uint8_t)(norm32>>_NORM_CC_SHIFT); + + combineFlags=norm32&_NORM_COMBINES_ANY; + if(combineFlags!=0) { + combiningIndex=*(_getExtraData(norm32)-1); + } + return combineFlags; + } +} + +/* + * given a composition-result starter (c, c2) - which means its cc==0, + * it combines forward, it has extra data, its norm32!=0, + * it is not a Hangul or Jamo, + * get just its combineFwdIndex + * + * norm32(c) is special if and only if c2!=0 + */ +static inline uint16_t +_getCombiningIndexFromStarter(UChar c, UChar c2) { + uint32_t norm32; + + norm32=_getNorm32(c); + if(c2!=0) { + norm32=_getNorm32FromSurrogatePair(norm32, c2); + } + return *(_getExtraData(norm32)-1); +} + +/* + * Find the recomposition result for + * a forward-combining character + * (specified with a pointer to its part of the combiningTable[]) + * and a backward-combining character + * (specified with its combineBackIndex). + * + * If these two characters combine, then set (value, value2) + * with the code unit(s) of the composition character. + * + * Return value: + * 0 do not combine + * 1 combine + * >1 combine, and the composition is a forward-combining starter + * + * See unormimp.h for a description of the composition table format. + */ +static inline uint16_t +_combine(const uint16_t *table, uint16_t combineBackIndex, + uint16_t &value, uint16_t &value2) { + uint16_t key; + + /* search in the starter's composition table */ + for(;;) { + key=*table++; + if(key>=combineBackIndex) { + break; + } + table+= *table&0x8000 ? 2 : 1; + } + + /* mask off bit 15, the last-entry-in-the-list flag */ + if((key&0x7fff)==combineBackIndex) { + /* found! combine! */ + value=*table; + + /* is the composition a starter that combines forward? */ + key=(uint16_t)((value&0x2000)+1); + + /* get the composition result code point from the variable-length result value */ + if(value&0x8000) { + if(value&0x4000) { + /* surrogate pair composition result */ + value=(uint16_t)((value&0x3ff)|0xd800); + value2=*(table+1); + } else { + /* BMP composition result U+2000..U+ffff */ + value=*(table+1); + value2=0; + } + } else { + /* BMP composition result U+0000..U+1fff */ + value&=0x1fff; + value2=0; + } + + return key; + } else { + /* not found */ + return 0; + } +} + +static inline UBool +_composeHangul(UChar prev, UChar c, uint32_t norm32, const UChar *&src, const UChar *limit, + UBool compat, UChar *dest, const UnicodeSet *nx) { + if(isJamoVTNorm32JamoV(norm32)) { + /* c is a Jamo V, compose with previous Jamo L and following Jamo T */ + prev=(UChar)(prev-JAMO_L_BASE); + if(prev + * the rest of the loop body will reset starter to NULL; + * technically, a composed Hangul syllable is a starter, but it + * does not combine forward now that we have consumed all eligible Jamos; + * for Jamo V/T, combineFlags does not contain _NORM_COMBINES_FWD + */ + + } else if( + /* the starter is not a Hangul LV or Jamo V/T and */ + !(combineFwdIndex&0x8000) && + /* the combining mark is not blocked and */ + ((options&UNORM_BEFORE_PRI_29) ? + (prevCC!=cc || prevCC==0) : + (prevCC1) { + combineFwdIndex=_getCombiningIndexFromStarter((UChar)value, (UChar)value2); + } else { + starter=NULL; + } + + /* we combined; continue with looking for compositions */ + continue; + } + } + + /* no combination this time */ + prevCC=cc; + if(p==limit) { + return prevCC; + } + + /* if (c, c2) did not combine, then check if it is a starter */ + if(cc==0) { + /* found a new starter; combineFlags==0 if (c, c2) is excluded */ + if(combineFlags&_NORM_COMBINES_FWD) { + /* it may combine with something, prepare for it */ + if(c2==0) { + starterIsSupplementary=FALSE; + starter=p-1; + } else { + starterIsSupplementary=TRUE; + starter=p-2; + } + combineFwdIndex=combineBackIndex; + } else { + /* it will not combine with anything */ + starter=NULL; + } + } else if(options&_NORM_OPTIONS_COMPOSE_CONTIGUOUS) { + /* FCC: no discontiguous compositions; any intervening character blocks */ + starter=NULL; + } + } +} + +/* decompose and recompose [prevStarter..src[ */ +static const UChar * +_composePart(UChar *stackBuffer, UChar *&buffer, int32_t &bufferCapacity, int32_t &length, + const UChar *prevStarter, const UChar *src, + uint8_t &prevCC, + int32_t options, const UnicodeSet *nx, + UErrorCode *pErrorCode) { + UChar *recomposeLimit; + uint8_t trailCC; + UBool compat; + + compat=(UBool)((options&_NORM_OPTIONS_COMPAT)!=0); + + /* decompose [prevStarter..src[ */ + length=_decompose(buffer, bufferCapacity, + prevStarter, (int32_t)(src-prevStarter), + compat, nx, + trailCC); + if(length>bufferCapacity) { + if(!u_growBufferFromStatic(stackBuffer, &buffer, &bufferCapacity, 2*length, 0)) { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + length=_decompose(buffer, bufferCapacity, + prevStarter, (int32_t)(src-prevStarter), + compat, nx, + trailCC); + } + + /* recompose the decomposition */ + recomposeLimit=buffer+length; + if(length>=2) { + prevCC=_recompose(buffer, recomposeLimit, options, nx); + } + + /* return with a pointer to the recomposition and its length */ + length=(int32_t)(recomposeLimit-buffer); + return buffer; +} + +static int32_t +_compose(UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + int32_t options, const UnicodeSet *nx, + UErrorCode *pErrorCode) { + UChar stackBuffer[_STACK_BUFFER_CAPACITY]; + UChar *buffer; + int32_t bufferCapacity; + + const UChar *limit, *prevSrc, *prevStarter; + uint32_t norm32, ccOrQCMask, qcMask; + int32_t destIndex, reorderStartIndex, length; + UChar c, c2, minNoMaybe; + uint8_t cc, prevCC; + + if(options&_NORM_OPTIONS_COMPAT) { + minNoMaybe=(UChar)indexes[_NORM_INDEX_MIN_NFKC_NO_MAYBE]; + qcMask=_NORM_QC_NFKC; + } else { + minNoMaybe=(UChar)indexes[_NORM_INDEX_MIN_NFC_NO_MAYBE]; + qcMask=_NORM_QC_NFC; + } + + /* initialize */ + buffer=stackBuffer; + bufferCapacity=_STACK_BUFFER_CAPACITY; + + /* + * prevStarter points to the last character before the current one + * that is a "true" starter with cc==0 and quick check "yes". + * + * prevStarter will be used instead of looking for a true starter + * while incrementally decomposing [prevStarter..prevSrc[ + * in _composePart(). Having a good prevStarter allows to just decompose + * the entire [prevStarter..prevSrc[. + * + * When _composePart() backs out from prevSrc back to prevStarter, + * then it also backs out destIndex by the same amount. + * Therefore, at all times, the (prevSrc-prevStarter) source units + * must correspond 1:1 to destination units counted with destIndex, + * except for reordering. + * This is true for the qc "yes" characters copied in the fast loop, + * and for pure reordering. + * prevStarter must be set forward to src when this is not true: + * In _composePart() and after composing a Hangul syllable. + * + * This mechanism relies on the assumption that the decomposition of a true starter + * also begins with a true starter. gennorm/store.c checks for this. + */ + prevStarter=src; + + ccOrQCMask=_NORM_CC_MASK|qcMask; + destIndex=reorderStartIndex=0; + prevCC=0; + + /* avoid compiler warnings */ + norm32=0; + c=0; + + if(srcLength>=0) { + /* string with length */ + limit=src+srcLength; + } else /* srcLength==-1 */ { + /* zero-terminated string */ + limit=NULL; + } + + U_ALIGN_CODE(16); + + for(;;) { + /* count code units below the minimum or with irrelevant data for the quick check */ + prevSrc=src; + if(limit==NULL) { + while((c=*src)0 && + _composeHangul( + *(prevSrc-1), c, norm32, src, limit, (UBool)((options&_NORM_OPTIONS_COMPAT)!=0), + destIndex<=destCapacity ? dest+(destIndex-1) : 0, + nx) + ) { + prevStarter=src; + continue; + } + + /* the Jamo V/T did not compose into a Hangul syllable, just append to dest */ + c2=0; + length=1; + prevStarter=prevSrc; + } else { + if(isNorm32Regular(norm32)) { + c2=0; + length=1; + } else { + /* c is a lead surrogate, get the real norm32 */ + if(src!=limit && UTF_IS_SECOND_SURROGATE(c2=*src)) { + ++src; + length=2; + norm32=_getNorm32FromSurrogatePair(norm32, c2); + } else { + /* c is an unpaired lead surrogate, nothing to do */ + c2=0; + length=1; + norm32=0; + } + } + + /* we are looking at the character (c, c2) at [prevSrc..src[ */ + if(nx_contains(nx, c, c2)) { + /* excluded: norm32==0 */ + cc=0; + } else if((norm32&qcMask)==0) { + cc=(uint8_t)(norm32>>_NORM_CC_SHIFT); + } else { + const UChar *p; + uint32_t decompQCMask; + + /* + * find appropriate boundaries around this character, + * decompose the source text from between the boundaries, + * and recompose it + * + * this puts the intermediate text into the side buffer because + * it might be longer than the recomposition end result, + * or the destination buffer may be too short or missing + * + * note that destIndex may be adjusted backwards to account + * for source text that passed the quick check but needed to + * take part in the recomposition + */ + decompQCMask=(qcMask<<2)&0xf; /* decomposition quick check mask */ + + /* + * find the last true starter in [prevStarter..src[ + * it is either the decomposition of the current character (at prevSrc), + * or prevStarter + */ + if(_isTrueStarter(norm32, ccOrQCMask, decompQCMask)) { + prevStarter=prevSrc; + } else { + /* adjust destIndex: back out what had been copied with qc "yes" */ + destIndex-=(int32_t)(prevSrc-prevStarter); + } + + /* find the next true starter in [src..limit[ - modifies src to point to the next starter */ + src=_findNextStarter(src, limit, qcMask, decompQCMask, minNoMaybe); + + /* compose [prevStarter..src[ */ + p=_composePart(stackBuffer, buffer, bufferCapacity, + length, /* output */ + prevStarter, src, + prevCC, /* output */ + options, nx, + pErrorCode); + + if(p==NULL) { + destIndex=0; /* an error occurred (out of memory) */ + break; + } + + /* append the recomposed buffer contents to the destination buffer */ + if((destIndex+length)<=destCapacity) { + while(length>0) { + dest[destIndex++]=*p++; + --length; + } + } else { + /* buffer overflow */ + /* keep incrementing the destIndex for preflighting */ + destIndex+=length; + } + + /* set the next starter */ + prevStarter=src; + + continue; + } + } + + /* append the single code point (c, c2) to the destination buffer */ + if((destIndex+length)<=destCapacity) { + if(cc!=0 && cc>_NORM_CC_SHIFT); + p=NULL; + } else { + /* c decomposes, get everything from the variable-length extra data */ + p=_decompose(norm32, length, cc, trailCC); + if(length==1) { + /* fastpath a single code unit from decomposition */ + c=*p; + c2=0; + p=NULL; + } + } + + /* append the decomposition to the destination buffer, assume length>0 */ + if((destIndex+length)<=destCapacity) { + UChar *reorderSplit=dest+destIndex; + if(p==NULL) { + /* fastpath: single code point */ + if(cc!=0 && cc0); + } + } + } else { + /* buffer overflow */ + /* keep incrementing the destIndex for preflighting */ + destIndex+=length; + } + + prevCC=trailCC; + if(prevCC==0) { + reorderStartIndex=destIndex; + } + } + + return prevCC; +} + +static int32_t +unorm_makeFCD(UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + const UnicodeSet *nx, + UErrorCode *pErrorCode) { + const UChar *limit, *prevSrc, *decompStart; + int32_t destIndex, length; + UChar c, c2; + uint16_t fcd16; + int16_t prevCC, cc; + + if(!_haveData(*pErrorCode)) { + return 0; + } + + /* initialize */ + decompStart=src; + destIndex=0; + prevCC=0; + + /* avoid compiler warnings */ + c=0; + fcd16=0; + + if(srcLength>=0) { + /* string with length */ + limit=src+srcLength; + } else /* srcLength==-1 */ { + /* zero-terminated string */ + limit=NULL; + } + + U_ALIGN_CODE(16); + + for(;;) { + /* skip a run of code units below the minimum or with irrelevant data for the FCD check */ + prevSrc=src; + if(limit==NULL) { + for(;;) { + c=*src; + if(c<_NORM_MIN_WITH_LEAD_CC) { + if(c==0) { + break; + } + prevCC=(int16_t)-c; + } else if((fcd16=_getFCD16(c))==0) { + prevCC=0; + } else { + break; + } + ++src; + } + } else { + for(;;) { + if(src==limit) { + break; + } else if((c=*src)<_NORM_MIN_WITH_LEAD_CC) { + prevCC=(int16_t)-c; + } else if((fcd16=_getFCD16(c))==0) { + prevCC=0; + } else { + break; + } + ++src; + } + } + + /* + * prevCC has values from the following ranges: + * 0..0xff - the previous trail combining class + * <0 - the negative value of the previous code unit; + * that code unit was <_NORM_MIN_WITH_LEAD_CC and its _getFCD16() + * was deferred so that average text is checked faster + */ + + /* copy these code units all at once */ + if(src!=prevSrc) { + length=(int32_t)(src-prevSrc); + if((destIndex+length)<=destCapacity) { + uprv_memcpy(dest+destIndex, prevSrc, length*U_SIZEOF_UCHAR); + } + destIndex+=length; + prevSrc=src; + + /* prevCC<0 is only possible from the above loop, i.e., only if prevSrc=0 + */ + + /* end of source reached? */ + if(limit==NULL ? c==0 : src==limit) { + break; + } + + /* set a pointer to after the last source position where prevCC==0 */ + if(prevCC==0) { + decompStart=prevSrc; + } + + /* c already contains *src and fcd16 is set for it, increment src */ + ++src; + + /* check one above-minimum, relevant code unit */ + if(UTF_IS_FIRST_SURROGATE(c)) { + /* c is a lead surrogate, get the real fcd16 */ + if(src!=limit && UTF_IS_SECOND_SURROGATE(c2=*src)) { + ++src; + fcd16=_getFCD16FromSurrogatePair(fcd16, c2); + } else { + c2=0; + fcd16=0; + } + } else { + c2=0; + } + + /* we are looking at the character (c, c2) at [prevSrc..src[ */ + if(nx_contains(nx, c, c2)) { + fcd16=0; /* excluded: fcd16==0 */ + } + + /* check the combining order, get the lead cc */ + cc=(int16_t)(fcd16>>8); + if(cc==0 || cc>=prevCC) { + /* the order is ok */ + if(cc==0) { + decompStart=prevSrc; + } + prevCC=(int16_t)(fcd16&0xff); + + /* just append (c, c2) */ + length= c2==0 ? 1 : 2; + if((destIndex+length)<=destCapacity) { + dest[destIndex++]=c; + if(c2!=0) { + dest[destIndex++]=c2; + } + } else { + destIndex+=length; + } + } else { + /* + * back out the part of the source that we copied already but + * is now going to be decomposed; + * prevSrc is set to after what was copied + */ + destIndex-=(int32_t)(prevSrc-decompStart); + + /* + * find the part of the source that needs to be decomposed; + * to be safe and simple, decompose to before the next character with lead cc==0 + */ + src=_findSafeFCD(src, limit, fcd16); + + /* + * the source text does not fulfill the conditions for FCD; + * decompose and reorder a limited piece of the text + */ + prevCC=_decomposeFCD(decompStart, src, + dest, destIndex, destCapacity, + nx); + decompStart=src; + } + } + + return u_terminateUChars(dest, destCapacity, destIndex, pErrorCode); +} + +/* quick check functions ---------------------------------------------------- */ + +static UBool +unorm_checkFCD(const UChar *src, int32_t srcLength, const UnicodeSet *nx) { + const UChar *limit; + UChar c, c2; + uint16_t fcd16; + int16_t prevCC, cc; + + /* initialize */ + prevCC=0; + + if(srcLength>=0) { + /* string with length */ + limit=src+srcLength; + } else /* srcLength==-1 */ { + /* zero-terminated string */ + limit=NULL; + } + + U_ALIGN_CODE(16); + + for(;;) { + /* skip a run of code units below the minimum or with irrelevant data for the FCD check */ + if(limit==NULL) { + for(;;) { + c=*src++; + if(c<_NORM_MIN_WITH_LEAD_CC) { + if(c==0) { + return TRUE; + } + /* + * delay _getFCD16(c) for any character <_NORM_MIN_WITH_LEAD_CC + * because chances are good that the next one will have + * a leading cc of 0; + * _getFCD16(-prevCC) is later called when necessary - + * -c fits into int16_t because it is <_NORM_MIN_WITH_LEAD_CC==0x300 + */ + prevCC=(int16_t)-c; + } else if((fcd16=_getFCD16(c))==0) { + prevCC=0; + } else { + break; + } + } + } else { + for(;;) { + if(src==limit) { + return TRUE; + } else if((c=*src++)<_NORM_MIN_WITH_LEAD_CC) { + prevCC=(int16_t)-c; + } else if((fcd16=_getFCD16(c))==0) { + prevCC=0; + } else { + break; + } + } + } + + /* check one above-minimum, relevant code unit */ + if(UTF_IS_FIRST_SURROGATE(c)) { + /* c is a lead surrogate, get the real fcd16 */ + if(src!=limit && UTF_IS_SECOND_SURROGATE(c2=*src)) { + ++src; + fcd16=_getFCD16FromSurrogatePair(fcd16, c2); + } else { + c2=0; + fcd16=0; + } + } else { + c2=0; + } + + if(nx_contains(nx, c, c2)) { + prevCC=0; /* excluded: fcd16==0 */ + continue; + } + + /* + * prevCC has values from the following ranges: + * 0..0xff - the previous trail combining class + * <0 - the negative value of the previous code unit; + * that code unit was <_NORM_MIN_WITH_LEAD_CC and its _getFCD16() + * was deferred so that average text is checked faster + */ + + /* check the combining order */ + cc=(int16_t)(fcd16>>8); + if(cc!=0) { + if(prevCC<0) { + /* the previous character was <_NORM_MIN_WITH_LEAD_CC, we need to get its trail cc */ + if(!nx_contains(nx, (UChar32)-prevCC)) { + prevCC=(int16_t)(_getFCD16((UChar)-prevCC)&0xff); + } else { + prevCC=0; /* excluded: fcd16==0 */ + } + } + + if(cc=0) { + /* string with length */ + limit=src+srcLength; + } else /* srcLength==-1 */ { + /* zero-terminated string */ + limit=NULL; + } + + U_ALIGN_CODE(16); + + for(;;) { + /* skip a run of code units below the minimum or with irrelevant data for the quick check */ + if(limit==NULL) { + for(;;) { + c=*src++; + if(c=minNoMaybe && ((norm32=_getNorm32(c))&ccOrQCMask)!=0) { + break; + } + prevCC=0; + } + } + + /* check one above-minimum, relevant code unit */ + if(isNorm32LeadSurrogate(norm32)) { + /* c is a lead surrogate, get the real norm32 */ + if(src!=limit && UTF_IS_SECOND_SURROGATE(c2=*src)) { + ++src; + norm32=_getNorm32FromSurrogatePair(norm32, c2); + } else { + c2=0; + norm32=0; + } + } else { + c2=0; + } + + if(nx_contains(nx, c, c2)) { + /* excluded: norm32==0 */ + norm32=0; + } + + /* check the combining order */ + cc=(uint8_t)(norm32>>_NORM_CC_SHIFT); + if(cc!=0 && cc0 && srcLength<=destCapacity) { + uprv_memcpy(dest, src, srcLength*U_SIZEOF_UCHAR); + } + destLength=srcLength; + break; + default: + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + return u_terminateUChars(dest, destCapacity, destLength, pErrorCode); +} + +/** + * Internal API for normalizing. + * Does not check for bad input. + * @internal + */ +U_CAPI int32_t U_EXPORT2 +unorm_internalNormalize(UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + UNormalizationMode mode, int32_t options, + UErrorCode *pErrorCode) { + const UnicodeSet *nx; + + if(!_haveData(*pErrorCode)) { + return 0; + } + + nx=getNX(options, *pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return 0; + } + + /* reset options bits that should only be set inside unorm_internalNormalizeWithNX() */ + options&=~(_NORM_OPTIONS_SETS_MASK|_NORM_OPTIONS_COMPAT|_NORM_OPTIONS_COMPOSE_CONTIGUOUS); + + return unorm_internalNormalizeWithNX(dest, destCapacity, + src, srcLength, + mode, options, nx, + pErrorCode); +} + +/** Public API for normalizing. */ +U_CAPI int32_t U_EXPORT2 +unorm_normalize(const UChar *src, int32_t srcLength, + UNormalizationMode mode, int32_t options, + UChar *dest, int32_t destCapacity, + UErrorCode *pErrorCode) { + /* check argument values */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + if( destCapacity<0 || (dest==NULL && destCapacity>0) || + src==NULL || srcLength<-1 + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* check for overlapping src and destination */ + if( dest!=NULL && + ((src>=dest && src<(dest+destCapacity)) || + (srcLength>0 && dest>=src && dest<(src+srcLength))) + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + return unorm_internalNormalize(dest, destCapacity, + src, srcLength, + mode, options, + pErrorCode); +} + + +/* iteration functions ------------------------------------------------------ */ + +/* + * These iteration functions are the core implementations of the + * Normalizer class iteration API. + * They read from a UCharIterator into their own buffer + * and normalize into the Normalizer iteration buffer. + * Normalizer itself then iterates over its buffer until that needs to be + * filled again. + */ + +/* + * ### TODO: + * Now that UCharIterator.next/previous return (int32_t)-1 not (UChar)0xffff + * if iteration bounds are reached, + * try to not call hasNext/hasPrevious and instead check for >=0. + */ + +/* backward iteration ------------------------------------------------------- */ + +/* + * read backwards and get norm32 + * return 0 if the character is 0) || + src==NULL + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + if(!_haveData(*pErrorCode)) { + return 0; + } + + if(pNeededToNormalize!=NULL) { + *pNeededToNormalize=FALSE; + } + + switch(mode) { + case UNORM_FCD: + if(fcdTrie.index==NULL) { + *pErrorCode=U_UNSUPPORTED_ERROR; + return 0; + } + /* fall through to NFD */ + case UNORM_NFD: + isPreviousBoundary=_isPrevNFDSafe; + minC=_NORM_MIN_WITH_LEAD_CC; + mask=_NORM_CC_MASK|_NORM_QC_NFD; + break; + case UNORM_NFKD: + isPreviousBoundary=_isPrevNFDSafe; + minC=_NORM_MIN_WITH_LEAD_CC; + mask=_NORM_CC_MASK|_NORM_QC_NFKD; + break; + case UNORM_NFC: + isPreviousBoundary=_isPrevTrueStarter; + minC=(UChar)indexes[_NORM_INDEX_MIN_NFC_NO_MAYBE]; + mask=_NORM_CC_MASK|_NORM_QC_NFC; + break; + case UNORM_NFKC: + isPreviousBoundary=_isPrevTrueStarter; + minC=(UChar)indexes[_NORM_INDEX_MIN_NFKC_NO_MAYBE]; + mask=_NORM_CC_MASK|_NORM_QC_NFKC; + break; + case UNORM_NONE: + destLength=0; + if((c=src->previous(src))>=0) { + destLength=1; + if(UTF_IS_TRAIL(c) && (c2=src->previous(src))>=0) { + if(UTF_IS_LEAD(c2)) { + if(destCapacity>=2) { + dest[1]=(UChar)c; /* trail surrogate */ + destLength=2; + } + c=c2; /* lead surrogate to be written below */ + } else { + src->move(src, 1, UITER_CURRENT); + } + } + + if(destCapacity>0) { + dest[0]=(UChar)c; + } + } + return u_terminateUChars(dest, destCapacity, destLength, pErrorCode); + default: + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + buffer=stackBuffer; + bufferCapacity=(int32_t)(sizeof(stackBuffer)/U_SIZEOF_UCHAR); + bufferLength=_findPreviousIterationBoundary(*src, + isPreviousBoundary, minC, mask, + buffer, bufferCapacity, + startIndex, + pErrorCode); + if(bufferLength>0) { + if(doNormalize) { + destLength=unorm_internalNormalize(dest, destCapacity, + buffer+startIndex, bufferLength, + mode, options, + pErrorCode); + if(pNeededToNormalize!=0 && U_SUCCESS(*pErrorCode)) { + *pNeededToNormalize= + (UBool)(destLength!=bufferLength || + 0!=uprv_memcmp(dest, buffer+startIndex, destLength*U_SIZEOF_UCHAR)); + } + } else { + /* just copy the source characters */ + if(destCapacity>0) { + uprv_memcpy(dest, buffer+startIndex, uprv_min(bufferLength, destCapacity)*U_SIZEOF_UCHAR); + } + destLength=u_terminateUChars(dest, destCapacity, bufferLength, pErrorCode); + } + } else { + destLength=u_terminateUChars(dest, destCapacity, 0, pErrorCode); + } + + /* cleanup */ + if(buffer!=stackBuffer) { + uprv_free(buffer); + } + + return destLength; +} + +/* forward iteration -------------------------------------------------------- */ + +/* + * read forward and get norm32 + * return 0 if the character is 0) || + src==NULL + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + if(!_haveData(*pErrorCode)) { + return 0; + } + + if(pNeededToNormalize!=NULL) { + *pNeededToNormalize=FALSE; + } + + switch(mode) { + case UNORM_FCD: + if(fcdTrie.index==NULL) { + *pErrorCode=U_UNSUPPORTED_ERROR; + return 0; + } + /* fall through to NFD */ + case UNORM_NFD: + isNextBoundary=_isNextNFDSafe; + minC=_NORM_MIN_WITH_LEAD_CC; + mask=_NORM_CC_MASK|_NORM_QC_NFD; + break; + case UNORM_NFKD: + isNextBoundary=_isNextNFDSafe; + minC=_NORM_MIN_WITH_LEAD_CC; + mask=_NORM_CC_MASK|_NORM_QC_NFKD; + break; + case UNORM_NFC: + isNextBoundary=_isNextTrueStarter; + minC=(UChar)indexes[_NORM_INDEX_MIN_NFC_NO_MAYBE]; + mask=_NORM_CC_MASK|_NORM_QC_NFC; + break; + case UNORM_NFKC: + isNextBoundary=_isNextTrueStarter; + minC=(UChar)indexes[_NORM_INDEX_MIN_NFKC_NO_MAYBE]; + mask=_NORM_CC_MASK|_NORM_QC_NFKC; + break; + case UNORM_NONE: + destLength=0; + if((c=src->next(src))>=0) { + destLength=1; + if(UTF_IS_LEAD(c) && (c2=src->next(src))>=0) { + if(UTF_IS_TRAIL(c2)) { + if(destCapacity>=2) { + dest[1]=(UChar)c2; /* trail surrogate */ + destLength=2; + } + /* lead surrogate to be written below */ + } else { + src->move(src, -1, UITER_CURRENT); + } + } + + if(destCapacity>0) { + dest[0]=(UChar)c; + } + } + return u_terminateUChars(dest, destCapacity, destLength, pErrorCode); + default: + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + buffer=stackBuffer; + bufferCapacity=(int32_t)(sizeof(stackBuffer)/U_SIZEOF_UCHAR); + bufferLength=_findNextIterationBoundary(*src, + isNextBoundary, minC, mask, + buffer, bufferCapacity, + pErrorCode); + if(bufferLength>0) { + if(doNormalize) { + destLength=unorm_internalNormalize(dest, destCapacity, + buffer, bufferLength, + mode, options, + pErrorCode); + if(pNeededToNormalize!=0 && U_SUCCESS(*pErrorCode)) { + *pNeededToNormalize= + (UBool)(destLength!=bufferLength || + 0!=uprv_memcmp(dest, buffer, destLength*U_SIZEOF_UCHAR)); + } + } else { + /* just copy the source characters */ + if(destCapacity>0) { + uprv_memcpy(dest, buffer, uprv_min(bufferLength, destCapacity)*U_SIZEOF_UCHAR); + } + destLength=u_terminateUChars(dest, destCapacity, bufferLength, pErrorCode); + } + } else { + destLength=u_terminateUChars(dest, destCapacity, 0, pErrorCode); + } + + /* cleanup */ + if(buffer!=stackBuffer) { + uprv_free(buffer); + } + + return destLength; +} + +/* + * ### TODO: check if NF*D and FCD iteration finds optimal boundaries + * and if not, how hard it would be to improve it. + * For example, see _findSafeFCD(). + */ + +/* Concatenation of normalized strings -------------------------------------- */ + +U_CAPI int32_t U_EXPORT2 +unorm_concatenate(const UChar *left, int32_t leftLength, + const UChar *right, int32_t rightLength, + UChar *dest, int32_t destCapacity, + UNormalizationMode mode, int32_t options, + UErrorCode *pErrorCode) { + UChar stackBuffer[100]; + UChar *buffer; + int32_t bufferLength, bufferCapacity; + + UCharIterator iter; + int32_t leftBoundary, rightBoundary, destLength; + + /* check argument values */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + if( destCapacity<0 || (dest==NULL && destCapacity>0) || + left==NULL || leftLength<-1 || + right==NULL || rightLength<-1 + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* check for overlapping right and destination */ + if( dest!=NULL && + ((right>=dest && right<(dest+destCapacity)) || + (rightLength>0 && dest>=right && dest<(right+rightLength))) + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* allow left==dest */ + + /* set up intermediate buffer */ + buffer=stackBuffer; + bufferCapacity=(int32_t)(sizeof(stackBuffer)/U_SIZEOF_UCHAR); + + /* + * Input: left[0..leftLength[ + right[0..rightLength[ + * + * Find normalization-safe boundaries leftBoundary and rightBoundary + * and copy the end parts together: + * buffer=left[leftBoundary..leftLength[ + right[0..rightBoundary[ + * + * dest=left[0..leftBoundary[ + + * normalize(buffer) + + * right[rightBoundary..rightLength[ + */ + + /* + * find a normalization boundary at the end of the left string + * and copy the end part into the buffer + */ + uiter_setString(&iter, left, leftLength); + iter.index=leftLength=iter.length; /* end of left string */ + + bufferLength=unorm_previous(&iter, buffer, bufferCapacity, + mode, options, + FALSE, NULL, + pErrorCode); + leftBoundary=iter.index; + if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) { + *pErrorCode=U_ZERO_ERROR; + if(!u_growBufferFromStatic(stackBuffer, &buffer, &bufferCapacity, 2*bufferLength, 0)) { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + /* dont need to cleanup here since + * u_growBufferFromStatic frees buffer if(buffer!=stackBuffer) + */ + return 0; + } + + /* just copy from the left string: we know the boundary already */ + uprv_memcpy(buffer, left+leftBoundary, bufferLength*U_SIZEOF_UCHAR); + } + + /* + * find a normalization boundary at the beginning of the right string + * and concatenate the beginning part to the buffer + */ + uiter_setString(&iter, right, rightLength); + rightLength=iter.length; /* in case it was -1 */ + + rightBoundary=unorm_next(&iter, buffer+bufferLength, bufferCapacity-bufferLength, + mode, options, + FALSE, NULL, + pErrorCode); + if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) { + *pErrorCode=U_ZERO_ERROR; + if(!u_growBufferFromStatic(stackBuffer, &buffer, &bufferCapacity, bufferLength+rightBoundary, 0)) { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + /* dont need to cleanup here since + * u_growBufferFromStatic frees buffer if(buffer!=stackBuffer) + */ + return 0; + } + + /* just copy from the right string: we know the boundary already */ + uprv_memcpy(buffer+bufferLength, right, rightBoundary*U_SIZEOF_UCHAR); + } + + bufferLength+=rightBoundary; + + /* copy left[0..leftBoundary[ to dest */ + if(left!=dest && leftBoundary>0 && destCapacity>0) { + uprv_memcpy(dest, left, uprv_min(leftBoundary, destCapacity)*U_SIZEOF_UCHAR); + } + destLength=leftBoundary; + + /* concatenate the normalization of the buffer to dest */ + if(destCapacity>destLength) { + destLength+=unorm_internalNormalize(dest+destLength, destCapacity-destLength, + buffer, bufferLength, + mode, options, + pErrorCode); + } else { + destLength+=unorm_internalNormalize(NULL, 0, + buffer, bufferLength, + mode, options, + pErrorCode); + } + /* + * only errorCode that is expected is a U_BUFFER_OVERFLOW_ERROR + * so we dont check for the error code here..just let it pass through + */ + /* concatenate right[rightBoundary..rightLength[ to dest */ + right+=rightBoundary; + rightLength-=rightBoundary; + if(rightLength>0 && destCapacity>destLength) { + uprv_memcpy(dest+destLength, right, uprv_min(rightLength, destCapacity-destLength)*U_SIZEOF_UCHAR); + } + destLength+=rightLength; + + /* cleanup */ + if(buffer!=stackBuffer) { + uprv_free(buffer); + } + + return u_terminateUChars(dest, destCapacity, destLength, pErrorCode); +} + +#endif /* #if !UCONFIG_NO_NORMALIZATION */ diff --git a/Build/source/libs/icu/icu-xetex/common/unorm_it.c b/Build/source/libs/icu/icu-xetex/common/unorm_it.c new file mode 100644 index 00000000000..cb18cf19af5 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unorm_it.c @@ -0,0 +1,643 @@ +/* +******************************************************************************* +* +* Copyright (C) 2003-2004, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: unorm_it.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2003jan21 +* created by: Markus W. Scherer +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION && !UCONFIG_NO_NORMALIZATION + +#include "unicode/uiter.h" +#include "unicode/unorm.h" +#include "unorm_it.h" +#include "cmemory.h" + +/* UNormIterator ------------------------------------------------------------ */ + +enum { + INITIAL_CAPACITY=100 +}; + +struct UNormIterator { + UCharIterator api; + UCharIterator *iter; + + /* + * chars and states either use the static buffers + * or are allocated in the same memory block + * + * They are parallel arrays with states[] holding the getState() values + * from normalization boundaries, and UITER_NO_STATE in between. + */ + UChar *chars; + uint32_t *states; + + /* + * api.start: first valid character & state in the arrays + * api.index: current position + * api.limit: one past the last valid character in chars[], but states[limit] is valid + * capacity: length of allocated arrays + */ + int32_t capacity; + + /* the current iter->getState(), saved to avoid unnecessary setState() calls; may not correspond to api->index! */ + uint32_t state; + + /* there are UChars available before start or after limit? */ + UBool hasPrevious, hasNext, isStackAllocated; + + UNormalizationMode mode; + + UChar charsBuffer[INITIAL_CAPACITY]; + uint32_t statesBuffer[INITIAL_CAPACITY+1]; /* one more than charsBuffer[]! */ +}; + +static void +initIndexes(UNormIterator *uni, UCharIterator *iter) { + /* do not pass api so that the compiler knows it's an alias pointer to uni itself */ + UCharIterator *api=&uni->api; + + if(!iter->hasPrevious(iter)) { + /* set indexes to the beginning of the arrays */ + api->start=api->index=api->limit=0; + uni->hasPrevious=FALSE; + uni->hasNext=iter->hasNext(iter); + } else if(!iter->hasNext(iter)) { + /* set indexes to the end of the arrays */ + api->start=api->index=api->limit=uni->capacity; + uni->hasNext=FALSE; + uni->hasPrevious=iter->hasPrevious(iter); + } else { + /* set indexes into the middle of the arrays */ + api->start=api->index=api->limit=uni->capacity/2; + uni->hasPrevious=uni->hasNext=TRUE; + } +} + +static UBool +reallocArrays(UNormIterator *uni, int32_t capacity, UBool addAtStart) { + /* do not pass api so that the compiler knows it's an alias pointer to uni itself */ + UCharIterator *api=&uni->api; + + uint32_t *states; + UChar *chars; + int32_t start, limit; + + states=(uint32_t *)uprv_malloc((capacity+1)*4+capacity*2); + if(states==NULL) { + return FALSE; + } + + chars=(UChar *)(states+(capacity+1)); + uni->capacity=capacity; + + start=api->start; + limit=api->limit; + + if(addAtStart) { + /* copy old contents to the end of the new arrays */ + int32_t delta; + + delta=capacity-uni->capacity; + uprv_memcpy(states+delta+start, uni->states+start, (limit-start+1)*4); + uprv_memcpy(chars+delta+start, uni->chars+start, (limit-start)*4); + + api->start=start+delta; + api->index+=delta; + api->limit=limit+delta; + } else { + /* copy old contents to the beginning of the new arrays */ + uprv_memcpy(states+start, uni->states+start, (limit-start+1)*4); + uprv_memcpy(chars+start, uni->chars+start, (limit-start)*4); + } + + uni->chars=chars; + uni->states=states; + + return TRUE; +} + +static void +moveContentsTowardStart(UCharIterator *api, UChar chars[], uint32_t states[], int32_t delta) { + /* move array contents up to make room */ + int32_t srcIndex, destIndex, limit; + + limit=api->limit; + srcIndex=delta; + if(srcIndex>api->start) { + /* look for a position in the arrays with a known state */ + while(srcIndexstart=destIndex=0; + while(srcIndexlimit=destIndex; +} + +static void +moveContentsTowardEnd(UCharIterator *api, UChar chars[], uint32_t states[], int32_t delta) { + /* move array contents up to make room */ + int32_t srcIndex, destIndex, start; + + start=api->start; + destIndex=((UNormIterator *)api)->capacity; + srcIndex=destIndex-delta; + if(srcIndexlimit) { + /* look for a position in the arrays with a known state */ + while(srcIndex>start && states[srcIndex]==UITER_NO_STATE) { + --srcIndex; + } + } + + /* now actually move the array contents */ + api->limit=destIndex; + + /* copy states[limit] as well! */ + states[destIndex]=states[srcIndex]; + + while(srcIndex>start) { + chars[--destIndex]=chars[--srcIndex]; + states[destIndex]=states[srcIndex]; + } + + api->start=destIndex; +} + +/* normalize forward from the limit, assume hasNext is true */ +static UBool +readNext(UNormIterator *uni, UCharIterator *iter) { + /* do not pass api so that the compiler knows it's an alias pointer to uni itself */ + UCharIterator *api=&uni->api; + + /* make capacity/4 room at the end of the arrays */ + int32_t limit, capacity, room; + UErrorCode errorCode; + + limit=api->limit; + capacity=uni->capacity; + room=capacity/4; + if(room>(capacity-limit)) { + /* move array contents to make room */ + moveContentsTowardStart(api, uni->chars, uni->states, room); + api->index=limit=api->limit; + uni->hasPrevious=TRUE; + } + + /* normalize starting from the limit position */ + errorCode=U_ZERO_ERROR; + if(uni->state!=uni->states[limit]) { + uiter_setState(iter, uni->states[limit], &errorCode); + if(U_FAILURE(errorCode)) { + uni->state=UITER_NO_STATE; + uni->hasNext=FALSE; + return FALSE; + } + } + + room=unorm_next(iter, uni->chars+limit, capacity-limit, uni->mode, 0, TRUE, NULL, &errorCode); + if(errorCode==U_BUFFER_OVERFLOW_ERROR) { + if(room<=capacity) { + /* empty and re-use the arrays */ + uni->states[0]=uni->states[limit]; + api->start=api->index=api->limit=limit=0; + uni->hasPrevious=TRUE; + } else { + capacity+=room+100; + if(!reallocArrays(uni, capacity, FALSE)) { + uni->state=UITER_NO_STATE; + uni->hasNext=FALSE; + return FALSE; + } + limit=api->limit; + } + + errorCode=U_ZERO_ERROR; + uiter_setState(iter, uni->states[limit], &errorCode); + room=unorm_next(iter, uni->chars+limit, capacity-limit, uni->mode, 0, TRUE, NULL, &errorCode); + } + if(U_FAILURE(errorCode) || room==0) { + uni->state=UITER_NO_STATE; + uni->hasNext=FALSE; + return FALSE; + } + + /* room>0 */ + ++limit; /* leave the known states[limit] alone */ + for(--room; room>0; --room) { + /* set unknown states for all but the normalization boundaries */ + uni->states[limit++]=UITER_NO_STATE; + } + uni->states[limit]=uni->state=uiter_getState(iter); + uni->hasNext=iter->hasNext(iter); + api->limit=limit; + return TRUE; +} + +/* normalize backward from the start, assume hasPrevious is true */ +static UBool +readPrevious(UNormIterator *uni, UCharIterator *iter) { + /* do not pass api so that the compiler knows it's an alias pointer to uni itself */ + UCharIterator *api=&uni->api; + + /* make capacity/4 room at the start of the arrays */ + int32_t start, capacity, room; + UErrorCode errorCode; + + start=api->start; + capacity=uni->capacity; + room=capacity/4; + if(room>start) { + /* move array contents to make room */ + moveContentsTowardEnd(api, uni->chars, uni->states, room); + api->index=start=api->start; + uni->hasNext=TRUE; + } + + /* normalize ending at the start position */ + errorCode=U_ZERO_ERROR; + if(uni->state!=uni->states[start]) { + uiter_setState(iter, uni->states[start], &errorCode); + if(U_FAILURE(errorCode)) { + uni->state=UITER_NO_STATE; + uni->hasPrevious=FALSE; + return FALSE; + } + } + + room=unorm_previous(iter, uni->chars, start, uni->mode, 0, TRUE, NULL, &errorCode); + if(errorCode==U_BUFFER_OVERFLOW_ERROR) { + if(room<=capacity) { + /* empty and re-use the arrays */ + uni->states[capacity]=uni->states[start]; + api->start=api->index=api->limit=start=capacity; + uni->hasNext=TRUE; + } else { + capacity+=room+100; + if(!reallocArrays(uni, capacity, TRUE)) { + uni->state=UITER_NO_STATE; + uni->hasPrevious=FALSE; + return FALSE; + } + start=api->start; + } + + errorCode=U_ZERO_ERROR; + uiter_setState(iter, uni->states[start], &errorCode); + room=unorm_previous(iter, uni->chars, start, uni->mode, 0, TRUE, NULL, &errorCode); + } + if(U_FAILURE(errorCode) || room==0) { + uni->state=UITER_NO_STATE; + uni->hasPrevious=FALSE; + return FALSE; + } + + /* room>0 */ + do { + /* copy the UChars from chars[0..room[ to chars[(start-room)..start[ */ + uni->chars[--start]=uni->chars[--room]; + /* set unknown states for all but the normalization boundaries */ + uni->states[start]=UITER_NO_STATE; + } while(room>0); + uni->states[start]=uni->state=uiter_getState(iter); + uni->hasPrevious=iter->hasPrevious(iter); + api->start=start; + return TRUE; +} + +/* Iterator runtime API functions ------------------------------------------- */ + +static int32_t U_CALLCONV +unormIteratorGetIndex(UCharIterator *api, UCharIteratorOrigin origin) { + switch(origin) { + case UITER_ZERO: + case UITER_START: + return 0; + case UITER_CURRENT: + case UITER_LIMIT: + case UITER_LENGTH: + return UITER_UNKNOWN_INDEX; + default: + /* not a valid origin */ + /* Should never get here! */ + return -1; + } +} + +static int32_t U_CALLCONV +unormIteratorMove(UCharIterator *api, int32_t delta, UCharIteratorOrigin origin) { + UNormIterator *uni=(UNormIterator *)api; + UCharIterator *iter=uni->iter; + int32_t pos; + + switch(origin) { + case UITER_ZERO: + case UITER_START: + /* restart from the beginning */ + if(uni->hasPrevious) { + iter->move(iter, 0, UITER_START); + api->start=api->index=api->limit=0; + uni->states[api->limit]=uni->state=uiter_getState(iter); + uni->hasPrevious=FALSE; + uni->hasNext=iter->hasNext(iter); + } else { + /* we already have the beginning of the normalized text */ + api->index=api->start; + } + break; + case UITER_CURRENT: + break; + case UITER_LIMIT: + case UITER_LENGTH: + /* restart from the end */ + if(uni->hasNext) { + iter->move(iter, 0, UITER_LIMIT); + api->start=api->index=api->limit=uni->capacity; + uni->states[api->limit]=uni->state=uiter_getState(iter); + uni->hasPrevious=iter->hasPrevious(iter); + uni->hasNext=FALSE; + } else { + /* we already have the end of the normalized text */ + api->index=api->limit; + } + break; + default: + return -1; /* Error */ + } + + /* move relative to the current position by delta normalized UChars */ + if(delta==0) { + /* nothing to do */ + } else if(delta>0) { + /* go forward until the requested position is in the buffer */ + for(;;) { + pos=api->index+delta; /* requested position */ + delta=pos-api->limit; /* remainder beyond buffered text */ + if(delta<=0) { + api->index=pos; /* position reached */ + break; + } + + /* go to end of buffer and normalize further */ + api->index=api->limit; + if(!uni->hasNext || !readNext(uni, iter)) { + break; /* reached end of text */ + } + } + } else /* delta<0 */ { + /* go backward until the requested position is in the buffer */ + for(;;) { + pos=api->index+delta; /* requested position */ + delta=pos-api->start; /* remainder beyond buffered text */ + if(delta>=0) { + api->index=pos; /* position reached */ + break; + } + + /* go to start of buffer and normalize further */ + api->index=api->start; + if(!uni->hasPrevious || !readPrevious(uni, iter)) { + break; /* reached start of text */ + } + } + } + + if(api->index==api->start && !uni->hasPrevious) { + return 0; + } else { + return UITER_UNKNOWN_INDEX; + } +} + +static UBool U_CALLCONV +unormIteratorHasNext(UCharIterator *api) { + return api->indexlimit || ((UNormIterator *)api)->hasNext; +} + +static UBool U_CALLCONV +unormIteratorHasPrevious(UCharIterator *api) { + return api->index>api->start || ((UNormIterator *)api)->hasPrevious; +} + +static UChar32 U_CALLCONV +unormIteratorCurrent(UCharIterator *api) { + UNormIterator *uni=(UNormIterator *)api; + + if( api->indexlimit || + (uni->hasNext && readNext(uni, uni->iter)) + ) { + return uni->chars[api->index]; + } else { + return U_SENTINEL; + } +} + +static UChar32 U_CALLCONV +unormIteratorNext(UCharIterator *api) { + UNormIterator *uni=(UNormIterator *)api; + + if( api->indexlimit || + (uni->hasNext && readNext(uni, uni->iter)) + ) { + return uni->chars[api->index++]; + } else { + return U_SENTINEL; + } +} + +static UChar32 U_CALLCONV +unormIteratorPrevious(UCharIterator *api) { + UNormIterator *uni=(UNormIterator *)api; + + if( api->index>api->start || + (uni->hasPrevious && readPrevious(uni, uni->iter)) + ) { + return uni->chars[--api->index]; + } else { + return U_SENTINEL; + } +} + +static uint32_t U_CALLCONV +unormIteratorGetState(const UCharIterator *api) { + /* not uni->state because that may not be at api->index */ + return ((UNormIterator *)api)->states[api->index]; +} + +static void U_CALLCONV +unormIteratorSetState(UCharIterator *api, uint32_t state, UErrorCode *pErrorCode) { + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + /* do nothing */ + } else if(api==NULL) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + } else if(state==UITER_NO_STATE) { + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + } else { + UNormIterator *uni=(UNormIterator *)api; + UCharIterator *iter=((UNormIterator *)api)->iter; + if(state!=uni->state) { + uni->state=state; + uiter_setState(iter, state, pErrorCode); + } + + /* + * Try shortcuts: If the requested state is in the array contents + * then just set the index there. + * + * We assume that the state is unique per position! + */ + if(state==uni->states[api->index]) { + return; + } else if(state==uni->states[api->limit]) { + api->index=api->limit; + return; + } else { + /* search for the index with this state */ + int32_t i; + + for(i=api->start; ilimit; ++i) { + if(state==uni->states[i]) { + api->index=i; + return; + } + } + } + + /* there is no array index for this state, reset for fresh contents */ + initIndexes((UNormIterator *)api, iter); + uni->states[api->limit]=state; + } +} + +static const UCharIterator unormIterator={ + NULL, 0, 0, 0, 0, 0, + unormIteratorGetIndex, + unormIteratorMove, + unormIteratorHasNext, + unormIteratorHasPrevious, + unormIteratorCurrent, + unormIteratorNext, + unormIteratorPrevious, + NULL, + unormIteratorGetState, + unormIteratorSetState +}; + +/* Setup functions ---------------------------------------------------------- */ + +U_CAPI UNormIterator * U_EXPORT2 +unorm_openIter(void *stackMem, int32_t stackMemSize, UErrorCode *pErrorCode) { + UNormIterator *uni; + + /* argument checking */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return NULL; + } + + /* allocate */ + uni=NULL; + if(stackMem!=NULL && stackMemSize>=sizeof(UNormIterator)) { + size_t align=U_ALIGNMENT_OFFSET(stackMem); + if(align==0) { + /* already aligned */ + uni=(UNormIterator *)stackMem; + } else if((stackMemSize-=(int32_t)align)>=(int32_t)sizeof(UNormIterator)) { + /* needs alignment */ + uni=(UNormIterator *)((char *)stackMem+align); + } + /* else does not fit */ + } + + if(uni!=NULL) { + uni->isStackAllocated=TRUE; + } else { + uni=(UNormIterator *)uprv_malloc(sizeof(UNormIterator)); + if(uni==NULL) { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + uni->isStackAllocated=FALSE; + } + + /* + * initialize + * do not memset because that would unnecessarily initialize the arrays + */ + uni->iter=NULL; + uni->chars=uni->charsBuffer; + uni->states=uni->statesBuffer; + uni->capacity=INITIAL_CAPACITY; + uni->state=UITER_NO_STATE; + uni->hasPrevious=uni->hasNext=FALSE; + uni->mode=UNORM_NONE; + + /* set a no-op iterator into the api */ + uiter_setString(&uni->api, NULL, 0); + return uni; +} + +U_CAPI void U_EXPORT2 +unorm_closeIter(UNormIterator *uni) { + if(uni!=NULL) { + if(uni->states!=uni->statesBuffer) { + /* chars and states are allocated in the same memory block */ + uprv_free(uni->states); + } + if(!uni->isStackAllocated) { + uprv_free(uni); + } + } +} + +U_CAPI UCharIterator * U_EXPORT2 +unorm_setIter(UNormIterator *uni, UCharIterator *iter, UNormalizationMode mode, UErrorCode *pErrorCode) { + /* argument checking */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return NULL; + } + if(uni==NULL) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + if( iter==NULL || iter->getState==NULL || iter->setState==NULL || + modeapi, NULL, 0); + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + + /* set the iterator and initialize */ + uprv_memcpy(&uni->api, &unormIterator, sizeof(unormIterator)); + + uni->iter=iter; + uni->mode=mode; + + initIndexes(uni, iter); + uni->states[uni->api.limit]=uni->state=uiter_getState(iter); + + return &uni->api; +} + +#endif /* uconfig.h switches */ diff --git a/Build/source/libs/icu/icu-xetex/common/unorm_it.h b/Build/source/libs/icu/icu-xetex/common/unorm_it.h new file mode 100644 index 00000000000..384b7d4469e --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unorm_it.h @@ -0,0 +1,148 @@ +/* +******************************************************************************* +* +* Copyright (C) 2003, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: unorm_it.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2003jan21 +* created by: Markus W. Scherer +*/ + +#ifndef __UNORM_IT_H__ +#define __UNORM_IT_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION && !UCONFIG_NO_NORMALIZATION + +#include "unicode/uiter.h" +#include "unicode/unorm.h" + +/** + * Normalizing UCharIterator wrapper. + * This internal API basically duplicates the functionality of the C++ Normalizer + * but + * - it actually implements a character iterator (UCharIterator) + * with few restrictions (see unorm_setIter()) + * - it supports UCharIterator getState()/setState() + * - it uses lower-level APIs and buffers more text and states, + * hopefully resulting in higher performance + * + * Usage example: + * \code + * function(UCharIterator *srcIter) { + * UNormIterator *uni; + * UCharIterator *iter; + * UErrorCode errorCode; + * + * errorCode=U_ZERO_ERROR; + * uni=unorm_openIter(&errorCode); + * if(U_FAILURE(errorCode)) { + * // report error + * return; + * } + * + * iter=unorm_setIter(uni, srcIter, UNORM_FCD, &errorCode); + * if(U_FAILURE(errorCode)) { + * // report error + * } else { + * // use iter to iterate over the canonically ordered + * // version of srcIter's text + * uint32_t state; + * + * ... + * + * state=uiter_getState(iter); + * if(state!=UITER_NO_STATE) { + * // use valid state, store it, use iter some more + * ... + * + * // later restore iter to the saved state: + * uiter_setState(iter, state, &errorCode); + * + * ... + * } + * + * ... + * } + * unorm_closeIter(uni); + * } + * \endcode + * + * See also the ICU test suites. + * + * @internal + */ +struct UNormIterator; +typedef struct UNormIterator UNormIterator; + +/** + * Size of a stack buffer to hold a UNormIterator, see the stackMem parameter + * of unorm_openIter(). + * + * @internal + */ +#define UNORM_ITER_SIZE 1024 + +/** + * Open a normalizing iterator. Must be closed later. + * Use unorm_setIter(). + * + * @param stackMem Pointer to preallocated (stack-allocated) buffer to hold + * the UNormIterator if possible; can be NULL. + * @param stackMemSize Number of bytes at stackMem; can be 0, + * or should be >= UNORM_ITER_SIZE for a non-NULL stackMem. + * @param pErrorCode ICU error code + * @return an allocated and pre-initialized UNormIterator + * @internal + */ +U_CAPI UNormIterator * U_EXPORT2 +unorm_openIter(void *stackMem, int32_t stackMemSize, UErrorCode *pErrorCode); + +/** + * Close a normalizing iterator. + * + * @param uni UNormIterator from unorm_openIter() + * @internal + */ +U_CAPI void U_EXPORT2 +unorm_closeIter(UNormIterator *uni); + +/** + * Set a UCharIterator and a normalization mode for the normalizing iterator + * to wrap. The normalizing iterator will read from the character iterator, + * normalize the text, and in turn deliver it with its own wrapper UCharIterator + * interface which it returns. + * + * The source iterator remains at its current position through the unorm_setIter() + * call but will be used and moved as soon as the + * the returned normalizing iterator is. + * + * The returned interface pointer is valid for as long as the normalizing iterator + * is open and until another unorm_setIter() call is made on it. + * + * The normalizing iterator's UCharIterator interface has the following properties: + * - getIndex() and move() will almost always return UITER_UNKNOWN_INDEX + * - getState() will return UITER_NO_STATE for unknown states for positions + * that are not at normalization boundaries + * + * @param uni UNormIterator from unorm_openIter() + * @param iter The source text UCharIterator to be wrapped. It is aliases into the normalizing iterator. + * Must support getState() and setState(). + * @param mode The normalization mode. + * @param pErrorCode ICU error code + * @return an alias to the normalizing iterator's UCharIterator interface + * @internal + */ +U_CAPI UCharIterator * U_EXPORT2 +unorm_setIter(UNormIterator *uni, UCharIterator *iter, UNormalizationMode mode, UErrorCode *pErrorCode); + +#endif /* uconfig.h switches */ + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/unorm_props_data.c b/Build/source/libs/icu/icu-xetex/common/unorm_props_data.c new file mode 100644 index 00000000000..16047b75bdc --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unorm_props_data.c @@ -0,0 +1,3092 @@ +/* + * Copyright (C) 1999-2006, International Business Machines + * Corporation and others. All Rights Reserved. + * + * file name: unorm_props_data.c + * + * machine-generated on: 2006-06-13 + */ + +static const UVersionInfo formatVersion={ 2,3,5,2 }; + +static const UVersionInfo dataVersion={ 5,0,0,0 }; + +static const int32_t indexes[_NORM_INDEX_TOP]={ +0xa1a0,0x3e4b,0x7a7,0x16f,0,0x36,0x300,0xa0,0xc0,0xa0,0x2b00,0x2cf8,0x1e2c,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +}; + +static const uint16_t normTrie_index[2208]={ +0,0x8b4,0x7ff,0x807,0,0x54d,0x336,0x33e,0x346,0x34e,0x356,0x35e,0,0x366,0x36d,0x375, +0x37d,0x385,0,0,0x80e,0x555,0x55c,0x564,0x4ff,0x507,0x175,8,0x17d,0x38d,0x10,0x16, +0x395,0x39d,0x3a5,0x3ad,0x816,0,0x3b5,0x3bd,0,0,0,0,0x56b,0x81e,0x826,0, +0x82a,0x3c5,0x50f,0x573,0,0,0x3cd,0x832,0x836,0x83b,0x843,0,0,0,0,0x849, +0,0,0,0,0,0,0,0,0,0x3d5,0x185,0,0,0x517,0x18d,0, +0,0x195,0x19d,0,0,0x84e,0x856,0,0,0x51f,0x1a5,0,0x3dd,0x527,0x3e4,0, +0,0,0x3eb,0,0,0x85a,0x3f3,0,0,0x52f,0x3fa,0,0,0,0x400,0, +0,0x57a,0x862,0,0,0x581,0x588,0,0x590,0x865,0x1ad,0x1b5,0x1bd,0x1c5,0x86c,0, +0,0x408,0,0,0,0,0,0x597,0x8dd,0,0,0x537,0,0x53d,0x545,0, +0,0,0,0,0,0,0,0,0,0,0x86e,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0x876,0x876,0,0,0,0,0x87c,0, +0,0,0,0,0,0x884,0,0,0,0x887,0,0,0,0,0,0, +0x88e,0,0,0,0,0,0,0,0x40f,0x414,0x41c,0x895,0,0,0,0, +0,0,0,0,0,0,0,0,0,0x1d,0x25,0x59f,0x5a6,0x5ae,0x89d,0x8a0, +0x424,0x42c,0x434,0x43c,0x444,0x44c,0x454,0x45c,0x464,0x46c,0x474,0x1cd,0x47c,0x1d5,0x1dd,0x1e5, +0x1ed,0x5b6,0x5be,0x5c6,0x5ce,0x2d,0x8a8,0x8b0,0x35,0x3d,0x45,0x5d6,0x484,0x48b,0x490,0, +0x498,0x4a0,0x4a8,0x4b0,0x4b8,0x4c0,0,0x4c8,0,0x1f3,0,0,0,0,0,0, +0,0,0,0x5de,0x5e6,0x5ee,0x5f6,0x5fe,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0x603,0,0,0x607,0,0,0x1f6,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0x60d,0,0,0,0, +0,0,0,0,0x611,0,0,0x619,0x621,0x629,0x631,0x639,0x641,0x649,0x651,0, +0x659,0x65f,0x4cf,0x4d7,0x4df,0x4e7,0x4ef,0x4f7,0,0x666,0x66e,0x676,0x67e,0,0,0, +0x686,0x68e,0x4d,0x696,0x69e,0x6a6,0x55,0x6ae,0x6b6,0x6be,0x6c6,0x5d,0x65,0x6d,0x75,0x6ce, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0x8bc,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5, +0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5, +0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5, +0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5, +0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5, +0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5, +0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5, +0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5, +0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5, +0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5, +0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5, +0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5, +0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5, +0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5, +0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5, +0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5, +0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5, +0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5, +0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5, +0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5, +0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5, +0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8e5,0x8ec,0,0, +0x8f4,0x8f7,0,0x8fd,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0x1fe,0x206,0x20e,0x216,0x21e,0x226,0x22e,0x236, +0x23e,0x246,0x24e,0x256,0x25e,0x266,0x26e,0,0x276,0x27e,0x286,0x6d6,0x6de,0x6e6,0x6eb,0x6f3, +0x6fb,0x703,0x70b,0x713,0x71b,0x723,0x72b,0x733,0x73b,0x743,0x74b,0x753,0x75b,0x763,0x76b,0x76f, +0x777,0x77f,0x787,0x78f,0x797,0x79f,0x7a7,0x7af,0x7b7,0x7bf,0x7c7,0x7cf,0x7d7,0x7df,0x7e7,0x7ef, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0x8c1,0x8c5,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0x28e,0x296,0x8cd,0x29e,0x2a6,0, +0,0,0x8d5,0,0,0,0,0,0,0,0,0,0,0,0,0, +0x7d,0x85,0x8d,0x95,0x9d,0xa5,0xad,0xb5,0xbd,0xc5,0xcd,0xd5,0xdd,0xe5,0xed,0xf5, +0xfd,0x105,0x10d,0x115,0x11d,0x125,0x12d,0x135,0x13d,0x145,0x14d,0x155,0x15d,0x165,0x16d,0x7f7, +0x2ae,0x2b6,0x2be,0x2c6,0x2ce,0x2d6,0x2de,0x2e6,0x2ee,0x2f6,0x2fe,0x306,0x30e,0x316,0x31e,0x326, +0x32e,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +}; + +static const uint32_t normTrie_data32[9236]={ +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0xea00,0xea00,0xe900,0xe600,0xe600,0xe600,0xe600,0xe600,0xe600,0xe600,0xe600,0xe600,0xe600,0xe600,0xe600,0xe600, +0,0,0,0,0x612000f,0,0,0,0,0,0x13b000a,0,0,0,0x614000f,0, +0,0x3dee0040,0,0,0,0x3d9f0040,0,0,0,0x3da10040,0x14a5004c,0x14aa004c,0x14ae000c,0x14b2000c,0x14b7004c,0, +0x2123000a,0x2125000a,0x140004a,0x142000e,0x149000e,0x2127000a,0x2129000a,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0x212b000a,0x212d000a,0x150000a,0,0x212f000a,0x2131000a,0,0, +0,0x152000a,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0x154000a,0x156000a,0x158000a,0,0x15a000a,0x15c000a,0x15e000a,0x160000a,0x162000a,0x164000a,0x166000a,0x168000a,0x16a000a,0x16c000a,0x16e000a,0, +0x170000a,0x172000a,0x174000a,0x176000a,0x178000a,0x17a000a,0x17c000a,0x215c000a,0x215e000a,0x2160000a,0x2162000a,0x2164000a,0x2166000a,0x2168000a,0x216a000a,0x216c000a, +0x216e000a,0x2170000a,0,0x2172000a,0x2174000a,0x2176000a,0x2178000a,0x217a000a,0x217c000a,0x217e000a,0x2180000a,0x2182000a,0x2184000a,0x2186000a,0x2188000a,0x218a000a, +0x218c000a,0x218e000a,0x2190000a,0x2192000a,0,0,0,0,0,0,0,0,0x17e000a,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0x2297000a,0x229b000a,0x181000a,0x183000a,0,0x229f000a,0x22a3000a,0x186000a,0,0x188000a,0x22a7000a,0x18b000a, +0x18d000a,0x18f000a,0x22a9000a,0x22ab000a,0x191000a,0x193000a,0x195000a,0x22ad000a,0,0x197000a,0x199000a,0,0,0x19c000a,0x19e000a,0x1a0000a, +0x1a2000a,0x1a4000a,0,0,0x1a6000a,0x1a9000a,0x1ad000a,0,0x1b0000a,0,0x6f5000f,0,0x1b2000a,0,0x6f7000f,0x6f9000f, +0x1b4000a,0x1b6000a,0,0x22af000a,0x1b8000a,0x1ba000a,0,0x1bc000a,0x22b1000a,0x22b3000a,0x22b5000a,0x22b7000a,0x22b9000a,0x22bb000a,0,0x1be000a, +0x22bd000a,0x22bf000a,0x1c2000a,0x1c4000a,0x22c1000a,0,0,0,0,0x1c6000a,0x22c3000a,0x22c5000a,0x22c7000a,0x22c9000a,0,0, +0,0,0,0,0,0,0,0x22cb000a,0x22cf000a,0x22d3000a,0x22d7000a,0x22db000a,0x22df000a,0x22e3000a,0x22e7000a,0x22eb000a, +0x22ef000a,0x22f3000a,0x22f7000a,0x22fb000a,0x28d3000a,0x28d7000a,0x28db000a,0x28df000a,0,0,0,0,0,0,0,0, +0,0,0,0,0x1c8000a,0x28e3000a,0x28e6000a,0x28e9000a,0x28ec000a,0x28ef000a,0x28f2000a,0x28f5000a,0x28f8000a,0x28fb000a,0x28fe000a,0x2901000a, +0x2904000a,0x2907000a,0x290a000a,0x290d000a,0x29f3000a,0x29f6000a,0x29f9000a,0x29fc000a,0x29ff000a,0x2a02000a,0x2a05000a,0x2a08000a,0x2a0b000a,0x2a0e000a,0x2a12000a,0x2a16000a, +0x1cc000a,0x2a1a000a,0x1cf000a,0x1d2000a,0x2a1e000a,0x2a20000a,0x2a22000a,0x2a24000a,0x2a26000a,0x2a28000a,0x2a2a000a,0x2a2c000a,0x2a2e000a,0x2a30000a,0x2a32000a,0x2a34000a, +0x2a36000a,0x2a38000a,0x2a3a000a,0x2a3c000a,0x2c52000a,0x2c55000a,0x2c58000a,0x2c5c000a,0x2c60000a,0x2c64000a,0x2c68000a,0x2c6c000a,0x2c70000a,0x2c74000a,0x2c78000a,0x2c7c000a, +0x2c80000a,0x2c84000a,0x2c88000a,0x2c8c000a,0x2c90000a,0x1d6000a,0x2c94000a,0x1da000a,0x2c97000a,0x1dd000a,0x2c9b000a,0x2c9e000a,0x2ca1000a,0x2ca5000a,0x1e0000a,0x2ca9000a, +0x2cac000a,0x2caf000a,0x2cb2000a,0x2cb5000a,0x1e3000a,0x1e6000a,0x1e9000a,0x1ec000a,0x1ef000a,0x1f2000a,0x1f5000a,0x1f8000a,0x2cba000a,0x2cbe000a,0x1fb000a,0x1fe000a, +0x201000a,0x2cc3000a,0x2cc6000a,0x2cc9000a,0x204000a,0x207000a,0x20b000a,0x20f000a,0x213000a,0x2ccc000a,0x2ccf000a,0x2cd2000a,0x2cd5000a,0x2cd8000a,0x2cdb000a,0x2cde000a, +0x2ce1000a,0x2ce4000a,0x2ce7000a,0x2cea000a,0x2cee000a,0x2cf2000a,0x2cf5000a,0x2cf9000a,0x2cfd000a,0x2d01000a,0x2d04000a,0x2d08000a,0x2d0c000a,0x217000a,0x21a000a,0x21e000a, +0x222000a,0x2d11000a,0x2d15000a,0x2d1b000a,0x2d22000a,0x2d25000a,0x2d28000a,0x2d2b000a,0x226000a,0x229000a,0x22c000a,0x22f000a,0x232000a,0x235000a,0x238000a,0x23b000a, +0x23e000a,0x241000a,0x244000a,0x247000a,0x24a000a,0x24d000a,0x2d2e000a,0x250000a,0x2d33000a,0x2d36000a,0x253000a,0x258000a,0x25c000a,0x25f000a,0x2d39000a,0x262000a, +0x2d3c000a,0x265000a,0x268000a,0x2d3f000a,0x2d42000a,0x2d45000a,0x2d48000a,0x2d4c000a,0x2d4f000a,0x2d52000a,0x2d56000a,0x26b000a,0x2d5a000a,0x26e000a,0x272000a,0x2d5f000a, +0x275000a,0x278000a,0x27b000a,0x27f000a,0x283000a,0x285000a,0x287000a,0x289000a,0x28b000a,0x28d000a,0x28f000a,0x291000a,0x293000a,0x295000a,0x297000a,0x299000a, +0x29b000a,0x29d000a,0x29f000a,0x2a1000a,0x2a3000a,0x2a5000a,0x2a7000a,0x2a9000a,0x2ab000a,0x2ad000a,0x2af000a,0x2b1000a,0x2b3000a,0x2b5000a,0x3921000a,0x3923000a, +0x3925000a,0x3927000a,0x3929000a,0x392b000a,0x392d000a,0x392f000a,0x3931000a,0x3933000a,0x3935000a,0x3937000a,0x3939000a,0x393b000a,0x393d000a,0x393f000a,0x3941000a,0x3943000a, +0x3945000a,0x3947000a,0x3949000a,0x394b000a,0x394d000a,0x394f000a,0x3951000a,0x3953000a,0x2b7000a,0x2b9000a,0x2bb000a,0x2bd000a,0x2bf000a,0x2c1000a,0x2c3000a,0x2c5000a, +0x2c7000a,0x2c9000a,0x2cb000a,0x2cd000a,0x2cf000a,0x2d1000a,0x2d3000a,0x2d5000a,0x2d7000a,0x2d9000a,0x2db000a,0x2dd000a,0x2df000a,0x2e1000a,0x2e3000a,0x2e5000a, +0x2e7000a,0x2e9000a,0x3955000a,0x3957000a,0x3959000a,0x395b000a,0x395d000a,0x395f000a,0x3961000a,0,0x3963000a,0x3965000a,0x3967000a,0x3969000a,0x396b000a,0x396d000a, +0x396f000a,0x3971000a,0x3973000a,0x3975000a,0x3977000a,0x3979000a,0x397b000a,0x397d000a,0x397f000a,0x3981000a,0x3983000a,0x3985000a,0x2eb000a,0x2ed000a,0x2ef000a,0x2f1000a, +0x2f3000a,0x2f5000a,0x2f7000a,0x2f9000a,0x2fb000a,0x2fd000a,0x2ff000a,0x301000a,0x303000a,0x305000a,0x307000a,0x309000a,0x30b000a,0x30d000a,0x30f000a,0x311000a, +0x313000a,0x315000a,0x317000a,0x319000a,0x31b000a,0x31d000a,0x3987000a,0x3989000a,0x398b000a,0x398d000a,0x398f000a,0x3991000a,0x3993000a,0x3995000a,0x3997000a,0x3999000a, +0x399b000a,0x399d000a,0x399f000a,0x39a1000a,0x39a3000a,0x39a5000a,0x39a7000a,0x39a9000a,0x39ab000a,0x39ad000a,0x39af000a,0x39b1000a,0x39b3000a,0x39b5000a,0x39b7000a,0x39b9000a, +0x31f000a,0,0x321000a,0x323000a,0,0,0x325000a,0,0,0x327000a,0x329000a,0,0,0x32b000a,0x32d000a,0x32f000a, +0x331000a,0,0x333000a,0x335000a,0x337000a,0x339000a,0x33b000a,0x33d000a,0x33f000a,0x341000a,0x39bb000a,0x39bd000a,0x39bf000a,0x39c1000a,0,0x39c3000a, +0,0x39c5000a,0x39c7000a,0x39c9000a,0x39cb000a,0x39cd000a,0x39cf000a,0x39d1000a,0,0x39d3000a,0x39d5000a,0x39d7000a,0x39d9000a,0x39db000a,0x39dd000a,0x39df000a, +0x39e1000a,0x39e3000a,0x39e5000a,0x39e7000a,0x343000a,0x345000a,0x347000a,0x349000a,0x34b000a,0x34d000a,0x34f000a,0x351000a,0x353000a,0x355000a,0x357000a,0x359000a, +0x35b000a,0x35d000a,0x35f000a,0x361000a,0x363000a,0x365000a,0x367000a,0x369000a,0x36b000a,0x36d000a,0x36f000a,0x371000a,0x373000a,0x375000a,0x39e9000a,0x39eb000a, +0x39ed000a,0x39ef000a,0x39f1000a,0x39f3000a,0x39f5000a,0x39f7000a,0x39f9000a,0x39fb000a,0x39fd000a,0x39ff000a,0x3a01000a,0x3a03000a,0x3a05000a,0x3a07000a,0x3a09000a,0x3a0b000a, +0x3a0d000a,0x3a0f000a,0x3a11000a,0x3a13000a,0x3a15000a,0x3a17000a,0x3a19000a,0x3a1b000a,0x377000a,0x379000a,0,0x37b000a,0x37d000a,0x37f000a,0x381000a,0, +0,0x383000a,0x385000a,0x387000a,0x389000a,0x38b000a,0x38d000a,0x38f000a,0x391000a,0,0x393000a,0x395000a,0x397000a,0x399000a,0x39b000a,0x39d000a, +0x39f000a,0,0x3a1d000a,0x3a1f000a,0x3a21000a,0x3a23000a,0x3a25000a,0x3a27000a,0x3a29000a,0x3a2b000a,0x3a2d000a,0x3a2f000a,0x3a31000a,0x3a33000a,0x3a35000a,0x3a37000a, +0x3a39000a,0x3a3b000a,0x3a3d000a,0x3a3f000a,0x3a41000a,0x3a43000a,0x3a45000a,0x3a47000a,0x3a49000a,0x3a4b000a,0x3a4d000a,0x3a4f000a,0x3a1000a,0x3a3000a,0,0x3a5000a, +0x3a7000a,0x3a9000a,0x3ab000a,0,0x3ad000a,0x3af000a,0x3b1000a,0x3b3000a,0x3b5000a,0,0x3b7000a,0,0,0,0x3b9000a,0x3bb000a, +0x3bd000a,0x3bf000a,0x3c1000a,0x3c3000a,0x3c5000a,0,0x3a51000a,0x3a53000a,0x3a55000a,0x3a57000a,0x3a59000a,0x3a5b000a,0x3a5d000a,0x3a5f000a,0x3a61000a,0x3a63000a, +0x3a65000a,0x3a67000a,0x3a69000a,0x3a6b000a,0x3a6d000a,0x3a6f000a,0x3a71000a,0x3a73000a,0x3a75000a,0x3a77000a,0x3a79000a,0x3a7b000a,0x3a7d000a,0x3a7f000a,0x3a81000a,0x3a83000a, +0x3c7000a,0x3c9000a,0x3cb000a,0x3cd000a,0x3cf000a,0x3d1000a,0x3d3000a,0x3d5000a,0x3d7000a,0x3d9000a,0x3db000a,0x3dd000a,0x3df000a,0x3e1000a,0x3e3000a,0x3e5000a, +0x3e7000a,0x3e9000a,0x3eb000a,0x3ed000a,0x3ef000a,0x3f1000a,0x3f3000a,0x3f5000a,0x3f7000a,0x3f9000a,0x3a85000a,0x3a87000a,0x3a89000a,0x3a8b000a,0x3a8d000a,0x3a8f000a, +0x3a91000a,0x3a93000a,0x3a95000a,0x3a97000a,0x3a99000a,0x3a9b000a,0x3a9d000a,0x3a9f000a,0x3aa1000a,0x3aa3000a,0x3aa5000a,0x3aa7000a,0x3aa9000a,0x3aab000a,0x3aad000a,0x3aaf000a, +0x3ab1000a,0x3ab3000a,0x3ab5000a,0x3ab7000a,0x3fb000a,0x3fd000a,0x3ff000a,0x401000a,0x403000a,0x405000a,0x407000a,0x409000a,0x40b000a,0x40d000a,0x40f000a,0x411000a, +0x413000a,0x415000a,0x417000a,0x419000a,0x41b000a,0x41d000a,0x41f000a,0x421000a,0x423000a,0x425000a,0x427000a,0x429000a,0x42b000a,0x42d000a,0x3ab9000a,0x3abb000a, +0x3abd000a,0x3abf000a,0x3ac1000a,0x3ac3000a,0x3ac5000a,0x3ac7000a,0x3ac9000a,0x3acb000a,0x3acd000a,0x3acf000a,0x3ad1000a,0x3ad3000a,0x3ad5000a,0x3ad7000a,0x3ad9000a,0x3adb000a, +0x3add000a,0x3adf000a,0x3ae1000a,0x3ae3000a,0x3ae5000a,0x3ae7000a,0x3ae9000a,0x3aeb000a,0x42f000a,0x431000a,0x433000a,0x435000a,0x437000a,0x439000a,0x43b000a,0x43d000a, +0x43f000a,0x441000a,0x443000a,0x445000a,0x447000a,0x449000a,0x44b000a,0x44d000a,0x44f000a,0x451000a,0x453000a,0x455000a,0x457000a,0x459000a,0x45b000a,0x45d000a, +0x45f000a,0x461000a,0x3aed000a,0x3aef000a,0x3af1000a,0x3af3000a,0x3af5000a,0x3af7000a,0x3af9000a,0x3afb000a,0x3afd000a,0x3aff000a,0x3b01000a,0x3b03000a,0x3b05000a,0x3b07000a, +0x3b09000a,0x3b0b000a,0x3b0d000a,0x3b0f000a,0x3b11000a,0x3b13000a,0x3b15000a,0x3b17000a,0x3b19000a,0x3b1b000a,0x3b1d000a,0x3b1f000a,0x463000a,0x465000a,0x467000a,0x469000a, +0x46b000a,0x46d000a,0x46f000a,0x471000a,0x473000a,0x475000a,0x477000a,0x479000a,0x47b000a,0x47d000a,0x47f000a,0x481000a,0x483000a,0x485000a,0x487000a,0x489000a, +0x48b000a,0x48d000a,0x48f000a,0x491000a,0x493000a,0x495000a,0x3b21000a,0x3b23000a,0x3b25000a,0x3b27000a,0x3b29000a,0x3b2b000a,0x3b2d000a,0x3b2f000a,0x3b31000a,0x3b33000a, +0x3b35000a,0x3b37000a,0x3b39000a,0x3b3b000a,0x3b3d000a,0x3b3f000a,0x3b41000a,0x3b43000a,0x3b45000a,0x3b47000a,0x3b49000a,0x3b4b000a,0x3b4d000a,0x3b4f000a,0x3b51000a,0x3b53000a, +0x497000a,0x499000a,0x49b000a,0x49d000a,0x49f000a,0x4a1000a,0x4a3000a,0x4a5000a,0x4a7000a,0x4a9000a,0x4ab000a,0x4ad000a,0x4af000a,0x4b1000a,0x4b3000a,0x4b5000a, +0x4b7000a,0x4b9000a,0x4bb000a,0x4bd000a,0x4bf000a,0x4c1000a,0x4c3000a,0x4c5000a,0x4c7000a,0x4c9000a,0x3b55000a,0x3b57000a,0x3b59000a,0x3b5b000a,0x3b5d000a,0x3b5f000a, +0x3b61000a,0x3b63000a,0x3b65000a,0x3b67000a,0x3b69000a,0x3b6b000a,0x3b6d000a,0x3b6f000a,0x3b71000a,0x3b73000a,0x3b75000a,0x3b77000a,0x3b79000a,0x3b7b000a,0x3b7d000a,0x3b7f000a, +0x3b81000a,0x3b83000a,0x3b85000a,0x3b87000a,0x4cb000a,0x4cd000a,0x4cf000a,0x4d1000a,0x4d3000a,0x4d5000a,0x4d7000a,0x4d9000a,0x4db000a,0x4dd000a,0x4df000a,0x4e1000a, +0x4e3000a,0x4e5000a,0x4e7000a,0x4e9000a,0x4eb000a,0x4ed000a,0x4ef000a,0x4f1000a,0x4f3000a,0x4f5000a,0x4f7000a,0x4f9000a,0x4fb000a,0x4fd000a,0x3b89000a,0x3b8b000a, +0x3b8d000a,0x3b8f000a,0x3b91000a,0x3b93000a,0x3b95000a,0x3b97000a,0x3b99000a,0x3b9b000a,0x3b9d000a,0x3b9f000a,0x3ba1000a,0x3ba3000a,0x3ba5000a,0x3ba7000a,0x3ba9000a,0x3bab000a, +0x3bad000a,0x3baf000a,0x3bb1000a,0x3bb3000a,0x3bb5000a,0x3bb7000a,0x3bb9000a,0x3bbb000a,0x3bbd000a,0x3bbf000a,0,0,0x4ff000a,0x501000a,0x503000a,0x505000a, +0x507000a,0x509000a,0x50b000a,0x50d000a,0x50f000a,0x511000a,0x513000a,0x515000a,0x517000a,0x519000a,0x51b000a,0x51d000a,0x51f000a,0x521000a,0x523000a,0x525000a, +0x527000a,0x529000a,0x52b000a,0x52d000a,0x52f000a,0x3bc1000a,0x3bc3000a,0x3bc5000a,0x3bc7000a,0x3bc9000a,0x3bcb000a,0x3bcd000a,0x3bcf000a,0x3bd1000a,0x3bd3000a,0x3bd5000a, +0x3bd7000a,0x3bd9000a,0x3bdb000a,0x3bdd000a,0x3bdf000a,0x3be1000a,0x3be3000a,0x531000a,0x3be5000a,0x3be7000a,0x3be9000a,0x3beb000a,0x3bed000a,0x3bef000a,0x3bf1000a,0x3bf3000a, +0x3bf5000a,0x3bf7000a,0x3bf9000a,0x3bfb000a,0x3bfd000a,0x3bff000a,0x533000a,0x535000a,0x537000a,0x539000a,0x53b000a,0x53d000a,0x53f000a,0x541000a,0x543000a,0x545000a, +0x547000a,0x549000a,0x54b000a,0x54d000a,0x54f000a,0x551000a,0x553000a,0x555000a,0x557000a,0x559000a,0x55b000a,0x55d000a,0x55f000a,0x561000a,0x563000a,0x3c01000a, +0x3c03000a,0x3c05000a,0x3c07000a,0x3c09000a,0x3c0b000a,0x3c0d000a,0x3c0f000a,0x3c11000a,0x3c13000a,0x3c15000a,0x3c17000a,0x3c19000a,0x3c1b000a,0x3c1d000a,0x3c1f000a,0x3c21000a, +0x3c23000a,0x565000a,0x3c25000a,0x3c27000a,0x3c29000a,0x3c2b000a,0x3c2d000a,0x3c2f000a,0x3c31000a,0x3c33000a,0x3c35000a,0x3c37000a,0x3c39000a,0x3c3b000a,0x3c3d000a,0x3c3f000a, +0x567000a,0x569000a,0x56b000a,0x56d000a,0x56f000a,0x571000a,0x573000a,0x575000a,0x577000a,0x579000a,0x57b000a,0x57d000a,0x57f000a,0x581000a,0x583000a,0x585000a, +0x587000a,0x589000a,0x58b000a,0x58d000a,0x58f000a,0x591000a,0x593000a,0x595000a,0x597000a,0x3c41000a,0x3c43000a,0x3c45000a,0x3c47000a,0x3c49000a,0x3c4b000a,0x3c4d000a, +0x3c4f000a,0x3c51000a,0x3c53000a,0x3c55000a,0x3c57000a,0x3c59000a,0x3c5b000a,0x3c5d000a,0x3c5f000a,0x3c61000a,0x3c63000a,0x599000a,0x3c65000a,0x3c67000a,0x3c69000a,0x3c6b000a, +0x3c6d000a,0x3c6f000a,0x3c71000a,0x3c73000a,0x3c75000a,0x3c77000a,0x3c79000a,0x3c7b000a,0x3c7d000a,0x3c7f000a,0x59b000a,0x59d000a,0x59f000a,0x5a1000a,0x5a3000a,0x5a5000a, +0x5a7000a,0x5a9000a,0x5ab000a,0x5ad000a,0x5af000a,0x5b1000a,0x5b3000a,0x5b5000a,0x5b7000a,0x5b9000a,0x5bb000a,0x5bd000a,0x5bf000a,0x5c1000a,0x5c3000a,0x5c5000a, +0x5c7000a,0x5c9000a,0x5cb000a,0x3c81000a,0x3c83000a,0x3c85000a,0x3c87000a,0x3c89000a,0x3c8b000a,0x3c8d000a,0x3c8f000a,0x3c91000a,0x3c93000a,0x3c95000a,0x3c97000a,0x3c99000a, +0x3c9b000a,0x3c9d000a,0x3c9f000a,0x3ca1000a,0x3ca3000a,0x5cd000a,0x3ca5000a,0x3ca7000a,0x3ca9000a,0x3cab000a,0x3cad000a,0x3caf000a,0x3cb1000a,0x3cb3000a,0x3cb5000a,0x3cb7000a, +0x3cb9000a,0x3cbb000a,0x3cbd000a,0x3cbf000a,0x5cf000a,0x5d1000a,0x5d3000a,0x5d5000a,0x5d7000a,0x5d9000a,0x5db000a,0x5dd000a,0x5df000a,0x5e1000a,0x5e3000a,0x5e5000a, +0x5e7000a,0x5e9000a,0x5eb000a,0x5ed000a,0x5ef000a,0x5f1000a,0x5f3000a,0x5f5000a,0x5f7000a,0x5f9000a,0x5fb000a,0x5fd000a,0x5ff000a,0x3cc1000a,0x3cc3000a,0x3cc5000a, +0x3cc7000a,0x3cc9000a,0x3ccb000a,0x3ccd000a,0x3ccf000a,0x3cd1000a,0x3cd3000a,0x3cd5000a,0x3cd7000a,0x3cd9000a,0x3cdb000a,0x3cdd000a,0x3cdf000a,0x3ce1000a,0x3ce3000a,0x601000a, +0x3ce5000a,0x3ce7000a,0x3ce9000a,0x3ceb000a,0x3ced000a,0x3cef000a,0x3cf1000a,0x3cf3000a,0x3cf5000a,0x3cf7000a,0x3cf9000a,0x3cfb000a,0x3cfd000a,0x3cff000a,0x603000a,0x3d01000a, +0,0,0x3d03000a,0x3d05000a,0x3d07000a,0x3d09000a,0x3d0b000a,0x3d0d000a,0x3d0f000a,0x3d11000a,0x3d13000a,0x3d15000a,0x3d17000a,0x3d19000a,0x3d1b000a,0x3d1d000a, +0x3d1f000a,0x3d21000a,0x3d23000a,0x3d25000a,0x605e60f,0x608e60f,0x206be6b0,0x60be60f,0x60ee60f,0x206cf0b0,0xe600,0xdc00,0xdc00,0xdc00,0xe600,0xe600, +0xe600,0xdc00,0xdc00,0,0xe600,0xe600,0xe600,0xdc00,0xdc00,0xdc00,0xdc00,0xe600,0xe800,0xdc00,0xdc00,0xe600, +0xe900,0xea00,0xea00,0xe900,0,0,0,0,0x211f000a,0x145c000e,0x1464000c,0x616000f,0x1468000c,0x146c000c,0x1470000c,0, +0x1474000c,0,0x1478000c,0x147c000c,0x1480000c,0x3d940040,0,0,0,0x3d950040,0,0x3d960040,0,0x3d970040,0,0, +0,0,0,0x3d980040,0,0,0,0,0,0,0,0,0,0,0,0, +0,0x900,0,0,0,0xe600,0xdc00,0xe600,0xe600,0,0,0,0x618000f,0x61c000f,0x620000f,0x624000f, +0x628000f,0x62c000f,0x630000f,0x634000f,0,0,0,0,0,0,0,0x3dcb0040,0,0,0,0x15b7000c, +0x15ba000c,0x900,0,0,0,0,0,0,0,0,0,0x207200b0,0,0,0,0, +0x638000f,0x63c000f,0,0x640000f,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0x644000f,0,0,0x648000f,0,0,0,0,0, +0x700,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0x900,0,0,0,0,0,0,0,0,0,0,0,0x64c000f,0x650000f,0x654000f, +0,0,0x658000f,0,0,0,0,0,0,0,0,0x3dcc0040,0x15bd000c,0,0,0x15c0000c, +0x15c3000c,0x900,0,0,0,0,0,0,0,0,0x207400b0,0x207500b0,0,0,0,0, +0x65c000f,0x660000f,0,0,0,0,0,0x664000f,0,0,0,0,0,0,0,0, +0,0x667000f,0,0,0,0,0x66a000f,0,0,0,0,0x66d000f,0,0,0,0, +0x670000f,0,0,0,0,0,0,0,0,0,0,0,0,0x673000f,0,0, +0,0,0,0,0,0x8100,0x8200,0x676000f,0x8400,0x67a000f,0x67e000f,0x2150000a,0x682000f,0x2155000a,0x8200,0x8200, +0x8200,0x8200,0,0,0x8200,0x686000f,0xe600,0xe600,0x900,0,0xe600,0xe600,0,0,0,0, +0,0,0,0,0,0,0,0x68a000f,0,0,0,0,0,0,0,0, +0,0x68d000f,0,0,0,0,0x690000f,0,0,0,0,0x693000f,0,0,0,0, +0x696000f,0,0,0,0,0,0,0,0,0,0,0,0,0x699000f,0,0, +0,0,0,0,0x1c20004c,0x1c25004c,0x1c2a004c,0x1c30004c,0x1c36004c,0x1c3c004c,0x1c42004c,0x1c48004c,0x1c4e004c,0x1c53004c,0x1c58004c,0x1c5e004c, +0x1c64004c,0x1c6a004c,0x1c70004c,0x1c76004c,0x1c7c004c,0x69c000f,0x1c80000c,0x6a0000f,0x1c85004c,0x6a4000f,0x1c89000c,0x6a8000f,0x1c8d000c,0x6ac000f,0x1c91000c,0x6b0000f, +0x1c96004c,0x6b4000f,0,0,0x1d52000c,0x1d57000c,0x1d5c000c,0x1d62000c,0x1d68000c,0x1d6e000c,0x1d74000c,0x1d7a000c,0x1d80000c,0x1d85000c,0x1d8a000c,0x1d90000c, +0x1d96000c,0x1d9c000c,0x1da2000c,0x1da8000c,0x1dae000c,0x1db2000c,0x1db6000c,0x1dbb000c,0x1dbf000c,0,0x1dc5004c,0x1dc9000c,0x1dce000c,0x1dd2000c,0x1dd6000c,0x6b8000f, +0x1dda000c,0x21f9000a,0x6bc000f,0x21fe004a,0x2202000a,0x1dde000e,0x1de6000c,0x1deb000c,0x1def000c,0,0x1df5004c,0x1df9000c,0x1dfe000c,0x6be000f,0x1e02000c,0x6c2000f, +0x1e06000c,0x1e0a000e,0x1e12000e,0x1e1a000e,0x1e22000c,0x1e26000c,0x1e2a000c,0x6c6000f,0,0,0x1e2f000c,0x1e33000c,0x1e38000c,0x1e3c000c,0x1e40000c,0x6cb000f, +0,0x1e44000e,0x1e4c000e,0x1e54000e,0x1e5c000c,0x1e60000c,0x1e64000c,0x6cf000f,0x1e69000c,0x1e6d000c,0x1e71000c,0x1e75000c,0x1e7a000c,0x1e7e000c,0x1e82000c,0x6d4000f, +0x1e86000c,0x1e8a000e,0x6d8000f,0x6e0000f,0,0,0x1e92000c,0x1e97000c,0x1e9b000c,0,0x1ea1004c,0x1ea5000c,0x1eaa000c,0x6e2000f,0x1eae000c,0x6e6000f, +0x1eb2000c,0x6ea000f,0x2207004a,0,0x6ef000f,0x6f2000f,0x220b000a,0x220d000a,0x220f000a,0x2211000a,0x2213000a,0x2215000a,0x2217000a,0x2219000a,0x221b000a,0, +0,0,0,0,0,0x221d000a,0,0,0,0,0,0x221f000a,0,0,0,0, +0,0,0,0,0,0x6fd000f,0x6ff000f,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0x701000f,0,0,0,0x705000f,0x707000f,0x709000f,0x70b000f,0x70d000f,0x70f000f,0x711000f,0x713000f, +0x715000f,0x717000f,0x719000f,0x71b000f,0x71d000f,0x71f000f,0x721000f,0x723000f,0x725000f,0x727000f,0x729000f,0x72b000f,0x72d000f,0x72f000f,0x731000f,0x733000f, +0x735000f,0x737000f,0x739000f,0x73b000f,0x73d000f,0x73f000f,0x741000f,0x743000f,0x745000f,0x747000f,0x749000f,0x74b000f,0x74d000f,0x74f000f,0x751000f,0x753000f, +0x755000f,0x757000f,0x759000f,0x75b000f,0x75d000f,0x75f000f,0x761000f,0x763000f,0x765000f,0x767000f,0x769000f,0x76b000f,0x76d000f,0x76f000f,0x771000f,0x773000f, +0x775000f,0x777000f,0x779000f,0x77b000f,0x77d000f,0x77f000f,0x781000f,0x783000f,0x785000f,0x787000f,0x789000f,0x78b000f,0x78d000f,0x78f000f,0x791000f,0x793000f, +0x795000f,0x797000f,0x799000f,0x79b000f,0x79d000f,0x79f000f,0x7a1000f,0x7a3000f,0x7a5000f,0x7a7000f,0x7a9000f,0x7ab000f,0x7ad000f,0x7af000f,0x7b1000f,0x7b3000f, +0x7b5000f,0x7b7000f,0x7b9000f,0x7bb000f,0x7bd000f,0x7bf000f,0x7c1000f,0x7c3000f,0x7c5000f,0x7c7000f,0x7c9000f,0x7cb000f,0x7cd000f,0x7cf000f,0x7d1000f,0x7d3000f, +0x7d5000f,0x7d7000f,0x7d9000f,0x7db000f,0x7dd000f,0x7df000f,0x7e1000f,0x7e3000f,0x7e5000f,0x7e7000f,0x7e9000f,0x7eb000f,0x7ed000f,0x7ef000f,0x7f1000f,0x7f3000f, +0x7f5000f,0x7f7000f,0x7f9000f,0x7fb000f,0x7fd000f,0x7ff000f,0x801000f,0x803000f,0x805000f,0x807000f,0x809000f,0x80b000f,0x80d000f,0x80f000f,0x811000f,0x813000f, +0x815000f,0x817000f,0x819000f,0x81b000f,0x81d000f,0x81f000f,0x821000f,0x823000f,0x825000f,0x827000f,0x829000f,0x82b000f,0x82d000f,0x82f000f,0x831000f,0x833000f, +0x835000f,0x837000f,0x839000f,0x83b000f,0x83d000f,0x83f000f,0x841000f,0x843000f,0x845000f,0x847000f,0x849000f,0x84b000f,0x84d000f,0x84f000f,0x851000f,0x853000f, +0x855000f,0x857000f,0x859000f,0x85b000f,0x85d000f,0x85f000f,0x861000f,0x863000f,0x865000f,0x867000f,0x869000f,0x86b000f,0x86d000f,0x86f000f,0x871000f,0x873000f, +0x875000f,0x877000f,0x879000f,0x87b000f,0x87d000f,0x87f000f,0x881000f,0x883000f,0x885000f,0x887000f,0x889000f,0x88b000f,0x88d000f,0x88f000f,0x891000f,0x893000f, +0x895000f,0x897000f,0x899000f,0x89b000f,0x89d000f,0x89f000f,0x8a1000f,0x8a3000f,0x8a5000f,0x8a7000f,0x8a9000f,0x8ab000f,0x8ad000f,0x8af000f,0x8b1000f,0x8b3000f, +0x8b5000f,0x8b7000f,0x8b9000f,0x8bb000f,0x8bd000f,0x8bf000f,0x8c1000f,0x8c3000f,0x8c5000f,0x8c7000f,0x8c9000f,0x8cb000f,0x8cd000f,0x8cf000f,0x8d1000f,0x8d3000f, +0x8d5000f,0x8d7000f,0x8d9000f,0x8db000f,0x8dd000f,0x8df000f,0x8e1000f,0x8e3000f,0x8e5000f,0x8e7000f,0x8e9000f,0x8eb000f,0x8ed000f,0x8ef000f,0x8f1000f,0x8f3000f, +0x8f5000f,0x8f7000f,0x8f9000f,0x8fb000f,0x8fd000f,0x8ff000f,0x901000f,0x903000f,0x905000f,0x907000f,0x909000f,0x90b000f,0x90d000f,0x90f000f,0x911000f,0x913000f, +0x915000f,0x917000f,0x919000f,0x91b000f,0x91d000f,0x91f000f,0,0,0x921000f,0,0x923000f,0,0,0x925000f,0x927000f,0x929000f, +0x92b000f,0x92d000f,0x92f000f,0x931000f,0x933000f,0x935000f,0x937000f,0,0x939000f,0,0x93b000f,0,0,0x93d000f,0x93f000f,0, +0,0,0x941000f,0x943000f,0x945000f,0x947000f,0,0,0x949000f,0x94b000f,0x94d000f,0x94f000f,0x951000f,0x953000f,0x955000f,0x957000f, +0x959000f,0x95b000f,0x95d000f,0x95f000f,0x961000f,0x963000f,0x965000f,0x967000f,0x969000f,0x96b000f,0x96d000f,0x96f000f,0x971000f,0x973000f,0x975000f,0x977000f, +0x979000f,0x97b000f,0x97d000f,0x97f000f,0x981000f,0x983000f,0x985000f,0x987000f,0x989000f,0x98b000f,0x98d000f,0x98f000f,0x991000f,0x993000f,0x995000f,0x997000f, +0x999000f,0x99b000f,0x99d000f,0x99f000f,0x9a1000f,0x9a3000f,0x9a5000f,0x9a7000f,0x9a9000f,0x9ab000f,0x9ad000f,0x9af000f,0x9b1000f,0x9b3000f,0x9b5000f,0x9b7000f, +0x9b9000f,0x9bb000f,0x9bd000f,0,0,0,0,0,0x9bf000f,0x9c1000f,0x9c3000f,0x9c5000f,0x9c7000f,0x9c9000f,0x9cb000f,0x9cd000f, +0x9cf000f,0x9d1000f,0x9d3000f,0x9d5000f,0x9d7000f,0x9d9000f,0x9db000f,0x9dd000f,0x9df000f,0x9e1000f,0x9e3000f,0x9e5000f,0x9e7000f,0x9e9000f,0x9eb000f,0x9ed000f, +0x9ef000f,0x9f1000f,0x9f3000f,0x9f5000f,0x9f7000f,0x9f9000f,0x9fb000f,0x9fd000f,0x9ff000f,0xa01000f,0xa03000f,0xa05000f,0xa07000f,0xa09000f,0xa0b000f,0xa0d000f, +0xa0f000f,0xa11000f,0xa13000f,0xa15000f,0xa17000f,0xa19000f,0xa1b000f,0xa1d000f,0xa1f000f,0xa21000f,0xa23000f,0xa25000f,0xa27000f,0xa29000f,0xa2b000f,0xa2d000f, +0xa2f000f,0xa31000f,0xa33000f,0xa35000f,0xa37000f,0xa39000f,0xa3b000f,0xa3d000f,0xa3f000f,0xa41000f,0xa43000f,0xa45000f,0xa47000f,0xa49000f,0xa4b000f,0xa4d000f, +0xa4f000f,0xa51000f,0xa53000f,0xa55000f,0xa57000f,0xa59000f,0xa5b000f,0xa5d000f,0xa5f000f,0xa61000f,0xa63000f,0xa65000f,0xa67000f,0xa69000f,0xa6b000f,0xa6d000f, +0xa6f000f,0xa71000f,0xa73000f,0xa75000f,0xa77000f,0xa79000f,0xa7b000f,0xa7d000f,0xa80000f,0xa83000f,0xa86000f,0xa88000f,0xa8a000f,0xa8c000f,0xa8f000f,0xa92000f, +0xa95000f,0xa97000f,0,0,0,0,0,0,0x2dd9000a,0x2ddc000a,0x2ddf000a,0x2de2000a,0x2de6000a,0x2dea000a,0x2ded000a,0, +0,0,0,0,0,0,0,0,0,0,0,0x2df0000a,0x2df3000a,0x2df6000a,0x2df9000a,0x2dfc000a, +0,0,0,0,0,0xa99000f,0x1a00,0xa9d000f,0x2dff000a,0x2e01000a,0x2e03000a,0x2e05000a,0x2e07000a,0x2e09000a,0x2e0b000a,0x2e0d000a, +0x2e0f000a,0x2e11000a,0xaa1000f,0xaa5000f,0xaa9000f,0xaae000f,0xab3000f,0xab7000f,0xabb000f,0xabf000f,0xac3000f,0xac7000f,0xacb000f,0xacf000f,0xad3000f,0, +0xad7000f,0xadb000f,0xadf000f,0xae3000f,0xae7000f,0,0xaeb000f,0,0xaef000f,0xaf3000f,0,0xaf7000f,0xafb000f,0,0xaff000f,0xb03000f, +0xb07000f,0xb0b000f,0xb0f000f,0xb13000f,0xb17000f,0xb1b000f,0xb1f000f,0x2e13000a,0x2e16000a,0x2e18000a,0x2e1a000a,0x2e1c000a,0x2e1e000a,0x2e20000a,0x2e22000a,0x2e24000a, +0x2e26000a,0x2e28000a,0x2e2a000a,0x2e2c000a,0x2e2e000a,0x2e30000a,0x2e32000a,0x2e34000a,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0xb23000f,0xb29000f,0xb2f000f,0xb37000f,0xb3f000f,0xb47000f,0xb4f000f,0xd800,0xd800,0x100, +0x100,0x100,0,0,0,0xe200,0xd800,0xd800,0xd800,0xd800,0xd800,0,0,0,0,0, +0,0,0,0xdc00,0xdc00,0xdc00,0xdc00,0xdc00,0,0,0,0,0,0,0,0, +0,0,0xe600,0xe600,0xe600,0xe600,0,0,0,0,0,0,0,0,0,0, +0,0,0,0xb57000f,0xb5d000f,0xb63000f,0xb6b000f,0xb73000f,0xb7b000f,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0xb83000f,0xb85000f,0xb87000f,0xb89000f,0xb8c000f,0xb8e000f,0xb90000f,0xb92000f, +0xb94000f,0xb96000f,0xb98000f,0xb9a000f,0xb9c000f,0xb9e000f,0xba1000f,0xba3000f,0xba5000f,0xba7000f,0xba9000f,0xbac000f,0xbae000f,0xbb0000f,0xbb2000f,0xbb5000f, +0xbb7000f,0xbb9000f,0xbbb000f,0xbbd000f,0xbbf000f,0xbc2000f,0xbc4000f,0xbc6000f,0xbc8000f,0xbca000f,0xbcc000f,0xbce000f,0xbd0000f,0xbd2000f,0xbd4000f,0xbd6000f, +0xbd8000f,0xbda000f,0xbdc000f,0xbde000f,0xbe0000f,0xbe2000f,0xbe4000f,0xbe6000f,0xbe8000f,0xbea000f,0xbec000f,0xbee000f,0xbf0000f,0xbf3000f,0xbf5000f,0xbf7000f, +0xbf9000f,0xbfc000f,0xbfe000f,0xc00000f,0xc02000f,0xc04000f,0xc06000f,0xc08000f,0xc0a000f,0xc0c000f,0xc0e000f,0xc10000f,0xc12000f,0xc14000f,0xc16000f,0xc18000f, +0xc1a000f,0xc1c000f,0xc1e000f,0xc20000f,0xc22000f,0xc24000f,0xc26000f,0xc28000f,0xc2a000f,0xc2c000f,0xc2e000f,0xc30000f,0xc32000f,0xc34000f,0xc36000f,0xc38000f, +0xc3a000f,0xc3c000f,0xc3f000f,0xc41000f,0xc43000f,0xc45000f,0xc47000f,0xc49000f,0xc4b000f,0xc4e000f,0xc51000f,0xc53000f,0xc55000f,0xc57000f,0xc59000f,0xc5b000f, +0xc5d000f,0xc5f000f,0xc61000f,0xc63000f,0xc65000f,0xc68000f,0xc6a000f,0xc6c000f,0xc6e000f,0xc70000f,0xc73000f,0xc75000f,0xc77000f,0xc79000f,0xc7b000f,0xc7d000f, +0xc7f000f,0xc81000f,0xc83000f,0xc85000f,0xc88000f,0xc8a000f,0xc8d000f,0xc8f000f,0xc91000f,0xc93000f,0xc95000f,0xc97000f,0xc99000f,0xc9b000f,0xc9d000f,0xc9f000f, +0xca1000f,0xca3000f,0xca6000f,0xca8000f,0xcaa000f,0xcac000f,0xcae000f,0xcb0000f,0xcb3000f,0xcb5000f,0xcb8000f,0xcbb000f,0xcbd000f,0xcbf000f,0xcc1000f,0xcc3000f, +0xcc6000f,0xcc9000f,0xccb000f,0xccd000f,0xccf000f,0xcd1000f,0xcd3000f,0xcd5000f,0xcd7000f,0xcd9000f,0xcdb000f,0xcdd000f,0xcdf000f,0xce2000f,0xce4000f,0xce6000f, +0xce8000f,0xcea000f,0xcec000f,0xcee000f,0xcf0000f,0xcf2000f,0xcf4000f,0xcf6000f,0xcf8000f,0xcfa000f,0xcfc000f,0xcfe000f,0xd00000f,0xd02000f,0xd04000f,0xd06000f, +0xd08000f,0xd0b000f,0xd0d000f,0xd0f000f,0xd11000f,0xd13000f,0xd15000f,0xd18000f,0xd1a000f,0xd1c000f,0xd1e000f,0xd20000f,0xd22000f,0xd24000f,0xd26000f,0xd28000f, +0xd2a000f,0xd2c000f,0xd2e000f,0xd31000f,0xd33000f,0xd35000f,0xd37000f,0xd39000f,0xd3b000f,0xd3d000f,0xd3f000f,0xd41000f,0xd43000f,0xd45000f,0xd47000f,0xd49000f, +0xd4b000f,0xd4d000f,0xd4f000f,0xd51000f,0xd53000f,0xd55000f,0xd58000f,0xd5a000f,0xd5c000f,0xd5e000f,0xd60000f,0xd62000f,0xd65000f,0xd67000f,0xd69000f,0xd6b000f, +0xd6d000f,0xd6f000f,0xd71000f,0xd73000f,0xd75000f,0xd78000f,0xd7a000f,0xd7c000f,0xd7e000f,0xd81000f,0xd83000f,0xd85000f,0xd87000f,0xd89000f,0xd8b000f,0xd8d000f, +0xd90000f,0xd93000f,0xd96000f,0xd98000f,0xd9b000f,0xd9d000f,0xd9f000f,0xda1000f,0xda3000f,0xda5000f,0xda7000f,0xda9000f,0xdab000f,0xdad000f,0xdaf000f,0xdb2000f, +0xdb4000f,0xdb6000f,0xdb8000f,0xdba000f,0xdbc000f,0xdbe000f,0xdc1000f,0xdc3000f,0xdc5000f,0xdc8000f,0xdcb000f,0xdcd000f,0xdcf000f,0xdd1000f,0xdd3000f,0xdd5000f, +0xdd7000f,0xdd9000f,0xddb000f,0xddd000f,0xde0000f,0xde2000f,0xde5000f,0xde7000f,0xdea000f,0xdec000f,0xdee000f,0xdf0000f,0xdf3000f,0xdf5000f,0xdf7000f,0xdfa000f, +0xdfd000f,0xdff000f,0xe01000f,0xe03000f,0xe05000f,0xe07000f,0xe09000f,0xe0b000f,0xe0d000f,0xe0f000f,0xe11000f,0xe13000f,0xe15000f,0xe17000f,0xe1a000f,0xe1c000f, +0xe1f000f,0xe21000f,0xe24000f,0xe26000f,0xe29000f,0xe2c000f,0xe2f000f,0xe31000f,0xe33000f,0xe35000f,0xe38000f,0xe3b000f,0xe3e000f,0xe41000f,0xe43000f,0xe45000f, +0xe47000f,0xe49000f,0xe4b000f,0xe4d000f,0xe4f000f,0xe51000f,0xe54000f,0xe56000f,0xe58000f,0xe5a000f,0xe5c000f,0xe5f000f,0xe61000f,0xe64000f,0xe67000f,0xe69000f, +0xe6b000f,0xe6d000f,0xe6f000f,0xe71000f,0xe73000f,0xe76000f,0xe79000f,0xe7c000f,0xe7e000f,0xe80000f,0xe83000f,0xe85000f,0xe87000f,0xe89000f,0xe8c000f,0xe8e000f, +0xe90000f,0xe92000f,0xe94000f,0xe96000f,0xe99000f,0xe9b000f,0xe9d000f,0xe9f000f,0xea1000f,0xea3000f,0xea5000f,0xea8000f,0xeab000f,0xead000f,0xeb0000f,0xeb2000f, +0xeb5000f,0xeb7000f,0xeb9000f,0xebb000f,0xebe000f,0xec1000f,0xec3000f,0xec6000f,0xec8000f,0xecb000f,0xecd000f,0xecf000f,0xed1000f,0xed3000f,0xed5000f,0xed7000f, +0xeda000f,0xedd000f,0xee0000f,0xee3000f,0xee5000f,0xee7000f,0xee9000f,0xeeb000f,0xeed000f,0xeef000f,0xef1000f,0xef3000f,0xef5000f,0xef7000f,0xef9000f,0xefb000f, +0xefe000f,0xf00000f,0xf02000f,0xf04000f,0xf06000f,0xf08000f,0xf0a000f,0xf0c000f,0xf0e000f,0xf10000f,0xf12000f,0xf14000f,0xf16000f,0xf19000f,0xf1c000f,0xf1f000f, +0xf21000f,0xf23000f,0xf25000f,0xf27000f,0xf2a000f,0xf2c000f,0xf2f000f,0xf31000f,0xf33000f,0xf36000f,0xf39000f,0xf3b000f,0xf3d000f,0xf3f000f,0xf41000f,0xf43000f, +0xf45000f,0xf47000f,0xf49000f,0xf4b000f,0xf4d000f,0xf4f000f,0xf51000f,0xf53000f,0xf55000f,0xf57000f,0xf59000f,0xf5b000f,0xf5d000f,0xf5f000f,0xf62000f,0xf64000f, +0xf66000f,0xf68000f,0xf6a000f,0xf6c000f,0xf6f000f,0xf72000f,0xf74000f,0xf76000f,0xf78000f,0xf7a000f,0xf7c000f,0xf7e000f,0xf81000f,0xf83000f,0xf85000f,0xf87000f, +0xf89000f,0xf8c000f,0xf8f000f,0xf91000f,0xf93000f,0xf95000f,0xf98000f,0xf9a000f,0xf9c000f,0xf9f000f,0xfa2000f,0xfa4000f,0xfa6000f,0xfa8000f,0xfab000f,0xfad000f, +0xfaf000f,0xfb1000f,0xfb3000f,0xfb5000f,0xfb7000f,0xfb9000f,0xfbc000f,0xfbe000f,0xfc0000f,0xfc2000f,0xfc5000f,0xfc7000f,0xfc9000f,0xfcb000f,0xfcd000f,0xfd0000f, +0xfd3000f,0xfd5000f,0xfd7000f,0xfd9000f,0xfdc000f,0xfde000f,0xfe1000f,0xfe3000f,0xfe5000f,0xfe7000f,0xfea000f,0xfec000f,0xfee000f,0xff0000f,0xff2000f,0xff4000f, +0xff6000f,0xff8000f,0xffb000f,0xffd000f,0xfff000f,0x1001000f,0x1003000f,0x1005000f,0x1007000f,0x100a000f,0x100c000f,0x100f000f,0x1012000f,0x1015000f,0x1017000f,0x1019000f, +0x101b000f,0x101d000f,0x101f000f,0x1021000f,0x1023000f,0x1025000f,0,0,0x1028000c,0x102c000c,0x1031004c,0x1035000c,0x103a004c,0x103f004c,0x3d8e0040,0x1044004c, +0x1048000c,0x104c000c,0x1051004c,0x1055000c,0x1059000c,0x105d000c,0x1061000c,0x1066004c,0,0x106a000c,0x106e000c,0x1072000c,0x1077004c,0x107c004c,0x1081004c,0, +0x3d920040,0x1085000c,0x1089000c,0x108d000c,0x1092004c,0x1096000c,0,0,0x109a000c,0x109e000c,0x10a3004c,0x10a7000c,0x10ac004c,0x10b1004c,0x3d8f0040,0x10b6004c, +0x10ba000c,0x10be000c,0x10c3004c,0x10c7000c,0x10cb000c,0x10cf000c,0x10d3000c,0x10d8004c,0,0x10dc000c,0x10e0000c,0x10e4000c,0x10e9004c,0x10ee004c,0x10f3004c,0, +0x3d930040,0x10f7000c,0x10fb000c,0x10ff000c,0x1104004c,0x1108000c,0,0x110c000c,0x1110000c,0x1114000c,0x1119004c,0x111e004c,0x1122000c,0x1126000c,0x112a000c,0x112e000c, +0x1132000c,0x1136000c,0x113a000c,0x113e000c,0x1142000c,0x1146000c,0x114a000c,0x114e000c,0,0,0x1153004c,0x1158004c,0x115c000c,0x1160000c,0x1164000c,0x1168000c, +0x116c000c,0x1170000c,0x1174000c,0x1178000c,0x117c000c,0x1180000c,0x1184000c,0x1188000c,0x118c000c,0x1190000c,0x1194000c,0x1198000c,0x119c000c,0x11a0000c,0,0, +0x11a4000c,0x11a8000c,0x11ac000c,0x11b0000c,0x11b4000c,0x11b8000c,0x11bc000c,0x11c0000c,0x11c4000c,0,0x20af000a,0x20b2000a,0x11c8000c,0x11cc000c,0x11d0000c,0x11d4000c, +0,0x11d8000c,0x11dc000c,0x11e0000c,0x11e4000c,0x11e8000c,0x11ec000c,0x20b5000a,0x20b8000a,0,0,0x11f0000c,0x11f4000c,0x11f8000c,0x11fc000c,0x1200000c, +0x1204000c,0x20bb000a,0,0,0x1209004c,0x120e004c,0x1212000c,0x1216000c,0x121a000c,0x121e000c,0,0,0x1222000c,0x1226000c,0x122a000c,0x122e000c, +0x1232000c,0x1236000c,0x123b004c,0x1240004c,0x1244000c,0x1248000c,0x124c000c,0x1250000c,0x1255004c,0x125a004c,0x125e000c,0x1262000c,0x1266000c,0x126a000c,0,0, +0x126f004c,0x1274004c,0x1279004c,0x127e004c,0x1282000c,0x1286000c,0x128a000c,0x128e000c,0x1292000c,0x1296000c,0x129a000c,0x129e000c,0x12a2000c,0x12a6000c,0x12aa000c,0x12ae000c, +0x12b2000c,0x12b6000c,0x12ba000c,0x12be000c,0x12c2000c,0x12c6000c,0x12ca000c,0x20bf004a,0x12cf004c,0x12d4004c,0,0,0,0,0,0, +0,0,0,0,0,0,0,0x12d9004c,0x12de004c,0,0,0,0,0,0,0x3d900040, +0,0,0,0,0,0,0,0,0x20c1000a,0x20c6000a,0x20cb000a,0x20d0000a,0x20d3000a,0x20d6000a,0x20d9000a,0x20dc000a, +0x20df000a,0x12e2000c,0x12e6000c,0x12ea000c,0x12ee000c,0x12f2000c,0x12f6000c,0x12fa000c,0x12fe000c,0x1302000c,0x1307000c,0x130c000c,0x1311000c,0x1316000c,0x131b000c,0x1320000c, +0x1325000c,0,0x132a000c,0x132f000c,0x1334000c,0x1339000c,0x133e000c,0x1342000c,0,0,0x1346000c,0x134a000c,0x134e000c,0x1352000c,0x1357004c,0x135c004c, +0x1360000c,0x1365000c,0x136a000c,0x136e000c,0x1372000c,0x20e2000a,0x20e5000a,0x20e8000a,0x1376000c,0x137a000c,0,0,0x137e000c,0x1382000c,0x1386000c,0x138b000c, +0x1390000c,0x1394000c,0x1398000c,0x139c000c,0x13a0000c,0x13a4000c,0x13a8000c,0x13ac000c,0x13b0000c,0x13b4000c,0x13b8000c,0x13bc000c,0x13c0000c,0x13c4000c,0x13c8000c,0x13cc000c, +0x13d0000c,0x13d4000c,0x13d8000c,0x13dc000c,0x13e0000c,0x13e4000c,0x13e8000c,0x13ec000c,0x13f0000c,0x13f4000c,0x13f8000c,0x13fc000c,0x1400000c,0x1404000c,0x1408000c,0x140c000c, +0,0,0x1410000c,0x1414000c,0,0,0,0,0,0,0x1419004c,0x141e004c,0x1423004c,0x1428004c,0x142c000c,0x1431000c, +0x1436000c,0x143b000c,0x1441004c,0x1446004c,0x144a000c,0x144f000c,0x1454000c,0x1458000c,0,0,0,0,0,0,0,0, +0,0,0,0,0,0x3def0040,0,0,0,0x3d990040,0,0,0,0x3d9a0040,0x1485000c,0x1489000c, +0x148e004c,0x1492000c,0x1497004c,0x149b000c,0x149f000c,0x3d9b0040,0,0,0,0x3d9c0040,0,0x3d9d0040,0,0x3d9e0040,0,0, +0,0,0,0x3da00040,0x14bb000c,0x14bf000c,0,0x14c3000c,0,0,0x3da40040,0x14c7000c,0,0,0,0, +0x14cb000c,0x14cf000c,0x14d3000c,0,0x3db20040,0,0,0x3da30040,0,0x3da20040,0x3db00040,0x3db60040,0x3da60040,0x14d7000c,0x3da50040,0, +0,0,0x3db80040,0,0,0,0,0x3da70040,0,0,0,0x3dbe0040,0,0,0,0x3dc00040, +0,0x3dbc0040,0,0,0x3db30040,0,0,0x3daa0040,0,0x3da90040,0x3db10040,0x3db70040,0x3da80040,0x14db000c,0x3dac0040,0, +0,0,0x3db90040,0,0,0,0,0x3dad0040,0,0,0,0x3dbf0040,0,0,0,0x3dc10040, +0,0x3dbd0040,0,0,0x14df000c,0x14e3000c,0,0x14e7000c,0,0,0x3dab0040,0x14eb000c,0,0,0,0, +0x14ef000c,0x14f3000c,0x14f7000c,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0x3dae0040,0x3daf0040,0x14fb000c,0x14ff000c,0,0,0,0, +0,0,0,0,0,0x1503000c,0x1507000c,0,0,0,0,0,0,0,0,0, +0,0,0,0,0x150b000c,0x150f000c,0x1513000c,0x1517000c,0,0,0x151b000c,0x151f000c,0x3db40040,0x3db50040,0x1523000c,0x1527000c, +0x152b000c,0x152f000c,0x1533000c,0x1537000c,0,0,0x153b000c,0x153f000c,0x1543000c,0x1547000c,0x154b000c,0x154f000c,0x3dba0040,0x3dbb0040,0x1553000c,0x1557000c, +0x155b000c,0x155f000c,0x1563000c,0x1567000c,0x156b000c,0x156f000c,0x1573000c,0x1577000c,0x157b000c,0x157f000c,0,0,0x1583000c,0x1587000c,0,0, +0,0,0,0,0,0,0x158b000c,0x158f000c,0x1593000c,0x1597000c,0x159b000c,0x3dc20040,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0x159f000c,0x3dc60040,0x15a3000c,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0x3dc70040,0x15a7000c,0,0x3dc50040,0xe600,0xe600,0xe600,0xe600,0xe600,0xe600, +0xe600,0,0,0xe600,0,0,0,0,0,0,0,0,0x3dc80040,0x15ab000c,0,0, +0,0,0,0,0x3dc90040,0x15af000c,0,0x3dca0040,0x15b3000c,0,0,0,0,0,0,0, +0x207007b0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0x3dcd0040,0,0x15c6000c,0,0,0,0,0,0,0, +0,0,0,0,0,0,0x3dce0040,0x3dcf0040,0,0,0x15c9000c,0x15cc000c,0x15cf000c,0x900,0,0, +0,0,0,0,0,0,0,0x207700b0,0,0,0,0,0,0,0,0, +0,0,0x3dd00040,0,0x15d2000c,0,0,0,0,0x900,0,0,0,0,0,0, +0,0x5400,0x20785bb0,0,0,0,0,0,0,0,0,0,0x15d6000c,0,0x207900b0,0, +0,0,0x3dd20040,0x15d9000c,0x15dc000c,0,0x15e0004c,0x15e3000c,0,0x900,0,0,0,0,0,0, +0,0x207a00b0,0x207b00b0,0,0,0,0,0,0,0,0,0,0,0,0x3dd30040,0x3dd40040, +0,0,0x15e7000c,0x15ea000c,0x15ed000c,0x900,0,0,0,0,0,0,0,0,0,0x207d00b0, +0,0,0,0,0,0,0,0,0,0,0x207e09b0,0,0,0,0,0x207f00b0, +0,0,0,0,0,0,0,0,0,0x3dd50040,0x15f0000c,0,0x15f5004c,0x15f8000c,0x15fd000c,0x208000b0, +0,0,0,0,0,0x3dd60040,0x1600000c,0,0,0,0,0,0,0,0x208100b0,0, +0,0,0,0,0,0,0,0x700,0,0x900,0,0,0,0,0,0, +0,0x3dd70040,0x1603000c,0x3dd80040,0x1606000c,0x3dd90040,0x1609000c,0x3dda0040,0x160c000c,0x3ddb0040,0x160f000c,0,0,0x3ddc0040,0x1612000c,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0x700,0x208200b0,0,0,0,0,0x3ddd0040,0x1615000c,0x3dde0040,0x1618000c,0x3ddf0040,0x3de00040, +0x161b000c,0x161e000c,0x3de10040,0x1621000c,0x900,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0x1624000c,0x1628000c,0x162c000c,0x1630000c,0x1634000c,0x1638000c,0x163c000c,0x1640000c,0x1644000c,0x1649000c,0x164e000c,0x1652000c,0x1656000c,0x165a000c,0x165e000c,0x1662000c, +0x1666000c,0x166a000c,0x166e000c,0x1672000c,0x1676000c,0x167b000c,0x1680000c,0x1685000c,0x168a000c,0x168e000c,0x1692000c,0x1696000c,0x169a000c,0x169f000c,0x16a4000c,0x16a8000c, +0x16ac000c,0x16b0000c,0x16b4000c,0x16b8000c,0x16bc000c,0x16c0000c,0x16c4000c,0x16c8000c,0x16cc000c,0x16d0000c,0x16d4000c,0x16d8000c,0x16dc000c,0x16e0000c,0x16e4000c,0x16e9000c, +0x16ee000c,0x16f2000c,0x16f6000c,0x16fa000c,0x16fe000c,0x1702000c,0x1707004c,0x170c004c,0x1710000c,0x1715000c,0x171a000c,0x171e000c,0x1722000c,0x1726000c,0x172a000c,0x172e000c, +0x1732000c,0x1736000c,0x173a000c,0x173e000c,0x1742000c,0x1746000c,0x174a000c,0x174e000c,0x1752000c,0x1756000c,0x175a000c,0x175e000c,0x1762000c,0x1767000c,0x176c000c,0x1771000c, +0x1776000c,0x177b000c,0x1780000c,0x1785000c,0x178a000c,0x178e000c,0x1792000c,0x1796000c,0x179a000c,0x179e000c,0x17a3004c,0x17a8004c,0x17ac000c,0x17b1000c,0x17b6000c,0x17ba000c, +0x17be000c,0x17c2000c,0x17c7004c,0x17cc004c,0x17d0000c,0x17d5000c,0x17da000c,0x17df000c,0x17e4000c,0x17e9000c,0x17ee000c,0x17f2000c,0x17f6000c,0x17fa000c,0x17fe000c,0x1802000c, +0x1806000c,0x180a000c,0x180e000c,0x1812000c,0x1816000c,0x181a000c,0x181e000c,0x1822000c,0x1826000c,0x182b000c,0x1830000c,0x1835000c,0x183a000c,0x183e000c,0x1842000c,0x1846000c, +0x184a000c,0x184e000c,0x1852000c,0x1856000c,0x185a000c,0x185e000c,0x1862000c,0x1866000c,0x186a000c,0x186e000c,0x1872000c,0x1876000c,0x187a000c,0x187e000c,0x1882000c,0x1886000c, +0x188a000c,0x188e000c,0x1892000c,0x1896000c,0x189a000c,0x189e000c,0x18a2000c,0x18a6000c,0x18aa000c,0x18ae000c,0x21f6000a,0x18b2000e,0,0,0,0, +0x18ba004c,0x18bf004c,0x18c3000c,0x18c7000c,0x18cb000c,0x18d0000c,0x18d5000c,0x18da000c,0x18df000c,0x18e4000c,0x18e9000c,0x18ee000c,0x18f3000c,0x18f8000c,0x18fd000c,0x1902000c, +0x1907000c,0x190c000c,0x1911000c,0x1916000c,0x191b000c,0x1920000c,0x1925000c,0x192a000c,0x1930004c,0x1935004c,0x1939000c,0x193d000c,0x1941000c,0x1945000c,0x1949000c,0x194e000c, +0x1953000c,0x1958000c,0x195d000c,0x1962000c,0x1967000c,0x196c000c,0x1971000c,0x1976000c,0x197b000c,0x197f000c,0x1983000c,0x1987000c,0x198c004c,0x1991004c,0x1995000c,0x1999000c, +0x199d000c,0x19a2000c,0x19a7000c,0x19ac000c,0x19b1000c,0x19b6000c,0x19bb000c,0x19c0000c,0x19c5000c,0x19ca000c,0x19cf000c,0x19d4000c,0x19d9000c,0x19de000c,0x19e3000c,0x19e8000c, +0x19ed000c,0x19f2000c,0x19f7000c,0x19fc000c,0x1a01000c,0x1a05000c,0x1a09000c,0x1a0d000c,0x1a11000c,0x1a16000c,0x1a1b000c,0x1a20000c,0x1a25000c,0x1a2a000c,0x1a2f000c,0x1a34000c, +0x1a39000c,0x1a3e000c,0x1a43000c,0x1a47000c,0x1a4b000c,0x1a4f000c,0x1a53000c,0x1a57000c,0x1a5b000c,0x1a5f000c,0,0,0,0,0,0, +0x1a64004c,0x1a69004c,0x1a6e004c,0x1a74004c,0x1a7a004c,0x1a80004c,0x1a86004c,0x1a8c004c,0x1a92004c,0x1a97004c,0x1a9c004c,0x1aa2004c,0x1aa8004c,0x1aae004c,0x1ab4004c,0x1aba004c, +0x1ac0004c,0x1ac5004c,0x1ac9000c,0x1ace000c,0x1ad3000c,0x1ad8000c,0,0,0x1ade004c,0x1ae3004c,0x1ae7000c,0x1aec000c,0x1af1000c,0x1af6000c,0,0, +0x1afc004c,0x1b01004c,0x1b06004c,0x1b0c004c,0x1b12004c,0x1b18004c,0x1b1e004c,0x1b24004c,0x1b2a004c,0x1b2f004c,0x1b34004c,0x1b3a004c,0x1b40004c,0x1b46004c,0x1b4c004c,0x1b52004c, +0x1b58004c,0x1b5d004c,0x1b61000c,0x1b66000c,0x1b6b000c,0x1b70000c,0x1b75000c,0x1b7a000c,0x1b80004c,0x1b85004c,0x1b89000c,0x1b8e000c,0x1b93000c,0x1b98000c,0x1b9d000c,0x1ba2000c, +0x1ba8004c,0x1bad004c,0x1bb1000c,0x1bb6000c,0x1bbb000c,0x1bc0000c,0,0,0x1bc6004c,0x1bcb004c,0x1bcf000c,0x1bd4000c,0x1bd9000c,0x1bde000c,0,0, +0x1be4004c,0x1be9004c,0x1bed000c,0x1bf2000c,0x1bf7000c,0x1bfc000c,0x1c01000c,0x1c06000c,0,0x1c0c004c,0,0x1c10000c,0,0x1c15000c,0,0x1c1a000c, +0x1c9a000c,0x1c9f000c,0x1ca4000c,0x1caa000c,0x1cb0000c,0x1cb6000c,0x1cbc000c,0x1cc2000c,0x1cc8000c,0x1ccd000c,0x1cd2000c,0x1cd8000c,0x1cde000c,0x1ce4000c,0x1cea000c,0x1cf0000c, +0x1cf6000c,0x1cfb000c,0x1d00000c,0x1d06000c,0x1d0c000c,0x1d12000c,0x1d18000c,0x1d1e000c,0x1d24000c,0x1d29000c,0x1d2e000c,0x1d34000c,0x1d3a000c,0x1d40000c,0x1d46000c,0x1d4c000c, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0x3df00040,0,0x3df10040,0,0x3df20040,0,0,0,0,0,0x1eb6000c,0x1eba000c,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0x1ebe000c,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0x1ec2000c,0x1ec6000c,0x1eca000c, +0x3df30040,0,0x3df50040,0,0x3df40040,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0x3df60040,0x1ece000c,0,0,0,0x3df70040,0x1ed2000c,0,0x3df80040,0x1ed6000c,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0x3df90040,0x1eda000c,0x3dfa0040,0x1ede000c,0,0,0,0,0,0x235a000a,0x235d000a,0,0x2361000a, +0x2364000a,0,0,0,0,0,0,0,0,0,0,0,0x3dfb0040,0,0,0, +0,0x1ee2000c,0,0x3dfc0040,0x1ee6000c,0x3dfd0040,0,0x1eea000c,0x3dfe0040,0x1eee000c,0,0,0,0x3e010040,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0x1ef2000c,0x3e000040,0x1ef6000c,0,0x3e040040,0x3e050040,0,0,0,0,0,0,0,0x1efa000c,0x1efe000c,0x1f02000c, +0x1f06000c,0x1f0a000c,0x3e060040,0x3e070040,0x1f0e000c,0x1f12000c,0x3e080040,0x3e090040,0x1f16000c,0x1f1a000c,0x3e0a0040,0x3e0b0040,0x3e140040,0x3e150040,0,0, +0x1f1e000c,0x1f22000c,0x3e0c0040,0x3e0d0040,0x1f26000c,0x1f2a000c,0x3e0e0040,0x3e0f0040,0x1f2e000c,0x1f32000c,0,0,0,0,0,0, +0,0x3e160040,0x3e170040,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0x3e100040,0,0,0,0,0,0x3e110040,0x3e120040,0,0x3e130040,0x1f36000c,0x1f3a000c,0x1f3e000c,0x1f42000c, +0,0,0x3e180040,0x3e190040,0x3e1a0040,0x3e1b0040,0,0,0,0,0,0,0,0,0,0, +0x1f46000c,0x1f4a000c,0x1f4e000c,0x1f52000c,0,0,0,0,0,0,0x1f56000c,0x1f5a000c,0x1f5e000c,0x1f62000c,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0x3e300040,0,0,0,0,0x3e1c0040,0x1f66000c,0x3e1d0040,0x1f6a000c,0x3e1e0040,0x1f6e000c,0x3e1f0040,0x1f72000c,0x3e200040, +0x1f76000c,0x3e210040,0x1f7a000c,0x3e220040,0x1f7e000c,0x3e230040,0x1f82000c,0x3e240040,0x1f86000c,0x3e250040,0x1f8a000c,0x3e260040,0x1f8e000c,0x3e270040,0x1f92000c,0, +0x3e280040,0x1f96000c,0x3e290040,0x1f9a000c,0x3e2a0040,0x1f9e000c,0,0,0,0,0,0x3e2b0040,0x1fa2000c,0x1fa6000c,0x3e2c0040,0x1faa000c, +0x1fae000c,0x3e2d0040,0x1fb2000c,0x1fb6000c,0x3e2e0040,0x1fba000c,0x1fbe000c,0x3e2f0040,0x1fc2000c,0x1fc6000c,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0x1fca000c,0,0,0,0,0x208308b0,0x208408b0,0x26db000a,0x26df000a,0x3e310040,0x1fce000c,0x26e3000a,0,0,0,0, +0,0,0x3e460040,0,0,0,0,0x3e320040,0x1fd2000c,0x3e330040,0x1fd6000c,0x3e340040,0x1fda000c,0x3e350040,0x1fde000c,0x3e360040, +0x1fe2000c,0x3e370040,0x1fe6000c,0x3e380040,0x1fea000c,0x3e390040,0x1fee000c,0x3e3a0040,0x1ff2000c,0x3e3b0040,0x1ff6000c,0x3e3c0040,0x1ffa000c,0x3e3d0040,0x1ffe000c,0, +0x3e3e0040,0x2002000c,0x3e3f0040,0x2006000c,0x3e400040,0x200a000c,0,0,0,0,0,0x3e410040,0x200e000c,0x2012000c,0x3e420040,0x2016000c, +0x201a000c,0x3e430040,0x201e000c,0x2022000c,0x3e440040,0x2026000c,0x202a000c,0x3e450040,0x202e000c,0x2032000c,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0x3e470040,0x3e480040,0x3e490040,0x3e4a0040,0, +0x2036000c,0,0,0x203a000c,0x203e000c,0x2042000c,0x2046000c,0,0,0x3e4b0040,0x204a000c,0x26e6000a,0x204fe6b0,0x2050e6b0,0x2051e6b0,0x2052e6b0, +0x2053e6b0,0xe600,0x2054e6b0,0x2055e6b0,0x2056e6b0,0x2057e6b0,0x2058e6b0,0x2059e6b0,0x205ae6b0,0xe600,0xe600,0x205be6b0,0xe600,0x205ce6b0,0xe600,0x205de6b0, +0x205ee6b0,0xe800,0xdc00,0xdc00,0xdc00,0xdc00,0xe800,0x205fd8b0,0xdc00,0xdc00,0xdc00,0xdc00,0xdc00,0xca00,0xca00,0x2060dcb0, +0x2061dcb0,0x2062dcb0,0x2063dcb0,0x2064cab0,0x2065cab0,0xdc00,0xdc00,0xdc00,0xdc00,0x2066dcb0,0x2067dcb0,0xdc00,0x2068dcb0,0x2069dcb0,0xdc00,0xdc00, +0x100,0x100,0x100,0x100,0x206a01b0,0xdc00,0xdc00,0xdc00,0xdc00,0xe600,0xe600,0xe600,0,0,0,0, +0,0,0,0,0x3dc30040,0,0x3dc40040,0x1b00,0x1c00,0x1d00,0x1e00,0x1f00,0x2000,0x2100,0x2200,0x206de6b0, +0x206ee6b0,0x206fdcb0,0xdc00,0xe600,0xe600,0xe600,0xe600,0xe600,0xdc00,0xe600,0xe600,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0x700,0,0x207100b0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0x700,0,0x207300b0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0x207600b0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0x207c00b0,0,0,0xfff200b0,0xfff200b0,0xfff200b0, +0xfff200b0,0xfff200b0,0xfff200b0,0xfff200b0,0xfff200b0,0xfff200b0,0xfff200b0,0xfff200b0,0xfff200b0,0xfff200b0,0xfff200b0,0xfff200b0,0xfff200b0,0xfff200b0,0xfff200b0,0xfff200b0, +0xfff200b0,0xfff200b0,0,0,0,0,0,0,0,0,0,0,0xfff300b0,0xfff300b0,0xfff300b0,0xfff300b0, +0xfff300b0,0xfff300b0,0xfff300b0,0xfff300b0,0xfff300b0,0xfff300b0,0xfff300b0,0xfff300b0,0xfff300b0,0xfff300b0,0xfff300b0,0xfff300b0,0xfff300b0,0xfff300b0,0xfff300b0,0xfff300b0, +0xfff300b0,0xfff300b0,0xfff300b0,0xfff300b0,0xfff300b0,0xfff300b0,0xfff300b0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0x2084000a,0,0,0,0,0,0,0,0x2087004a,0,0x208b000a,0, +0,0,0,0x208d000a,0,0,0x2091000a,0x2093000a,0x2095000a,0x2099000a,0,0,0x209b000a,0x209f000a,0x20a1000a,0, +0x20a3000a,0x20a7000a,0x20ab000a,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0x20eb000a,0x20ed000a,0x20ef000a,0x20f1000a,0x20f3000a,0x20f5000a,0x20f7000a,0x20f9000a,0x20fb000a,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0x20fd000a,0x2101000a,0x2105000a,0x2109000a,0x210d000a,0x2111000a,0,0, +0x2115000a,0x2117000a,0x2119000a,0x211b000a,0x211d000a,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0x2133000a,0,0,0,0,0,0,0,0,0,0xdc00,0xe600,0xe600, +0xe600,0xe600,0xdc00,0xe600,0xe600,0xe600,0xde00,0xdc00,0xe600,0xe600,0xe600,0xe600,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0x2300,0,0,0, +0,0x2136000a,0x2139000a,0x213c000a,0x213f000a,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0x2142000a,0,0,0,0, +0x6700,0x6700,0x900,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0x2145000a,0,0,0,0,0x7600,0x7600,0,0, +0,0,0,0,0,0,0,0,0x7a00,0x7a00,0x7a00,0x7a00,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0x2148000a,0x214b000a,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0x214e000a,0,0,0, +0,0,0,0,0,0,0,0,0xdc00,0xdc00,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0x215a000a,0,0,0,0x2194000a,0x2196000a,0x2198000a,0x219a000a, +0x219c000a,0x219e000a,0x21a0000a,0x21a2000a,0x21a4000a,0x21a6000a,0x21a8000a,0,0,0,0,0,0,0,0,0, +0,0,0,0,0x21aa000a,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0x21ac000a,0x21ae000a,0x21b0000a,0x21b2000a,0x21b4000a,0x21b6000a,0x21b8000a,0x21ba000a,0x21bc000a,0x21be000a,0x21c0000a,0x21c2000a,0x21c4000a, +0x21c6000a,0x21c8000a,0x21ca000a,0x21cc000a,0x21ce000a,0x21d0000a,0x21d2000a,0x21d4000a,0x21d6000a,0x21d8000a,0x21da000a,0x21dc000a,0x21de000a,0x21e0000a,0x21e2000a,0x21e4000a, +0x21e6000a,0x21e8000a,0x21ea000a,0x21ec000a,0x21ee000a,0x21f0000a,0x21f2000a,0x21f4000a,0,0,0,0,0x2223000a,0x2225000a,0x2228000a,0, +0,0,0,0,0,0,0,0x222c000a,0,0,0,0x222e000a,0x2231000a,0,0x2235000a,0x2238000a, +0,0,0,0,0x223c000a,0,0x223f000a,0,0,0,0,0,0,0,0,0x2243000a, +0x2246000a,0x2249000a,0,0,0,0,0,0,0,0,0,0,0,0,0,0x224c000a, +0,0,0,0,0,0,0,0x2251000a,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0x2253000a,0x2255000a,0,0,0x2257000a,0x2259000a,0x225b000a,0x225d000a, +0x225f000a,0x2261000a,0x2263000a,0x2265000a,0x2267000a,0x2269000a,0x226b000a,0x226d000a,0x226f000a,0x2271000a,0x2273000a,0x2275000a,0x2277000a,0x2279000a,0x227b000a,0x227d000a, +0x227f000a,0x2281000a,0x2283000a,0x2285000a,0x2287000a,0x2289000a,0x228b000a,0,0x228d000a,0x228f000a,0x2291000a,0x2293000a,0x2295000a,0,0,0, +0,0,0,0,0,0,0,0,0x22fe000a,0x2300000a,0x2303000a,0x2307000a,0x230a000a,0x230c000a,0x230f000a,0x2313000a, +0x2318000a,0x231b000a,0x231d000a,0x2320000a,0x2324000a,0x2326000a,0x2328000a,0x232a000a,0x232c000a,0x232e000a,0x2331000a,0x2335000a,0x2338000a,0x233a000a,0x233d000a,0x2341000a, +0x2346000a,0x2349000a,0x234b000a,0x234e000a,0x2352000a,0x2354000a,0x2356000a,0x2358000a,0x2368000a,0x236a000a,0x236c000a,0x236e000a,0x2370000a,0x2372000a,0x2374000a,0x2376000a, +0x2378000a,0x237a000a,0x237d000a,0x2380000a,0x2383000a,0x2386000a,0x2389000a,0x238c000a,0x238f000a,0x2392000a,0x2395000a,0x2398000a,0x239b000a,0x239f000a,0x23a3000a,0x23a7000a, +0x23ab000a,0x23af000a,0x23b3000a,0x23b7000a,0x23bb000a,0x23bf000a,0x23c4000a,0x23c9000a,0x23ce000a,0x23d3000a,0x23d8000a,0x23dd000a,0x23e2000a,0x23e7000a,0x23ec000a,0x23f1000a, +0x23f6000a,0x23f9000a,0x23fc000a,0x23ff000a,0x2402000a,0x2405000a,0x2408000a,0x240b000a,0x240e000a,0x2411000a,0x2415000a,0x2419000a,0x241d000a,0x2421000a,0x2425000a,0x2429000a, +0x242d000a,0x2431000a,0x2435000a,0x2439000a,0x243d000a,0x2441000a,0x2445000a,0x2449000a,0x244d000a,0x2451000a,0x2455000a,0x2459000a,0x245d000a,0x2461000a,0x2465000a,0x2469000a, +0x246d000a,0x2471000a,0x2475000a,0x2479000a,0x247d000a,0x2481000a,0x2485000a,0x2489000a,0x248d000a,0x2491000a,0x2495000a,0x2499000a,0x249d000a,0x24a1000a,0x24a5000a,0x24a7000a, +0x24a9000a,0x24ab000a,0x24ad000a,0x24af000a,0x24b1000a,0x24b3000a,0x24b5000a,0x24b7000a,0x24b9000a,0x24bb000a,0x24bd000a,0x24bf000a,0x24c1000a,0x24c3000a,0x24c5000a,0x24c7000a, +0x24c9000a,0x24cb000a,0x24cd000a,0x24cf000a,0x24d1000a,0x24d3000a,0x24d5000a,0x24d7000a,0x24d9000a,0x24db000a,0x24dd000a,0x24df000a,0x24e1000a,0x24e3000a,0x24e5000a,0x24e7000a, +0x24e9000a,0x24eb000a,0x24ed000a,0x24ef000a,0x24f1000a,0x24f3000a,0x24f5000a,0x24f7000a,0x24f9000a,0x24fb000a,0x24fd000a,0x24ff000a,0x2501000a,0x2503000a,0x2505000a,0x2507000a, +0x2509000a,0x250b000a,0x250d000a,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0x250f000a,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0x2514000a,0x2518000a,0x251b000a,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0x251f000a,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0x2521000a,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0x2523000a,0,0,0,0,0,0,0,0, +0,0,0,0,0x2525000a,0x2527000a,0x2529000a,0x252b000a,0x252d000a,0x252f000a,0x2531000a,0x2533000a,0x2535000a,0x2537000a,0x2539000a,0x253b000a, +0x253d000a,0x253f000a,0x2541000a,0x2543000a,0x2545000a,0x2547000a,0x2549000a,0x254b000a,0x254d000a,0x254f000a,0x2551000a,0x2553000a,0x2555000a,0x2557000a,0x2559000a,0x255b000a, +0x255d000a,0x255f000a,0x2561000a,0x2563000a,0x2565000a,0x2567000a,0x2569000a,0x256b000a,0x256d000a,0x256f000a,0x2571000a,0x2573000a,0x2575000a,0x2577000a,0x2579000a,0x257b000a, +0x257d000a,0x257f000a,0x2581000a,0x2583000a,0x2585000a,0x2587000a,0x2589000a,0x258b000a,0x258d000a,0x258f000a,0x2591000a,0x2593000a,0x2595000a,0x2597000a,0x2599000a,0x259b000a, +0x259d000a,0x259f000a,0x25a1000a,0x25a3000a,0x25a5000a,0x25a7000a,0x25a9000a,0x25ab000a,0x25ad000a,0x25af000a,0x25b1000a,0x25b3000a,0x25b5000a,0x25b7000a,0x25b9000a,0x25bb000a, +0x25bd000a,0x25bf000a,0x25c1000a,0x25c3000a,0x25c5000a,0x25c7000a,0x25c9000a,0x25cb000a,0x25cd000a,0x25cf000a,0x25d1000a,0x25d3000a,0x25d5000a,0x25d7000a,0x25d9000a,0x25db000a, +0x25dd000a,0x25df000a,0x25e1000a,0x25e3000a,0x25e5000a,0x25e7000a,0x25e9000a,0x25eb000a,0x25ed000a,0x25ef000a,0x25f1000a,0x25f3000a,0x25f5000a,0x25f7000a,0x25f9000a,0x25fb000a, +0x25fd000a,0x25ff000a,0x2601000a,0x2603000a,0x2605000a,0x2607000a,0x2609000a,0x260b000a,0x260d000a,0x260f000a,0x2611000a,0x2613000a,0x2615000a,0x2617000a,0x2619000a,0x261b000a, +0x261d000a,0x261f000a,0x2621000a,0x2623000a,0x2625000a,0x2627000a,0x2629000a,0x262b000a,0x262d000a,0x262f000a,0x2631000a,0x2633000a,0x2635000a,0x2637000a,0x2639000a,0x263b000a, +0x263d000a,0x263f000a,0x2641000a,0x2643000a,0x2645000a,0x2647000a,0x2649000a,0x264b000a,0x264d000a,0x264f000a,0x2651000a,0x2653000a,0x2655000a,0x2657000a,0x2659000a,0x265b000a, +0x265d000a,0x265f000a,0x2661000a,0x2663000a,0x2665000a,0x2667000a,0x2669000a,0x266b000a,0x266d000a,0x266f000a,0x2671000a,0x2673000a,0x2675000a,0x2677000a,0x2679000a,0x267b000a, +0x267d000a,0x267f000a,0x2681000a,0x2683000a,0x2685000a,0x2687000a,0x2689000a,0x268b000a,0x268d000a,0x268f000a,0x2691000a,0x2693000a,0x2695000a,0x2697000a,0x2699000a,0x269b000a, +0x269d000a,0x269f000a,0x26a1000a,0x26a3000a,0x26a5000a,0x26a7000a,0x26a9000a,0x26ab000a,0x26ad000a,0x26af000a,0x26b1000a,0x26b3000a,0x26b5000a,0x26b7000a,0x26b9000a,0x26bb000a, +0x26bd000a,0x26bf000a,0x26c1000a,0x26c3000a,0x26c5000a,0x26c7000a,0x26c9000a,0x26cb000a,0x26cd000a,0x26cf000a,0,0,0,0,0,0, +0,0,0,0,0x26d1000a,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0xda00,0xe400,0xe800,0xde00,0xe000,0xe000,0,0,0,0, +0,0,0x26d3000a,0,0x26d5000a,0x26d7000a,0x26d9000a,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0x26e9000a,0x26eb000a,0x26ed000a,0x26ef000a,0x26f1000a,0x26f3000a,0x26f5000a, +0x26f7000a,0x26f9000a,0x26fb000a,0x26fd000a,0x26ff000a,0x2701000a,0x2703000a,0x2705000a,0x2707000a,0x2709000a,0x270b000a,0x270d000a,0x270f000a,0x2711000a,0x2713000a,0x2715000a, +0x2717000a,0x2719000a,0x271b000a,0x271d000a,0x271f000a,0x2721000a,0x2723000a,0x2725000a,0x2727000a,0x2729000a,0x272b000a,0x272d000a,0x272f000a,0x2731000a,0x2733000a,0x2735000a, +0x2737000a,0x2739000a,0x273b000a,0x273d000a,0x273f000a,0x2741000a,0x2743000a,0x2745000a,0x2747000a,0x2749000a,0x274b000a,0x274d000a,0x274f000a,0x2751000a,0x2753000a,0x2755000a, +0x2757000a,0x2759000a,0x275b000a,0x275d000a,0x275f000a,0x2761000a,0x2763000a,0x2765000a,0x2767000a,0x2769000a,0x276b000a,0x276d000a,0x276f000a,0x2771000a,0x2773000a,0x2775000a, +0x2777000a,0x2779000a,0x277b000a,0x277d000a,0x277f000a,0x2781000a,0x2783000a,0x2785000a,0x2787000a,0x2789000a,0x278b000a,0x278d000a,0x278f000a,0x2791000a,0x2793000a,0x2795000a, +0x2797000a,0x2799000a,0x279b000a,0x279d000a,0x279f000a,0x27a1000a,0x27a3000a,0,0,0,0x27a5000a,0x27a7000a,0x27a9000a,0x27ab000a,0x27ad000a,0x27af000a, +0x27b1000a,0x27b3000a,0x27b5000a,0x27b7000a,0x27b9000a,0x27bb000a,0x27bd000a,0x27bf000a,0x27c1000a,0x27c5000a,0x27c9000a,0x27cd000a,0x27d1000a,0x27d5000a,0x27d9000a,0x27dd000a, +0x27e1000a,0x27e5000a,0x27e9000a,0x27ed000a,0x27f1000a,0x27f5000a,0x27f9000a,0x27fe000a,0x2803000a,0x2808000a,0x280d000a,0x2812000a,0x2817000a,0x281c000a,0x2821000a,0x2826000a, +0x282b000a,0x2830000a,0x2835000a,0x283a000a,0x283f000a,0x2844000a,0x284c000a,0,0x2853000a,0x2857000a,0x285b000a,0x285f000a,0x2863000a,0x2867000a,0x286b000a,0x286f000a, +0x2873000a,0x2877000a,0x287b000a,0x287f000a,0x2883000a,0x2887000a,0x288b000a,0x288f000a,0x2893000a,0x2897000a,0x289b000a,0x289f000a,0x28a3000a,0x28a7000a,0x28ab000a,0x28af000a, +0x28b3000a,0x28b7000a,0x28bb000a,0x28bf000a,0x28c3000a,0x28c7000a,0x28cb000a,0x28cf000a,0x2910000a,0x2912000a,0x2914000a,0x2916000a,0x2918000a,0x291a000a,0x291c000a,0x291e000a, +0x2920000a,0x2922000a,0x2924000a,0x2926000a,0x2928000a,0x292a000a,0x292c000a,0x292f000a,0x2932000a,0x2935000a,0x2938000a,0x293b000a,0x293e000a,0x2941000a,0x2944000a,0x2947000a, +0x294a000a,0x294d000a,0x2950000a,0x2953000a,0x2956000a,0x295c000a,0x2961000a,0,0x2964000a,0x2966000a,0x2968000a,0x296a000a,0x296c000a,0x296e000a,0x2970000a,0x2972000a, +0x2974000a,0x2976000a,0x2978000a,0x297a000a,0x297c000a,0x297e000a,0x2980000a,0x2982000a,0x2984000a,0x2986000a,0x2988000a,0x298a000a,0x298c000a,0x298e000a,0x2990000a,0x2992000a, +0x2994000a,0x2996000a,0x2998000a,0x299a000a,0x299c000a,0x299e000a,0x29a0000a,0x29a2000a,0x29a4000a,0x29a6000a,0x29a8000a,0x29aa000a,0x29ac000a,0x29ae000a,0x29b0000a,0x29b2000a, +0x29b4000a,0x29b6000a,0x29b8000a,0x29ba000a,0x29bc000a,0x29be000a,0x29c0000a,0x29c2000a,0x29c4000a,0x29c6000a,0x29c9000a,0x29cc000a,0x29cf000a,0x29d2000a,0x29d5000a,0x29d8000a, +0x29db000a,0x29de000a,0x29e1000a,0x29e4000a,0x29e7000a,0x29ea000a,0x29ed000a,0x29f0000a,0x2a3e000a,0x2a40000a,0x2a42000a,0x2a44000a,0x2a46000a,0x2a48000a,0x2a4a000a,0x2a4c000a, +0x2a4e000a,0x2a50000a,0x2a52000a,0x2a54000a,0x2a56000a,0x2a58000a,0x2a5a000a,0x2a5c000a,0x2a5e000a,0x2a60000a,0x2a62000a,0x2a64000a,0x2a66000a,0x2a68000a,0x2a6a000a,0x2a6c000a, +0x2a6e000a,0x2a70000a,0x2a72000a,0x2a74000a,0x2a76000a,0x2a78000a,0x2a7a000a,0,0x2a7c000a,0x2a82000a,0x2a87000a,0x2a8d000a,0x2a91000a,0x2a98000a,0x2a9c000a,0x2aa0000a, +0x2aa8000a,0x2aad000a,0x2ab1000a,0x2ab5000a,0x2ab9000a,0x2abe000a,0x2ac3000a,0x2ac8000a,0x2acd000a,0x2ad3000a,0x2ad8000a,0x2add000a,0x2ae4000a,0x2ae7000a,0x2aee000a,0x2af5000a, +0x2afb000a,0x2b00000a,0x2b07000a,0x2b0e000a,0x2b13000a,0x2b17000a,0x2b1b000a,0x2b21000a,0x2b26000a,0x2b2c000a,0x2b33000a,0x2b37000a,0x2b3b000a,0x2b40000a,0x2b44000a,0x2b48000a, +0x2b4b000a,0x2b4e000a,0x2b52000a,0x2b56000a,0x2b5d000a,0x2b62000a,0x2b68000a,0x2b6f000a,0x2b74000a,0x2b78000a,0x2b7c000a,0x2b84000a,0x2b89000a,0x2b90000a,0x2b94000a,0x2b9a000a, +0x2b9e000a,0x2ba3000a,0x2ba7000a,0x2bac000a,0x2bb3000a,0x2bb8000a,0x2bbe000a,0x2bc3000a,0x2bc6000a,0x2bcd000a,0x2bd1000a,0x2bd5000a,0x2bda000a,0x2bde000a,0x2be2000a,0x2be6000a, +0x2bec000a,0x2bf1000a,0x2bf4000a,0x2bfb000a,0x2c00000a,0x2c06000a,0x2c0b000a,0x2c11000a,0x2c15000a,0x2c19000a,0x2c1e000a,0x2c21000a,0x2c26000a,0x2c2c000a,0x2c2f000a,0x2c36000a, +0x2c3a000a,0x2c3d000a,0x2c40000a,0x2c43000a,0x2c46000a,0x2c49000a,0x2c4c000a,0x2c4f000a,0x2d62000a,0x2d65000a,0x2d68000a,0x2d6b000a,0x2d6e000a,0x2d71000a,0x2d74000a,0x2d77000a, +0x2d7a000a,0x2d7d000a,0x2d81000a,0x2d85000a,0x2d89000a,0x2d8d000a,0x2d91000a,0x2d95000a,0x2d99000a,0x2d9d000a,0x2da1000a,0x2da5000a,0x2da9000a,0x2dad000a,0x2db1000a,0x2db5000a, +0x2db9000a,0x2dbd000a,0x2dc1000a,0x2dc5000a,0x2dc9000a,0x2dcd000a,0x2dd1000a,0x2dd5000a,0x2e36000a,0x2e38000a,0x2e3a000a,0x2e3c000a,0x2e3e000a,0x2e40000a,0x2e42000a,0x2e44000a, +0x2e46000a,0x2e48000a,0x2e4a000a,0x2e4c000a,0x2e4e000a,0x2e50000a,0x2e52000a,0x2e54000a,0x2e56000a,0x2e58000a,0x2e5a000a,0x2e5c000a,0x2e5e000a,0x2e60000a,0x2e62000a,0x2e64000a, +0x2e66000a,0x2e68000a,0x2e6a000a,0x2e6c000a,0x2e6e000a,0x2e70000a,0x2e72000a,0x2e74000a,0x2e76000a,0x2e78000a,0x2e7a000a,0x2e7c000a,0x2e7e000a,0x2e80000a,0x2e82000a,0x2e84000a, +0x2e86000a,0x2e88000a,0x2e8a000a,0x2e8c000a,0x2e8e000a,0x2e90000a,0x2e92000a,0x2e94000a,0x2e96000a,0x2e98000a,0x2e9a000a,0x2e9c000a,0x2e9e000a,0x2ea0000a,0x2ea2000a,0x2ea4000a, +0x2ea6000a,0x2ea8000a,0x2eaa000a,0x2eac000a,0x2eae000a,0x2eb0000a,0x2eb2000a,0x2eb4000a,0x2eb6000a,0x2eb8000a,0x2eba000a,0x2ebc000a,0x2ebe000a,0x2ec2000a,0x2ec6000a,0x2ec8000a, +0x2eca000a,0x2ecc000a,0x2ece000a,0x2ed0000a,0x2ed2000a,0x2ed4000a,0x2ed6000a,0x2ed8000a,0x2eda000a,0x2ede000a,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0x2ee2000a, +0x2ee4000a,0x2ee6000a,0x2ee8000a,0x2eea000a,0x2eec000a,0x2eee000a,0x2ef0000a,0x2ef2000a,0x2ef4000a,0x2ef6000a,0x2ef9000a,0x2efb000a,0x2efd000a,0x2eff000a,0x2f01000a,0x2f03000a, +0x2f05000a,0x2f07000a,0x2f09000a,0x2f0b000a,0x2f0d000a,0x2f0f000a,0x2f11000a,0x2f15000a,0x2f19000a,0x2f1d000a,0x2f21000a,0x2f25000a,0x2f29000a,0x2f2d000a,0x2f31000a,0x2f35000a, +0x2f39000a,0x2f3d000a,0x2f41000a,0x2f45000a,0x2f49000a,0x2f4d000a,0x2f51000a,0x2f55000a,0x2f59000a,0x2f5b000a,0x2f5d000a,0x2f5f000a,0x2f61000a,0x2f65000a,0x2f69000a,0x2f6d000a, +0x2f71000a,0x2f75000a,0x2f78000a,0x2f7b000a,0x2f7e000a,0x2f81000a,0x2f84000a,0x2f87000a,0x2f8a000a,0x2f8d000a,0x2f90000a,0x2f93000a,0x2f96000a,0x2f99000a,0x2f9c000a,0x2f9f000a, +0x2fa2000a,0x2fa5000a,0x2fa8000a,0x2fab000a,0x2fae000a,0x2fb1000a,0x2fb4000a,0x2fb7000a,0x2fba000a,0x2fbd000a,0x2fc0000a,0x2fc3000a,0x2fc6000a,0x2fc9000a,0x2fcc000a,0x2fcf000a, +0x2fd2000a,0x2fd5000a,0x2fd8000a,0x2fdb000a,0x2fde000a,0x2fe1000a,0x2fe4000a,0x2fe7000a,0x2fea000a,0x2fed000a,0x2ff0000a,0x2ff3000a,0x2ff6000a,0x2ff9000a,0x2ffc000a,0x2fff000a, +0x3002000a,0x3005000a,0x3008000a,0x300b000a,0x300e000a,0x3011000a,0x3014000a,0x3017000a,0x301a000a,0x301d000a,0x3020000a,0x3023000a,0x3026000a,0x3029000a,0x302c000a,0x302f000a, +0x3032000a,0x3035000a,0x3038000a,0x303b000a,0x303e000a,0x3041000a,0x3044000a,0x3047000a,0x304a000a,0x304d000a,0x3050000a,0x3053000a,0x3056000a,0x3059000a,0x305c000a,0x305f000a, +0x3062000a,0x3065000a,0x3068000a,0x306b000a,0x306e000a,0x3071000a,0x3074000a,0x3077000a,0x307b000a,0x307f000a,0x3083000a,0x3088000a,0x308d000a,0x3092000a,0x3097000a,0x309c000a, +0x30a1000a,0x30a5000a,0x30a9000a,0x30ad000a,0x30b1000a,0x30b5000a,0x30b9000a,0x30bc000a,0x30bf000a,0x30c2000a,0x30c5000a,0x30c8000a,0x30cb000a,0x30ce000a,0x30d1000a,0x30d4000a, +0x30d7000a,0x30da000a,0x30dd000a,0x30e0000a,0x30e3000a,0x30e6000a,0x30e9000a,0x30ec000a,0x30ef000a,0x30f2000a,0x30f5000a,0x30f8000a,0x30fb000a,0x30fe000a,0x3101000a,0x3104000a, +0x3107000a,0x310a000a,0x310d000a,0x3110000a,0x3113000a,0x3116000a,0x3119000a,0x311c000a,0x311f000a,0x3122000a,0x3125000a,0x3128000a,0x312b000a,0x312f000a,0x3132000a,0x3135000a, +0x3138000a,0x313b000a,0x313e000a,0x3141000a,0x3145000a,0x3149000a,0x314d000a,0x3151000a,0x3155000a,0x3158000a,0x315b000a,0x315e000a,0x3161000a,0x3164000a,0x3167000a,0x316a000a, +0x316d000a,0x3170000a,0x3173000a,0x3176000a,0x3179000a,0x317c000a,0x317f000a,0x3182000a,0x3185000a,0x3188000a,0x318b000a,0x318e000a,0x3191000a,0x3194000a,0x3197000a,0x319a000a, +0x319d000a,0x31a0000a,0x31a3000a,0x31a6000a,0x31a9000a,0x31ac000a,0x31af000a,0x31b2000a,0x31b5000a,0x31b8000a,0x31bb000a,0x31be000a,0x31c1000a,0x31c4000a,0x31c7000a,0x31ca000a, +0x31cd000a,0x31d0000a,0x31d3000a,0x31d6000a,0x31d9000a,0x31dc000a,0x31df000a,0x31e2000a,0x31e5000a,0x31e8000a,0x31eb000a,0x31ee000a,0x31f1000a,0x31f4000a,0x31f7000a,0x31fa000a, +0x31fd000a,0x3200000a,0x3203000a,0x3206000a,0x3209000a,0x320c000a,0x3210000a,0x3213000a,0x3216000a,0x3219000a,0x321c000a,0x321f000a,0x3223000a,0x3227000a,0x322a000a,0x322d000a, +0x3230000a,0x3233000a,0x3236000a,0x3239000a,0x323c000a,0x323f000a,0x3242000a,0x3245000a,0x3248000a,0x324b000a,0x324e000a,0x3251000a,0x3254000a,0x3257000a,0x325a000a,0x325f000a, +0x3264000a,0x3269000a,0x326c000a,0x326f000a,0x3272000a,0x3275000a,0x3278000a,0x327b000a,0x327e000a,0x3281000a,0x3284000a,0x3287000a,0x328a000a,0x328d000a,0x3290000a,0x3293000a, +0x3296000a,0x3299000a,0x329c000a,0x329f000a,0x32a2000a,0x32a5000a,0x32a8000a,0x32ab000a,0x32ae000a,0x32b1000a,0x32b4000a,0x32b7000a,0x32ba000a,0x32bd000a,0x32c0000a,0x32c3000a, +0x32c6000a,0x32c9000a,0x32cc000a,0x32cf000a,0x32d2000a,0x32d5000a,0x32d8000a,0x32db000a,0x32de000a,0x32e1000a,0x32e4000a,0x32e7000a,0x32ea000a,0x32ed000a,0x32f0000a,0x32f3000a, +0x32f6000a,0x32f9000a,0x32fc000a,0x32ff000a,0x3302000a,0x3305000a,0x3308000a,0x330b000a,0x330e000a,0x3311000a,0x3314000a,0x3317000a,0x331a000a,0x331d000a,0x3320000a,0x3323000a, +0x3326000a,0x3329000a,0x332c000a,0x332f000a,0x3332000a,0x3335000a,0x3338000a,0x333b000a,0x333e000a,0x3342000a,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0x3346000a,0x334a000a,0x334e000a,0x3352000a, +0x3356000a,0x335a000a,0x335e000a,0x3362000a,0x3366000a,0x336a000a,0x336e000a,0x3372000a,0x3376000a,0x337a000a,0x337e000a,0x3382000a,0x3386000a,0x338a000a,0x338e000a,0x3392000a, +0x3396000a,0x339a000a,0x339e000a,0x33a2000a,0x33a6000a,0x33aa000a,0x33ae000a,0x33b2000a,0x33b6000a,0x33ba000a,0x33be000a,0x33c2000a,0x33c6000a,0x33ca000a,0x33ce000a,0x33d2000a, +0x33d6000a,0x33da000a,0x33de000a,0x33e2000a,0x33e6000a,0x33ea000a,0x33ee000a,0x33f2000a,0x33f6000a,0x33fa000a,0x33fe000a,0x3402000a,0x3406000a,0x340a000a,0x340e000a,0x3412000a, +0x3416000a,0x341a000a,0x341e000a,0x3422000a,0x3426000a,0x342a000a,0x342e000a,0x3432000a,0x3436000a,0x343a000a,0x343e000a,0x3442000a,0,0,0x3446000a,0x344a000a, +0x344e000a,0x3452000a,0x3456000a,0x345a000a,0x345e000a,0x3462000a,0x3466000a,0x346a000a,0x346e000a,0x3472000a,0x3476000a,0x347a000a,0x347e000a,0x3482000a,0x3486000a,0x348a000a, +0x348e000a,0x3492000a,0x3496000a,0x349a000a,0x349e000a,0x34a2000a,0x34a6000a,0x34aa000a,0x34ae000a,0x34b2000a,0x34b6000a,0x34ba000a,0x34be000a,0x34c2000a,0x34c6000a,0x34ca000a, +0x34ce000a,0x34d2000a,0x34d6000a,0x34da000a,0x34de000a,0x34e2000a,0x34e6000a,0x34ea000a,0x34ee000a,0x34f2000a,0x34f6000a,0x34fa000a,0x34fe000a,0x3502000a,0x3506000a,0x350a000a, +0x350e000a,0x3512000a,0x3516000a,0x351a000a,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0x351e000a,0x3522000a,0x3526000a,0x352b000a, +0x3530000a,0x3535000a,0x353a000a,0x353f000a,0x3544000a,0x3549000a,0x354d000a,0x3560000a,0x3569000a,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0x356e000a,0x3570000a,0x3572000a,0x3574000a, +0x3576000a,0x3578000a,0x357a000a,0x357c000a,0x357e000a,0x3580000a,0,0,0,0,0,0,0xe600,0xe600,0xe600,0xe600, +0,0,0,0,0,0,0,0,0,0,0,0,0x3584000a,0x3587000a,0x3589000a,0x358b000a, +0x358d000a,0x358f000a,0x3591000a,0x3593000a,0x3595000a,0x3597000a,0x3599000a,0x359b000a,0x359d000a,0x359f000a,0x35a1000a,0x35a3000a,0x35a5000a,0x35a7000a,0x35a9000a,0x35ab000a, +0x35ad000a,0,0,0x35af000a,0x35b1000a,0x35b3000a,0x35b7000a,0x35bb000a,0x35bf000a,0x35c3000a,0x35c5000a,0x35c7000a,0x35c9000a,0x35cb000a,0x35cd000a,0, +0x35cf000a,0x35d1000a,0x35d3000a,0x35d5000a,0x35d7000a,0x35d9000a,0x35db000a,0x35dd000a,0x35df000a,0x35e1000a,0x35e3000a,0x35e5000a,0x35e7000a,0x35e9000a,0x35eb000a,0x35ed000a, +0x35ef000a,0x35f1000a,0x35f3000a,0,0x35f5000a,0x35f7000a,0x35f9000a,0x35fb000a,0,0,0,0,0x35fd000a,0x3601000a,0x3605000a,0, +0x3609000a,0,0x360d000a,0x3611000a,0x3615000a,0x3619000a,0x361d000a,0x3621000a,0x3625000a,0x3629000a,0x362d000a,0x3631000a,0x3635000a,0x3637000a,0x363b000a,0x363f000a, +0x3643000a,0x3647000a,0x364b000a,0x364f000a,0x3653000a,0x3657000a,0x365b000a,0x365f000a,0x3663000a,0x3667000a,0x3669000a,0x366b000a,0x366d000a,0x366f000a,0x3671000a,0x3673000a, +0x3675000a,0x3677000a,0x3679000a,0x367b000a,0x367d000a,0x367f000a,0x3681000a,0x3683000a,0x3685000a,0x3687000a,0x3689000a,0x368b000a,0x368d000a,0x368f000a,0x3691000a,0x3693000a, +0x3695000a,0x3697000a,0x3699000a,0x369b000a,0x369d000a,0x369f000a,0x36a1000a,0x36a3000a,0x36a5000a,0x36a7000a,0x36a9000a,0x36ab000a,0x36ad000a,0x36af000a,0x36b1000a,0x36b3000a, +0x36b5000a,0x36b7000a,0x36b9000a,0x36bb000a,0x36bd000a,0x36bf000a,0x36c1000a,0x36c3000a,0x36c5000a,0x36c7000a,0x36c9000a,0x36cb000a,0x36cd000a,0x36cf000a,0x36d1000a,0x36d3000a, +0x36d5000a,0x36d7000a,0x36d9000a,0x36db000a,0x36dd000a,0x36df000a,0x36e1000a,0x36e3000a,0x36e5000a,0x36e7000a,0x36e9000a,0x36eb000a,0x36ed000a,0x36ef000a,0x36f1000a,0x36f3000a, +0x36f5000a,0x36f7000a,0x36f9000a,0x36fb000a,0x36fd000a,0x36ff000a,0x3701000a,0x3703000a,0x3705000a,0x3707000a,0x3709000a,0x370b000a,0x370d000a,0x370f000a,0x3711000a,0x3713000a, +0x3715000a,0x3717000a,0x3719000a,0x371b000a,0x371d000a,0x371f000a,0x3721000a,0x3723000a,0x3725000a,0x3727000a,0x3729000a,0x372b000a,0x372d000a,0x372f000a,0x3731000a,0x3733000a, +0x3735000a,0x3737000a,0x373c000a,0x3741000a,0x3746000a,0x374b000a,0x3750000a,0x3755000a,0x3758000a,0,0,0,0,0x375b000a,0x375d000a,0x375f000a, +0x3761000a,0x3763000a,0x3765000a,0x3767000a,0x3769000a,0x376b000a,0x376d000a,0x376f000a,0x3771000a,0x3773000a,0x3775000a,0x3777000a,0x3779000a,0x377b000a,0x377d000a,0x377f000a, +0x3781000a,0x3783000a,0x3785000a,0x3787000a,0x3789000a,0x378b000a,0x378d000a,0x378f000a,0x3791000a,0x3793000a,0x3795000a,0x3797000a,0x3799000a,0x379b000a,0x379d000a,0x379f000a, +0x37a1000a,0x37a3000a,0x37a5000a,0x37a7000a,0x37a9000a,0x37ab000a,0x37ad000a,0x37af000a,0x37b1000a,0x37b3000a,0x37b5000a,0x37b7000a,0x37b9000a,0x37bb000a,0x37bd000a,0x37bf000a, +0x37c1000a,0x37c3000a,0x37c5000a,0x37c7000a,0x37c9000a,0x37cb000a,0x37cd000a,0x37cf000a,0x37d1000a,0x37d3000a,0x37d5000a,0x37d7000a,0x37d9000a,0x37db000a,0x37dd000a,0x37df000a, +0x37e1000a,0x37e3000a,0x37e5000a,0x37e7000a,0x37e9000a,0x37eb000a,0x37ed000a,0x37ef000a,0x37f1000a,0x37f3000a,0x37f5000a,0x37f7000a,0x37f9000a,0x37fb000a,0x37fd000a,0x37ff000a, +0x3801000a,0x3803000a,0x3805000a,0x3807000a,0x3809000a,0x380b000a,0x380d000a,0x380f000a,0x3811000a,0x3813000a,0x3815000a,0x3817000a,0x3819000a,0x381b000a,0x381d000a,0x381f000a, +0x3821000a,0x3823000a,0x3825000a,0x3827000a,0x3829000a,0x382b000a,0x382d000a,0x382f000a,0x3831000a,0x3833000a,0x3835000a,0x3837000a,0x3839000a,0x383b000a,0x383d000a,0x383f000a, +0x3841000a,0x3843000a,0x3845000a,0x3847000a,0x3849000a,0x384b000a,0x384d000a,0x384f000a,0x3851000a,0x3853000a,0x3855000a,0x3857000a,0x3859000a,0x385b000a,0x385d000a,0x385f000a, +0x3861000a,0x3863000a,0x3865000a,0x3867000a,0x3869000a,0x386b000a,0x386d000a,0x386f000a,0x3871000a,0x3873000a,0x3875000a,0x3877000a,0x3879000a,0x387b000a,0x387d000a,0x387f000a, +0x3881000a,0x3883000a,0x3885000a,0x3887000a,0x3889000a,0x388b000a,0x388d000a,0x388f000a,0x3891000a,0x3893000a,0x3895000a,0x3898000a,0x389b000a,0x389d000a,0x389f000a,0x38a1000a, +0x38a3000a,0x38a5000a,0x38a7000a,0x38a9000a,0x38ab000a,0x38ad000a,0x38af000a,0x38b1000a,0x38b3000a,0x38b5000a,0x38b7000a,0x38b9000a,0x38bb000a,0x38bd000a,0x38bf000a,0x38c1000a, +0x38c3000a,0x38c5000a,0x38c7000a,0x38c9000a,0x38cb000a,0x38cd000a,0x38cf000a,0x38d1000a,0x38d3000a,0x38d5000a,0x38d7000a,0,0,0,0x38d9000a,0x38db000a, +0x38dd000a,0x38df000a,0x38e1000a,0x38e3000a,0,0,0x38e5000a,0x38e7000a,0x38e9000a,0x38eb000a,0x38ed000a,0x38ef000a,0,0,0x38f1000a,0x38f3000a, +0x38f5000a,0x38f7000a,0x38f9000a,0x38fb000a,0,0,0x38fd000a,0x38ff000a,0x3901000a,0,0,0,0x3903000a,0x3905000a,0x3907000a,0x3909000a, +0x390d000a,0x390f000a,0x3911000a,0,0x3913000a,0x3915000a,0x3917000a,0x3919000a,0x391b000a,0x391d000a,0x391f000a,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0x3d27000a,0x3d29000a,0x3d2b000a,0x3d2d000a, +0x3d2f000a,0x3d31000a,0x3d33000a,0x3d35000a,0x3d37000a,0x3d39000a,0x3d3b000a,0x3d3d000a,0x3d3f000a,0x3d41000a,0x3d43000a,0x3d45000a,0x3d47000a,0x3d49000a,0x3d4b000a,0x3d4d000a, +0x3d4f000a,0x3d51000a,0x3d53000a,0x3d55000a,0x3d57000a,0x3d59000a,0x3d5b000a,0x3d5d000a,0x3d5f000a,0x3d61000a,0x3d63000a,0x3d65000a,0,0x3d680040,0x3de20040,0x3d690040, +0x3d780040,0x3d6a0040,0x3de40040,0x3d7a0040,0x3d7c0040,0x3d6b0040,0x3d7e0040,0x3d800040,0x3d820040,0x3de60040,0x3d6c0040,0x3d6d0040,0x3de80040,0,0x3d840040,0x3d860040, +0x3d880040,0x3d6e0040,0x3dea0040,0x3d8a0040,0x3dec0040,0x3d6f0040,0x3d8c0040,0,0,0,0,0,0,0x3d700040,0x3de30040,0x3d710040, +0x3d790040,0x3d720040,0x3de50040,0x3d7b0040,0x3d7d0040,0x3d730040,0x3d7f0040,0x3d810040,0x3d830040,0x3de70040,0x3d740040,0x3d750040,0x3de90040,0,0x3d850040,0x3d870040, +0x3d890040,0x3d760040,0x3deb0040,0x3d8b0040,0x3ded0040,0x3d770040,0x3d8d0040,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0x3d910040,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0xe600,0xe600,0xe600,0xe600,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0xe600,0xe600,0xdc00,0xdc00,0xdc00,0xdc00,0xdc00,0xdc00, +0xe600,0xe600,0xdc00,0xe600,0xe600,0xde00,0xe400,0xe600,0xa00,0xb00,0xc00,0xd00,0xe00,0xf00,0x1000,0x1100, +0x1200,0x1300,0x1300,0x1400,0x1500,0x1600,0,0x1700,0,0x1800,0x1900,0,0xe600,0xdc00,0,0x1200, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0xe600,0xe600,0xe600,0xe600,0xe600,0xe600,0,0, +0,0,0,0,0,0,0,0,0xe600,0xe600,0xe600,0xdc00,0xe600,0,0,0xe600, +0xe600,0,0xdc00,0xe600,0xe600,0xdc00,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0x2400,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0xe600,0xdc00,0xe600,0xe600, +0xdc00,0xe600,0xe600,0xdc00,0xdc00,0xdc00,0xe600,0xdc00,0xdc00,0xe600,0xdc00,0xe600,0xe600,0xe600,0xdc00,0xe600, +0xdc00,0xe600,0xdc00,0xe600,0xdc00,0xe600,0xe600,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xe600, +0xe600,0xe600,0xe600,0xe600,0xe600,0xe600,0xdc00,0xe600,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0x700,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0x900,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0x700,0,0,0x3dd10040,0,0,0,0,0,0,0,0, +0x6b00,0x6b00,0x6b00,0x6b00,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0xdc00,0,0xdc00,0,0xd800,0,0, +0,0,0,0,0,0,0xdc00,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0xe600,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0x900,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0x900,0,0,0,0,0,0,0,0,0,0,0xe600,0,0, +0,0,0,0,0,0,0,0,0,0xe400,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0xde00,0xe600,0xdc00,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xe600, +0xdc00,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xe600, +0xdc00,0xe600,0xe600,0xe600,0xe600,0xe600,0xe600,0xe600,0,0,0,0,0,0,0,0, +0,0,0,0,0xe600,0xe600,0xdc00,0xe600,0xe600,0xe600,0xe600,0xe600,0xe600,0xe600,0xdc00,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xe600,0xdc00, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0xe600,0xe600,0x100,0x100,0xe600,0xe600,0xe600,0xe600,0x100,0x100,0x100,0xe600,0xe600,0,0,0, +0,0xe600,0,0,0,0x100,0x100,0xe600,0xdc00,0xe600,0x100,0x100,0xdc00,0xdc00,0xdc00,0xdc00, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0x3e020040,0x3dff0040,0x3e030040,0, +0,0,0,0,0,0,0x900,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0xdc00,0,0xe600,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0xe600,0x100,0xdc00,0, +0,0,0,0x900,0xdc00,0xdc00,0xdc00,0,0,0xe600,0xe600,0xe600,0xe600,0xe600,0xdc00,0xdc00, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0xe600,0xe600,0xe600,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0xfff10040,0xfff10040,0xfff10040,0xfff10040,0xfff10040,0xfff10040,0xfff10040,0xfff10040,0xfff10040,0xfff10040,0xfff10040,0xfff10040, +0xfff10040,0xfff10040,0xfff10040,0xfff10040,0xfff10040,0xfff10040,0xfff10040,0,0,0,0,0,0,0,0,0, +0,0,0,0,0xfff0000c,0xfff0000c,0xfff0000c,0xfff0000c,0xfff0000c,0xfff0000c,0xfff0000c,0xfff0000c,0xfff0000c,0xfff0000c,0xfff0000c,0xfff0000c, +0xfff0000c,0xfff0000c,0xfff0000c,0xfff0000c,0xfff0000c,0xfff0000c,0xfff0000c,0xfff0000c,0xfff0000c,0xfff0000c,0xfff0000c,0xfff0000c,0xfff0000c,0xfff0000c,0xfff0000c,0xfff0000c, +0xfff0000c,0xfff0000c,0xfff0000c,0xfff0000c,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0xfc01ff00,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0xfc02ff0f,0xfc03000a,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0xfc04000f,0 +}; + +static const UTrie normTrie={ + normTrie_index, + normTrie_data32, + getFoldingNormOffset, + 2208, + 9236, + 0, + FALSE +}; + +static const uint16_t extraData[15947]={ +0x13b,0xff02,0x20,0x3b9,0xff01,0x3c5,0xff01,0x3cd,0xff01,0x3cb,0xff01,0x3c3,0xff01,0x61,0xff01,0xe6, +0xff01,0x62,0xff01,0x64,0xff01,0x65,0xff01,0x1dd,0xff01,0x67,0xff01,0x68,0xff01,0x69,0xff01,0x6a, +0xff01,0x6b,0xff01,0x6c,0xff01,0x6d,0xff01,0x6e,0xff01,0x6f,0xff01,0x223,0xff01,0x70,0xff01,0x72, +0xff01,0x74,0xff01,0x75,0xff01,0x77,0xff02,0x72,0x73,0xff01,0x63,0xff02,0xb0,0x63,0xff01,0x25b, +0xff02,0xb0,0x66,0xff02,0x6e,0x6f,0xff01,0x71,0xff02,0x73,0x6d,0xff03,0x74,0x65,0x6c,0xff02, +0x74,0x6d,0xff01,0x7a,0xff03,0x66,0x61,0x78,0xff01,0x3b3,0xff01,0x3c0,0xff03,0x70,0x74,0x65, +0xff02,0x68,0x67,0xff02,0x65,0x76,0xff03,0x6c,0x74,0x64,0xff03,0x68,0x70,0x61,0xff02,0x61, +0x75,0xff02,0x6f,0x76,0xff02,0x69,0x75,0xff02,0x70,0x61,0xff02,0x6e,0x61,0xff02,0x3bc,0x61, +0xff02,0x6d,0x61,0xff02,0x6b,0x61,0xff02,0x6b,0x62,0xff02,0x6d,0x62,0xff02,0x67,0x62,0xff02, +0x70,0x66,0xff02,0x6e,0x66,0xff02,0x3bc,0x66,0xff02,0x68,0x7a,0xff03,0x6b,0x68,0x7a,0xff03, +0x6d,0x68,0x7a,0xff03,0x67,0x68,0x7a,0xff03,0x74,0x68,0x7a,0xff02,0x70,0x61,0xff03,0x6b, +0x70,0x61,0xff03,0x6d,0x70,0x61,0xff03,0x67,0x70,0x61,0xff02,0x70,0x76,0xff02,0x6e,0x76, +0xff02,0x3bc,0x76,0xff02,0x6d,0x76,0xff02,0x6b,0x76,0xff02,0x6d,0x76,0xff02,0x70,0x77,0xff02, +0x6e,0x77,0xff02,0x3bc,0x77,0xff02,0x6d,0x77,0xff02,0x6b,0x77,0xff02,0x6d,0x77,0xff02,0x6b, +0x3c9,0xff02,0x6d,0x3c9,0xff02,0x62,0x71,0xff04,0x63,0x2215,0x6b,0x67,0xff03,0x63,0x6f,0x2e, +0xff02,0x64,0x62,0xff02,0x67,0x79,0xff02,0x68,0x70,0xff02,0x6b,0x6b,0xff02,0x6b,0x6d,0xff02, +0x70,0x68,0xff03,0x70,0x70,0x6d,0xff02,0x70,0x72,0xff02,0x73,0x76,0xff02,0x77,0x62,0xff03, +0x76,0x2215,0x6d,0xff03,0x61,0x2215,0x6d,0xff01,0x3b1,0xff01,0x3b2,0xff01,0x3b4,0xff01,0x3b5,0xff01, +0x3b6,0xff01,0x3b7,0xff01,0x3b8,0xff01,0x3ba,0xff01,0x3bb,0xff01,0x3bd,0xff01,0x3be,0xff01,0x3bf,0xff01, +0x3c1,0xff01,0x3c4,0xff01,0x3c6,0xff01,0x3c7,0xff01,0x3c8,0xff01,0x3dd,0x8200,0xf0,0x20,0x345,0x3b2, +0x100,0x3a5,0x8282,0xe6,0x3d2,0x301,0xe6,0x3a5,0x301,0x8282,0xe6,0x3d2,0x308,0xe6,0x3a5,0x308, +0x100,0x3c2,0x100,0x3a3,0x100,0x41,0x100,0xc6,0x100,0x42,0x100,0x44,0x100,0x45,0x100,0x18e, +0x100,0x47,0x100,0x48,0x100,0x49,0x100,0x4a,0x100,0x4b,0x100,0x4c,0x100,0x4d,0x100,0x4e, +0x100,0x4f,0x100,0x222,0x100,0x50,0x100,0x52,0x100,0x54,0x100,0x55,0x100,0x57,0x200,0x52, +0x73,0x100,0x43,0x200,0xb0,0x43,0x100,0x190,0x200,0xb0,0x46,0x100,0x48,0x100,0x48,0x100, +0x48,0x100,0x49,0x100,0x49,0x100,0x4c,0x100,0x4e,0x200,0x4e,0x6f,0x100,0x50,0x100,0x51, +0x100,0x52,0x100,0x52,0x100,0x52,0x200,0x53,0x4d,0x300,0x54,0x45,0x4c,0x200,0x54,0x4d, +0x100,0x5a,0x100,0x5a,0x100,0x42,0x100,0x43,0x100,0x45,0x100,0x46,0x100,0x4d,0x300,0x46, +0x41,0x58,0x100,0x393,0x100,0x3a0,0x100,0x44,0x300,0x50,0x54,0x45,0x200,0x48,0x67,0x200, +0x65,0x56,0x300,0x4c,0x54,0x44,0x300,0x68,0x50,0x61,0x200,0x41,0x55,0x200,0x6f,0x56, +0x200,0x49,0x55,0x200,0x70,0x41,0x200,0x6e,0x41,0x200,0x3bc,0x41,0x200,0x6d,0x41,0x200, +0x6b,0x41,0x200,0x4b,0x42,0x200,0x4d,0x42,0x200,0x47,0x42,0x200,0x70,0x46,0x200,0x6e, +0x46,0x200,0x3bc,0x46,0x200,0x48,0x7a,0x300,0x6b,0x48,0x7a,0x300,0x4d,0x48,0x7a,0x300, +0x47,0x48,0x7a,0x300,0x54,0x48,0x7a,0x200,0x50,0x61,0x300,0x6b,0x50,0x61,0x300,0x4d, +0x50,0x61,0x300,0x47,0x50,0x61,0x200,0x70,0x56,0x200,0x6e,0x56,0x200,0x3bc,0x56,0x200, +0x6d,0x56,0x200,0x6b,0x56,0x200,0x4d,0x56,0x200,0x70,0x57,0x200,0x6e,0x57,0x200,0x3bc, +0x57,0x200,0x6d,0x57,0x200,0x6b,0x57,0x200,0x4d,0x57,0x200,0x6b,0x3a9,0x200,0x4d,0x3a9, +0x200,0x42,0x71,0x400,0x43,0x2215,0x6b,0x67,0x300,0x43,0x6f,0x2e,0x200,0x64,0x42,0x200, +0x47,0x79,0x200,0x48,0x50,0x200,0x4b,0x4b,0x200,0x4b,0x4d,0x200,0x50,0x48,0x300,0x50, +0x50,0x4d,0x200,0x50,0x52,0x200,0x53,0x76,0x200,0x57,0x62,0x300,0x56,0x2215,0x6d,0x300, +0x41,0x2215,0x6d,0x100,0x41,0x100,0x42,0x100,0x43,0x100,0x44,0x100,0x45,0x100,0x46,0x100, +0x47,0x100,0x48,0x100,0x49,0x100,0x4a,0x100,0x4b,0x100,0x4c,0x100,0x4d,0x100,0x4e,0x100, +0x4f,0x100,0x50,0x100,0x51,0x100,0x52,0x100,0x53,0x100,0x54,0x100,0x55,0x100,0x56,0x100, +0x57,0x100,0x58,0x100,0x59,0x100,0x5a,0x100,0x41,0x100,0x42,0x100,0x43,0x100,0x44,0x100, +0x45,0x100,0x46,0x100,0x47,0x100,0x48,0x100,0x49,0x100,0x4a,0x100,0x4b,0x100,0x4c,0x100, +0x4d,0x100,0x4e,0x100,0x4f,0x100,0x50,0x100,0x51,0x100,0x52,0x100,0x53,0x100,0x54,0x100, +0x55,0x100,0x56,0x100,0x57,0x100,0x58,0x100,0x59,0x100,0x5a,0x100,0x41,0x100,0x42,0x100, +0x43,0x100,0x44,0x100,0x45,0x100,0x46,0x100,0x47,0x100,0x48,0x100,0x49,0x100,0x4a,0x100, +0x4b,0x100,0x4c,0x100,0x4d,0x100,0x4e,0x100,0x4f,0x100,0x50,0x100,0x51,0x100,0x52,0x100, +0x53,0x100,0x54,0x100,0x55,0x100,0x56,0x100,0x57,0x100,0x58,0x100,0x59,0x100,0x5a,0x100, +0x41,0x100,0x43,0x100,0x44,0x100,0x47,0x100,0x4a,0x100,0x4b,0x100,0x4e,0x100,0x4f,0x100, +0x50,0x100,0x51,0x100,0x53,0x100,0x54,0x100,0x55,0x100,0x56,0x100,0x57,0x100,0x58,0x100, +0x59,0x100,0x5a,0x100,0x41,0x100,0x42,0x100,0x43,0x100,0x44,0x100,0x45,0x100,0x46,0x100, +0x47,0x100,0x48,0x100,0x49,0x100,0x4a,0x100,0x4b,0x100,0x4c,0x100,0x4d,0x100,0x4e,0x100, +0x4f,0x100,0x50,0x100,0x51,0x100,0x52,0x100,0x53,0x100,0x54,0x100,0x55,0x100,0x56,0x100, +0x57,0x100,0x58,0x100,0x59,0x100,0x5a,0x100,0x41,0x100,0x42,0x100,0x44,0x100,0x45,0x100, +0x46,0x100,0x47,0x100,0x4a,0x100,0x4b,0x100,0x4c,0x100,0x4d,0x100,0x4e,0x100,0x4f,0x100, +0x50,0x100,0x51,0x100,0x53,0x100,0x54,0x100,0x55,0x100,0x56,0x100,0x57,0x100,0x58,0x100, +0x59,0x100,0x41,0x100,0x42,0x100,0x44,0x100,0x45,0x100,0x46,0x100,0x47,0x100,0x49,0x100, +0x4a,0x100,0x4b,0x100,0x4c,0x100,0x4d,0x100,0x4f,0x100,0x53,0x100,0x54,0x100,0x55,0x100, +0x56,0x100,0x57,0x100,0x58,0x100,0x59,0x100,0x41,0x100,0x42,0x100,0x43,0x100,0x44,0x100, +0x45,0x100,0x46,0x100,0x47,0x100,0x48,0x100,0x49,0x100,0x4a,0x100,0x4b,0x100,0x4c,0x100, +0x4d,0x100,0x4e,0x100,0x4f,0x100,0x50,0x100,0x51,0x100,0x52,0x100,0x53,0x100,0x54,0x100, +0x55,0x100,0x56,0x100,0x57,0x100,0x58,0x100,0x59,0x100,0x5a,0x100,0x41,0x100,0x42,0x100, +0x43,0x100,0x44,0x100,0x45,0x100,0x46,0x100,0x47,0x100,0x48,0x100,0x49,0x100,0x4a,0x100, +0x4b,0x100,0x4c,0x100,0x4d,0x100,0x4e,0x100,0x4f,0x100,0x50,0x100,0x51,0x100,0x52,0x100, +0x53,0x100,0x54,0x100,0x55,0x100,0x56,0x100,0x57,0x100,0x58,0x100,0x59,0x100,0x5a,0x100, +0x41,0x100,0x42,0x100,0x43,0x100,0x44,0x100,0x45,0x100,0x46,0x100,0x47,0x100,0x48,0x100, +0x49,0x100,0x4a,0x100,0x4b,0x100,0x4c,0x100,0x4d,0x100,0x4e,0x100,0x4f,0x100,0x50,0x100, +0x51,0x100,0x52,0x100,0x53,0x100,0x54,0x100,0x55,0x100,0x56,0x100,0x57,0x100,0x58,0x100, +0x59,0x100,0x5a,0x100,0x41,0x100,0x42,0x100,0x43,0x100,0x44,0x100,0x45,0x100,0x46,0x100, +0x47,0x100,0x48,0x100,0x49,0x100,0x4a,0x100,0x4b,0x100,0x4c,0x100,0x4d,0x100,0x4e,0x100, +0x4f,0x100,0x50,0x100,0x51,0x100,0x52,0x100,0x53,0x100,0x54,0x100,0x55,0x100,0x56,0x100, +0x57,0x100,0x58,0x100,0x59,0x100,0x5a,0x100,0x41,0x100,0x42,0x100,0x43,0x100,0x44,0x100, +0x45,0x100,0x46,0x100,0x47,0x100,0x48,0x100,0x49,0x100,0x4a,0x100,0x4b,0x100,0x4c,0x100, +0x4d,0x100,0x4e,0x100,0x4f,0x100,0x50,0x100,0x51,0x100,0x52,0x100,0x53,0x100,0x54,0x100, +0x55,0x100,0x56,0x100,0x57,0x100,0x58,0x100,0x59,0x100,0x5a,0x100,0x41,0x100,0x42,0x100, +0x43,0x100,0x44,0x100,0x45,0x100,0x46,0x100,0x47,0x100,0x48,0x100,0x49,0x100,0x4a,0x100, +0x4b,0x100,0x4c,0x100,0x4d,0x100,0x4e,0x100,0x4f,0x100,0x50,0x100,0x51,0x100,0x52,0x100, +0x53,0x100,0x54,0x100,0x55,0x100,0x56,0x100,0x57,0x100,0x58,0x100,0x59,0x100,0x5a,0x100, +0x391,0x100,0x392,0x100,0x393,0x100,0x394,0x100,0x395,0x100,0x396,0x100,0x397,0x100,0x398,0x100, +0x399,0x100,0x39a,0x100,0x39b,0x100,0x39c,0x100,0x39d,0x100,0x39e,0x100,0x39f,0x100,0x3a0,0x100, +0x3a1,0x100,0x398,0x100,0x3a3,0x100,0x3a4,0x100,0x3a5,0x100,0x3a6,0x100,0x3a7,0x100,0x3a8,0x100, +0x3a9,0x100,0x3c2,0x100,0x391,0x100,0x392,0x100,0x393,0x100,0x394,0x100,0x395,0x100,0x396,0x100, +0x397,0x100,0x398,0x100,0x399,0x100,0x39a,0x100,0x39b,0x100,0x39c,0x100,0x39d,0x100,0x39e,0x100, +0x39f,0x100,0x3a0,0x100,0x3a1,0x100,0x398,0x100,0x3a3,0x100,0x3a4,0x100,0x3a5,0x100,0x3a6,0x100, +0x3a7,0x100,0x3a8,0x100,0x3a9,0x100,0x3c2,0x100,0x391,0x100,0x392,0x100,0x393,0x100,0x394,0x100, +0x395,0x100,0x396,0x100,0x397,0x100,0x398,0x100,0x399,0x100,0x39a,0x100,0x39b,0x100,0x39c,0x100, +0x39d,0x100,0x39e,0x100,0x39f,0x100,0x3a0,0x100,0x3a1,0x100,0x398,0x100,0x3a3,0x100,0x3a4,0x100, +0x3a5,0x100,0x3a6,0x100,0x3a7,0x100,0x3a8,0x100,0x3a9,0x100,0x3c2,0x100,0x391,0x100,0x392,0x100, +0x393,0x100,0x394,0x100,0x395,0x100,0x396,0x100,0x397,0x100,0x398,0x100,0x399,0x100,0x39a,0x100, +0x39b,0x100,0x39c,0x100,0x39d,0x100,0x39e,0x100,0x39f,0x100,0x3a0,0x100,0x3a1,0x100,0x398,0x100, +0x3a3,0x100,0x3a4,0x100,0x3a5,0x100,0x3a6,0x100,0x3a7,0x100,0x3a8,0x100,0x3a9,0x100,0x3c2,0x100, +0x391,0x100,0x392,0x100,0x393,0x100,0x394,0x100,0x395,0x100,0x396,0x100,0x397,0x100,0x398,0x100, +0x399,0x100,0x39a,0x100,0x39b,0x100,0x39c,0x100,0x39d,0x100,0x39e,0x100,0x39f,0x100,0x3a0,0x100, +0x3a1,0x100,0x398,0x100,0x3a3,0x100,0x3a4,0x100,0x3a5,0x100,0x3a6,0x100,0x3a7,0x100,0x3a8,0x100, +0x3a9,0x100,0x3c2,0x100,0x3dc,0x81,0xe6e6,0x300,0x81,0xe6e6,0x301,0x81,0xe6e6,0x313,0x82,0xe6e6, +0x308,0x301,1,0x2b9,1,0x3b,1,0xb7,0x82,7,0x915,0x93c,0x82,7,0x916,0x93c, +0x82,7,0x917,0x93c,0x82,7,0x91c,0x93c,0x82,7,0x921,0x93c,0x82,7,0x922,0x93c, +0x82,7,0x92b,0x93c,0x82,7,0x92f,0x93c,0x82,7,0x9a1,0x9bc,0x82,7,0x9a2,0x9bc, +0x82,7,0x9af,0x9bc,0x82,7,0xa32,0xa3c,0x82,7,0xa38,0xa3c,0x82,7,0xa16,0xa3c, +0x82,7,0xa17,0xa3c,0x82,7,0xa1c,0xa3c,0x82,7,0xa2b,0xa3c,0x82,7,0xb21,0xb3c, +0x82,7,0xb22,0xb3c,2,0xf42,0xfb7,2,0xf4c,0xfb7,2,0xf51,0xfb7,2,0xf56,0xfb7, +2,0xf5b,0xfb7,2,0xf40,0xfb5,0x82,0x8182,0xf71,0xf72,0x82,0x8184,0xf71,0xf74,0x82,0x82, +0xfb2,0xf80,0x82,0x82,0xfb3,0xf80,0x82,0x8182,0xf71,0xf80,2,0xf92,0xfb7,2,0xf9c,0xfb7, +2,0xfa1,0xfb7,2,0xfa6,0xfb7,2,0xfab,0xfb7,2,0xf90,0xfb5,0x82,0xe6,0x3b1,0x301, +0x82,0xe6,0x3b5,0x301,0x82,0xe6,0x3b7,0x301,0x82,0xe6,0x3b9,0x301,0x82,0xe6,0x3bf,0x301, +0x82,0xe6,0x3c5,0x301,0x82,0xe6,0x3c9,0x301,0x82,0xe6,0x391,0x301,1,0x3b9,0x82,0xe6, +0x395,0x301,0x82,0xe6,0x397,0x301,0x83,0xe6,0x3b9,0x308,0x301,0x82,0xe6,0x399,0x301,0x83, +0xe6,0x3c5,0x308,0x301,0x82,0xe6,0x3a5,0x301,0x8382,0xe6,0xa8,0x301,0xe6,0x20,0x308,0x301, +1,0x60,0x82,0xe6,0x39f,0x301,0x82,0xe6,0x3a9,0x301,0x8201,0xb4,0xe6,0x20,0x301,0x101, +0x2002,0x20,0x101,0x2003,0x20,1,0x3a9,1,0x4b,0x82,0xe6,0x41,0x30a,1,0x3008,1, +0x3009,0x82,1,0x2add,0x338,1,0x8c48,1,0x66f4,1,0x8eca,1,0x8cc8,1,0x6ed1,1, +0x4e32,1,0x53e5,1,0x9f9c,1,0x9f9c,1,0x5951,1,0x91d1,1,0x5587,1,0x5948,1, +0x61f6,1,0x7669,1,0x7f85,1,0x863f,1,0x87ba,1,0x88f8,1,0x908f,1,0x6a02,1, +0x6d1b,1,0x70d9,1,0x73de,1,0x843d,1,0x916a,1,0x99f1,1,0x4e82,1,0x5375,1, +0x6b04,1,0x721b,1,0x862d,1,0x9e1e,1,0x5d50,1,0x6feb,1,0x85cd,1,0x8964,1, +0x62c9,1,0x81d8,1,0x881f,1,0x5eca,1,0x6717,1,0x6d6a,1,0x72fc,1,0x90ce,1, +0x4f86,1,0x51b7,1,0x52de,1,0x64c4,1,0x6ad3,1,0x7210,1,0x76e7,1,0x8001,1, +0x8606,1,0x865c,1,0x8def,1,0x9732,1,0x9b6f,1,0x9dfa,1,0x788c,1,0x797f,1, +0x7da0,1,0x83c9,1,0x9304,1,0x9e7f,1,0x8ad6,1,0x58df,1,0x5f04,1,0x7c60,1, +0x807e,1,0x7262,1,0x78ca,1,0x8cc2,1,0x96f7,1,0x58d8,1,0x5c62,1,0x6a13,1, +0x6dda,1,0x6f0f,1,0x7d2f,1,0x7e37,1,0x964b,1,0x52d2,1,0x808b,1,0x51dc,1, +0x51cc,1,0x7a1c,1,0x7dbe,1,0x83f1,1,0x9675,1,0x8b80,1,0x62cf,1,0x6a02,1, +0x8afe,1,0x4e39,1,0x5be7,1,0x6012,1,0x7387,1,0x7570,1,0x5317,1,0x78fb,1, +0x4fbf,1,0x5fa9,1,0x4e0d,1,0x6ccc,1,0x6578,1,0x7d22,1,0x53c3,1,0x585e,1, +0x7701,1,0x8449,1,0x8aaa,1,0x6bba,1,0x8fb0,1,0x6c88,1,0x62fe,1,0x82e5,1, +0x63a0,1,0x7565,1,0x4eae,1,0x5169,1,0x51c9,1,0x6881,1,0x7ce7,1,0x826f,1, +0x8ad2,1,0x91cf,1,0x52f5,1,0x5442,1,0x5973,1,0x5eec,1,0x65c5,1,0x6ffe,1, +0x792a,1,0x95ad,1,0x9a6a,1,0x9e97,1,0x9ece,1,0x529b,1,0x66c6,1,0x6b77,1, +0x8f62,1,0x5e74,1,0x6190,1,0x6200,1,0x649a,1,0x6f23,1,0x7149,1,0x7489,1, +0x79ca,1,0x7df4,1,0x806f,1,0x8f26,1,0x84ee,1,0x9023,1,0x934a,1,0x5217,1, +0x52a3,1,0x54bd,1,0x70c8,1,0x88c2,1,0x8aaa,1,0x5ec9,1,0x5ff5,1,0x637b,1, +0x6bae,1,0x7c3e,1,0x7375,1,0x4ee4,1,0x56f9,1,0x5be7,1,0x5dba,1,0x601c,1, +0x73b2,1,0x7469,1,0x7f9a,1,0x8046,1,0x9234,1,0x96f6,1,0x9748,1,0x9818,1, +0x4f8b,1,0x79ae,1,0x91b4,1,0x96b8,1,0x60e1,1,0x4e86,1,0x50da,1,0x5bee,1, +0x5c3f,1,0x6599,1,0x6a02,1,0x71ce,1,0x7642,1,0x84fc,1,0x907c,1,0x9f8d,1, +0x6688,1,0x962e,1,0x5289,1,0x677b,1,0x67f3,1,0x6d41,1,0x6e9c,1,0x7409,1, +0x7559,1,0x786b,1,0x7d10,1,0x985e,1,0x516d,1,0x622e,1,0x9678,1,0x502b,1, +0x5d19,1,0x6dea,1,0x8f2a,1,0x5f8b,1,0x6144,1,0x6817,1,0x7387,1,0x9686,1, +0x5229,1,0x540f,1,0x5c65,1,0x6613,1,0x674e,1,0x68a8,1,0x6ce5,1,0x7406,1, +0x75e2,1,0x7f79,1,0x88cf,1,0x88e1,1,0x91cc,1,0x96e2,1,0x533f,1,0x6eba,1, +0x541d,1,0x71d0,1,0x7498,1,0x85fa,1,0x96a3,1,0x9c57,1,0x9e9f,1,0x6797,1, +0x6dcb,1,0x81e8,1,0x7acb,1,0x7b20,1,0x7c92,1,0x72c0,1,0x7099,1,0x8b58,1, +0x4ec0,1,0x8336,1,0x523a,1,0x5207,1,0x5ea6,1,0x62d3,1,0x7cd6,1,0x5b85,1, +0x6d1e,1,0x66b4,1,0x8f3b,1,0x884c,1,0x964d,1,0x898b,1,0x5ed3,1,0x5140,1, +0x55c0,1,0x585a,1,0x6674,1,0x51de,1,0x732a,1,0x76ca,1,0x793c,1,0x795e,1, +0x7965,1,0x798f,1,0x9756,1,0x7cbe,1,0x7fbd,1,0x8612,1,0x8af8,1,0x9038,1, +0x90fd,1,0x98ef,1,0x98fc,1,0x9928,1,0x9db4,1,0x4fae,1,0x50e7,1,0x514d,1, +0x52c9,1,0x52e4,1,0x5351,1,0x559d,1,0x5606,1,0x5668,1,0x5840,1,0x58a8,1, +0x5c64,1,0x5c6e,1,0x6094,1,0x6168,1,0x618e,1,0x61f2,1,0x654f,1,0x65e2,1, +0x6691,1,0x6885,1,0x6d77,1,0x6e1a,1,0x6f22,1,0x716e,1,0x722b,1,0x7422,1, +0x7891,1,0x793e,1,0x7949,1,0x7948,1,0x7950,1,0x7956,1,0x795d,1,0x798d,1, +0x798e,1,0x7a40,1,0x7a81,1,0x7bc0,1,0x7df4,1,0x7e09,1,0x7e41,1,0x7f72,1, +0x8005,1,0x81ed,1,0x8279,1,0x8279,1,0x8457,1,0x8910,1,0x8996,1,0x8b01,1, +0x8b39,1,0x8cd3,1,0x8d08,1,0x8fb6,1,0x9038,1,0x96e3,1,0x97ff,1,0x983b,1, +0x4e26,1,0x51b5,1,0x5168,1,0x4f80,1,0x5145,1,0x5180,1,0x52c7,1,0x52fa,1, +0x559d,1,0x5555,1,0x5599,1,0x55e2,1,0x585a,1,0x58b3,1,0x5944,1,0x5954,1, +0x5a62,1,0x5b28,1,0x5ed2,1,0x5ed9,1,0x5f69,1,0x5fad,1,0x60d8,1,0x614e,1, +0x6108,1,0x618e,1,0x6160,1,0x61f2,1,0x6234,1,0x63c4,1,0x641c,1,0x6452,1, +0x6556,1,0x6674,1,0x6717,1,0x671b,1,0x6756,1,0x6b79,1,0x6bba,1,0x6d41,1, +0x6edb,1,0x6ecb,1,0x6f22,1,0x701e,1,0x716e,1,0x77a7,1,0x7235,1,0x72af,1, +0x732a,1,0x7471,1,0x7506,1,0x753b,1,0x761d,1,0x761f,1,0x76ca,1,0x76db,1, +0x76f4,1,0x774a,1,0x7740,1,0x78cc,1,0x7ab1,1,0x7bc0,1,0x7c7b,1,0x7d5b,1, +0x7df4,1,0x7f3e,1,0x8005,1,0x8352,1,0x83ef,1,0x8779,1,0x8941,1,0x8986,1, +0x8996,1,0x8abf,1,0x8af8,1,0x8acb,1,0x8b01,1,0x8afe,1,0x8aed,1,0x8b39,1, +0x8b8a,1,0x8d08,1,0x8f38,1,0x9072,1,0x9199,1,0x9276,1,0x967c,1,0x96e3,1, +0x9756,1,0x97db,1,0x97ff,1,0x980b,1,0x983b,1,0x9b12,1,0x9f9c,2,0xd84a,0xdc4a, +2,0xd84a,0xdc44,2,0xd84c,0xdfd5,1,0x3b9d,1,0x4018,1,0x4039,2,0xd854,0xde49,2, +0xd857,0xdcd0,2,0xd85f,0xded3,1,0x9f43,1,0x9f8e,0x82,0xe,0x5d9,0x5b4,0x82,0x11,0x5f2, +0x5b7,0x82,0x18,0x5e9,0x5c1,0x82,0x19,0x5e9,0x5c2,0x83,0x18,0x5e9,0x5bc,0x5c1,0x83,0x19, +0x5e9,0x5bc,0x5c2,0x82,0x11,0x5d0,0x5b7,0x82,0x12,0x5d0,0x5b8,0x82,0x15,0x5d0,0x5bc,0x82, +0x15,0x5d1,0x5bc,0x82,0x15,0x5d2,0x5bc,0x82,0x15,0x5d3,0x5bc,0x82,0x15,0x5d4,0x5bc,0x82, +0x15,0x5d5,0x5bc,0x82,0x15,0x5d6,0x5bc,0x82,0x15,0x5d8,0x5bc,0x82,0x15,0x5d9,0x5bc,0x82, +0x15,0x5da,0x5bc,0x82,0x15,0x5db,0x5bc,0x82,0x15,0x5dc,0x5bc,0x82,0x15,0x5de,0x5bc,0x82, +0x15,0x5e0,0x5bc,0x82,0x15,0x5e1,0x5bc,0x82,0x15,0x5e3,0x5bc,0x82,0x15,0x5e4,0x5bc,0x82, +0x15,0x5e6,0x5bc,0x82,0x15,0x5e7,0x5bc,0x82,0x15,0x5e8,0x5bc,0x82,0x15,0x5e9,0x5bc,0x82, +0x15,0x5ea,0x5bc,0x82,0x13,0x5d5,0x5b9,0x82,0x17,0x5d1,0x5bf,0x82,0x17,0x5db,0x5bf,0x82, +0x17,0x5e4,0x5bf,0x84,0xd8,0xd834,0xdd57,0xd834,0xdd65,0x84,0xd8,0xd834,0xdd58,0xd834,0xdd65,0x86, +0xd8,0xd834,0xdd58,0xd834,0xdd65,0xd834,0xdd6e,0x86,0xd8,0xd834,0xdd58,0xd834,0xdd65,0xd834,0xdd6f,0x86, +0xd8,0xd834,0xdd58,0xd834,0xdd65,0xd834,0xdd70,0x86,0xd8,0xd834,0xdd58,0xd834,0xdd65,0xd834,0xdd71,0x86, +0xd8,0xd834,0xdd58,0xd834,0xdd65,0xd834,0xdd72,0x84,0xd8,0xd834,0xddb9,0xd834,0xdd65,0x84,0xd8,0xd834, +0xddba,0xd834,0xdd65,0x86,0xd8,0xd834,0xddb9,0xd834,0xdd65,0xd834,0xdd6e,0x86,0xd8,0xd834,0xddba,0xd834, +0xdd65,0xd834,0xdd6e,0x86,0xd8,0xd834,0xddb9,0xd834,0xdd65,0xd834,0xdd6f,0x86,0xd8,0xd834,0xddba,0xd834, +0xdd65,0xd834,0xdd6f,1,0x4e3d,1,0x4e38,1,0x4e41,2,0xd840,0xdd22,1,0x4f60,1,0x4fae, +1,0x4fbb,1,0x5002,1,0x507a,1,0x5099,1,0x50e7,1,0x50cf,1,0x349e,2,0xd841, +0xde3a,1,0x514d,1,0x5154,1,0x5164,1,0x5177,2,0xd841,0xdd1c,1,0x34b9,1,0x5167, +1,0x518d,2,0xd841,0xdd4b,1,0x5197,1,0x51a4,1,0x4ecc,1,0x51ac,1,0x51b5,2, +0xd864,0xdddf,1,0x51f5,1,0x5203,1,0x34df,1,0x523b,1,0x5246,1,0x5272,1,0x5277, +1,0x3515,1,0x52c7,1,0x52c9,1,0x52e4,1,0x52fa,1,0x5305,1,0x5306,1,0x5317, +1,0x5349,1,0x5351,1,0x535a,1,0x5373,1,0x537d,1,0x537f,1,0x537f,1,0x537f, +2,0xd842,0xde2c,1,0x7070,1,0x53ca,1,0x53df,2,0xd842,0xdf63,1,0x53eb,1,0x53f1, +1,0x5406,1,0x549e,1,0x5438,1,0x5448,1,0x5468,1,0x54a2,1,0x54f6,1,0x5510, +1,0x5553,1,0x5563,1,0x5584,1,0x5584,1,0x5599,1,0x55ab,1,0x55b3,1,0x55c2, +1,0x5716,1,0x5606,1,0x5717,1,0x5651,1,0x5674,1,0x5207,1,0x58ee,1,0x57ce, +1,0x57f4,1,0x580d,1,0x578b,1,0x5832,1,0x5831,1,0x58ac,2,0xd845,0xdce4,1, +0x58f2,1,0x58f7,1,0x5906,1,0x591a,1,0x5922,1,0x5962,2,0xd845,0xdea8,2,0xd845, +0xdeea,1,0x59ec,1,0x5a1b,1,0x5a27,1,0x59d8,1,0x5a66,1,0x36ee,1,0x36fc,1, +0x5b08,1,0x5b3e,1,0x5b3e,2,0xd846,0xddc8,1,0x5bc3,1,0x5bd8,1,0x5be7,1,0x5bf3, +2,0xd846,0xdf18,1,0x5bff,1,0x5c06,1,0x5f53,1,0x5c22,1,0x3781,1,0x5c60,1, +0x5c6e,1,0x5cc0,1,0x5c8d,2,0xd847,0xdde4,1,0x5d43,2,0xd847,0xdde6,1,0x5d6e,1, +0x5d6b,1,0x5d7c,1,0x5de1,1,0x5de2,1,0x382f,1,0x5dfd,1,0x5e28,1,0x5e3d,1, +0x5e69,1,0x3862,2,0xd848,0xdd83,1,0x387c,1,0x5eb0,1,0x5eb3,1,0x5eb6,1,0x5eca, +2,0xd868,0xdf92,1,0x5efe,2,0xd848,0xdf31,2,0xd848,0xdf31,1,0x8201,1,0x5f22,1, +0x5f22,1,0x38c7,2,0xd84c,0xdeb8,2,0xd858,0xddda,1,0x5f62,1,0x5f6b,1,0x38e3,1, +0x5f9a,1,0x5fcd,1,0x5fd7,1,0x5ff9,1,0x6081,1,0x393a,1,0x391c,1,0x6094,2, +0xd849,0xded4,1,0x60c7,1,0x6148,1,0x614c,1,0x614e,1,0x614c,1,0x617a,1,0x618e, +1,0x61b2,1,0x61a4,1,0x61af,1,0x61de,1,0x61f2,1,0x61f6,1,0x6210,1,0x621b, +1,0x625d,1,0x62b1,1,0x62d4,1,0x6350,2,0xd84a,0xdf0c,1,0x633d,1,0x62fc,1, +0x6368,1,0x6383,1,0x63e4,2,0xd84a,0xdff1,1,0x6422,1,0x63c5,1,0x63a9,1,0x3a2e, +1,0x6469,1,0x647e,1,0x649d,1,0x6477,1,0x3a6c,1,0x654f,1,0x656c,2,0xd84c, +0xdc0a,1,0x65e3,1,0x66f8,1,0x6649,1,0x3b19,1,0x6691,1,0x3b08,1,0x3ae4,1, +0x5192,1,0x5195,1,0x6700,1,0x669c,1,0x80ad,1,0x43d9,1,0x6717,1,0x671b,1, +0x6721,1,0x675e,1,0x6753,2,0xd84c,0xdfc3,1,0x3b49,1,0x67fa,1,0x6785,1,0x6852, +1,0x6885,2,0xd84d,0xdc6d,1,0x688e,1,0x681f,1,0x6914,1,0x3b9d,1,0x6942,1, +0x69a3,1,0x69ea,1,0x6aa8,2,0xd84d,0xdea3,1,0x6adb,1,0x3c18,1,0x6b21,2,0xd84e, +0xdca7,1,0x6b54,1,0x3c4e,1,0x6b72,1,0x6b9f,1,0x6bba,1,0x6bbb,2,0xd84e,0xde8d, +2,0xd847,0xdd0b,2,0xd84e,0xdefa,1,0x6c4e,2,0xd84f,0xdcbc,1,0x6cbf,1,0x6ccd,1, +0x6c67,1,0x6d16,1,0x6d3e,1,0x6d77,1,0x6d41,1,0x6d69,1,0x6d78,1,0x6d85,2, +0xd84f,0xdd1e,1,0x6d34,1,0x6e2f,1,0x6e6e,1,0x3d33,1,0x6ecb,1,0x6ec7,2,0xd84f, +0xded1,1,0x6df9,1,0x6f6e,2,0xd84f,0xdf5e,2,0xd84f,0xdf8e,1,0x6fc6,1,0x7039,1, +0x701e,1,0x701b,1,0x3d96,1,0x704a,1,0x707d,1,0x7077,1,0x70ad,2,0xd841,0xdd25, +1,0x7145,2,0xd850,0xde63,1,0x719c,2,0xd850,0xdfab,1,0x7228,1,0x7235,1,0x7250, +2,0xd851,0xde08,1,0x7280,1,0x7295,2,0xd851,0xdf35,2,0xd852,0xdc14,1,0x737a,1, +0x738b,1,0x3eac,1,0x73a5,1,0x3eb8,1,0x3eb8,1,0x7447,1,0x745c,1,0x7471,1, +0x7485,1,0x74ca,1,0x3f1b,1,0x7524,2,0xd853,0xdc36,1,0x753e,2,0xd853,0xdc92,1, +0x7570,2,0xd848,0xdd9f,1,0x7610,2,0xd853,0xdfa1,2,0xd853,0xdfb8,2,0xd854,0xdc44,1, +0x3ffc,1,0x4008,1,0x76f4,2,0xd854,0xdcf3,2,0xd854,0xdcf2,2,0xd854,0xdd19,2,0xd854, +0xdd33,1,0x771e,1,0x771f,1,0x771f,1,0x774a,1,0x4039,1,0x778b,1,0x4046,1, +0x4096,2,0xd855,0xdc1d,1,0x784e,1,0x788c,1,0x78cc,1,0x40e3,2,0xd855,0xde26,1, +0x7956,2,0xd855,0xde9a,2,0xd855,0xdec5,1,0x798f,1,0x79eb,1,0x412f,1,0x7a40,1, +0x7a4a,1,0x7a4f,2,0xd856,0xdd7c,2,0xd856,0xdea7,2,0xd856,0xdea7,1,0x7aee,1,0x4202, +2,0xd856,0xdfab,1,0x7bc6,1,0x7bc9,1,0x4227,2,0xd857,0xdc80,1,0x7cd2,1,0x42a0, +1,0x7ce8,1,0x7ce3,1,0x7d00,2,0xd857,0xdf86,1,0x7d63,1,0x4301,1,0x7dc7,1, +0x7e02,1,0x7e45,1,0x4334,2,0xd858,0xde28,2,0xd858,0xde47,1,0x4359,2,0xd858,0xded9, +1,0x7f7a,2,0xd858,0xdf3e,1,0x7f95,1,0x7ffa,1,0x8005,2,0xd859,0xdcda,2,0xd859, +0xdd23,1,0x8060,2,0xd859,0xdda8,1,0x8070,2,0xd84c,0xdf5f,1,0x43d5,1,0x80b2,1, +0x8103,1,0x440b,1,0x813e,1,0x5ab5,2,0xd859,0xdfa7,2,0xd859,0xdfb5,2,0xd84c,0xdf93, +2,0xd84c,0xdf9c,1,0x8201,1,0x8204,1,0x8f9e,1,0x446b,1,0x8291,1,0x828b,1, +0x829d,1,0x52b3,1,0x82b1,1,0x82b3,1,0x82bd,1,0x82e6,2,0xd85a,0xdf3c,1,0x82e5, +1,0x831d,1,0x8363,1,0x83ad,1,0x8323,1,0x83bd,1,0x83e7,1,0x8457,1,0x8353, +1,0x83ca,1,0x83cc,1,0x83dc,2,0xd85b,0xdc36,2,0xd85b,0xdd6b,2,0xd85b,0xdcd5,1, +0x452b,1,0x84f1,1,0x84f3,1,0x8516,2,0xd85c,0xdfca,1,0x8564,2,0xd85b,0xdf2c,1, +0x455d,1,0x4561,2,0xd85b,0xdfb1,2,0xd85c,0xdcd2,1,0x456b,1,0x8650,1,0x865c,1, +0x8667,1,0x8669,1,0x86a9,1,0x8688,1,0x870e,1,0x86e2,1,0x8779,1,0x8728,1, +0x876b,1,0x8786,1,0x45d7,1,0x87e1,1,0x8801,1,0x45f9,1,0x8860,1,0x8863,2, +0xd85d,0xde67,1,0x88d7,1,0x88de,1,0x4635,1,0x88fa,1,0x34bb,2,0xd85e,0xdcae,2, +0xd85e,0xdd66,1,0x46be,1,0x46c7,1,0x8aa0,1,0x8aed,1,0x8b8a,1,0x8c55,2,0xd85f, +0xdca8,1,0x8cab,1,0x8cc1,1,0x8d1b,1,0x8d77,2,0xd85f,0xdf2f,2,0xd842,0xdc04,1, +0x8dcb,1,0x8dbc,1,0x8df0,2,0xd842,0xdcde,1,0x8ed4,1,0x8f38,2,0xd861,0xddd2,2, +0xd861,0xdded,1,0x9094,1,0x90f1,1,0x9111,2,0xd861,0xdf2e,1,0x911b,1,0x9238,1, +0x92d7,1,0x92d8,1,0x927c,1,0x93f9,1,0x9415,2,0xd862,0xdffa,1,0x958b,1,0x4995, +1,0x95b7,2,0xd863,0xdd77,1,0x49e6,1,0x96c3,1,0x5db2,1,0x9723,2,0xd864,0xdd45, +2,0xd864,0xde1a,1,0x4a6e,1,0x4a76,1,0x97e0,2,0xd865,0xdc0a,1,0x4ab2,2,0xd865, +0xdc96,1,0x980b,1,0x980b,1,0x9829,2,0xd865,0xddb6,1,0x98e2,1,0x4b33,1,0x9929, +1,0x99a7,1,0x99c2,1,0x99fe,1,0x4bce,2,0xd866,0xdf30,1,0x9b12,1,0x9c40,1, +0x9cfd,1,0x4cce,1,0x4ced,1,0x9d67,2,0xd868,0xdcce,1,0x4cf8,2,0xd868,0xdd05,2, +0xd868,0xde0e,2,0xd868,0xde91,1,0x9ebb,1,0x4d56,1,0x9ef9,1,0x9efe,1,0x9f05,1, +0x9f0f,1,0x9f16,1,0x9f3b,2,0xd869,0xde00,0x82,0xe6,0x41,0x300,0x82,0xe6,0x41,0x301, +0x4e6,0x82,0xe6,0x41,0x302,0x82,0xe6,0x41,0x303,0x2c8,0x82,0xe6,0x41,0x308,0x2e0,0x82, +0xe6,0x41,0x30a,0x484,0x82,0xca,0x43,0x327,0x82,0xe6,0x45,0x300,0x82,0xe6,0x45,0x301, +0x50e,0x82,0xe6,0x45,0x302,0x82,0xe6,0x45,0x308,0x82,0xe6,0x49,0x300,0x82,0xe6,0x49, +0x301,0x82,0xe6,0x49,0x302,0x498,0x82,0xe6,0x49,0x308,0x82,0xe6,0x4e,0x303,0x82,0xe6, +0x4f,0x300,0x82,0xe6,0x4f,0x301,0x522,0x82,0xe6,0x4f,0x302,0x2ec,0x82,0xe6,0x4f,0x303, +0x2e8,0x82,0xe6,0x4f,0x308,0x82,0xe6,0x55,0x300,0x82,0xe6,0x55,0x301,0x82,0xe6,0x55, +0x302,0x2b8,0x82,0xe6,0x55,0x308,0x82,0xe6,0x59,0x301,0x82,0xe6,0x61,0x300,0x82,0xe6, +0x61,0x301,0x4ee,0x82,0xe6,0x61,0x302,0x82,0xe6,0x61,0x303,0x2ca,0x82,0xe6,0x61,0x308, +0x2e2,0x82,0xe6,0x61,0x30a,0x486,0x82,0xca,0x63,0x327,0x82,0xe6,0x65,0x300,0x82,0xe6, +0x65,0x301,0x516,0x82,0xe6,0x65,0x302,0x82,0xe6,0x65,0x308,0x82,0xe6,0x69,0x300,0x82, +0xe6,0x69,0x301,0x82,0xe6,0x69,0x302,0x49a,0x82,0xe6,0x69,0x308,0x82,0xe6,0x6e,0x303, +0x82,0xe6,0x6f,0x300,0x82,0xe6,0x6f,0x301,0x52a,0x82,0xe6,0x6f,0x302,0x2f2,0x82,0xe6, +0x6f,0x303,0x2ea,0x82,0xe6,0x6f,0x308,0x82,0xe6,0x75,0x300,0x82,0xe6,0x75,0x301,0x82, +0xe6,0x75,0x302,0x2c0,0x82,0xe6,0x75,0x308,0x82,0xe6,0x79,0x301,0x82,0xe6,0x79,0x308, +0x82,0xe6,0x41,0x304,0x82,0xe6,0x61,0x304,0x4fe,0x82,0xe6,0x41,0x306,0x506,0x82,0xe6, +0x61,0x306,0x82,0xca,0x41,0x328,0x82,0xca,0x61,0x328,0x82,0xe6,0x43,0x301,0x82,0xe6, +0x63,0x301,0x82,0xe6,0x43,0x302,0x82,0xe6,0x63,0x302,0x82,0xe6,0x43,0x307,0x82,0xe6, +0x63,0x307,0x82,0xe6,0x43,0x30c,0x82,0xe6,0x63,0x30c,0x82,0xe6,0x44,0x30c,0x82,0xe6, +0x64,0x30c,0x488,0x82,0xe6,0x45,0x304,0x48c,0x82,0xe6,0x65,0x304,0x82,0xe6,0x45,0x306, +0x82,0xe6,0x65,0x306,0x82,0xe6,0x45,0x307,0x82,0xe6,0x65,0x307,0x82,0xca,0x45,0x328, +0x82,0xca,0x65,0x328,0x82,0xe6,0x45,0x30c,0x82,0xe6,0x65,0x30c,0x82,0xe6,0x47,0x302, +0x82,0xe6,0x67,0x302,0x82,0xe6,0x47,0x306,0x82,0xe6,0x67,0x306,0x82,0xe6,0x47,0x307, +0x82,0xe6,0x67,0x307,0x82,0xca,0x47,0x327,0x82,0xca,0x67,0x327,0x82,0xe6,0x48,0x302, +0x82,0xe6,0x68,0x302,0x82,0xe6,0x49,0x303,0x82,0xe6,0x69,0x303,0x82,0xe6,0x49,0x304, +0x82,0xe6,0x69,0x304,0x82,0xe6,0x49,0x306,0x82,0xe6,0x69,0x306,0x82,0xca,0x49,0x328, +0x82,0xca,0x69,0x328,0x82,0xe6,0x49,0x307,0x82,0xe6,0x4a,0x302,0x82,0xe6,0x6a,0x302, +0x82,0xca,0x4b,0x327,0x82,0xca,0x6b,0x327,0x82,0xe6,0x4c,0x301,0x82,0xe6,0x6c,0x301, +0x82,0xca,0x4c,0x327,0x82,0xca,0x6c,0x327,0x82,0xe6,0x4c,0x30c,0x82,0xe6,0x6c,0x30c, +0x82,0xe6,0x4e,0x301,0x82,0xe6,0x6e,0x301,0x82,0xca,0x4e,0x327,0x82,0xca,0x6e,0x327, +0x82,0xe6,0x4e,0x30c,0x82,0xe6,0x6e,0x30c,0x4ac,0x82,0xe6,0x4f,0x304,0x4b0,0x82,0xe6, +0x6f,0x304,0x82,0xe6,0x4f,0x306,0x82,0xe6,0x6f,0x306,0x82,0xe6,0x4f,0x30b,0x82,0xe6, +0x6f,0x30b,0x82,0xe6,0x52,0x301,0x82,0xe6,0x72,0x301,0x82,0xca,0x52,0x327,0x82,0xca, +0x72,0x327,0x82,0xe6,0x52,0x30c,0x82,0xe6,0x72,0x30c,0x4c0,0x82,0xe6,0x53,0x301,0x4c2, +0x82,0xe6,0x73,0x301,0x82,0xe6,0x53,0x302,0x82,0xe6,0x73,0x302,0x82,0xca,0x53,0x327, +0x82,0xca,0x73,0x327,0x4c4,0x82,0xe6,0x53,0x30c,0x4c6,0x82,0xe6,0x73,0x30c,0x82,0xca, +0x54,0x327,0x82,0xca,0x74,0x327,0x82,0xe6,0x54,0x30c,0x82,0xe6,0x74,0x30c,0x4cc,0x82, +0xe6,0x55,0x303,0x4ce,0x82,0xe6,0x75,0x303,0x4d0,0x82,0xe6,0x55,0x304,0x4d2,0x82,0xe6, +0x75,0x304,0x82,0xe6,0x55,0x306,0x82,0xe6,0x75,0x306,0x82,0xe6,0x55,0x30a,0x82,0xe6, +0x75,0x30a,0x82,0xe6,0x55,0x30b,0x82,0xe6,0x75,0x30b,0x82,0xca,0x55,0x328,0x82,0xca, +0x75,0x328,0x82,0xe6,0x57,0x302,0x82,0xe6,0x77,0x302,0x82,0xe6,0x59,0x302,0x82,0xe6, +0x79,0x302,0x82,0xe6,0x59,0x308,0x82,0xe6,0x5a,0x301,0x82,0xe6,0x7a,0x301,0x82,0xe6, +0x5a,0x307,0x82,0xe6,0x7a,0x307,0x82,0xe6,0x5a,0x30c,0x82,0xe6,0x7a,0x30c,0x536,0x82, +0xd8,0x4f,0x31b,0x540,0x82,0xd8,0x6f,0x31b,0x54a,0x82,0xd8,0x55,0x31b,0x554,0x82,0xd8, +0x75,0x31b,0x82,0xe6,0x41,0x30c,0x82,0xe6,0x61,0x30c,0x82,0xe6,0x49,0x30c,0x82,0xe6, +0x69,0x30c,0x82,0xe6,0x4f,0x30c,0x82,0xe6,0x6f,0x30c,0x82,0xe6,0x55,0x30c,0x82,0xe6, +0x75,0x30c,0x83,0xe6,0x55,0x308,0x304,0x83,0xe6,0x75,0x308,0x304,0x83,0xe6,0x55,0x308, +0x301,0x83,0xe6,0x75,0x308,0x301,0x83,0xe6,0x55,0x308,0x30c,0x83,0xe6,0x75,0x308,0x30c, +0x83,0xe6,0x55,0x308,0x300,0x83,0xe6,0x75,0x308,0x300,0x83,0xe6,0x41,0x308,0x304,0x83, +0xe6,0x61,0x308,0x304,0x83,0xe6,0x41,0x307,0x304,0x83,0xe6,0x61,0x307,0x304,0x82,0xe6, +0xc6,0x304,0x82,0xe6,0xe6,0x304,0x82,0xe6,0x47,0x30c,0x82,0xe6,0x67,0x30c,0x82,0xe6, +0x4b,0x30c,0x82,0xe6,0x6b,0x30c,0x2d8,0x82,0xca,0x4f,0x328,0x2da,0x82,0xca,0x6f,0x328, +0x83,0xe6,0x4f,0x328,0x304,0x83,0xe6,0x6f,0x328,0x304,0x82,0xe6,0x1b7,0x30c,0x82,0xe6, +0x292,0x30c,0x82,0xe6,0x6a,0x30c,0x82,0xe6,0x47,0x301,0x82,0xe6,0x67,0x301,0x82,0xe6, +0x4e,0x300,0x82,0xe6,0x6e,0x300,0x83,0xe6,0x41,0x30a,0x301,0x83,0xe6,0x61,0x30a,0x301, +0x82,0xe6,0xc6,0x301,0x82,0xe6,0xe6,0x301,0x82,0xe6,0xd8,0x301,0x82,0xe6,0xf8,0x301, +0x82,0xe6,0x41,0x30f,0x82,0xe6,0x61,0x30f,0x82,0xe6,0x41,0x311,0x82,0xe6,0x61,0x311, +0x82,0xe6,0x45,0x30f,0x82,0xe6,0x65,0x30f,0x82,0xe6,0x45,0x311,0x82,0xe6,0x65,0x311, +0x82,0xe6,0x49,0x30f,0x82,0xe6,0x69,0x30f,0x82,0xe6,0x49,0x311,0x82,0xe6,0x69,0x311, +0x82,0xe6,0x4f,0x30f,0x82,0xe6,0x6f,0x30f,0x82,0xe6,0x4f,0x311,0x82,0xe6,0x6f,0x311, +0x82,0xe6,0x52,0x30f,0x82,0xe6,0x72,0x30f,0x82,0xe6,0x52,0x311,0x82,0xe6,0x72,0x311, +0x82,0xe6,0x55,0x30f,0x82,0xe6,0x75,0x30f,0x82,0xe6,0x55,0x311,0x82,0xe6,0x75,0x311, +0x82,0xdc,0x53,0x326,0x82,0xdc,0x73,0x326,0x82,0xdc,0x54,0x326,0x82,0xdc,0x74,0x326, +0x82,0xe6,0x48,0x30c,0x82,0xe6,0x68,0x30c,0x2cc,0x82,0xe6,0x41,0x307,0x2ce,0x82,0xe6, +0x61,0x307,0x490,0x82,0xca,0x45,0x327,0x492,0x82,0xca,0x65,0x327,0x83,0xe6,0x4f,0x308, +0x304,0x83,0xe6,0x6f,0x308,0x304,0x83,0xe6,0x4f,0x303,0x304,0x83,0xe6,0x6f,0x303,0x304, +0x2f8,0x82,0xe6,0x4f,0x307,0x2fa,0x82,0xe6,0x6f,0x307,0x83,0xe6,0x4f,0x307,0x304,0x83, +0xe6,0x6f,0x307,0x304,0x82,0xe6,0x59,0x304,0x82,0xe6,0x79,0x304,0x8382,0xe6,0xa8,0x301, +0xe6,0x20,0x308,0x301,0x82,0xe6,0x391,0x301,0x82,0xe6,0x395,0x301,0x82,0xe6,0x397,0x301, +0x82,0xe6,0x399,0x301,0x82,0xe6,0x39f,0x301,0x82,0xe6,0x3a5,0x301,0x82,0xe6,0x3a9,0x301, +0x83,0xe6,0x3b9,0x308,0x301,0x82,0xe6,0x399,0x308,0x82,0xe6,0x3a5,0x308,0x652,0x82,0xe6, +0x3b1,0x301,0x82,0xe6,0x3b5,0x301,0x658,0x82,0xe6,0x3b7,0x301,0x82,0xe6,0x3b9,0x301,0x83, +0xe6,0x3c5,0x308,0x301,0x34e,0x82,0xe6,0x3b9,0x308,0x388,0x82,0xe6,0x3c5,0x308,0x82,0xe6, +0x3bf,0x301,0x82,0xe6,0x3c5,0x301,0x670,0x82,0xe6,0x3c9,0x301,0x82,0xe6,0x415,0x300,0x82, +0xe6,0x415,0x308,0x82,0xe6,0x413,0x301,0x82,0xe6,0x406,0x308,0x82,0xe6,0x41a,0x301,0x82, +0xe6,0x418,0x300,0x82,0xe6,0x423,0x306,0x82,0xe6,0x418,0x306,0x82,0xe6,0x438,0x306,0x82, +0xe6,0x435,0x300,0x82,0xe6,0x435,0x308,0x82,0xe6,0x433,0x301,0x82,0xe6,0x456,0x308,0x82, +0xe6,0x43a,0x301,0x82,0xe6,0x438,0x300,0x82,0xe6,0x443,0x306,0x82,0xe6,0x474,0x30f,0x82, +0xe6,0x475,0x30f,0x82,0xe6,0x416,0x306,0x82,0xe6,0x436,0x306,0x82,0xe6,0x410,0x306,0x82, +0xe6,0x430,0x306,0x82,0xe6,0x410,0x308,0x82,0xe6,0x430,0x308,0x82,0xe6,0x415,0x306,0x82, +0xe6,0x435,0x306,0x82,0xe6,0x4d8,0x308,0x82,0xe6,0x4d9,0x308,0x82,0xe6,0x416,0x308,0x82, +0xe6,0x436,0x308,0x82,0xe6,0x417,0x308,0x82,0xe6,0x437,0x308,0x82,0xe6,0x418,0x304,0x82, +0xe6,0x438,0x304,0x82,0xe6,0x418,0x308,0x82,0xe6,0x438,0x308,0x82,0xe6,0x41e,0x308,0x82, +0xe6,0x43e,0x308,0x82,0xe6,0x4e8,0x308,0x82,0xe6,0x4e9,0x308,0x82,0xe6,0x42d,0x308,0x82, +0xe6,0x44d,0x308,0x82,0xe6,0x423,0x304,0x82,0xe6,0x443,0x304,0x82,0xe6,0x423,0x308,0x82, +0xe6,0x443,0x308,0x82,0xe6,0x423,0x30b,0x82,0xe6,0x443,0x30b,0x82,0xe6,0x427,0x308,0x82, +0xe6,0x447,0x308,0x82,0xe6,0x42b,0x308,0x82,0xe6,0x44b,0x308,0x82,0xe6,0x627,0x653,0x82, +0xe6,0x627,0x654,0x82,0xe6,0x648,0x654,0x82,0xdc,0x627,0x655,0x82,0xe6,0x64a,0x654,0x82, +0xe6,0x6d5,0x654,0x82,0xe6,0x6c1,0x654,0x82,0xe6,0x6d2,0x654,0x82,7,0x928,0x93c,0x82, +7,0x930,0x93c,0x82,7,0x933,0x93c,2,0x9c7,0x9be,2,0x9c7,0x9d7,2,0xb47,0xb56, +2,0xb47,0xb3e,2,0xb47,0xb57,2,0xb92,0xbd7,2,0xbc6,0xbbe,2,0xbc7,0xbbe,2, +0xbc6,0xbd7,0x82,0x5b,0xc46,0xc56,2,0xcbf,0xcd5,2,0xcc6,0xcd5,2,0xcc6,0xcd6,0x450, +2,0xcc6,0xcc2,3,0xcc6,0xcc2,0xcd5,2,0xd46,0xd3e,2,0xd47,0xd3e,2,0xd46,0xd57, +0x82,9,0xdd9,0xdca,0x45e,2,0xdd9,0xdcf,0x83,9,0xdd9,0xdcf,0xdca,2,0xdd9,0xddf, +2,0x1025,0x102e,2,0x1b05,0x1b35,2,0x1b07,0x1b35,2,0x1b09,0x1b35,2,0x1b0b,0x1b35,2, +0x1b0d,0x1b35,2,0x1b11,0x1b35,2,0x1b3a,0x1b35,2,0x1b3c,0x1b35,2,0x1b3e,0x1b35,2,0x1b3f, +0x1b35,2,0x1b42,0x1b35,0x82,0xdc,0x41,0x325,0x82,0xdc,0x61,0x325,0x82,0xe6,0x42,0x307, +0x82,0xe6,0x62,0x307,0x82,0xdc,0x42,0x323,0x82,0xdc,0x62,0x323,0x82,0xdc,0x42,0x331, +0x82,0xdc,0x62,0x331,0x83,0xe6,0x43,0x327,0x301,0x83,0xe6,0x63,0x327,0x301,0x82,0xe6, +0x44,0x307,0x82,0xe6,0x64,0x307,0x82,0xdc,0x44,0x323,0x82,0xdc,0x64,0x323,0x82,0xdc, +0x44,0x331,0x82,0xdc,0x64,0x331,0x82,0xca,0x44,0x327,0x82,0xca,0x64,0x327,0x82,0xdc, +0x44,0x32d,0x82,0xdc,0x64,0x32d,0x83,0xe6,0x45,0x304,0x300,0x83,0xe6,0x65,0x304,0x300, +0x83,0xe6,0x45,0x304,0x301,0x83,0xe6,0x65,0x304,0x301,0x82,0xdc,0x45,0x32d,0x82,0xdc, +0x65,0x32d,0x82,0xdc,0x45,0x330,0x82,0xdc,0x65,0x330,0x83,0xe6,0x45,0x327,0x306,0x83, +0xe6,0x65,0x327,0x306,0x82,0xe6,0x46,0x307,0x82,0xe6,0x66,0x307,0x82,0xe6,0x47,0x304, +0x82,0xe6,0x67,0x304,0x82,0xe6,0x48,0x307,0x82,0xe6,0x68,0x307,0x82,0xdc,0x48,0x323, +0x82,0xdc,0x68,0x323,0x82,0xe6,0x48,0x308,0x82,0xe6,0x68,0x308,0x82,0xca,0x48,0x327, +0x82,0xca,0x68,0x327,0x82,0xdc,0x48,0x32e,0x82,0xdc,0x68,0x32e,0x82,0xdc,0x49,0x330, +0x82,0xdc,0x69,0x330,0x83,0xe6,0x49,0x308,0x301,0x83,0xe6,0x69,0x308,0x301,0x82,0xe6, +0x4b,0x301,0x82,0xe6,0x6b,0x301,0x82,0xdc,0x4b,0x323,0x82,0xdc,0x6b,0x323,0x82,0xdc, +0x4b,0x331,0x82,0xdc,0x6b,0x331,0x49c,0x82,0xdc,0x4c,0x323,0x49e,0x82,0xdc,0x6c,0x323, +0x83,0xe6,0x4c,0x323,0x304,0x83,0xe6,0x6c,0x323,0x304,0x82,0xdc,0x4c,0x331,0x82,0xdc, +0x6c,0x331,0x82,0xdc,0x4c,0x32d,0x82,0xdc,0x6c,0x32d,0x82,0xe6,0x4d,0x301,0x82,0xe6, +0x6d,0x301,0x82,0xe6,0x4d,0x307,0x82,0xe6,0x6d,0x307,0x82,0xdc,0x4d,0x323,0x82,0xdc, +0x6d,0x323,0x82,0xe6,0x4e,0x307,0x82,0xe6,0x6e,0x307,0x82,0xdc,0x4e,0x323,0x82,0xdc, +0x6e,0x323,0x82,0xdc,0x4e,0x331,0x82,0xdc,0x6e,0x331,0x82,0xdc,0x4e,0x32d,0x82,0xdc, +0x6e,0x32d,0x83,0xe6,0x4f,0x303,0x301,0x83,0xe6,0x6f,0x303,0x301,0x83,0xe6,0x4f,0x303, +0x308,0x83,0xe6,0x6f,0x303,0x308,0x83,0xe6,0x4f,0x304,0x300,0x83,0xe6,0x6f,0x304,0x300, +0x83,0xe6,0x4f,0x304,0x301,0x83,0xe6,0x6f,0x304,0x301,0x82,0xe6,0x50,0x301,0x82,0xe6, +0x70,0x301,0x82,0xe6,0x50,0x307,0x82,0xe6,0x70,0x307,0x82,0xe6,0x52,0x307,0x82,0xe6, +0x72,0x307,0x4bc,0x82,0xdc,0x52,0x323,0x4be,0x82,0xdc,0x72,0x323,0x83,0xe6,0x52,0x323, +0x304,0x83,0xe6,0x72,0x323,0x304,0x82,0xdc,0x52,0x331,0x82,0xdc,0x72,0x331,0x82,0xe6, +0x53,0x307,0x82,0xe6,0x73,0x307,0x4c8,0x82,0xdc,0x53,0x323,0x4ca,0x82,0xdc,0x73,0x323, +0x83,0xe6,0x53,0x301,0x307,0x83,0xe6,0x73,0x301,0x307,0x83,0xe6,0x53,0x30c,0x307,0x83, +0xe6,0x73,0x30c,0x307,0x83,0xe6,0x53,0x323,0x307,0x83,0xe6,0x73,0x323,0x307,0x82,0xe6, +0x54,0x307,0x82,0xe6,0x74,0x307,0x82,0xdc,0x54,0x323,0x82,0xdc,0x74,0x323,0x82,0xdc, +0x54,0x331,0x82,0xdc,0x74,0x331,0x82,0xdc,0x54,0x32d,0x82,0xdc,0x74,0x32d,0x82,0xdc, +0x55,0x324,0x82,0xdc,0x75,0x324,0x82,0xdc,0x55,0x330,0x82,0xdc,0x75,0x330,0x82,0xdc, +0x55,0x32d,0x82,0xdc,0x75,0x32d,0x83,0xe6,0x55,0x303,0x301,0x83,0xe6,0x75,0x303,0x301, +0x83,0xe6,0x55,0x304,0x308,0x83,0xe6,0x75,0x304,0x308,0x82,0xe6,0x56,0x303,0x82,0xe6, +0x76,0x303,0x82,0xdc,0x56,0x323,0x82,0xdc,0x76,0x323,0x82,0xe6,0x57,0x300,0x82,0xe6, +0x77,0x300,0x82,0xe6,0x57,0x301,0x82,0xe6,0x77,0x301,0x82,0xe6,0x57,0x308,0x82,0xe6, +0x77,0x308,0x82,0xe6,0x57,0x307,0x82,0xe6,0x77,0x307,0x82,0xdc,0x57,0x323,0x82,0xdc, +0x77,0x323,0x82,0xe6,0x58,0x307,0x82,0xe6,0x78,0x307,0x82,0xe6,0x58,0x308,0x82,0xe6, +0x78,0x308,0x82,0xe6,0x59,0x307,0x82,0xe6,0x79,0x307,0x82,0xe6,0x5a,0x302,0x82,0xe6, +0x7a,0x302,0x82,0xdc,0x5a,0x323,0x82,0xdc,0x7a,0x323,0x82,0xdc,0x5a,0x331,0x82,0xdc, +0x7a,0x331,0x82,0xdc,0x68,0x331,0x82,0xe6,0x74,0x308,0x82,0xe6,0x77,0x30a,0x82,0xe6, +0x79,0x30a,0x8282,0xe6,0x17f,0x307,0xe6,0x73,0x307,0x4f6,0x82,0xdc,0x41,0x323,0x4fa,0x82, +0xdc,0x61,0x323,0x82,0xe6,0x41,0x309,0x82,0xe6,0x61,0x309,0x83,0xe6,0x41,0x302,0x301, +0x83,0xe6,0x61,0x302,0x301,0x83,0xe6,0x41,0x302,0x300,0x83,0xe6,0x61,0x302,0x300,0x83, +0xe6,0x41,0x302,0x309,0x83,0xe6,0x61,0x302,0x309,0x83,0xe6,0x41,0x302,0x303,0x83,0xe6, +0x61,0x302,0x303,0x83,0xe6,0x41,0x323,0x302,0x83,0xe6,0x61,0x323,0x302,0x83,0xe6,0x41, +0x306,0x301,0x83,0xe6,0x61,0x306,0x301,0x83,0xe6,0x41,0x306,0x300,0x83,0xe6,0x61,0x306, +0x300,0x83,0xe6,0x41,0x306,0x309,0x83,0xe6,0x61,0x306,0x309,0x83,0xe6,0x41,0x306,0x303, +0x83,0xe6,0x61,0x306,0x303,0x83,0xe6,0x41,0x323,0x306,0x83,0xe6,0x61,0x323,0x306,0x51e, +0x82,0xdc,0x45,0x323,0x520,0x82,0xdc,0x65,0x323,0x82,0xe6,0x45,0x309,0x82,0xe6,0x65, +0x309,0x82,0xe6,0x45,0x303,0x82,0xe6,0x65,0x303,0x83,0xe6,0x45,0x302,0x301,0x83,0xe6, +0x65,0x302,0x301,0x83,0xe6,0x45,0x302,0x300,0x83,0xe6,0x65,0x302,0x300,0x83,0xe6,0x45, +0x302,0x309,0x83,0xe6,0x65,0x302,0x309,0x83,0xe6,0x45,0x302,0x303,0x83,0xe6,0x65,0x302, +0x303,0x83,0xe6,0x45,0x323,0x302,0x83,0xe6,0x65,0x323,0x302,0x82,0xe6,0x49,0x309,0x82, +0xe6,0x69,0x309,0x82,0xdc,0x49,0x323,0x82,0xdc,0x69,0x323,0x532,0x82,0xdc,0x4f,0x323, +0x534,0x82,0xdc,0x6f,0x323,0x82,0xe6,0x4f,0x309,0x82,0xe6,0x6f,0x309,0x83,0xe6,0x4f, +0x302,0x301,0x83,0xe6,0x6f,0x302,0x301,0x83,0xe6,0x4f,0x302,0x300,0x83,0xe6,0x6f,0x302, +0x300,0x83,0xe6,0x4f,0x302,0x309,0x83,0xe6,0x6f,0x302,0x309,0x83,0xe6,0x4f,0x302,0x303, +0x83,0xe6,0x6f,0x302,0x303,0x83,0xe6,0x4f,0x323,0x302,0x83,0xe6,0x6f,0x323,0x302,0x83, +0xe6,0x4f,0x31b,0x301,0x83,0xe6,0x6f,0x31b,0x301,0x83,0xe6,0x4f,0x31b,0x300,0x83,0xe6, +0x6f,0x31b,0x300,0x83,0xe6,0x4f,0x31b,0x309,0x83,0xe6,0x6f,0x31b,0x309,0x83,0xe6,0x4f, +0x31b,0x303,0x83,0xe6,0x6f,0x31b,0x303,0x83,0xdc,0x4f,0x31b,0x323,0x83,0xdc,0x6f,0x31b, +0x323,0x82,0xdc,0x55,0x323,0x82,0xdc,0x75,0x323,0x82,0xe6,0x55,0x309,0x82,0xe6,0x75, +0x309,0x83,0xe6,0x55,0x31b,0x301,0x83,0xe6,0x75,0x31b,0x301,0x83,0xe6,0x55,0x31b,0x300, +0x83,0xe6,0x75,0x31b,0x300,0x83,0xe6,0x55,0x31b,0x309,0x83,0xe6,0x75,0x31b,0x309,0x83, +0xe6,0x55,0x31b,0x303,0x83,0xe6,0x75,0x31b,0x303,0x83,0xdc,0x55,0x31b,0x323,0x83,0xdc, +0x75,0x31b,0x323,0x82,0xe6,0x59,0x300,0x82,0xe6,0x79,0x300,0x82,0xdc,0x59,0x323,0x82, +0xdc,0x79,0x323,0x82,0xe6,0x59,0x309,0x82,0xe6,0x79,0x309,0x82,0xe6,0x59,0x303,0x82, +0xe6,0x79,0x303,0x55e,0x82,0xe6,0x3b1,0x313,0x566,0x82,0xe6,0x3b1,0x314,0x608,0x83,0xe6, +0x3b1,0x313,0x300,0x60a,0x83,0xe6,0x3b1,0x314,0x300,0x60c,0x83,0xe6,0x3b1,0x313,0x301,0x60e, +0x83,0xe6,0x3b1,0x314,0x301,0x610,0x83,0xe6,0x3b1,0x313,0x342,0x612,0x83,0xe6,0x3b1,0x314, +0x342,0x56e,0x82,0xe6,0x391,0x313,0x576,0x82,0xe6,0x391,0x314,0x614,0x83,0xe6,0x391,0x313, +0x300,0x616,0x83,0xe6,0x391,0x314,0x300,0x618,0x83,0xe6,0x391,0x313,0x301,0x61a,0x83,0xe6, +0x391,0x314,0x301,0x61c,0x83,0xe6,0x391,0x313,0x342,0x61e,0x83,0xe6,0x391,0x314,0x342,0x57e, +0x82,0xe6,0x3b5,0x313,0x582,0x82,0xe6,0x3b5,0x314,0x83,0xe6,0x3b5,0x313,0x300,0x83,0xe6, +0x3b5,0x314,0x300,0x83,0xe6,0x3b5,0x313,0x301,0x83,0xe6,0x3b5,0x314,0x301,0x586,0x82,0xe6, +0x395,0x313,0x58a,0x82,0xe6,0x395,0x314,0x83,0xe6,0x395,0x313,0x300,0x83,0xe6,0x395,0x314, +0x300,0x83,0xe6,0x395,0x313,0x301,0x83,0xe6,0x395,0x314,0x301,0x58e,0x82,0xe6,0x3b7,0x313, +0x596,0x82,0xe6,0x3b7,0x314,0x620,0x83,0xe6,0x3b7,0x313,0x300,0x622,0x83,0xe6,0x3b7,0x314, +0x300,0x624,0x83,0xe6,0x3b7,0x313,0x301,0x626,0x83,0xe6,0x3b7,0x314,0x301,0x628,0x83,0xe6, +0x3b7,0x313,0x342,0x62a,0x83,0xe6,0x3b7,0x314,0x342,0x59e,0x82,0xe6,0x397,0x313,0x5a6,0x82, +0xe6,0x397,0x314,0x62c,0x83,0xe6,0x397,0x313,0x300,0x62e,0x83,0xe6,0x397,0x314,0x300,0x630, +0x83,0xe6,0x397,0x313,0x301,0x632,0x83,0xe6,0x397,0x314,0x301,0x634,0x83,0xe6,0x397,0x313, +0x342,0x636,0x83,0xe6,0x397,0x314,0x342,0x5ae,0x82,0xe6,0x3b9,0x313,0x5b4,0x82,0xe6,0x3b9, +0x314,0x83,0xe6,0x3b9,0x313,0x300,0x83,0xe6,0x3b9,0x314,0x300,0x83,0xe6,0x3b9,0x313,0x301, +0x83,0xe6,0x3b9,0x314,0x301,0x83,0xe6,0x3b9,0x313,0x342,0x83,0xe6,0x3b9,0x314,0x342,0x5ba, +0x82,0xe6,0x399,0x313,0x5c0,0x82,0xe6,0x399,0x314,0x83,0xe6,0x399,0x313,0x300,0x83,0xe6, +0x399,0x314,0x300,0x83,0xe6,0x399,0x313,0x301,0x83,0xe6,0x399,0x314,0x301,0x83,0xe6,0x399, +0x313,0x342,0x83,0xe6,0x399,0x314,0x342,0x5c6,0x82,0xe6,0x3bf,0x313,0x5ca,0x82,0xe6,0x3bf, +0x314,0x83,0xe6,0x3bf,0x313,0x300,0x83,0xe6,0x3bf,0x314,0x300,0x83,0xe6,0x3bf,0x313,0x301, +0x83,0xe6,0x3bf,0x314,0x301,0x5ce,0x82,0xe6,0x39f,0x313,0x5d2,0x82,0xe6,0x39f,0x314,0x83, +0xe6,0x39f,0x313,0x300,0x83,0xe6,0x39f,0x314,0x300,0x83,0xe6,0x39f,0x313,0x301,0x83,0xe6, +0x39f,0x314,0x301,0x5d6,0x82,0xe6,0x3c5,0x313,0x5dc,0x82,0xe6,0x3c5,0x314,0x83,0xe6,0x3c5, +0x313,0x300,0x83,0xe6,0x3c5,0x314,0x300,0x83,0xe6,0x3c5,0x313,0x301,0x83,0xe6,0x3c5,0x314, +0x301,0x83,0xe6,0x3c5,0x313,0x342,0x83,0xe6,0x3c5,0x314,0x342,0x5e2,0x82,0xe6,0x3a5,0x314, +0x83,0xe6,0x3a5,0x314,0x300,0x83,0xe6,0x3a5,0x314,0x301,0x83,0xe6,0x3a5,0x314,0x342,0x5e8, +0x82,0xe6,0x3c9,0x313,0x5f0,0x82,0xe6,0x3c9,0x314,0x638,0x83,0xe6,0x3c9,0x313,0x300,0x63a, +0x83,0xe6,0x3c9,0x314,0x300,0x63c,0x83,0xe6,0x3c9,0x313,0x301,0x63e,0x83,0xe6,0x3c9,0x314, +0x301,0x640,0x83,0xe6,0x3c9,0x313,0x342,0x642,0x83,0xe6,0x3c9,0x314,0x342,0x5f8,0x82,0xe6, +0x3a9,0x313,0x600,0x82,0xe6,0x3a9,0x314,0x644,0x83,0xe6,0x3a9,0x313,0x300,0x646,0x83,0xe6, +0x3a9,0x314,0x300,0x648,0x83,0xe6,0x3a9,0x313,0x301,0x64a,0x83,0xe6,0x3a9,0x314,0x301,0x64c, +0x83,0xe6,0x3a9,0x313,0x342,0x64e,0x83,0xe6,0x3a9,0x314,0x342,0x650,0x82,0xe6,0x3b1,0x300, +0x82,0xe6,0x3b5,0x300,0x656,0x82,0xe6,0x3b7,0x300,0x82,0xe6,0x3b9,0x300,0x82,0xe6,0x3bf, +0x300,0x82,0xe6,0x3c5,0x300,0x66e,0x82,0xe6,0x3c9,0x300,0x83,0xf0,0x3b1,0x313,0x345,0x83, +0xf0,0x3b1,0x314,0x345,0x84,0xf0,0x3b1,0x313,0x300,0x345,0x84,0xf0,0x3b1,0x314,0x300,0x345, +0x84,0xf0,0x3b1,0x313,0x301,0x345,0x84,0xf0,0x3b1,0x314,0x301,0x345,0x84,0xf0,0x3b1,0x313, +0x342,0x345,0x84,0xf0,0x3b1,0x314,0x342,0x345,0x83,0xf0,0x391,0x313,0x345,0x83,0xf0,0x391, +0x314,0x345,0x84,0xf0,0x391,0x313,0x300,0x345,0x84,0xf0,0x391,0x314,0x300,0x345,0x84,0xf0, +0x391,0x313,0x301,0x345,0x84,0xf0,0x391,0x314,0x301,0x345,0x84,0xf0,0x391,0x313,0x342,0x345, +0x84,0xf0,0x391,0x314,0x342,0x345,0x83,0xf0,0x3b7,0x313,0x345,0x83,0xf0,0x3b7,0x314,0x345, +0x84,0xf0,0x3b7,0x313,0x300,0x345,0x84,0xf0,0x3b7,0x314,0x300,0x345,0x84,0xf0,0x3b7,0x313, +0x301,0x345,0x84,0xf0,0x3b7,0x314,0x301,0x345,0x84,0xf0,0x3b7,0x313,0x342,0x345,0x84,0xf0, +0x3b7,0x314,0x342,0x345,0x83,0xf0,0x397,0x313,0x345,0x83,0xf0,0x397,0x314,0x345,0x84,0xf0, +0x397,0x313,0x300,0x345,0x84,0xf0,0x397,0x314,0x300,0x345,0x84,0xf0,0x397,0x313,0x301,0x345, +0x84,0xf0,0x397,0x314,0x301,0x345,0x84,0xf0,0x397,0x313,0x342,0x345,0x84,0xf0,0x397,0x314, +0x342,0x345,0x83,0xf0,0x3c9,0x313,0x345,0x83,0xf0,0x3c9,0x314,0x345,0x84,0xf0,0x3c9,0x313, +0x300,0x345,0x84,0xf0,0x3c9,0x314,0x300,0x345,0x84,0xf0,0x3c9,0x313,0x301,0x345,0x84,0xf0, +0x3c9,0x314,0x301,0x345,0x84,0xf0,0x3c9,0x313,0x342,0x345,0x84,0xf0,0x3c9,0x314,0x342,0x345, +0x83,0xf0,0x3a9,0x313,0x345,0x83,0xf0,0x3a9,0x314,0x345,0x84,0xf0,0x3a9,0x313,0x300,0x345, +0x84,0xf0,0x3a9,0x314,0x300,0x345,0x84,0xf0,0x3a9,0x313,0x301,0x345,0x84,0xf0,0x3a9,0x314, +0x301,0x345,0x84,0xf0,0x3a9,0x313,0x342,0x345,0x84,0xf0,0x3a9,0x314,0x342,0x345,0x82,0xe6, +0x3b1,0x306,0x82,0xe6,0x3b1,0x304,0x83,0xf0,0x3b1,0x300,0x345,0x82,0xf0,0x3b1,0x345,0x83, +0xf0,0x3b1,0x301,0x345,0x654,0x82,0xe6,0x3b1,0x342,0x83,0xf0,0x3b1,0x342,0x345,0x82,0xe6, +0x391,0x306,0x82,0xe6,0x391,0x304,0x82,0xe6,0x391,0x300,0x82,0xf0,0x391,0x345,0x8382,0xe6, +0xa8,0x342,0xe6,0x20,0x308,0x342,0x83,0xf0,0x3b7,0x300,0x345,0x82,0xf0,0x3b7,0x345,0x83, +0xf0,0x3b7,0x301,0x345,0x65a,0x82,0xe6,0x3b7,0x342,0x83,0xf0,0x3b7,0x342,0x345,0x82,0xe6, +0x395,0x300,0x82,0xe6,0x397,0x300,0x82,0xf0,0x397,0x345,0x8382,0xe6,0x1fbf,0x300,0xe6,0x20, +0x313,0x300,0x8382,0xe6,0x1fbf,0x301,0xe6,0x20,0x313,0x301,0x8382,0xe6,0x1fbf,0x342,0xe6,0x20, +0x313,0x342,0x82,0xe6,0x3b9,0x306,0x82,0xe6,0x3b9,0x304,0x83,0xe6,0x3b9,0x308,0x300,0x82, +0xe6,0x3b9,0x342,0x83,0xe6,0x3b9,0x308,0x342,0x82,0xe6,0x399,0x306,0x82,0xe6,0x399,0x304, +0x82,0xe6,0x399,0x300,0x8382,0xe6,0x1ffe,0x300,0xe6,0x20,0x314,0x300,0x8382,0xe6,0x1ffe,0x301, +0xe6,0x20,0x314,0x301,0x8382,0xe6,0x1ffe,0x342,0xe6,0x20,0x314,0x342,0x82,0xe6,0x3c5,0x306, +0x82,0xe6,0x3c5,0x304,0x83,0xe6,0x3c5,0x308,0x300,0x82,0xe6,0x3c1,0x313,0x82,0xe6,0x3c1, +0x314,0x82,0xe6,0x3c5,0x342,0x83,0xe6,0x3c5,0x308,0x342,0x82,0xe6,0x3a5,0x306,0x82,0xe6, +0x3a5,0x304,0x82,0xe6,0x3a5,0x300,0x82,0xe6,0x3a1,0x314,0x8382,0xe6,0xa8,0x300,0xe6,0x20, +0x308,0x300,0x83,0xf0,0x3c9,0x300,0x345,0x82,0xf0,0x3c9,0x345,0x83,0xf0,0x3c9,0x301,0x345, +0x672,0x82,0xe6,0x3c9,0x342,0x83,0xf0,0x3c9,0x342,0x345,0x82,0xe6,0x39f,0x300,0x82,0xe6, +0x3a9,0x300,0x82,0xf0,0x3a9,0x345,0x82,1,0x2190,0x338,0x82,1,0x2192,0x338,0x82,1, +0x2194,0x338,0x82,1,0x21d0,0x338,0x82,1,0x21d4,0x338,0x82,1,0x21d2,0x338,0x82,1, +0x2203,0x338,0x82,1,0x2208,0x338,0x82,1,0x220b,0x338,0x82,1,0x2223,0x338,0x82,1, +0x2225,0x338,0x82,1,0x223c,0x338,0x82,1,0x2243,0x338,0x82,1,0x2245,0x338,0x82,1, +0x2248,0x338,0x82,1,0x3d,0x338,0x82,1,0x2261,0x338,0x82,1,0x224d,0x338,0x82,1, +0x3c,0x338,0x82,1,0x3e,0x338,0x82,1,0x2264,0x338,0x82,1,0x2265,0x338,0x82,1, +0x2272,0x338,0x82,1,0x2273,0x338,0x82,1,0x2276,0x338,0x82,1,0x2277,0x338,0x82,1, +0x227a,0x338,0x82,1,0x227b,0x338,0x82,1,0x2282,0x338,0x82,1,0x2283,0x338,0x82,1, +0x2286,0x338,0x82,1,0x2287,0x338,0x82,1,0x22a2,0x338,0x82,1,0x22a8,0x338,0x82,1, +0x22a9,0x338,0x82,1,0x22ab,0x338,0x82,1,0x227c,0x338,0x82,1,0x227d,0x338,0x82,1, +0x2291,0x338,0x82,1,0x2292,0x338,0x82,1,0x22b2,0x338,0x82,1,0x22b3,0x338,0x82,1, +0x22b4,0x338,0x82,1,0x22b5,0x338,0x82,8,0x304b,0x3099,0x82,8,0x304d,0x3099,0x82,8, +0x304f,0x3099,0x82,8,0x3051,0x3099,0x82,8,0x3053,0x3099,0x82,8,0x3055,0x3099,0x82,8, +0x3057,0x3099,0x82,8,0x3059,0x3099,0x82,8,0x305b,0x3099,0x82,8,0x305d,0x3099,0x82,8, +0x305f,0x3099,0x82,8,0x3061,0x3099,0x82,8,0x3064,0x3099,0x82,8,0x3066,0x3099,0x82,8, +0x3068,0x3099,0x82,8,0x306f,0x3099,0x82,8,0x306f,0x309a,0x82,8,0x3072,0x3099,0x82,8, +0x3072,0x309a,0x82,8,0x3075,0x3099,0x82,8,0x3075,0x309a,0x82,8,0x3078,0x3099,0x82,8, +0x3078,0x309a,0x82,8,0x307b,0x3099,0x82,8,0x307b,0x309a,0x82,8,0x3046,0x3099,0x82,8, +0x309d,0x3099,0x82,8,0x30ab,0x3099,0x82,8,0x30ad,0x3099,0x82,8,0x30af,0x3099,0x82,8, +0x30b1,0x3099,0x82,8,0x30b3,0x3099,0x82,8,0x30b5,0x3099,0x82,8,0x30b7,0x3099,0x82,8, +0x30b9,0x3099,0x82,8,0x30bb,0x3099,0x82,8,0x30bd,0x3099,0x82,8,0x30bf,0x3099,0x82,8, +0x30c1,0x3099,0x82,8,0x30c4,0x3099,0x82,8,0x30c6,0x3099,0x82,8,0x30c8,0x3099,0x82,8, +0x30cf,0x3099,0x82,8,0x30cf,0x309a,0x82,8,0x30d2,0x3099,0x82,8,0x30d2,0x309a,0x82,8, +0x30d5,0x3099,0x82,8,0x30d5,0x309a,0x82,8,0x30d8,0x3099,0x82,8,0x30d8,0x309a,0x82,8, +0x30db,0x3099,0x82,8,0x30db,0x309a,0x82,8,0x30a6,0x3099,0x82,8,0x30ef,0x3099,0x82,8, +0x30f0,0x3099,0x82,8,0x30f1,0x3099,0x82,8,0x30f2,0x3099,0x82,8,0x30fd,0x3099,0x7a6,0x7a7, +0x7a8,0x7a9,0x7ad,0x7ae,0x7b0,0x7aa,0x7d4,0x7ab,0x7b2,0x7b1,0x7b4,0x7b5,0x7d5,0x7d6,0x7b3,0x7ce, +0x7d3,0x7cd,0x7b6,0x7ac,0x7af,0x7d0,0x7d2,0x7d1,0x7cf,0x7d9,0x7d7,0x7d8,0x7b7,0x7b8,0x7b9,0x7ba, +0x7bb,0x7bc,0x7be,0x7bd,0x7bf,0x7c1,0x7c0,0x7c2,0x7c5,0x7c3,0x7c4,0x7c6,0x7c7,0x7c8,0x7c9,0x7ca, +0x7cb,0x7cc,0x7da,0x7db,0x100,0x20,0x2fc,0x8200,0xe6,0x20,0x308,0x100,0x61,0x8200,0xe6,0x20, +0x304,0x100,0x32,0x100,0x33,0x8200,0xe6,0x20,0x301,0x100,0x3bc,0x8200,0xca,0x20,0x327,0x100, +0x31,0x100,0x6f,0x300,0x31,0x2044,0x34,0x300,0x31,0x2044,0x32,0x300,0x33,0x2044,0x34,0x200, +0x49,0x4a,0x200,0x69,0x6a,0x200,0x4c,0xb7,0x200,0x6c,0xb7,0x200,0x2bc,0x6e,0x4e4,0x100, +0x73,0x8300,0xe6,0x44,0x5a,0x30c,0x8300,0xe6,0x44,0x7a,0x30c,0x8300,0xe6,0x64,0x7a,0x30c, +0x200,0x4c,0x4a,0x200,0x4c,0x6a,0x200,0x6c,0x6a,0x200,0x4e,0x4a,0x200,0x4e,0x6a,0x200, +0x6e,0x6a,0x200,0x44,0x5a,0x200,0x44,0x7a,0x200,0x64,0x7a,0x100,0x68,0x100,0x266,0x100, +0x6a,0x100,0x72,0x100,0x279,0x100,0x27b,0x100,0x281,0x100,0x77,0x100,0x79,0x8200,0xe6,0x20, +0x306,0x8200,0xe6,0x20,0x307,0x8200,0xe6,0x20,0x30a,0x8200,0xca,0x20,0x328,0x8200,0xe6,0x20, +0x303,0x8200,0xe6,0x20,0x30b,0x100,0x263,0x100,0x6c,0x100,0x73,0x100,0x78,0x100,0x295,0x8200, +0xe6,0x20,0x301,0x100,0x3b2,0x100,0x3b8,0x100,0x3c6,0x100,0x3c0,0x100,0x3ba,0x100,0x3c1,0x100, +0x398,0x100,0x3b5,0x200,0x565,0x582,0x200,0x627,0x674,0x200,0x648,0x674,0x200,0x6c7,0x674,0x200, +0x64a,0x674,0x200,0xe4d,0xe32,0x200,0xecd,0xeb2,0x200,0xeab,0xe99,0x200,0xeab,0xea1,0x100,0xf0b, +0x8300,0x82,0xfb2,0xf71,0xf80,0x8300,0x82,0xfb3,0xf71,0xf80,0x100,0x10dc,0x100,0x61,0x100,0x250, +0x100,0x251,0x100,0x1d02,0x100,0x62,0x100,0x64,0x100,0x65,0x100,0x259,0x100,0x25b,0x100,0x25c, +0x100,0x67,0x100,0x6b,0x100,0x6d,0x100,0x14b,0x100,0x6f,0x100,0x254,0x100,0x1d16,0x100,0x1d17, +0x100,0x70,0x100,0x74,0x100,0x75,0x100,0x1d1d,0x100,0x26f,0x100,0x76,0x100,0x1d25,0x100,0x3b2, +0x100,0x3b3,0x100,0x3b4,0x100,0x3c6,0x100,0x3c7,0x100,0x69,0x100,0x72,0x100,0x75,0x100,0x76, +0x100,0x3b2,0x100,0x3b3,0x100,0x3c1,0x100,0x3c6,0x100,0x3c7,0x100,0x43d,0x100,0x252,0x100,0x63, +0x100,0x255,0x100,0xf0,0x100,0x25c,0x100,0x66,0x100,0x25f,0x100,0x261,0x100,0x265,0x100,0x268, +0x100,0x269,0x100,0x26a,0x100,0x1d7b,0x100,0x29d,0x100,0x26d,0x100,0x1d85,0x100,0x29f,0x100,0x271, +0x100,0x270,0x100,0x272,0x100,0x273,0x100,0x274,0x100,0x275,0x100,0x278,0x100,0x282,0x100,0x283, +0x100,0x1ab,0x100,0x289,0x100,0x28a,0x100,0x1d1c,0x100,0x28b,0x100,0x28c,0x100,0x7a,0x100,0x290, +0x100,0x291,0x100,0x292,0x100,0x3b8,0x200,0x61,0x2be,0x8200,0xe6,0x20,0x313,0x65c,0x8200,0xe6, +0x20,0x313,0x8200,0xe6,0x20,0x342,0x662,0x8200,0xe6,0x20,0x314,0x100,0x20,0x100,0x20,0x100, +0x20,0x100,0x20,0x100,0x20,0x100,0x20,0x100,0x20,0x100,0x20,0x100,0x20,0x100,0x2010,0x8200, +0xdc,0x20,0x333,0x100,0x2e,0x200,0x2e,0x2e,0x300,0x2e,0x2e,0x2e,0x100,0x20,0x200,0x2032, +0x2032,0x300,0x2032,0x2032,0x2032,0x200,0x2035,0x2035,0x300,0x2035,0x2035,0x2035,0x200,0x21,0x21,0x8200, +0xe6,0x20,0x305,0x200,0x3f,0x3f,0x200,0x3f,0x21,0x200,0x21,0x3f,0x400,0x2032,0x2032,0x2032, +0x2032,0x100,0x20,0x100,0x30,0x100,0x69,0x100,0x34,0x100,0x35,0x100,0x36,0x100,0x37,0x100, +0x38,0x100,0x39,0x100,0x2b,0x100,0x2212,0x100,0x3d,0x100,0x28,0x100,0x29,0x100,0x6e,0x100, +0x30,0x100,0x31,0x100,0x32,0x100,0x33,0x100,0x34,0x100,0x35,0x100,0x36,0x100,0x37,0x100, +0x38,0x100,0x39,0x100,0x2b,0x100,0x2212,0x100,0x3d,0x100,0x28,0x100,0x29,0x100,0x61,0x100, +0x65,0x100,0x6f,0x100,0x78,0x100,0x259,0x300,0x61,0x2f,0x63,0x300,0x61,0x2f,0x73,0x300, +0x63,0x2f,0x6f,0x300,0x63,0x2f,0x75,0x100,0x67,0x100,0x68,0x100,0x127,0x100,0x6c,0x100, +0x65,0x100,0x6f,0x100,0x5d0,0x100,0x5d1,0x100,0x5d2,0x100,0x5d3,0x100,0x69,0x100,0x3c0,0x100, +0x3b3,0x100,0x2211,0x100,0x64,0x100,0x65,0x100,0x69,0x100,0x6a,0x300,0x31,0x2044,0x33,0x300, +0x32,0x2044,0x33,0x300,0x31,0x2044,0x35,0x300,0x32,0x2044,0x35,0x300,0x33,0x2044,0x35,0x300, +0x34,0x2044,0x35,0x300,0x31,0x2044,0x36,0x300,0x35,0x2044,0x36,0x300,0x31,0x2044,0x38,0x300, +0x33,0x2044,0x38,0x300,0x35,0x2044,0x38,0x300,0x37,0x2044,0x38,0x200,0x31,0x2044,0x100,0x49, +0x200,0x49,0x49,0x300,0x49,0x49,0x49,0x200,0x49,0x56,0x100,0x56,0x200,0x56,0x49,0x300, +0x56,0x49,0x49,0x400,0x56,0x49,0x49,0x49,0x200,0x49,0x58,0x100,0x58,0x200,0x58,0x49, +0x300,0x58,0x49,0x49,0x100,0x4c,0x100,0x43,0x100,0x44,0x100,0x4d,0x100,0x69,0x200,0x69, +0x69,0x300,0x69,0x69,0x69,0x200,0x69,0x76,0x100,0x76,0x200,0x76,0x69,0x300,0x76,0x69, +0x69,0x400,0x76,0x69,0x69,0x69,0x200,0x69,0x78,0x100,0x78,0x200,0x78,0x69,0x300,0x78, +0x69,0x69,0x100,0x6c,0x100,0x63,0x100,0x64,0x100,0x6d,0x200,0x222b,0x222b,0x300,0x222b,0x222b, +0x222b,0x200,0x222e,0x222e,0x300,0x222e,0x222e,0x222e,0x100,0x31,0x100,0x32,0x100,0x33,0x100,0x34, +0x100,0x35,0x100,0x36,0x100,0x37,0x100,0x38,0x100,0x39,0x200,0x31,0x30,0x200,0x31,0x31, +0x200,0x31,0x32,0x200,0x31,0x33,0x200,0x31,0x34,0x200,0x31,0x35,0x200,0x31,0x36,0x200, +0x31,0x37,0x200,0x31,0x38,0x200,0x31,0x39,0x200,0x32,0x30,0x300,0x28,0x31,0x29,0x300, +0x28,0x32,0x29,0x300,0x28,0x33,0x29,0x300,0x28,0x34,0x29,0x300,0x28,0x35,0x29,0x300, +0x28,0x36,0x29,0x300,0x28,0x37,0x29,0x300,0x28,0x38,0x29,0x300,0x28,0x39,0x29,0x400, +0x28,0x31,0x30,0x29,0x400,0x28,0x31,0x31,0x29,0x400,0x28,0x31,0x32,0x29,0x400,0x28, +0x31,0x33,0x29,0x400,0x28,0x31,0x34,0x29,0x400,0x28,0x31,0x35,0x29,0x400,0x28,0x31, +0x36,0x29,0x400,0x28,0x31,0x37,0x29,0x400,0x28,0x31,0x38,0x29,0x400,0x28,0x31,0x39, +0x29,0x400,0x28,0x32,0x30,0x29,0x200,0x31,0x2e,0x200,0x32,0x2e,0x200,0x33,0x2e,0x200, +0x34,0x2e,0x200,0x35,0x2e,0x200,0x36,0x2e,0x200,0x37,0x2e,0x200,0x38,0x2e,0x200,0x39, +0x2e,0x300,0x31,0x30,0x2e,0x300,0x31,0x31,0x2e,0x300,0x31,0x32,0x2e,0x300,0x31,0x33, +0x2e,0x300,0x31,0x34,0x2e,0x300,0x31,0x35,0x2e,0x300,0x31,0x36,0x2e,0x300,0x31,0x37, +0x2e,0x300,0x31,0x38,0x2e,0x300,0x31,0x39,0x2e,0x300,0x32,0x30,0x2e,0x300,0x28,0x61, +0x29,0x300,0x28,0x62,0x29,0x300,0x28,0x63,0x29,0x300,0x28,0x64,0x29,0x300,0x28,0x65, +0x29,0x300,0x28,0x66,0x29,0x300,0x28,0x67,0x29,0x300,0x28,0x68,0x29,0x300,0x28,0x69, +0x29,0x300,0x28,0x6a,0x29,0x300,0x28,0x6b,0x29,0x300,0x28,0x6c,0x29,0x300,0x28,0x6d, +0x29,0x300,0x28,0x6e,0x29,0x300,0x28,0x6f,0x29,0x300,0x28,0x70,0x29,0x300,0x28,0x71, +0x29,0x300,0x28,0x72,0x29,0x300,0x28,0x73,0x29,0x300,0x28,0x74,0x29,0x300,0x28,0x75, +0x29,0x300,0x28,0x76,0x29,0x300,0x28,0x77,0x29,0x300,0x28,0x78,0x29,0x300,0x28,0x79, +0x29,0x300,0x28,0x7a,0x29,0x100,0x41,0x100,0x42,0x100,0x43,0x100,0x44,0x100,0x45,0x100, +0x46,0x100,0x47,0x100,0x48,0x100,0x49,0x100,0x4a,0x100,0x4b,0x100,0x4c,0x100,0x4d,0x100, +0x4e,0x100,0x4f,0x100,0x50,0x100,0x51,0x100,0x52,0x100,0x53,0x100,0x54,0x100,0x55,0x100, +0x56,0x100,0x57,0x100,0x58,0x100,0x59,0x100,0x5a,0x100,0x61,0x100,0x62,0x100,0x63,0x100, +0x64,0x100,0x65,0x100,0x66,0x100,0x67,0x100,0x68,0x100,0x69,0x100,0x6a,0x100,0x6b,0x100, +0x6c,0x100,0x6d,0x100,0x6e,0x100,0x6f,0x100,0x70,0x100,0x71,0x100,0x72,0x100,0x73,0x100, +0x74,0x100,0x75,0x100,0x76,0x100,0x77,0x100,0x78,0x100,0x79,0x100,0x7a,0x100,0x30,0x400, +0x222b,0x222b,0x222b,0x222b,0x300,0x3a,0x3a,0x3d,0x200,0x3d,0x3d,0x300,0x3d,0x3d,0x3d,0x100, +0x2d61,0x100,0x6bcd,0x100,0x9f9f,0x100,0x4e00,0x100,0x4e28,0x100,0x4e36,0x100,0x4e3f,0x100,0x4e59,0x100, +0x4e85,0x100,0x4e8c,0x100,0x4ea0,0x100,0x4eba,0x100,0x513f,0x100,0x5165,0x100,0x516b,0x100,0x5182,0x100, +0x5196,0x100,0x51ab,0x100,0x51e0,0x100,0x51f5,0x100,0x5200,0x100,0x529b,0x100,0x52f9,0x100,0x5315,0x100, +0x531a,0x100,0x5338,0x100,0x5341,0x100,0x535c,0x100,0x5369,0x100,0x5382,0x100,0x53b6,0x100,0x53c8,0x100, +0x53e3,0x100,0x56d7,0x100,0x571f,0x100,0x58eb,0x100,0x5902,0x100,0x590a,0x100,0x5915,0x100,0x5927,0x100, +0x5973,0x100,0x5b50,0x100,0x5b80,0x100,0x5bf8,0x100,0x5c0f,0x100,0x5c22,0x100,0x5c38,0x100,0x5c6e,0x100, +0x5c71,0x100,0x5ddb,0x100,0x5de5,0x100,0x5df1,0x100,0x5dfe,0x100,0x5e72,0x100,0x5e7a,0x100,0x5e7f,0x100, +0x5ef4,0x100,0x5efe,0x100,0x5f0b,0x100,0x5f13,0x100,0x5f50,0x100,0x5f61,0x100,0x5f73,0x100,0x5fc3,0x100, +0x6208,0x100,0x6236,0x100,0x624b,0x100,0x652f,0x100,0x6534,0x100,0x6587,0x100,0x6597,0x100,0x65a4,0x100, +0x65b9,0x100,0x65e0,0x100,0x65e5,0x100,0x66f0,0x100,0x6708,0x100,0x6728,0x100,0x6b20,0x100,0x6b62,0x100, +0x6b79,0x100,0x6bb3,0x100,0x6bcb,0x100,0x6bd4,0x100,0x6bdb,0x100,0x6c0f,0x100,0x6c14,0x100,0x6c34,0x100, +0x706b,0x100,0x722a,0x100,0x7236,0x100,0x723b,0x100,0x723f,0x100,0x7247,0x100,0x7259,0x100,0x725b,0x100, +0x72ac,0x100,0x7384,0x100,0x7389,0x100,0x74dc,0x100,0x74e6,0x100,0x7518,0x100,0x751f,0x100,0x7528,0x100, +0x7530,0x100,0x758b,0x100,0x7592,0x100,0x7676,0x100,0x767d,0x100,0x76ae,0x100,0x76bf,0x100,0x76ee,0x100, +0x77db,0x100,0x77e2,0x100,0x77f3,0x100,0x793a,0x100,0x79b8,0x100,0x79be,0x100,0x7a74,0x100,0x7acb,0x100, +0x7af9,0x100,0x7c73,0x100,0x7cf8,0x100,0x7f36,0x100,0x7f51,0x100,0x7f8a,0x100,0x7fbd,0x100,0x8001,0x100, +0x800c,0x100,0x8012,0x100,0x8033,0x100,0x807f,0x100,0x8089,0x100,0x81e3,0x100,0x81ea,0x100,0x81f3,0x100, +0x81fc,0x100,0x820c,0x100,0x821b,0x100,0x821f,0x100,0x826e,0x100,0x8272,0x100,0x8278,0x100,0x864d,0x100, +0x866b,0x100,0x8840,0x100,0x884c,0x100,0x8863,0x100,0x897e,0x100,0x898b,0x100,0x89d2,0x100,0x8a00,0x100, +0x8c37,0x100,0x8c46,0x100,0x8c55,0x100,0x8c78,0x100,0x8c9d,0x100,0x8d64,0x100,0x8d70,0x100,0x8db3,0x100, +0x8eab,0x100,0x8eca,0x100,0x8f9b,0x100,0x8fb0,0x100,0x8fb5,0x100,0x9091,0x100,0x9149,0x100,0x91c6,0x100, +0x91cc,0x100,0x91d1,0x100,0x9577,0x100,0x9580,0x100,0x961c,0x100,0x96b6,0x100,0x96b9,0x100,0x96e8,0x100, +0x9751,0x100,0x975e,0x100,0x9762,0x100,0x9769,0x100,0x97cb,0x100,0x97ed,0x100,0x97f3,0x100,0x9801,0x100, +0x98a8,0x100,0x98db,0x100,0x98df,0x100,0x9996,0x100,0x9999,0x100,0x99ac,0x100,0x9aa8,0x100,0x9ad8,0x100, +0x9adf,0x100,0x9b25,0x100,0x9b2f,0x100,0x9b32,0x100,0x9b3c,0x100,0x9b5a,0x100,0x9ce5,0x100,0x9e75,0x100, +0x9e7f,0x100,0x9ea5,0x100,0x9ebb,0x100,0x9ec3,0x100,0x9ecd,0x100,0x9ed1,0x100,0x9ef9,0x100,0x9efd,0x100, +0x9f0e,0x100,0x9f13,0x100,0x9f20,0x100,0x9f3b,0x100,0x9f4a,0x100,0x9f52,0x100,0x9f8d,0x100,0x9f9c,0x100, +0x9fa0,0x100,0x20,0x100,0x3012,0x100,0x5341,0x100,0x5344,0x100,0x5345,0x8200,8,0x20,0x3099,0x8200, +8,0x20,0x309a,0x200,0x3088,0x308a,0x200,0x30b3,0x30c8,0x100,0x1100,0x100,0x1101,0x100,0x11aa,0x100, +0x1102,0x100,0x11ac,0x100,0x11ad,0x100,0x1103,0x100,0x1104,0x100,0x1105,0x100,0x11b0,0x100,0x11b1,0x100, +0x11b2,0x100,0x11b3,0x100,0x11b4,0x100,0x11b5,0x100,0x111a,0x100,0x1106,0x100,0x1107,0x100,0x1108,0x100, +0x1121,0x100,0x1109,0x100,0x110a,0x100,0x110b,0x100,0x110c,0x100,0x110d,0x100,0x110e,0x100,0x110f,0x100, +0x1110,0x100,0x1111,0x100,0x1112,0x100,0x1161,0x100,0x1162,0x100,0x1163,0x100,0x1164,0x100,0x1165,0x100, +0x1166,0x100,0x1167,0x100,0x1168,0x100,0x1169,0x100,0x116a,0x100,0x116b,0x100,0x116c,0x100,0x116d,0x100, +0x116e,0x100,0x116f,0x100,0x1170,0x100,0x1171,0x100,0x1172,0x100,0x1173,0x100,0x1174,0x100,0x1175,0x100, +0x1160,0x100,0x1114,0x100,0x1115,0x100,0x11c7,0x100,0x11c8,0x100,0x11cc,0x100,0x11ce,0x100,0x11d3,0x100, +0x11d7,0x100,0x11d9,0x100,0x111c,0x100,0x11dd,0x100,0x11df,0x100,0x111d,0x100,0x111e,0x100,0x1120,0x100, +0x1122,0x100,0x1123,0x100,0x1127,0x100,0x1129,0x100,0x112b,0x100,0x112c,0x100,0x112d,0x100,0x112e,0x100, +0x112f,0x100,0x1132,0x100,0x1136,0x100,0x1140,0x100,0x1147,0x100,0x114c,0x100,0x11f1,0x100,0x11f2,0x100, +0x1157,0x100,0x1158,0x100,0x1159,0x100,0x1184,0x100,0x1185,0x100,0x1188,0x100,0x1191,0x100,0x1192,0x100, +0x1194,0x100,0x119e,0x100,0x11a1,0x100,0x4e00,0x100,0x4e8c,0x100,0x4e09,0x100,0x56db,0x100,0x4e0a,0x100, +0x4e2d,0x100,0x4e0b,0x100,0x7532,0x100,0x4e59,0x100,0x4e19,0x100,0x4e01,0x100,0x5929,0x100,0x5730,0x100, +0x4eba,0x300,0x28,0x1100,0x29,0x300,0x28,0x1102,0x29,0x300,0x28,0x1103,0x29,0x300,0x28,0x1105, +0x29,0x300,0x28,0x1106,0x29,0x300,0x28,0x1107,0x29,0x300,0x28,0x1109,0x29,0x300,0x28,0x110b, +0x29,0x300,0x28,0x110c,0x29,0x300,0x28,0x110e,0x29,0x300,0x28,0x110f,0x29,0x300,0x28,0x1110, +0x29,0x300,0x28,0x1111,0x29,0x300,0x28,0x1112,0x29,0x400,0x28,0x1100,0x1161,0x29,0x400,0x28, +0x1102,0x1161,0x29,0x400,0x28,0x1103,0x1161,0x29,0x400,0x28,0x1105,0x1161,0x29,0x400,0x28,0x1106, +0x1161,0x29,0x400,0x28,0x1107,0x1161,0x29,0x400,0x28,0x1109,0x1161,0x29,0x400,0x28,0x110b,0x1161, +0x29,0x400,0x28,0x110c,0x1161,0x29,0x400,0x28,0x110e,0x1161,0x29,0x400,0x28,0x110f,0x1161,0x29, +0x400,0x28,0x1110,0x1161,0x29,0x400,0x28,0x1111,0x1161,0x29,0x400,0x28,0x1112,0x1161,0x29,0x400, +0x28,0x110c,0x116e,0x29,0x700,0x28,0x110b,0x1169,0x110c,0x1165,0x11ab,0x29,0x600,0x28,0x110b,0x1169, +0x1112,0x116e,0x29,0x300,0x28,0x4e00,0x29,0x300,0x28,0x4e8c,0x29,0x300,0x28,0x4e09,0x29,0x300, +0x28,0x56db,0x29,0x300,0x28,0x4e94,0x29,0x300,0x28,0x516d,0x29,0x300,0x28,0x4e03,0x29,0x300, +0x28,0x516b,0x29,0x300,0x28,0x4e5d,0x29,0x300,0x28,0x5341,0x29,0x300,0x28,0x6708,0x29,0x300, +0x28,0x706b,0x29,0x300,0x28,0x6c34,0x29,0x300,0x28,0x6728,0x29,0x300,0x28,0x91d1,0x29,0x300, +0x28,0x571f,0x29,0x300,0x28,0x65e5,0x29,0x300,0x28,0x682a,0x29,0x300,0x28,0x6709,0x29,0x300, +0x28,0x793e,0x29,0x300,0x28,0x540d,0x29,0x300,0x28,0x7279,0x29,0x300,0x28,0x8ca1,0x29,0x300, +0x28,0x795d,0x29,0x300,0x28,0x52b4,0x29,0x300,0x28,0x4ee3,0x29,0x300,0x28,0x547c,0x29,0x300, +0x28,0x5b66,0x29,0x300,0x28,0x76e3,0x29,0x300,0x28,0x4f01,0x29,0x300,0x28,0x8cc7,0x29,0x300, +0x28,0x5354,0x29,0x300,0x28,0x796d,0x29,0x300,0x28,0x4f11,0x29,0x300,0x28,0x81ea,0x29,0x300, +0x28,0x81f3,0x29,0x200,0x32,0x31,0x200,0x32,0x32,0x200,0x32,0x33,0x200,0x32,0x34,0x200, +0x32,0x35,0x200,0x32,0x36,0x200,0x32,0x37,0x200,0x32,0x38,0x200,0x32,0x39,0x200,0x33, +0x30,0x200,0x33,0x31,0x200,0x33,0x32,0x200,0x33,0x33,0x200,0x33,0x34,0x200,0x33,0x35, +0x100,0x1100,0x100,0x1102,0x100,0x1103,0x100,0x1105,0x100,0x1106,0x100,0x1107,0x100,0x1109,0x100,0x110b, +0x100,0x110c,0x100,0x110e,0x100,0x110f,0x100,0x1110,0x100,0x1111,0x100,0x1112,0x200,0x1100,0x1161,0x200, +0x1102,0x1161,0x200,0x1103,0x1161,0x200,0x1105,0x1161,0x200,0x1106,0x1161,0x200,0x1107,0x1161,0x200,0x1109, +0x1161,0x200,0x110b,0x1161,0x200,0x110c,0x1161,0x200,0x110e,0x1161,0x200,0x110f,0x1161,0x200,0x1110,0x1161, +0x200,0x1111,0x1161,0x200,0x1112,0x1161,0x500,0x110e,0x1161,0x11b7,0x1100,0x1169,0x400,0x110c,0x116e,0x110b, +0x1174,0x200,0x110b,0x116e,0x100,0x4e00,0x100,0x4e8c,0x100,0x4e09,0x100,0x56db,0x100,0x4e94,0x100,0x516d, +0x100,0x4e03,0x100,0x516b,0x100,0x4e5d,0x100,0x5341,0x100,0x6708,0x100,0x706b,0x100,0x6c34,0x100,0x6728, +0x100,0x91d1,0x100,0x571f,0x100,0x65e5,0x100,0x682a,0x100,0x6709,0x100,0x793e,0x100,0x540d,0x100,0x7279, +0x100,0x8ca1,0x100,0x795d,0x100,0x52b4,0x100,0x79d8,0x100,0x7537,0x100,0x5973,0x100,0x9069,0x100,0x512a, +0x100,0x5370,0x100,0x6ce8,0x100,0x9805,0x100,0x4f11,0x100,0x5199,0x100,0x6b63,0x100,0x4e0a,0x100,0x4e2d, +0x100,0x4e0b,0x100,0x5de6,0x100,0x53f3,0x100,0x533b,0x100,0x5b97,0x100,0x5b66,0x100,0x76e3,0x100,0x4f01, +0x100,0x8cc7,0x100,0x5354,0x100,0x591c,0x200,0x33,0x36,0x200,0x33,0x37,0x200,0x33,0x38,0x200, +0x33,0x39,0x200,0x34,0x30,0x200,0x34,0x31,0x200,0x34,0x32,0x200,0x34,0x33,0x200,0x34, +0x34,0x200,0x34,0x35,0x200,0x34,0x36,0x200,0x34,0x37,0x200,0x34,0x38,0x200,0x34,0x39, +0x200,0x35,0x30,0x200,0x31,0x6708,0x200,0x32,0x6708,0x200,0x33,0x6708,0x200,0x34,0x6708,0x200, +0x35,0x6708,0x200,0x36,0x6708,0x200,0x37,0x6708,0x200,0x38,0x6708,0x200,0x39,0x6708,0x300,0x31, +0x30,0x6708,0x300,0x31,0x31,0x6708,0x300,0x31,0x32,0x6708,0x300,0x65,0x72,0x67,0x100,0x30a2, +0x100,0x30a4,0x100,0x30a6,0x100,0x30a8,0x100,0x30aa,0x100,0x30ab,0x100,0x30ad,0x100,0x30af,0x100,0x30b1, +0x100,0x30b3,0x100,0x30b5,0x100,0x30b7,0x100,0x30b9,0x100,0x30bb,0x100,0x30bd,0x100,0x30bf,0x100,0x30c1, +0x100,0x30c4,0x100,0x30c6,0x100,0x30c8,0x100,0x30ca,0x100,0x30cb,0x100,0x30cc,0x100,0x30cd,0x100,0x30ce, +0x100,0x30cf,0x100,0x30d2,0x100,0x30d5,0x100,0x30d8,0x100,0x30db,0x100,0x30de,0x100,0x30df,0x100,0x30e0, +0x100,0x30e1,0x100,0x30e2,0x100,0x30e4,0x100,0x30e6,0x100,0x30e8,0x100,0x30e9,0x100,0x30ea,0x100,0x30eb, +0x100,0x30ec,0x100,0x30ed,0x100,0x30ef,0x100,0x30f0,0x100,0x30f1,0x100,0x30f2,0x500,0x30a2,0x30cf,0x309a, +0x30fc,0x30c8,0x400,0x30a2,0x30eb,0x30d5,0x30a1,0x500,0x30a2,0x30f3,0x30d8,0x309a,0x30a2,0x300,0x30a2,0x30fc, +0x30eb,0x8500,8,0x30a4,0x30cb,0x30f3,0x30af,0x3099,0x300,0x30a4,0x30f3,0x30c1,0x300,0x30a6,0x30a9,0x30f3, +0x8600,8,0x30a8,0x30b9,0x30af,0x30fc,0x30c8,0x3099,0x400,0x30a8,0x30fc,0x30ab,0x30fc,0x300,0x30aa,0x30f3, +0x30b9,0x300,0x30aa,0x30fc,0x30e0,0x300,0x30ab,0x30a4,0x30ea,0x400,0x30ab,0x30e9,0x30c3,0x30c8,0x400,0x30ab, +0x30ed,0x30ea,0x30fc,0x400,0x30ab,0x3099,0x30ed,0x30f3,0x400,0x30ab,0x3099,0x30f3,0x30de,0x8400,8,0x30ad, +0x3099,0x30ab,0x3099,0x400,0x30ad,0x3099,0x30cb,0x30fc,0x400,0x30ad,0x30e5,0x30ea,0x30fc,0x600,0x30ad,0x3099, +0x30eb,0x30bf,0x3099,0x30fc,0x200,0x30ad,0x30ed,0x600,0x30ad,0x30ed,0x30af,0x3099,0x30e9,0x30e0,0x600,0x30ad, +0x30ed,0x30e1,0x30fc,0x30c8,0x30eb,0x500,0x30ad,0x30ed,0x30ef,0x30c3,0x30c8,0x400,0x30af,0x3099,0x30e9,0x30e0, +0x600,0x30af,0x3099,0x30e9,0x30e0,0x30c8,0x30f3,0x600,0x30af,0x30eb,0x30bb,0x3099,0x30a4,0x30ed,0x400,0x30af, +0x30ed,0x30fc,0x30cd,0x300,0x30b1,0x30fc,0x30b9,0x300,0x30b3,0x30eb,0x30ca,0x8400,8,0x30b3,0x30fc,0x30db, +0x309a,0x400,0x30b5,0x30a4,0x30af,0x30eb,0x500,0x30b5,0x30f3,0x30c1,0x30fc,0x30e0,0x8500,8,0x30b7,0x30ea, +0x30f3,0x30af,0x3099,0x300,0x30bb,0x30f3,0x30c1,0x300,0x30bb,0x30f3,0x30c8,0x400,0x30bf,0x3099,0x30fc,0x30b9, +0x300,0x30c6,0x3099,0x30b7,0x300,0x30c8,0x3099,0x30eb,0x200,0x30c8,0x30f3,0x200,0x30ca,0x30ce,0x300,0x30ce, +0x30c3,0x30c8,0x300,0x30cf,0x30a4,0x30c4,0x600,0x30cf,0x309a,0x30fc,0x30bb,0x30f3,0x30c8,0x400,0x30cf,0x309a, +0x30fc,0x30c4,0x500,0x30cf,0x3099,0x30fc,0x30ec,0x30eb,0x600,0x30d2,0x309a,0x30a2,0x30b9,0x30c8,0x30eb,0x400, +0x30d2,0x309a,0x30af,0x30eb,0x300,0x30d2,0x309a,0x30b3,0x300,0x30d2,0x3099,0x30eb,0x8600,8,0x30d5,0x30a1, +0x30e9,0x30c3,0x30c8,0x3099,0x400,0x30d5,0x30a3,0x30fc,0x30c8,0x600,0x30d5,0x3099,0x30c3,0x30b7,0x30a7,0x30eb, +0x300,0x30d5,0x30e9,0x30f3,0x500,0x30d8,0x30af,0x30bf,0x30fc,0x30eb,0x300,0x30d8,0x309a,0x30bd,0x400,0x30d8, +0x309a,0x30cb,0x30d2,0x300,0x30d8,0x30eb,0x30c4,0x400,0x30d8,0x309a,0x30f3,0x30b9,0x8500,8,0x30d8,0x309a, +0x30fc,0x30b7,0x3099,0x400,0x30d8,0x3099,0x30fc,0x30bf,0x500,0x30db,0x309a,0x30a4,0x30f3,0x30c8,0x400,0x30db, +0x3099,0x30eb,0x30c8,0x200,0x30db,0x30f3,0x8500,8,0x30db,0x309a,0x30f3,0x30c8,0x3099,0x300,0x30db,0x30fc, +0x30eb,0x300,0x30db,0x30fc,0x30f3,0x400,0x30de,0x30a4,0x30af,0x30ed,0x300,0x30de,0x30a4,0x30eb,0x300,0x30de, +0x30c3,0x30cf,0x300,0x30de,0x30eb,0x30af,0x500,0x30de,0x30f3,0x30b7,0x30e7,0x30f3,0x400,0x30df,0x30af,0x30ed, +0x30f3,0x200,0x30df,0x30ea,0x600,0x30df,0x30ea,0x30cf,0x3099,0x30fc,0x30eb,0x8300,8,0x30e1,0x30ab,0x3099, +0x500,0x30e1,0x30ab,0x3099,0x30c8,0x30f3,0x400,0x30e1,0x30fc,0x30c8,0x30eb,0x8400,8,0x30e4,0x30fc,0x30c8, +0x3099,0x300,0x30e4,0x30fc,0x30eb,0x300,0x30e6,0x30a2,0x30f3,0x400,0x30ea,0x30c3,0x30c8,0x30eb,0x200,0x30ea, +0x30e9,0x400,0x30eb,0x30d2,0x309a,0x30fc,0x500,0x30eb,0x30fc,0x30d5,0x3099,0x30eb,0x200,0x30ec,0x30e0,0x600, +0x30ec,0x30f3,0x30c8,0x30b1,0x3099,0x30f3,0x300,0x30ef,0x30c3,0x30c8,0x200,0x30,0x70b9,0x200,0x31,0x70b9, +0x200,0x32,0x70b9,0x200,0x33,0x70b9,0x200,0x34,0x70b9,0x200,0x35,0x70b9,0x200,0x36,0x70b9,0x200, +0x37,0x70b9,0x200,0x38,0x70b9,0x200,0x39,0x70b9,0x300,0x31,0x30,0x70b9,0x300,0x31,0x31,0x70b9, +0x300,0x31,0x32,0x70b9,0x300,0x31,0x33,0x70b9,0x300,0x31,0x34,0x70b9,0x300,0x31,0x35,0x70b9, +0x300,0x31,0x36,0x70b9,0x300,0x31,0x37,0x70b9,0x300,0x31,0x38,0x70b9,0x300,0x31,0x39,0x70b9, +0x300,0x32,0x30,0x70b9,0x300,0x32,0x31,0x70b9,0x300,0x32,0x32,0x70b9,0x300,0x32,0x33,0x70b9, +0x300,0x32,0x34,0x70b9,0x200,0x64,0x61,0x300,0x62,0x61,0x72,0x200,0x70,0x63,0x200,0x64, +0x6d,0x300,0x64,0x6d,0x32,0x300,0x64,0x6d,0x33,0x200,0x5e73,0x6210,0x200,0x662d,0x548c,0x200, +0x5927,0x6b63,0x200,0x660e,0x6cbb,0x400,0x682a,0x5f0f,0x4f1a,0x793e,0x300,0x63,0x61,0x6c,0x400,0x6b, +0x63,0x61,0x6c,0x200,0x3bc,0x67,0x200,0x6d,0x67,0x200,0x6b,0x67,0x200,0x3bc,0x6c,0x200, +0x6d,0x6c,0x200,0x64,0x6c,0x200,0x6b,0x6c,0x200,0x66,0x6d,0x200,0x6e,0x6d,0x200,0x3bc, +0x6d,0x200,0x6d,0x6d,0x200,0x63,0x6d,0x200,0x6b,0x6d,0x300,0x6d,0x6d,0x32,0x300,0x63, +0x6d,0x32,0x200,0x6d,0x32,0x300,0x6b,0x6d,0x32,0x300,0x6d,0x6d,0x33,0x300,0x63,0x6d, +0x33,0x200,0x6d,0x33,0x300,0x6b,0x6d,0x33,0x300,0x6d,0x2215,0x73,0x400,0x6d,0x2215,0x73, +0x32,0x300,0x72,0x61,0x64,0x500,0x72,0x61,0x64,0x2215,0x73,0x600,0x72,0x61,0x64,0x2215, +0x73,0x32,0x200,0x70,0x73,0x200,0x6e,0x73,0x200,0x3bc,0x73,0x200,0x6d,0x73,0x400,0x61, +0x2e,0x6d,0x2e,0x200,0x63,0x63,0x200,0x63,0x64,0x200,0x68,0x61,0x200,0x69,0x6e,0x200, +0x6b,0x74,0x200,0x6c,0x6d,0x200,0x6c,0x6e,0x300,0x6c,0x6f,0x67,0x200,0x6c,0x78,0x200, +0x6d,0x62,0x300,0x6d,0x69,0x6c,0x300,0x6d,0x6f,0x6c,0x400,0x70,0x2e,0x6d,0x2e,0x200, +0x73,0x72,0x200,0x31,0x65e5,0x200,0x32,0x65e5,0x200,0x33,0x65e5,0x200,0x34,0x65e5,0x200,0x35, +0x65e5,0x200,0x36,0x65e5,0x200,0x37,0x65e5,0x200,0x38,0x65e5,0x200,0x39,0x65e5,0x300,0x31,0x30, +0x65e5,0x300,0x31,0x31,0x65e5,0x300,0x31,0x32,0x65e5,0x300,0x31,0x33,0x65e5,0x300,0x31,0x34, +0x65e5,0x300,0x31,0x35,0x65e5,0x300,0x31,0x36,0x65e5,0x300,0x31,0x37,0x65e5,0x300,0x31,0x38, +0x65e5,0x300,0x31,0x39,0x65e5,0x300,0x32,0x30,0x65e5,0x300,0x32,0x31,0x65e5,0x300,0x32,0x32, +0x65e5,0x300,0x32,0x33,0x65e5,0x300,0x32,0x34,0x65e5,0x300,0x32,0x35,0x65e5,0x300,0x32,0x36, +0x65e5,0x300,0x32,0x37,0x65e5,0x300,0x32,0x38,0x65e5,0x300,0x32,0x39,0x65e5,0x300,0x33,0x30, +0x65e5,0x300,0x33,0x31,0x65e5,0x300,0x67,0x61,0x6c,0x200,0x66,0x66,0x200,0x66,0x69,0x200, +0x66,0x6c,0x300,0x66,0x66,0x69,0x300,0x66,0x66,0x6c,0x200,0x73,0x74,0x200,0x73,0x74, +0x200,0x574,0x576,0x200,0x574,0x565,0x200,0x574,0x56b,0x200,0x57e,0x576,0x200,0x574,0x56d,0x100, +0x5e2,0x100,0x5d0,0x100,0x5d3,0x100,0x5d4,0x100,0x5db,0x100,0x5dc,0x100,0x5dd,0x100,0x5e8,0x100, +0x5ea,0x100,0x2b,0x200,0x5d0,0x5dc,0x100,0x671,0x100,0x671,0x100,0x67b,0x100,0x67b,0x100,0x67b, +0x100,0x67b,0x100,0x67e,0x100,0x67e,0x100,0x67e,0x100,0x67e,0x100,0x680,0x100,0x680,0x100,0x680, +0x100,0x680,0x100,0x67a,0x100,0x67a,0x100,0x67a,0x100,0x67a,0x100,0x67f,0x100,0x67f,0x100,0x67f, +0x100,0x67f,0x100,0x679,0x100,0x679,0x100,0x679,0x100,0x679,0x100,0x6a4,0x100,0x6a4,0x100,0x6a4, +0x100,0x6a4,0x100,0x6a6,0x100,0x6a6,0x100,0x6a6,0x100,0x6a6,0x100,0x684,0x100,0x684,0x100,0x684, +0x100,0x684,0x100,0x683,0x100,0x683,0x100,0x683,0x100,0x683,0x100,0x686,0x100,0x686,0x100,0x686, +0x100,0x686,0x100,0x687,0x100,0x687,0x100,0x687,0x100,0x687,0x100,0x68d,0x100,0x68d,0x100,0x68c, +0x100,0x68c,0x100,0x68e,0x100,0x68e,0x100,0x688,0x100,0x688,0x100,0x698,0x100,0x698,0x100,0x691, +0x100,0x691,0x100,0x6a9,0x100,0x6a9,0x100,0x6a9,0x100,0x6a9,0x100,0x6af,0x100,0x6af,0x100,0x6af, +0x100,0x6af,0x100,0x6b3,0x100,0x6b3,0x100,0x6b3,0x100,0x6b3,0x100,0x6b1,0x100,0x6b1,0x100,0x6b1, +0x100,0x6b1,0x100,0x6ba,0x100,0x6ba,0x100,0x6bb,0x100,0x6bb,0x100,0x6bb,0x100,0x6bb,0x8200,0xe6, +0x6d5,0x654,0x8200,0xe6,0x6d5,0x654,0x100,0x6c1,0x100,0x6c1,0x100,0x6c1,0x100,0x6c1,0x100,0x6be, +0x100,0x6be,0x100,0x6be,0x100,0x6be,0x100,0x6d2,0x100,0x6d2,0x8200,0xe6,0x6d2,0x654,0x8200,0xe6, +0x6d2,0x654,0x100,0x6ad,0x100,0x6ad,0x100,0x6ad,0x100,0x6ad,0x100,0x6c7,0x100,0x6c7,0x100,0x6c6, +0x100,0x6c6,0x100,0x6c8,0x100,0x6c8,0x200,0x6c7,0x674,0x100,0x6cb,0x100,0x6cb,0x100,0x6c5,0x100, +0x6c5,0x100,0x6c9,0x100,0x6c9,0x100,0x6d0,0x100,0x6d0,0x100,0x6d0,0x100,0x6d0,0x100,0x649,0x100, +0x649,0x300,0x64a,0x654,0x627,0x300,0x64a,0x654,0x627,0x300,0x64a,0x654,0x6d5,0x300,0x64a,0x654, +0x6d5,0x300,0x64a,0x654,0x648,0x300,0x64a,0x654,0x648,0x300,0x64a,0x654,0x6c7,0x300,0x64a,0x654, +0x6c7,0x300,0x64a,0x654,0x6c6,0x300,0x64a,0x654,0x6c6,0x300,0x64a,0x654,0x6c8,0x300,0x64a,0x654, +0x6c8,0x300,0x64a,0x654,0x6d0,0x300,0x64a,0x654,0x6d0,0x300,0x64a,0x654,0x6d0,0x300,0x64a,0x654, +0x649,0x300,0x64a,0x654,0x649,0x300,0x64a,0x654,0x649,0x100,0x6cc,0x100,0x6cc,0x100,0x6cc,0x100, +0x6cc,0x300,0x64a,0x654,0x62c,0x300,0x64a,0x654,0x62d,0x300,0x64a,0x654,0x645,0x300,0x64a,0x654, +0x649,0x300,0x64a,0x654,0x64a,0x200,0x628,0x62c,0x200,0x628,0x62d,0x200,0x628,0x62e,0x200,0x628, +0x645,0x200,0x628,0x649,0x200,0x628,0x64a,0x200,0x62a,0x62c,0x200,0x62a,0x62d,0x200,0x62a,0x62e, +0x200,0x62a,0x645,0x200,0x62a,0x649,0x200,0x62a,0x64a,0x200,0x62b,0x62c,0x200,0x62b,0x645,0x200, +0x62b,0x649,0x200,0x62b,0x64a,0x200,0x62c,0x62d,0x200,0x62c,0x645,0x200,0x62d,0x62c,0x200,0x62d, +0x645,0x200,0x62e,0x62c,0x200,0x62e,0x62d,0x200,0x62e,0x645,0x200,0x633,0x62c,0x200,0x633,0x62d, +0x200,0x633,0x62e,0x200,0x633,0x645,0x200,0x635,0x62d,0x200,0x635,0x645,0x200,0x636,0x62c,0x200, +0x636,0x62d,0x200,0x636,0x62e,0x200,0x636,0x645,0x200,0x637,0x62d,0x200,0x637,0x645,0x200,0x638, +0x645,0x200,0x639,0x62c,0x200,0x639,0x645,0x200,0x63a,0x62c,0x200,0x63a,0x645,0x200,0x641,0x62c, +0x200,0x641,0x62d,0x200,0x641,0x62e,0x200,0x641,0x645,0x200,0x641,0x649,0x200,0x641,0x64a,0x200, +0x642,0x62d,0x200,0x642,0x645,0x200,0x642,0x649,0x200,0x642,0x64a,0x200,0x643,0x627,0x200,0x643, +0x62c,0x200,0x643,0x62d,0x200,0x643,0x62e,0x200,0x643,0x644,0x200,0x643,0x645,0x200,0x643,0x649, +0x200,0x643,0x64a,0x200,0x644,0x62c,0x200,0x644,0x62d,0x200,0x644,0x62e,0x200,0x644,0x645,0x200, +0x644,0x649,0x200,0x644,0x64a,0x200,0x645,0x62c,0x200,0x645,0x62d,0x200,0x645,0x62e,0x200,0x645, +0x645,0x200,0x645,0x649,0x200,0x645,0x64a,0x200,0x646,0x62c,0x200,0x646,0x62d,0x200,0x646,0x62e, +0x200,0x646,0x645,0x200,0x646,0x649,0x200,0x646,0x64a,0x200,0x647,0x62c,0x200,0x647,0x645,0x200, +0x647,0x649,0x200,0x647,0x64a,0x200,0x64a,0x62c,0x200,0x64a,0x62d,0x200,0x64a,0x62e,0x200,0x64a, +0x645,0x200,0x64a,0x649,0x200,0x64a,0x64a,0x8200,0x23,0x630,0x670,0x8200,0x23,0x631,0x670,0x8200, +0x23,0x649,0x670,0x8300,0x21,0x20,0x64c,0x651,0x8300,0x21,0x20,0x64d,0x651,0x8300,0x21,0x20, +0x64e,0x651,0x8300,0x21,0x20,0x64f,0x651,0x8300,0x21,0x20,0x650,0x651,0x8300,0x23,0x20,0x651, +0x670,0x300,0x64a,0x654,0x631,0x300,0x64a,0x654,0x632,0x300,0x64a,0x654,0x645,0x300,0x64a,0x654, +0x646,0x300,0x64a,0x654,0x649,0x300,0x64a,0x654,0x64a,0x200,0x628,0x631,0x200,0x628,0x632,0x200, +0x628,0x645,0x200,0x628,0x646,0x200,0x628,0x649,0x200,0x628,0x64a,0x200,0x62a,0x631,0x200,0x62a, +0x632,0x200,0x62a,0x645,0x200,0x62a,0x646,0x200,0x62a,0x649,0x200,0x62a,0x64a,0x200,0x62b,0x631, +0x200,0x62b,0x632,0x200,0x62b,0x645,0x200,0x62b,0x646,0x200,0x62b,0x649,0x200,0x62b,0x64a,0x200, +0x641,0x649,0x200,0x641,0x64a,0x200,0x642,0x649,0x200,0x642,0x64a,0x200,0x643,0x627,0x200,0x643, +0x644,0x200,0x643,0x645,0x200,0x643,0x649,0x200,0x643,0x64a,0x200,0x644,0x645,0x200,0x644,0x649, +0x200,0x644,0x64a,0x200,0x645,0x627,0x200,0x645,0x645,0x200,0x646,0x631,0x200,0x646,0x632,0x200, +0x646,0x645,0x200,0x646,0x646,0x200,0x646,0x649,0x200,0x646,0x64a,0x8200,0x23,0x649,0x670,0x200, +0x64a,0x631,0x200,0x64a,0x632,0x200,0x64a,0x645,0x200,0x64a,0x646,0x200,0x64a,0x649,0x200,0x64a, +0x64a,0x300,0x64a,0x654,0x62c,0x300,0x64a,0x654,0x62d,0x300,0x64a,0x654,0x62e,0x300,0x64a,0x654, +0x645,0x300,0x64a,0x654,0x647,0x200,0x628,0x62c,0x200,0x628,0x62d,0x200,0x628,0x62e,0x200,0x628, +0x645,0x200,0x628,0x647,0x200,0x62a,0x62c,0x200,0x62a,0x62d,0x200,0x62a,0x62e,0x200,0x62a,0x645, +0x200,0x62a,0x647,0x200,0x62b,0x645,0x200,0x62c,0x62d,0x200,0x62c,0x645,0x200,0x62d,0x62c,0x200, +0x62d,0x645,0x200,0x62e,0x62c,0x200,0x62e,0x645,0x200,0x633,0x62c,0x200,0x633,0x62d,0x200,0x633, +0x62e,0x200,0x633,0x645,0x200,0x635,0x62d,0x200,0x635,0x62e,0x200,0x635,0x645,0x200,0x636,0x62c, +0x200,0x636,0x62d,0x200,0x636,0x62e,0x200,0x636,0x645,0x200,0x637,0x62d,0x200,0x638,0x645,0x200, +0x639,0x62c,0x200,0x639,0x645,0x200,0x63a,0x62c,0x200,0x63a,0x645,0x200,0x641,0x62c,0x200,0x641, +0x62d,0x200,0x641,0x62e,0x200,0x641,0x645,0x200,0x642,0x62d,0x200,0x642,0x645,0x200,0x643,0x62c, +0x200,0x643,0x62d,0x200,0x643,0x62e,0x200,0x643,0x644,0x200,0x643,0x645,0x200,0x644,0x62c,0x200, +0x644,0x62d,0x200,0x644,0x62e,0x200,0x644,0x645,0x200,0x644,0x647,0x200,0x645,0x62c,0x200,0x645, +0x62d,0x200,0x645,0x62e,0x200,0x645,0x645,0x200,0x646,0x62c,0x200,0x646,0x62d,0x200,0x646,0x62e, +0x200,0x646,0x645,0x200,0x646,0x647,0x200,0x647,0x62c,0x200,0x647,0x645,0x8200,0x23,0x647,0x670, +0x200,0x64a,0x62c,0x200,0x64a,0x62d,0x200,0x64a,0x62e,0x200,0x64a,0x645,0x200,0x64a,0x647,0x300, +0x64a,0x654,0x645,0x300,0x64a,0x654,0x647,0x200,0x628,0x645,0x200,0x628,0x647,0x200,0x62a,0x645, +0x200,0x62a,0x647,0x200,0x62b,0x645,0x200,0x62b,0x647,0x200,0x633,0x645,0x200,0x633,0x647,0x200, +0x634,0x645,0x200,0x634,0x647,0x200,0x643,0x644,0x200,0x643,0x645,0x200,0x644,0x645,0x200,0x646, +0x645,0x200,0x646,0x647,0x200,0x64a,0x645,0x200,0x64a,0x647,0x8300,0x21,0x640,0x64e,0x651,0x8300, +0x21,0x640,0x64f,0x651,0x8300,0x21,0x640,0x650,0x651,0x200,0x637,0x649,0x200,0x637,0x64a,0x200, +0x639,0x649,0x200,0x639,0x64a,0x200,0x63a,0x649,0x200,0x63a,0x64a,0x200,0x633,0x649,0x200,0x633, +0x64a,0x200,0x634,0x649,0x200,0x634,0x64a,0x200,0x62d,0x649,0x200,0x62d,0x64a,0x200,0x62c,0x649, +0x200,0x62c,0x64a,0x200,0x62e,0x649,0x200,0x62e,0x64a,0x200,0x635,0x649,0x200,0x635,0x64a,0x200, +0x636,0x649,0x200,0x636,0x64a,0x200,0x634,0x62c,0x200,0x634,0x62d,0x200,0x634,0x62e,0x200,0x634, +0x645,0x200,0x634,0x631,0x200,0x633,0x631,0x200,0x635,0x631,0x200,0x636,0x631,0x200,0x637,0x649, +0x200,0x637,0x64a,0x200,0x639,0x649,0x200,0x639,0x64a,0x200,0x63a,0x649,0x200,0x63a,0x64a,0x200, +0x633,0x649,0x200,0x633,0x64a,0x200,0x634,0x649,0x200,0x634,0x64a,0x200,0x62d,0x649,0x200,0x62d, +0x64a,0x200,0x62c,0x649,0x200,0x62c,0x64a,0x200,0x62e,0x649,0x200,0x62e,0x64a,0x200,0x635,0x649, +0x200,0x635,0x64a,0x200,0x636,0x649,0x200,0x636,0x64a,0x200,0x634,0x62c,0x200,0x634,0x62d,0x200, +0x634,0x62e,0x200,0x634,0x645,0x200,0x634,0x631,0x200,0x633,0x631,0x200,0x635,0x631,0x200,0x636, +0x631,0x200,0x634,0x62c,0x200,0x634,0x62d,0x200,0x634,0x62e,0x200,0x634,0x645,0x200,0x633,0x647, +0x200,0x634,0x647,0x200,0x637,0x645,0x200,0x633,0x62c,0x200,0x633,0x62d,0x200,0x633,0x62e,0x200, +0x634,0x62c,0x200,0x634,0x62d,0x200,0x634,0x62e,0x200,0x637,0x645,0x200,0x638,0x645,0x8200,0x1b, +0x627,0x64b,0x8200,0x1b,0x627,0x64b,0x300,0x62a,0x62c,0x645,0x300,0x62a,0x62d,0x62c,0x300,0x62a, +0x62d,0x62c,0x300,0x62a,0x62d,0x645,0x300,0x62a,0x62e,0x645,0x300,0x62a,0x645,0x62c,0x300,0x62a, +0x645,0x62d,0x300,0x62a,0x645,0x62e,0x300,0x62c,0x645,0x62d,0x300,0x62c,0x645,0x62d,0x300,0x62d, +0x645,0x64a,0x300,0x62d,0x645,0x649,0x300,0x633,0x62d,0x62c,0x300,0x633,0x62c,0x62d,0x300,0x633, +0x62c,0x649,0x300,0x633,0x645,0x62d,0x300,0x633,0x645,0x62d,0x300,0x633,0x645,0x62c,0x300,0x633, +0x645,0x645,0x300,0x633,0x645,0x645,0x300,0x635,0x62d,0x62d,0x300,0x635,0x62d,0x62d,0x300,0x635, +0x645,0x645,0x300,0x634,0x62d,0x645,0x300,0x634,0x62d,0x645,0x300,0x634,0x62c,0x64a,0x300,0x634, +0x645,0x62e,0x300,0x634,0x645,0x62e,0x300,0x634,0x645,0x645,0x300,0x634,0x645,0x645,0x300,0x636, +0x62d,0x649,0x300,0x636,0x62e,0x645,0x300,0x636,0x62e,0x645,0x300,0x637,0x645,0x62d,0x300,0x637, +0x645,0x62d,0x300,0x637,0x645,0x645,0x300,0x637,0x645,0x64a,0x300,0x639,0x62c,0x645,0x300,0x639, +0x645,0x645,0x300,0x639,0x645,0x645,0x300,0x639,0x645,0x649,0x300,0x63a,0x645,0x645,0x300,0x63a, +0x645,0x64a,0x300,0x63a,0x645,0x649,0x300,0x641,0x62e,0x645,0x300,0x641,0x62e,0x645,0x300,0x642, +0x645,0x62d,0x300,0x642,0x645,0x645,0x300,0x644,0x62d,0x645,0x300,0x644,0x62d,0x64a,0x300,0x644, +0x62d,0x649,0x300,0x644,0x62c,0x62c,0x300,0x644,0x62c,0x62c,0x300,0x644,0x62e,0x645,0x300,0x644, +0x62e,0x645,0x300,0x644,0x645,0x62d,0x300,0x644,0x645,0x62d,0x300,0x645,0x62d,0x62c,0x300,0x645, +0x62d,0x645,0x300,0x645,0x62d,0x64a,0x300,0x645,0x62c,0x62d,0x300,0x645,0x62c,0x645,0x300,0x645, +0x62e,0x62c,0x300,0x645,0x62e,0x645,0x300,0x645,0x62c,0x62e,0x300,0x647,0x645,0x62c,0x300,0x647, +0x645,0x645,0x300,0x646,0x62d,0x645,0x300,0x646,0x62d,0x649,0x300,0x646,0x62c,0x645,0x300,0x646, +0x62c,0x645,0x300,0x646,0x62c,0x649,0x300,0x646,0x645,0x64a,0x300,0x646,0x645,0x649,0x300,0x64a, +0x645,0x645,0x300,0x64a,0x645,0x645,0x300,0x628,0x62e,0x64a,0x300,0x62a,0x62c,0x64a,0x300,0x62a, +0x62c,0x649,0x300,0x62a,0x62e,0x64a,0x300,0x62a,0x62e,0x649,0x300,0x62a,0x645,0x64a,0x300,0x62a, +0x645,0x649,0x300,0x62c,0x645,0x64a,0x300,0x62c,0x62d,0x649,0x300,0x62c,0x645,0x649,0x300,0x633, +0x62e,0x649,0x300,0x635,0x62d,0x64a,0x300,0x634,0x62d,0x64a,0x300,0x636,0x62d,0x64a,0x300,0x644, +0x62c,0x64a,0x300,0x644,0x645,0x64a,0x300,0x64a,0x62d,0x64a,0x300,0x64a,0x62c,0x64a,0x300,0x64a, +0x645,0x64a,0x300,0x645,0x645,0x64a,0x300,0x642,0x645,0x64a,0x300,0x646,0x62d,0x64a,0x300,0x642, +0x645,0x62d,0x300,0x644,0x62d,0x645,0x300,0x639,0x645,0x64a,0x300,0x643,0x645,0x64a,0x300,0x646, +0x62c,0x62d,0x300,0x645,0x62e,0x64a,0x300,0x644,0x62c,0x645,0x300,0x643,0x645,0x645,0x300,0x644, +0x62c,0x645,0x300,0x646,0x62c,0x62d,0x300,0x62c,0x62d,0x64a,0x300,0x62d,0x62c,0x64a,0x300,0x645, +0x62c,0x64a,0x300,0x641,0x645,0x64a,0x300,0x628,0x62d,0x64a,0x300,0x643,0x645,0x645,0x300,0x639, +0x62c,0x645,0x300,0x635,0x645,0x645,0x300,0x633,0x62e,0x64a,0x300,0x646,0x62c,0x64a,0x300,0x635, +0x644,0x6d2,0x300,0x642,0x644,0x6d2,0x400,0x627,0x644,0x644,0x647,0x400,0x627,0x643,0x628,0x631, +0x400,0x645,0x62d,0x645,0x62f,0x400,0x635,0x644,0x639,0x645,0x400,0x631,0x633,0x648,0x644,0x400, +0x639,0x644,0x64a,0x647,0x400,0x648,0x633,0x644,0x645,0x300,0x635,0x644,0x649,0x1200,0x635,0x644, +0x649,0x20,0x627,0x644,0x644,0x647,0x20,0x639,0x644,0x64a,0x647,0x20,0x648,0x633,0x644,0x645, +0x800,0x62c,0x644,0x20,0x62c,0x644,0x627,0x644,0x647,0x400,0x631,0x6cc,0x627,0x644,0x100,0x2c, +0x100,0x3001,0x100,0x3002,0x100,0x3a,0x100,0x3b,0x100,0x21,0x100,0x3f,0x100,0x3016,0x100,0x3017, +0x300,0x2e,0x2e,0x2e,0x200,0x2e,0x2e,0x100,0x2014,0x100,0x2013,0x100,0x5f,0x100,0x5f,0x100, +0x28,0x100,0x29,0x100,0x7b,0x100,0x7d,0x100,0x3014,0x100,0x3015,0x100,0x3010,0x100,0x3011,0x100, +0x300a,0x100,0x300b,0x100,0x3008,0x100,0x3009,0x100,0x300c,0x100,0x300d,0x100,0x300e,0x100,0x300f,0x100, +0x5b,0x100,0x5d,0x8200,0xe6,0x20,0x305,0x8200,0xe6,0x20,0x305,0x8200,0xe6,0x20,0x305,0x8200, +0xe6,0x20,0x305,0x100,0x5f,0x100,0x5f,0x100,0x5f,0x100,0x2c,0x100,0x3001,0x100,0x2e,0x100, +0x3b,0x100,0x3a,0x100,0x3f,0x100,0x21,0x100,0x2014,0x100,0x28,0x100,0x29,0x100,0x7b,0x100, +0x7d,0x100,0x3014,0x100,0x3015,0x100,0x23,0x100,0x26,0x100,0x2a,0x100,0x2b,0x100,0x2d,0x100, +0x3c,0x100,0x3e,0x100,0x3d,0x100,0x5c,0x100,0x24,0x100,0x25,0x100,0x40,0x8200,0x1b,0x20, +0x64b,0x8200,0x1b,0x640,0x64b,0x8200,0x1c,0x20,0x64c,0x8200,0x1d,0x20,0x64d,0x8200,0x1e,0x20, +0x64e,0x8200,0x1e,0x640,0x64e,0x8200,0x1f,0x20,0x64f,0x8200,0x1f,0x640,0x64f,0x8200,0x20,0x20, +0x650,0x8200,0x20,0x640,0x650,0x8200,0x21,0x20,0x651,0x8200,0x21,0x640,0x651,0x8200,0x22,0x20, +0x652,0x8200,0x22,0x640,0x652,0x100,0x621,0x8200,0xe6,0x627,0x653,0x8200,0xe6,0x627,0x653,0x8200, +0xe6,0x627,0x654,0x8200,0xe6,0x627,0x654,0x8200,0xe6,0x648,0x654,0x8200,0xe6,0x648,0x654,0x8200, +0xdc,0x627,0x655,0x8200,0xdc,0x627,0x655,0x8200,0xe6,0x64a,0x654,0x8200,0xe6,0x64a,0x654,0x8200, +0xe6,0x64a,0x654,0x8200,0xe6,0x64a,0x654,0x100,0x627,0x100,0x627,0x100,0x628,0x100,0x628,0x100, +0x628,0x100,0x628,0x100,0x629,0x100,0x629,0x100,0x62a,0x100,0x62a,0x100,0x62a,0x100,0x62a,0x100, +0x62b,0x100,0x62b,0x100,0x62b,0x100,0x62b,0x100,0x62c,0x100,0x62c,0x100,0x62c,0x100,0x62c,0x100, +0x62d,0x100,0x62d,0x100,0x62d,0x100,0x62d,0x100,0x62e,0x100,0x62e,0x100,0x62e,0x100,0x62e,0x100, +0x62f,0x100,0x62f,0x100,0x630,0x100,0x630,0x100,0x631,0x100,0x631,0x100,0x632,0x100,0x632,0x100, +0x633,0x100,0x633,0x100,0x633,0x100,0x633,0x100,0x634,0x100,0x634,0x100,0x634,0x100,0x634,0x100, +0x635,0x100,0x635,0x100,0x635,0x100,0x635,0x100,0x636,0x100,0x636,0x100,0x636,0x100,0x636,0x100, +0x637,0x100,0x637,0x100,0x637,0x100,0x637,0x100,0x638,0x100,0x638,0x100,0x638,0x100,0x638,0x100, +0x639,0x100,0x639,0x100,0x639,0x100,0x639,0x100,0x63a,0x100,0x63a,0x100,0x63a,0x100,0x63a,0x100, +0x641,0x100,0x641,0x100,0x641,0x100,0x641,0x100,0x642,0x100,0x642,0x100,0x642,0x100,0x642,0x100, +0x643,0x100,0x643,0x100,0x643,0x100,0x643,0x100,0x644,0x100,0x644,0x100,0x644,0x100,0x644,0x100, +0x645,0x100,0x645,0x100,0x645,0x100,0x645,0x100,0x646,0x100,0x646,0x100,0x646,0x100,0x646,0x100, +0x647,0x100,0x647,0x100,0x647,0x100,0x647,0x100,0x648,0x100,0x648,0x100,0x649,0x100,0x649,0x100, +0x64a,0x100,0x64a,0x100,0x64a,0x100,0x64a,0x8300,0xe6,0x644,0x627,0x653,0x8300,0xe6,0x644,0x627, +0x653,0x8300,0xe6,0x644,0x627,0x654,0x8300,0xe6,0x644,0x627,0x654,0x8300,0xdc,0x644,0x627,0x655, +0x8300,0xdc,0x644,0x627,0x655,0x200,0x644,0x627,0x200,0x644,0x627,0x100,0x21,0x100,0x22,0x100, +0x23,0x100,0x24,0x100,0x25,0x100,0x26,0x100,0x27,0x100,0x28,0x100,0x29,0x100,0x2a,0x100, +0x2b,0x100,0x2c,0x100,0x2d,0x100,0x2e,0x100,0x2f,0x100,0x30,0x100,0x31,0x100,0x32,0x100, +0x33,0x100,0x34,0x100,0x35,0x100,0x36,0x100,0x37,0x100,0x38,0x100,0x39,0x100,0x3a,0x100, +0x3b,0x100,0x3c,0x100,0x3d,0x100,0x3e,0x100,0x3f,0x100,0x40,0x100,0x41,0x100,0x42,0x100, +0x43,0x100,0x44,0x100,0x45,0x100,0x46,0x100,0x47,0x100,0x48,0x100,0x49,0x100,0x4a,0x100, +0x4b,0x100,0x4c,0x100,0x4d,0x100,0x4e,0x100,0x4f,0x100,0x50,0x100,0x51,0x100,0x52,0x100, +0x53,0x100,0x54,0x100,0x55,0x100,0x56,0x100,0x57,0x100,0x58,0x100,0x59,0x100,0x5a,0x100, +0x5b,0x100,0x5c,0x100,0x5d,0x100,0x5e,0x100,0x5f,0x100,0x60,0x100,0x61,0x100,0x62,0x100, +0x63,0x100,0x64,0x100,0x65,0x100,0x66,0x100,0x67,0x100,0x68,0x100,0x69,0x100,0x6a,0x100, +0x6b,0x100,0x6c,0x100,0x6d,0x100,0x6e,0x100,0x6f,0x100,0x70,0x100,0x71,0x100,0x72,0x100, +0x73,0x100,0x74,0x100,0x75,0x100,0x76,0x100,0x77,0x100,0x78,0x100,0x79,0x100,0x7a,0x100, +0x7b,0x100,0x7c,0x100,0x7d,0x100,0x7e,0x100,0x2985,0x100,0x2986,0x100,0x3002,0x100,0x300c,0x100, +0x300d,0x100,0x3001,0x100,0x30fb,0x100,0x30f2,0x100,0x30a1,0x100,0x30a3,0x100,0x30a5,0x100,0x30a7,0x100, +0x30a9,0x100,0x30e3,0x100,0x30e5,0x100,0x30e7,0x100,0x30c3,0x100,0x30fc,0x100,0x30a2,0x100,0x30a4,0x100, +0x30a6,0x100,0x30a8,0x100,0x30aa,0x100,0x30ab,0x100,0x30ad,0x100,0x30af,0x100,0x30b1,0x100,0x30b3,0x100, +0x30b5,0x100,0x30b7,0x100,0x30b9,0x100,0x30bb,0x100,0x30bd,0x100,0x30bf,0x100,0x30c1,0x100,0x30c4,0x100, +0x30c6,0x100,0x30c8,0x100,0x30ca,0x100,0x30cb,0x100,0x30cc,0x100,0x30cd,0x100,0x30ce,0x100,0x30cf,0x100, +0x30d2,0x100,0x30d5,0x100,0x30d8,0x100,0x30db,0x100,0x30de,0x100,0x30df,0x100,0x30e0,0x100,0x30e1,0x100, +0x30e2,0x100,0x30e4,0x100,0x30e6,0x100,0x30e8,0x100,0x30e9,0x100,0x30ea,0x100,0x30eb,0x100,0x30ec,0x100, +0x30ed,0x100,0x30ef,0x100,0x30f3,0x8100,0x808,0x3099,0x8100,0x808,0x309a,0x100,0x1160,0x100,0x1100,0x100, +0x1101,0x100,0x11aa,0x100,0x1102,0x100,0x11ac,0x100,0x11ad,0x100,0x1103,0x100,0x1104,0x100,0x1105,0x100, +0x11b0,0x100,0x11b1,0x100,0x11b2,0x100,0x11b3,0x100,0x11b4,0x100,0x11b5,0x100,0x111a,0x100,0x1106,0x100, +0x1107,0x100,0x1108,0x100,0x1121,0x100,0x1109,0x100,0x110a,0x100,0x110b,0x100,0x110c,0x100,0x110d,0x100, +0x110e,0x100,0x110f,0x100,0x1110,0x100,0x1111,0x100,0x1112,0x100,0x1161,0x100,0x1162,0x100,0x1163,0x100, +0x1164,0x100,0x1165,0x100,0x1166,0x100,0x1167,0x100,0x1168,0x100,0x1169,0x100,0x116a,0x100,0x116b,0x100, +0x116c,0x100,0x116d,0x100,0x116e,0x100,0x116f,0x100,0x1170,0x100,0x1171,0x100,0x1172,0x100,0x1173,0x100, +0x1174,0x100,0x1175,0x100,0xa2,0x100,0xa3,0x100,0xac,0x8200,0xe6,0x20,0x304,0x100,0xa6,0x100, +0xa5,0x100,0x20a9,0x100,0x2502,0x100,0x2190,0x100,0x2191,0x100,0x2192,0x100,0x2193,0x100,0x25a0,0x100, +0x25cb,0x100,0x61,0x100,0x62,0x100,0x63,0x100,0x64,0x100,0x65,0x100,0x66,0x100,0x67,0x100, +0x68,0x100,0x69,0x100,0x6a,0x100,0x6b,0x100,0x6c,0x100,0x6d,0x100,0x6e,0x100,0x6f,0x100, +0x70,0x100,0x71,0x100,0x72,0x100,0x73,0x100,0x74,0x100,0x75,0x100,0x76,0x100,0x77,0x100, +0x78,0x100,0x79,0x100,0x7a,0x100,0x61,0x100,0x62,0x100,0x63,0x100,0x64,0x100,0x65,0x100, +0x66,0x100,0x67,0x100,0x69,0x100,0x6a,0x100,0x6b,0x100,0x6c,0x100,0x6d,0x100,0x6e,0x100, +0x6f,0x100,0x70,0x100,0x71,0x100,0x72,0x100,0x73,0x100,0x74,0x100,0x75,0x100,0x76,0x100, +0x77,0x100,0x78,0x100,0x79,0x100,0x7a,0x100,0x61,0x100,0x62,0x100,0x63,0x100,0x64,0x100, +0x65,0x100,0x66,0x100,0x67,0x100,0x68,0x100,0x69,0x100,0x6a,0x100,0x6b,0x100,0x6c,0x100, +0x6d,0x100,0x6e,0x100,0x6f,0x100,0x70,0x100,0x71,0x100,0x72,0x100,0x73,0x100,0x74,0x100, +0x75,0x100,0x76,0x100,0x77,0x100,0x78,0x100,0x79,0x100,0x7a,0x100,0x61,0x100,0x62,0x100, +0x63,0x100,0x64,0x100,0x66,0x100,0x68,0x100,0x69,0x100,0x6a,0x100,0x6b,0x100,0x6c,0x100, +0x6d,0x100,0x6e,0x100,0x70,0x100,0x71,0x100,0x72,0x100,0x73,0x100,0x74,0x100,0x75,0x100, +0x76,0x100,0x77,0x100,0x78,0x100,0x79,0x100,0x7a,0x100,0x61,0x100,0x62,0x100,0x63,0x100, +0x64,0x100,0x65,0x100,0x66,0x100,0x67,0x100,0x68,0x100,0x69,0x100,0x6a,0x100,0x6b,0x100, +0x6c,0x100,0x6d,0x100,0x6e,0x100,0x6f,0x100,0x70,0x100,0x71,0x100,0x72,0x100,0x73,0x100, +0x74,0x100,0x75,0x100,0x76,0x100,0x77,0x100,0x78,0x100,0x79,0x100,0x7a,0x100,0x61,0x100, +0x62,0x100,0x63,0x100,0x64,0x100,0x65,0x100,0x66,0x100,0x67,0x100,0x68,0x100,0x69,0x100, +0x6a,0x100,0x6b,0x100,0x6c,0x100,0x6d,0x100,0x6e,0x100,0x6f,0x100,0x70,0x100,0x71,0x100, +0x72,0x100,0x73,0x100,0x74,0x100,0x75,0x100,0x76,0x100,0x77,0x100,0x78,0x100,0x79,0x100, +0x7a,0x100,0x61,0x100,0x62,0x100,0x63,0x100,0x64,0x100,0x65,0x100,0x66,0x100,0x67,0x100, +0x68,0x100,0x69,0x100,0x6a,0x100,0x6b,0x100,0x6c,0x100,0x6d,0x100,0x6e,0x100,0x6f,0x100, +0x70,0x100,0x71,0x100,0x72,0x100,0x73,0x100,0x74,0x100,0x75,0x100,0x76,0x100,0x77,0x100, +0x78,0x100,0x79,0x100,0x7a,0x100,0x61,0x100,0x62,0x100,0x63,0x100,0x64,0x100,0x65,0x100, +0x66,0x100,0x67,0x100,0x68,0x100,0x69,0x100,0x6a,0x100,0x6b,0x100,0x6c,0x100,0x6d,0x100, +0x6e,0x100,0x6f,0x100,0x70,0x100,0x71,0x100,0x72,0x100,0x73,0x100,0x74,0x100,0x75,0x100, +0x76,0x100,0x77,0x100,0x78,0x100,0x79,0x100,0x7a,0x100,0x61,0x100,0x62,0x100,0x63,0x100, +0x64,0x100,0x65,0x100,0x66,0x100,0x67,0x100,0x68,0x100,0x69,0x100,0x6a,0x100,0x6b,0x100, +0x6c,0x100,0x6d,0x100,0x6e,0x100,0x6f,0x100,0x70,0x100,0x71,0x100,0x72,0x100,0x73,0x100, +0x74,0x100,0x75,0x100,0x76,0x100,0x77,0x100,0x78,0x100,0x79,0x100,0x7a,0x100,0x61,0x100, +0x62,0x100,0x63,0x100,0x64,0x100,0x65,0x100,0x66,0x100,0x67,0x100,0x68,0x100,0x69,0x100, +0x6a,0x100,0x6b,0x100,0x6c,0x100,0x6d,0x100,0x6e,0x100,0x6f,0x100,0x70,0x100,0x71,0x100, +0x72,0x100,0x73,0x100,0x74,0x100,0x75,0x100,0x76,0x100,0x77,0x100,0x78,0x100,0x79,0x100, +0x7a,0x100,0x61,0x100,0x62,0x100,0x63,0x100,0x64,0x100,0x65,0x100,0x66,0x100,0x67,0x100, +0x68,0x100,0x69,0x100,0x6a,0x100,0x6b,0x100,0x6c,0x100,0x6d,0x100,0x6e,0x100,0x6f,0x100, +0x70,0x100,0x71,0x100,0x72,0x100,0x73,0x100,0x74,0x100,0x75,0x100,0x76,0x100,0x77,0x100, +0x78,0x100,0x79,0x100,0x7a,0x100,0x61,0x100,0x62,0x100,0x63,0x100,0x64,0x100,0x65,0x100, +0x66,0x100,0x67,0x100,0x68,0x100,0x69,0x100,0x6a,0x100,0x6b,0x100,0x6c,0x100,0x6d,0x100, +0x6e,0x100,0x6f,0x100,0x70,0x100,0x71,0x100,0x72,0x100,0x73,0x100,0x74,0x100,0x75,0x100, +0x76,0x100,0x77,0x100,0x78,0x100,0x79,0x100,0x7a,0x100,0x61,0x100,0x62,0x100,0x63,0x100, +0x64,0x100,0x65,0x100,0x66,0x100,0x67,0x100,0x68,0x100,0x69,0x100,0x6a,0x100,0x6b,0x100, +0x6c,0x100,0x6d,0x100,0x6e,0x100,0x6f,0x100,0x70,0x100,0x71,0x100,0x72,0x100,0x73,0x100, +0x74,0x100,0x75,0x100,0x76,0x100,0x77,0x100,0x78,0x100,0x79,0x100,0x7a,0x100,0x131,0x100, +0x237,0x100,0x2207,0x100,0x3b1,0x100,0x3b2,0x100,0x3b3,0x100,0x3b4,0x100,0x3b5,0x100,0x3b6,0x100, +0x3b7,0x100,0x3b8,0x100,0x3b9,0x100,0x3ba,0x100,0x3bb,0x100,0x3bc,0x100,0x3bd,0x100,0x3be,0x100, +0x3bf,0x100,0x3c0,0x100,0x3c1,0x100,0x3c3,0x100,0x3c4,0x100,0x3c5,0x100,0x3c6,0x100,0x3c7,0x100, +0x3c8,0x100,0x3c9,0x100,0x2202,0x100,0x3b5,0x100,0x3b8,0x100,0x3ba,0x100,0x3c6,0x100,0x3c1,0x100, +0x3c0,0x100,0x2207,0x100,0x3b1,0x100,0x3b2,0x100,0x3b3,0x100,0x3b4,0x100,0x3b5,0x100,0x3b6,0x100, +0x3b7,0x100,0x3b8,0x100,0x3b9,0x100,0x3ba,0x100,0x3bb,0x100,0x3bc,0x100,0x3bd,0x100,0x3be,0x100, +0x3bf,0x100,0x3c0,0x100,0x3c1,0x100,0x3c3,0x100,0x3c4,0x100,0x3c5,0x100,0x3c6,0x100,0x3c7,0x100, +0x3c8,0x100,0x3c9,0x100,0x2202,0x100,0x3b5,0x100,0x3b8,0x100,0x3ba,0x100,0x3c6,0x100,0x3c1,0x100, +0x3c0,0x100,0x2207,0x100,0x3b1,0x100,0x3b2,0x100,0x3b3,0x100,0x3b4,0x100,0x3b5,0x100,0x3b6,0x100, +0x3b7,0x100,0x3b8,0x100,0x3b9,0x100,0x3ba,0x100,0x3bb,0x100,0x3bc,0x100,0x3bd,0x100,0x3be,0x100, +0x3bf,0x100,0x3c0,0x100,0x3c1,0x100,0x3c3,0x100,0x3c4,0x100,0x3c5,0x100,0x3c6,0x100,0x3c7,0x100, +0x3c8,0x100,0x3c9,0x100,0x2202,0x100,0x3b5,0x100,0x3b8,0x100,0x3ba,0x100,0x3c6,0x100,0x3c1,0x100, +0x3c0,0x100,0x2207,0x100,0x3b1,0x100,0x3b2,0x100,0x3b3,0x100,0x3b4,0x100,0x3b5,0x100,0x3b6,0x100, +0x3b7,0x100,0x3b8,0x100,0x3b9,0x100,0x3ba,0x100,0x3bb,0x100,0x3bc,0x100,0x3bd,0x100,0x3be,0x100, +0x3bf,0x100,0x3c0,0x100,0x3c1,0x100,0x3c3,0x100,0x3c4,0x100,0x3c5,0x100,0x3c6,0x100,0x3c7,0x100, +0x3c8,0x100,0x3c9,0x100,0x2202,0x100,0x3b5,0x100,0x3b8,0x100,0x3ba,0x100,0x3c6,0x100,0x3c1,0x100, +0x3c0,0x100,0x2207,0x100,0x3b1,0x100,0x3b2,0x100,0x3b3,0x100,0x3b4,0x100,0x3b5,0x100,0x3b6,0x100, +0x3b7,0x100,0x3b8,0x100,0x3b9,0x100,0x3ba,0x100,0x3bb,0x100,0x3bc,0x100,0x3bd,0x100,0x3be,0x100, +0x3bf,0x100,0x3c0,0x100,0x3c1,0x100,0x3c3,0x100,0x3c4,0x100,0x3c5,0x100,0x3c6,0x100,0x3c7,0x100, +0x3c8,0x100,0x3c9,0x100,0x2202,0x100,0x3b5,0x100,0x3b8,0x100,0x3ba,0x100,0x3c6,0x100,0x3c1,0x100, +0x3c0,0x100,0x3dd,0x100,0x30,0x100,0x31,0x100,0x32,0x100,0x33,0x100,0x34,0x100,0x35,0x100, +0x36,0x100,0x37,0x100,0x38,0x100,0x39,0x100,0x30,0x100,0x31,0x100,0x32,0x100,0x33,0x100, +0x34,0x100,0x35,0x100,0x36,0x100,0x37,0x100,0x38,0x100,0x39,0x100,0x30,0x100,0x31,0x100, +0x32,0x100,0x33,0x100,0x34,0x100,0x35,0x100,0x36,0x100,0x37,0x100,0x38,0x100,0x39,0x100, +0x30,0x100,0x31,0x100,0x32,0x100,0x33,0x100,0x34,0x100,0x35,0x100,0x36,0x100,0x37,0x100, +0x38,0x100,0x39,0x100,0x30,0x100,0x31,0x100,0x32,0x100,0x33,0x100,0x34,0x100,0x35,0x100, +0x36,0x100,0x37,0x100,0x38,0x100,0x39,0,0x20,0x2a,0x4c,0x6a,0x7c,0x9c,0xc2,0xd4, +0xf4,0xfe,0x120,0x13c,0x14e,0x16e,0x194,0x1a8,0x1b4,0x1c0,0x1ce,0x1dc,0x1ea,0x1fa,0x1fc,0x200, +0x20a,0x214,0x220,0x22c,0x23c,0x24c,0x25a,0x268,0x276,0x286,0x292,0x2a0,0x2ac,0x2d0,0x2d4,0x2dc, +0x2de,0x2e4,0x2e6,0x302,0x310,0x318,0x322,0x330,0x338,0x344,0x354,0x364,0x36c,0x378,0x38e,0x39e, +0x3a6,0x3b6,0x3bc,0x3be,0x3c0,0x3c2,0x3ca,0x3d2,0x3da,0x3e0,0x3e2,0x3e4,0x3e6,0x3ee,0x3f0,0x3f2, +0x3f6,0x3fa,0x3fe,0x402,0x404,0x406,0x408,0x40a,0x40c,0x40e,0x410,0x412,0x414,0x416,0x418,0x41a, +0x41c,0x41e,0x424,0x426,0x428,0x42a,0x42c,0x42e,0x430,0x432,0x434,0x438,0x43e,0x440,0x444,0x446, +0x448,0x44a,0x452,0x456,0x458,0x460,0x462,0x464,0x466,0x468,0x46a,0x46c,0x46e,0x470,0x472,0x474, +0x476,0x478,0x47e,0x494,0x496,0x4a0,0x4a6,0x4b4,0x4b8,0x4d4,0x4d8,0x4dc,0x4e0,0x668,0x66c,0x674, +0x677,0x67a,0x67d,0x680,0x683,0x686,0x689,0x68c,0x68f,0x692,0x695,0x698,0x69b,0x69e,0x6a1,0x6a4, +0x6a7,0x6aa,0x6ad,0x6b0,0x6b3,0x6b6,0x6b9,0x6bc,0x6bf,0x6c2,0x6c5,0x6c8,0x6cb,0x6ce,0x6d1,0x6d4, +0x6d7,0x6da,0x6dd,0x6e0,0x6e3,0x6e6,0x6e9,0x6ec,0x6ef,0x6f2,0x6f5,0x6f8,0x6fb,0x6fe,0x701,0x704, +0x707,0x70a,0x70d,0x710,0x713,0x716,0x719,0x71c,0x71f,0x722,0x725,0x72b,0x731,0x737,0x73d,0x743, +0x746,0x749,0x74c,0x74f,0x752,0x755,0x758,0x75b,0x75e,0x761,0x764,0x767,0x76a,0x76d,0x770,0x773, +0x776,0x77c,0x782,0x788,0x78e,0x794,0x797,0x79a,0x79d,0x7a0,0x7a3 +}; + +static const uint16_t combiningTable[1959]={ +0x7a6,0xc0,0x7a7,0xc1,0x7a8,0x20c2,0x7a9,0xc3,0x7aa,0x20c4,0x7ab,0x20c5,0x7ad,0x100,0x7ae,0x2102, +0x7af,0x104,0x7b0,0x2226,0x7b1,0x1cd,0x7b4,0x200,0x7b5,0x202,0x7cd,0x1e00,0x7ce,0x3ea0,0x87d4,0x1ea2, +0x7a7,0x106,0x7a8,0x108,0x7ac,0x20c7,0x7b0,0x10a,0x87b1,0x10c,0x7a6,0xc8,0x7a7,0xc9,0x7a8,0x20ca, +0x7a9,0x1ebc,0x7aa,0xcb,0x7ac,0x2228,0x7ad,0x2112,0x7ae,0x114,0x7af,0x118,0x7b0,0x116,0x7b1,0x11a, +0x7b4,0x204,0x7b5,0x206,0x7ce,0x3eb8,0x7d0,0x1e18,0x7d1,0x1e1a,0x87d4,0x1eba,0x7a6,0xcc,0x7a7,0xcd, +0x7a8,0xce,0x7a9,0x128,0x7aa,0x20cf,0x7ad,0x12a,0x7ae,0x12c,0x7af,0x12e,0x7b0,0x130,0x7b1,0x1cf, +0x7b4,0x208,0x7b5,0x20a,0x7ce,0x1eca,0x7d1,0x1e2c,0x87d4,0x1ec8,0x7a6,0x1f8,0x7a7,0x143,0x7a9,0xd1, +0x7ac,0x145,0x7b0,0x1e44,0x7b1,0x147,0x7ce,0x1e46,0x7cf,0x1e48,0x87d0,0x1e4a,0x7a6,0xd2,0x7a7,0xd3, +0x7a8,0x20d4,0x7a9,0x20d5,0x7aa,0x20d6,0x7ad,0x214c,0x7ae,0x14e,0x7af,0x21ea,0x7b0,0x222e,0x7b1,0x1d1, +0x7b2,0x150,0x7b3,0x21a0,0x7b4,0x20c,0x7b5,0x20e,0x7ce,0x3ecc,0x87d4,0x1ece,0x7a6,0xd9,0x7a7,0xda, +0x7a8,0xdb,0x7a9,0x2168,0x7aa,0x20dc,0x7ab,0x16e,0x7ad,0x216a,0x7ae,0x16c,0x7af,0x172,0x7b1,0x1d3, +0x7b2,0x170,0x7b3,0x21af,0x7b4,0x214,0x7b5,0x216,0x7ce,0x1ee4,0x7d0,0x1e76,0x7d1,0x1e74,0x7d3,0x1e72, +0x87d4,0x1ee6,0x7a6,0x1ef2,0x7a7,0xdd,0x7a8,0x176,0x7a9,0x1ef8,0x7aa,0x178,0x7ad,0x232,0x7b0,0x1e8e, +0x7ce,0x1ef4,0x87d4,0x1ef6,0x7a6,0xe0,0x7a7,0xe1,0x7a8,0x20e2,0x7a9,0xe3,0x7aa,0x20e4,0x7ab,0x20e5, +0x7ad,0x101,0x7ae,0x2103,0x7af,0x105,0x7b0,0x2227,0x7b1,0x1ce,0x7b4,0x201,0x7b5,0x203,0x7cd,0x1e01, +0x7ce,0x3ea1,0x87d4,0x1ea3,0x7a7,0x107,0x7a8,0x109,0x7ac,0x20e7,0x7b0,0x10b,0x87b1,0x10d,0x7a6,0xe8, +0x7a7,0xe9,0x7a8,0x20ea,0x7a9,0x1ebd,0x7aa,0xeb,0x7ac,0x2229,0x7ad,0x2113,0x7ae,0x115,0x7af,0x119, +0x7b0,0x117,0x7b1,0x11b,0x7b4,0x205,0x7b5,0x207,0x7ce,0x3eb9,0x7d0,0x1e19,0x7d1,0x1e1b,0x87d4,0x1ebb, +0x7a6,0xec,0x7a7,0xed,0x7a8,0xee,0x7a9,0x129,0x7aa,0x20ef,0x7ad,0x12b,0x7ae,0x12d,0x7af,0x12f, +0x7b1,0x1d0,0x7b4,0x209,0x7b5,0x20b,0x7ce,0x1ecb,0x7d1,0x1e2d,0x87d4,0x1ec9,0x7a6,0x1f9,0x7a7,0x144, +0x7a9,0xf1,0x7ac,0x146,0x7b0,0x1e45,0x7b1,0x148,0x7ce,0x1e47,0x7cf,0x1e49,0x87d0,0x1e4b,0x7a6,0xf2, +0x7a7,0xf3,0x7a8,0x20f4,0x7a9,0x20f5,0x7aa,0x20f6,0x7ad,0x214d,0x7ae,0x14f,0x7af,0x21eb,0x7b0,0x222f, +0x7b1,0x1d2,0x7b2,0x151,0x7b3,0x21a1,0x7b4,0x20d,0x7b5,0x20f,0x7ce,0x3ecd,0x87d4,0x1ecf,0x7a6,0xf9, +0x7a7,0xfa,0x7a8,0xfb,0x7a9,0x2169,0x7aa,0x20fc,0x7ab,0x16f,0x7ad,0x216b,0x7ae,0x16d,0x7af,0x173, +0x7b1,0x1d4,0x7b2,0x171,0x7b3,0x21b0,0x7b4,0x215,0x7b5,0x217,0x7ce,0x1ee5,0x7d0,0x1e77,0x7d1,0x1e75, +0x7d3,0x1e73,0x87d4,0x1ee7,0x7a6,0x1ef3,0x7a7,0xfd,0x7a8,0x177,0x7a9,0x1ef9,0x7aa,0xff,0x7ab,0x1e99, +0x7ad,0x233,0x7b0,0x1e8f,0x7ce,0x1ef5,0x87d4,0x1ef7,0x7ac,0x1e10,0x7b0,0x1e0a,0x7b1,0x10e,0x7ce,0x1e0c, +0x7cf,0x1e0e,0x87d0,0x1e12,0x7ac,0x1e11,0x7b0,0x1e0b,0x7b1,0x10f,0x7ce,0x1e0d,0x7cf,0x1e0f,0x87d0,0x1e13, +0x7a7,0x1f4,0x7a8,0x11c,0x7ac,0x122,0x7ad,0x1e20,0x7ae,0x11e,0x7b0,0x120,0x87b1,0x1e6,0x7a7,0x1f5, +0x7a8,0x11d,0x7ac,0x123,0x7ad,0x1e21,0x7ae,0x11f,0x7b0,0x121,0x87b1,0x1e7,0x7a8,0x124,0x7aa,0x1e26, +0x7ac,0x1e28,0x7b0,0x1e22,0x7b1,0x21e,0x7ce,0x1e24,0x87d2,0x1e2a,0x7a8,0x125,0x7aa,0x1e27,0x7ac,0x1e29, +0x7b0,0x1e23,0x7b1,0x21f,0x7ce,0x1e25,0x7cf,0x1e96,0x87d2,0x1e2b,0x87a8,0x134,0x7a8,0x135,0x87b1,0x1f0, +0x7a7,0x1e30,0x7ac,0x136,0x7b1,0x1e8,0x7ce,0x1e32,0x87cf,0x1e34,0x7a7,0x1e31,0x7ac,0x137,0x7b1,0x1e9, +0x7ce,0x1e33,0x87cf,0x1e35,0x7a7,0x139,0x7ac,0x13b,0x7b1,0x13d,0x7ce,0x3e36,0x7cf,0x1e3a,0x87d0,0x1e3c, +0x7a7,0x13a,0x7ac,0x13c,0x7b1,0x13e,0x7ce,0x3e37,0x7cf,0x1e3b,0x87d0,0x1e3d,0x7a7,0x154,0x7ac,0x156, +0x7b0,0x1e58,0x7b1,0x158,0x7b4,0x210,0x7b5,0x212,0x7ce,0x3e5a,0x87cf,0x1e5e,0x7a7,0x155,0x7ac,0x157, +0x7b0,0x1e59,0x7b1,0x159,0x7b4,0x211,0x7b5,0x213,0x7ce,0x3e5b,0x87cf,0x1e5f,0x7a7,0x215a,0x7a8,0x15c, +0x7ac,0x15e,0x7b0,0x1e60,0x7b1,0x2160,0x7b6,0x218,0x87ce,0x3e62,0x7a7,0x215b,0x7a8,0x15d,0x7ac,0x15f, +0x7b0,0x1e61,0x7b1,0x2161,0x7b6,0x219,0x87ce,0x3e63,0x7ac,0x162,0x7b0,0x1e6a,0x7b1,0x164,0x7b6,0x21a, +0x7ce,0x1e6c,0x7cf,0x1e6e,0x87d0,0x1e70,0x7aa,0x1e97,0x7ac,0x163,0x7b0,0x1e6b,0x7b1,0x165,0x7b6,0x21b, +0x7ce,0x1e6d,0x7cf,0x1e6f,0x87d0,0x1e71,0x7a6,0x1e80,0x7a7,0x1e82,0x7a8,0x174,0x7aa,0x1e84,0x7b0,0x1e86, +0x87ce,0x1e88,0x7a6,0x1e81,0x7a7,0x1e83,0x7a8,0x175,0x7aa,0x1e85,0x7ab,0x1e98,0x7b0,0x1e87,0x87ce,0x1e89, +0x7a7,0x179,0x7a8,0x1e90,0x7b0,0x17b,0x7b1,0x17d,0x7ce,0x1e92,0x87cf,0x1e94,0x7a7,0x17a,0x7a8,0x1e91, +0x7b0,0x17c,0x7b1,0x17e,0x7ce,0x1e93,0x87cf,0x1e95,0x7a6,0x1db,0x7a7,0x1d7,0x7ad,0x1d5,0x87b1,0x1d9, +0x7a6,0x1dc,0x7a7,0x1d8,0x7ad,0x1d6,0x87b1,0x1da,0x87ad,0x1de,0x87ad,0x1df,0x87ad,0x1e0,0x87ad,0x1e1, +0x7a7,0x1fc,0x87ad,0x1e2,0x7a7,0x1fd,0x87ad,0x1e3,0x87ad,0x1ec,0x87ad,0x1ed,0x87b1,0x1ee,0x87b1,0x1ef, +0x87a7,0x1fa,0x87a7,0x1fb,0x87a7,0x1fe,0x87a7,0x1ff,0x87ad,0x22a,0x87ad,0x22b,0x7a7,0x1e4c,0x7aa,0x1e4e, +0x87ad,0x22c,0x7a7,0x1e4d,0x7aa,0x1e4f,0x87ad,0x22d,0x87ad,0x230,0x87ad,0x231,0x7a6,0x1fed,0x7a7,0x385, +0x87d7,0x1fc1,0x7a6,0x1fba,0x7a7,0x386,0x7ad,0x1fb9,0x7ae,0x1fb8,0x7d5,0x3f08,0x7d6,0x3f09,0x87d8,0x1fbc, +0x7a6,0x1fc8,0x7a7,0x388,0x7d5,0x3f18,0x87d6,0x3f19,0x7a6,0x1fca,0x7a7,0x389,0x7d5,0x3f28,0x7d6,0x3f29, +0x87d8,0x1fcc,0x7a6,0x1fda,0x7a7,0x38a,0x7aa,0x3aa,0x7ad,0x1fd9,0x7ae,0x1fd8,0x7d5,0x3f38,0x87d6,0x3f39, +0x7a6,0x1ff8,0x7a7,0x38c,0x7d5,0x3f48,0x87d6,0x3f49,0x7a6,0x1fea,0x7a7,0x38e,0x7aa,0x3ab,0x7ad,0x1fe9, +0x7ae,0x1fe8,0x87d6,0x3f59,0x7a6,0x1ffa,0x7a7,0x38f,0x7d5,0x3f68,0x7d6,0x3f69,0x87d8,0x1ffc,0x7a6,0x1fd2, +0x7a7,0x390,0x87d7,0x1fd7,0x7a6,0x3f70,0x7a7,0x23ac,0x7ad,0x1fb1,0x7ae,0x1fb0,0x7d5,0x3f00,0x7d6,0x3f01, +0x7d7,0x3fb6,0x87d8,0x1fb3,0x7a6,0x1f72,0x7a7,0x3ad,0x7d5,0x3f10,0x87d6,0x3f11,0x7a6,0x3f74,0x7a7,0x23ae, +0x7d5,0x3f20,0x7d6,0x3f21,0x7d7,0x3fc6,0x87d8,0x1fc3,0x7a6,0x1f76,0x7a7,0x3af,0x7aa,0x23ca,0x7ad,0x1fd1, +0x7ae,0x1fd0,0x7d5,0x3f30,0x7d6,0x3f31,0x87d7,0x1fd6,0x7a6,0x1fe2,0x7a7,0x3b0,0x87d7,0x1fe7,0x7a6,0x1f7a, +0x7a7,0x3cd,0x7aa,0x23cb,0x7ad,0x1fe1,0x7ae,0x1fe0,0x7d5,0x3f50,0x7d6,0x3f51,0x87d7,0x1fe6,0x7a6,0x1f78, +0x7a7,0x3cc,0x7d5,0x3f40,0x87d6,0x3f41,0x7a6,0x3f7c,0x7a7,0x23ce,0x7d5,0x3f60,0x7d6,0x3f61,0x7d7,0x3ff6, +0x87d8,0x1ff3,0x7a7,0x3d3,0x87aa,0x3d4,0x7a6,0x400,0x7aa,0x401,0x87ae,0x4d6,0x87a7,0x403,0x87aa,0x407, +0x87a7,0x40c,0x7a6,0x40d,0x7aa,0x4e4,0x7ad,0x4e2,0x87ae,0x419,0x7aa,0x4f0,0x7ad,0x4ee,0x7ae,0x40e, +0x87b2,0x4f2,0x7a6,0x45d,0x7aa,0x4e5,0x7ad,0x4e3,0x87ae,0x439,0x7a6,0x450,0x7aa,0x451,0x87ae,0x4d7, +0x87a7,0x453,0x87aa,0x457,0x87a7,0x45c,0x7aa,0x4f1,0x7ad,0x4ef,0x7ae,0x45e,0x87b2,0x4f3,0x87b4,0x476, +0x87b4,0x477,0x7aa,0x4dc,0x87ae,0x4c1,0x7aa,0x4dd,0x87ae,0x4c2,0x7aa,0x4d2,0x87ae,0x4d0,0x7aa,0x4d3, +0x87ae,0x4d1,0x87aa,0x4da,0x87aa,0x4db,0x87aa,0x4de,0x87aa,0x4df,0x87aa,0x4e6,0x87aa,0x4e7,0x87aa,0x4ea, +0x87aa,0x4eb,0x87aa,0x4ec,0x87aa,0x4ed,0x87aa,0x4f4,0x87aa,0x4f5,0x87aa,0x4f8,0x87aa,0x4f9,0x7b7,0x622, +0x7b8,0x623,0x87b9,0x625,0x87b8,0x624,0x87b8,0x626,0x87b8,0x6c0,0x87b8,0x6c2,0x87b8,0x6d3,0x87ba,0x929, +0x87ba,0x931,0x87ba,0x934,0x7bb,0x9cb,0x87bc,0x9cc,0x7bd,0xb48,0x7be,0xb4b,0x87bf,0xb4c,0x87c0,0xb94, +0x7c0,0xbcc,0x87c1,0xbca,0x87c1,0xbcb,0x87c2,0xc48,0x87c3,0xcc0,0x7c3,0xcc7,0x7c4,0xcc8,0x87c5,0x2cca, +0x87c3,0xccb,0x7c6,0xd4a,0x87c7,0xd4c,0x87c6,0xd4b,0x7c8,0xdda,0x7c9,0x2ddc,0x87ca,0xdde,0x87c8,0xddd, +0x87cb,0x1026,0x87cc,0x1b06,0x87cc,0x1b08,0x87cc,0x1b0a,0x87cc,0x1b0c,0x87cc,0x1b0e,0x87cc,0x1b12,0x87cc,0x1b3b, +0x87cc,0x1b3d,0x87cc,0x1b40,0x87cc,0x1b41,0x87cc,0x1b43,0x7b0,0x1e02,0x7ce,0x1e04,0x87cf,0x1e06,0x7b0,0x1e03, +0x7ce,0x1e05,0x87cf,0x1e07,0x87a7,0x1e08,0x87a7,0x1e09,0x7a6,0x1e14,0x87a7,0x1e16,0x7a6,0x1e15,0x87a7,0x1e17, +0x87ae,0x1e1c,0x87ae,0x1e1d,0x87b0,0x1e1e,0x87b0,0x1e1f,0x87a7,0x1e2e,0x87a7,0x1e2f,0x87ad,0x1e38,0x87ad,0x1e39, +0x7a7,0x1e3e,0x7b0,0x1e40,0x87ce,0x1e42,0x7a7,0x1e3f,0x7b0,0x1e41,0x87ce,0x1e43,0x7a6,0x1e50,0x87a7,0x1e52, +0x7a6,0x1e51,0x87a7,0x1e53,0x7a7,0x1e54,0x87b0,0x1e56,0x7a7,0x1e55,0x87b0,0x1e57,0x87ad,0x1e5c,0x87ad,0x1e5d, +0x87b0,0x1e64,0x87b0,0x1e65,0x87b0,0x1e66,0x87b0,0x1e67,0x87b0,0x1e68,0x87b0,0x1e69,0x87a7,0x1e78,0x87a7,0x1e79, +0x87aa,0x1e7a,0x87aa,0x1e7b,0x7a9,0x1e7c,0x87ce,0x1e7e,0x7a9,0x1e7d,0x87ce,0x1e7f,0x7aa,0x1e8c,0x87b0,0x1e8a, +0x7aa,0x1e8d,0x87b0,0x1e8b,0x87b0,0x1e9b,0x7a6,0x1ea6,0x7a7,0x1ea4,0x7a9,0x1eaa,0x87d4,0x1ea8,0x7a6,0x1ea7, +0x7a7,0x1ea5,0x7a9,0x1eab,0x87d4,0x1ea9,0x7a8,0x1eac,0x87ae,0x1eb6,0x7a8,0x1ead,0x87ae,0x1eb7,0x7a6,0x1eb0, +0x7a7,0x1eae,0x7a9,0x1eb4,0x87d4,0x1eb2,0x7a6,0x1eb1,0x7a7,0x1eaf,0x7a9,0x1eb5,0x87d4,0x1eb3,0x7a6,0x1ec0, +0x7a7,0x1ebe,0x7a9,0x1ec4,0x87d4,0x1ec2,0x7a6,0x1ec1,0x7a7,0x1ebf,0x7a9,0x1ec5,0x87d4,0x1ec3,0x87a8,0x1ec6, +0x87a8,0x1ec7,0x7a6,0x1ed2,0x7a7,0x1ed0,0x7a9,0x1ed6,0x87d4,0x1ed4,0x7a6,0x1ed3,0x7a7,0x1ed1,0x7a9,0x1ed7, +0x87d4,0x1ed5,0x87a8,0x1ed8,0x87a8,0x1ed9,0x7a6,0x1edc,0x7a7,0x1eda,0x7a9,0x1ee0,0x7ce,0x1ee2,0x87d4,0x1ede, +0x7a6,0x1edd,0x7a7,0x1edb,0x7a9,0x1ee1,0x7ce,0x1ee3,0x87d4,0x1edf,0x7a6,0x1eea,0x7a7,0x1ee8,0x7a9,0x1eee, +0x7ce,0x1ef0,0x87d4,0x1eec,0x7a6,0x1eeb,0x7a7,0x1ee9,0x7a9,0x1eef,0x7ce,0x1ef1,0x87d4,0x1eed,0x7a6,0x3f02, +0x7a7,0x3f04,0x7d7,0x3f06,0x87d8,0x1f80,0x7a6,0x3f03,0x7a7,0x3f05,0x7d7,0x3f07,0x87d8,0x1f81,0x7a6,0x3f0a, +0x7a7,0x3f0c,0x7d7,0x3f0e,0x87d8,0x1f88,0x7a6,0x3f0b,0x7a7,0x3f0d,0x7d7,0x3f0f,0x87d8,0x1f89,0x7a6,0x1f12, +0x87a7,0x1f14,0x7a6,0x1f13,0x87a7,0x1f15,0x7a6,0x1f1a,0x87a7,0x1f1c,0x7a6,0x1f1b,0x87a7,0x1f1d,0x7a6,0x3f22, +0x7a7,0x3f24,0x7d7,0x3f26,0x87d8,0x1f90,0x7a6,0x3f23,0x7a7,0x3f25,0x7d7,0x3f27,0x87d8,0x1f91,0x7a6,0x3f2a, +0x7a7,0x3f2c,0x7d7,0x3f2e,0x87d8,0x1f98,0x7a6,0x3f2b,0x7a7,0x3f2d,0x7d7,0x3f2f,0x87d8,0x1f99,0x7a6,0x1f32, +0x7a7,0x1f34,0x87d7,0x1f36,0x7a6,0x1f33,0x7a7,0x1f35,0x87d7,0x1f37,0x7a6,0x1f3a,0x7a7,0x1f3c,0x87d7,0x1f3e, +0x7a6,0x1f3b,0x7a7,0x1f3d,0x87d7,0x1f3f,0x7a6,0x1f42,0x87a7,0x1f44,0x7a6,0x1f43,0x87a7,0x1f45,0x7a6,0x1f4a, +0x87a7,0x1f4c,0x7a6,0x1f4b,0x87a7,0x1f4d,0x7a6,0x1f52,0x7a7,0x1f54,0x87d7,0x1f56,0x7a6,0x1f53,0x7a7,0x1f55, +0x87d7,0x1f57,0x7a6,0x1f5b,0x7a7,0x1f5d,0x87d7,0x1f5f,0x7a6,0x3f62,0x7a7,0x3f64,0x7d7,0x3f66,0x87d8,0x1fa0, +0x7a6,0x3f63,0x7a7,0x3f65,0x7d7,0x3f67,0x87d8,0x1fa1,0x7a6,0x3f6a,0x7a7,0x3f6c,0x7d7,0x3f6e,0x87d8,0x1fa8, +0x7a6,0x3f6b,0x7a7,0x3f6d,0x7d7,0x3f6f,0x87d8,0x1fa9,0x87d8,0x1f82,0x87d8,0x1f83,0x87d8,0x1f84,0x87d8,0x1f85, +0x87d8,0x1f86,0x87d8,0x1f87,0x87d8,0x1f8a,0x87d8,0x1f8b,0x87d8,0x1f8c,0x87d8,0x1f8d,0x87d8,0x1f8e,0x87d8,0x1f8f, +0x87d8,0x1f92,0x87d8,0x1f93,0x87d8,0x1f94,0x87d8,0x1f95,0x87d8,0x1f96,0x87d8,0x1f97,0x87d8,0x1f9a,0x87d8,0x1f9b, +0x87d8,0x1f9c,0x87d8,0x1f9d,0x87d8,0x1f9e,0x87d8,0x1f9f,0x87d8,0x1fa2,0x87d8,0x1fa3,0x87d8,0x1fa4,0x87d8,0x1fa5, +0x87d8,0x1fa6,0x87d8,0x1fa7,0x87d8,0x1faa,0x87d8,0x1fab,0x87d8,0x1fac,0x87d8,0x1fad,0x87d8,0x1fae,0x87d8,0x1faf, +0x87d8,0x1fb2,0x87d8,0x1fb4,0x87d8,0x1fb7,0x87d8,0x1fc2,0x87d8,0x1fc4,0x87d8,0x1fc7,0x7a6,0x1fcd,0x7a7,0x1fce, +0x87d7,0x1fcf,0x7a6,0x1fdd,0x7a7,0x1fde,0x87d7,0x1fdf,0x7d5,0x1fe4,0x87d6,0x1fe5,0x87d6,0x1fec,0x87d8,0x1ff2, +0x87d8,0x1ff4,0x87d8,0x1ff7,0x87d9,0x8000,0x219a,0x87d9,0x8000,0x219b,0x87d9,0x8000,0x21ae,0x87d9,0x8000,0x21cd, +0x87d9,0x8000,0x21ce,0x87d9,0x8000,0x21cf,0x87d9,0x8000,0x2204,0x87d9,0x8000,0x2209,0x87d9,0x8000,0x220c,0x87d9, +0x8000,0x2224,0x87d9,0x8000,0x2226,0x87d9,0x8000,0x2241,0x87d9,0x8000,0x2244,0x87d9,0x8000,0x2247,0x87d9,0x8000, +0x2249,0x87d9,0x8000,0x2260,0x87d9,0x8000,0x2262,0x87d9,0x8000,0x226d,0x87d9,0x8000,0x226e,0x87d9,0x8000,0x226f, +0x87d9,0x8000,0x2270,0x87d9,0x8000,0x2271,0x87d9,0x8000,0x2274,0x87d9,0x8000,0x2275,0x87d9,0x8000,0x2278,0x87d9, +0x8000,0x2279,0x87d9,0x8000,0x2280,0x87d9,0x8000,0x2281,0x87d9,0x8000,0x2284,0x87d9,0x8000,0x2285,0x87d9,0x8000, +0x2288,0x87d9,0x8000,0x2289,0x87d9,0x8000,0x22ac,0x87d9,0x8000,0x22ad,0x87d9,0x8000,0x22ae,0x87d9,0x8000,0x22af, +0x87d9,0x8000,0x22e0,0x87d9,0x8000,0x22e1,0x87d9,0x8000,0x22e2,0x87d9,0x8000,0x22e3,0x87d9,0x8000,0x22ea,0x87d9, +0x8000,0x22eb,0x87d9,0x8000,0x22ec,0x87d9,0x8000,0x22ed,0x87da,0x8000,0x304c,0x87da,0x8000,0x304e,0x87da,0x8000, +0x3050,0x87da,0x8000,0x3052,0x87da,0x8000,0x3054,0x87da,0x8000,0x3056,0x87da,0x8000,0x3058,0x87da,0x8000,0x305a, +0x87da,0x8000,0x305c,0x87da,0x8000,0x305e,0x87da,0x8000,0x3060,0x87da,0x8000,0x3062,0x87da,0x8000,0x3065,0x87da, +0x8000,0x3067,0x87da,0x8000,0x3069,0x7da,0x8000,0x3070,0x87db,0x8000,0x3071,0x7da,0x8000,0x3073,0x87db,0x8000, +0x3074,0x7da,0x8000,0x3076,0x87db,0x8000,0x3077,0x7da,0x8000,0x3079,0x87db,0x8000,0x307a,0x7da,0x8000,0x307c, +0x87db,0x8000,0x307d,0x87da,0x8000,0x3094,0x87da,0x8000,0x309e,0x87da,0x8000,0x30ac,0x87da,0x8000,0x30ae,0x87da, +0x8000,0x30b0,0x87da,0x8000,0x30b2,0x87da,0x8000,0x30b4,0x87da,0x8000,0x30b6,0x87da,0x8000,0x30b8,0x87da,0x8000, +0x30ba,0x87da,0x8000,0x30bc,0x87da,0x8000,0x30be,0x87da,0x8000,0x30c0,0x87da,0x8000,0x30c2,0x87da,0x8000,0x30c5, +0x87da,0x8000,0x30c7,0x87da,0x8000,0x30c9,0x7da,0x8000,0x30d0,0x87db,0x8000,0x30d1,0x7da,0x8000,0x30d3,0x87db, +0x8000,0x30d4,0x7da,0x8000,0x30d6,0x87db,0x8000,0x30d7,0x7da,0x8000,0x30d9,0x87db,0x8000,0x30da,0x7da,0x8000, +0x30dc,0x87db,0x8000,0x30dd,0x87da,0x8000,0x30f4,0x87da,0x8000,0x30f7,0x87da,0x8000,0x30f8,0x87da,0x8000,0x30f9, +0x87da,0x8000,0x30fa,0x87da,0x8000,0x30fe,0x1234 +}; + +static const uint16_t fcdTrie_index[5496]={ +0x218,0x218,0x218,0x218,0x218,0x218,0x2d5,0x2dd,0x2e5,0x2ed,0x2f5,0x2fc,0x218,0x304,0x309,0x311, +0x317,0x31f,0x218,0x218,0x218,0x218,0x218,0x218,0x456,0x45e,0x234,0x220,0x23c,0x325,0x226,0x218, +0x32d,0x334,0x33b,0x343,0x4ab,0x218,0x34b,0x351,0x218,0x218,0x218,0x218,0x476,0x4b3,0x4bb,0x218, +0x4bf,0x359,0x466,0x47e,0x218,0x218,0x361,0x4c7,0x4cb,0x4d0,0x4d8,0x218,0x218,0x218,0x218,0x4de, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x369,0x241,0x218,0x218,0x46e,0x249,0x218, +0x218,0x251,0x259,0x218,0x218,0x46e,0x371,0x218,0x218,0x46e,0x261,0x218,0x218,0x218,0x371,0x218, +0x218,0x218,0x377,0x218,0x218,0x46e,0x371,0x218,0x218,0x218,0x371,0x218,0x218,0x218,0x37d,0x218, +0x218,0x483,0x4e4,0x218,0x218,0x48a,0x491,0x218,0x494,0x4e7,0x218,0x269,0x271,0x218,0x4ee,0x218, +0x218,0x385,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x4a4,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x4f1,0x4f1,0x218,0x218,0x218,0x218,0x4f7,0x218, +0x218,0x218,0x218,0x218,0x218,0x4ff,0x218,0x218,0x218,0x502,0x218,0x218,0x218,0x218,0x218,0x218, +0x509,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x38c,0x393,0x510,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x518,0x51b, +0x39b,0x3a2,0x3aa,0x3b1,0x3b9,0x3c1,0x3c8,0x3d0,0x3d8,0x3e0,0x3e7,0x279,0x3ef,0x281,0x289,0x291, +0x218,0x218,0x218,0x218,0x218,0x218,0x523,0x52b,0x218,0x22c,0x218,0x218,0x3f7,0x3fe,0x403,0x218, +0x40a,0x411,0x419,0x421,0x425,0x42a,0x218,0x432,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x299,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x49b,0x437,0x43e,0x446,0x437,0x43e,0x44e,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x532,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x553,0x556,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x2a1,0x2a9,0x2b1,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x4a3,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x537,0x53b,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x2b5,0x2bd,0x543,0x2c5,0x2cd,0x218, +0x218,0x218,0x54b,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218,0x218, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0xeaea,0xeaea,0xe9e9,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0xe6,0xe6,0xe6,0xe6,0xe6,0,0,0,0,0xe6,0xe6,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0xe6,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xf0f0,0xe6e6,0xdcdc,0xdcdc,0xdcdc,0xe6e6,0xe6e6,0xe6e6,0xdcdc,0xdcdc,0, +0xe6e6,0xe6e6,0xe6e6,0xdcdc,0xdcdc,0xdcdc,0xdcdc,0xe6e6,0xe8e8,0xdcdc,0xdcdc,0xe6e6,0xe9e9,0xeaea,0xeaea,0xe9e9, +0,0,0,0,0,0xe6,0xe6,0,0xe6,0xe6,0xe6,0,0xe6,0,0xe6,0xe6, +0xe6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0x909,0,0,0,0xe6e6,0xdcdc,0xe6e6,0xe6e6,0,0,0,7,7,7,7, +7,7,7,7,0,0,0,0,0,0,0,0,0,0,0,0, +0,0x909,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +7,7,0,7,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,7,0,0,7,0,0,0,0,0, +0x707,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0x909,0,0,0,0,0,0,0,0,0,0,0,7,7,7, +0,0,7,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0x909,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +7,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0x8181,0x8282,0x8182,0x8484,0x8184,0x82,0,0x82,0,0x8282,0x8282, +0x8282,0x8282,0,0,0x8282,0x8182,0xe6e6,0xe6e6,0x909,0,0xe6e6,0xe6e6,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6, +0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6, +0xe6,0xe6,0,0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0, +0xf0,0xf0,0xf0,0xf0,0xe6,0xe6,0xf0,0xf0,0xf0,0,0xe6,0xf0,0xe6,0xe6,0xe6,0xe6, +0xf0,0,0,0,0,0xe6,0xf0,0xf0,0xf0,0,0xe6,0xf0,0xe6,0xe6,0xe6,0xe6, +0xf0,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0,0,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6, +0,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6, +0xe6,0xe6,0xe6,0,0,0,0xf0,0xf0,0xf0,0,0xe6,0xf0,0xe6,0xe6,0xe6,0xe6, +0xf0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0xe,0x1a1a,0x11,0,0,0,0,0,0,0,0,0,0,0x18,0x19, +0x18,0x19,0x11,0x12,0x15,0x15,0x15,0x15,0x15,0x15,0x15,0,0x15,0x15,0x15,0x15, +0x15,0,0x15,0,0x15,0x15,0,0x15,0x15,0,0x15,0x15,0x15,0x15,0x15,0x13, +0x17,0x17,0x17,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0xd8,0xd8,0xd8,0xd8,0xd8,0xd8,0xd8,0xd8d8,0xd8d8,0x101,0x101,0x101,0,0, +0,0xe2e2,0xd8d8,0xd8d8,0xd8d8,0xd8d8,0xd8d8,0,0,0,0,0,0,0,0,0xdcdc, +0xdcdc,0xdcdc,0xdcdc,0xdcdc,0,0,0,0,0,0,0,0,0,0,0xe6e6,0xe6e6, +0xe6e6,0xe6e6,0,0,0,0,0,0,0,0,0,0,0,0,0,0xd8, +0xd8,0xd8,0xd8,0xd8,0xd8,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0,0xca,0xe6,0xe6,0xe6,0xe6, +0xe6,0xe6,0xe6,0xe6,0,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0,0,0xe6,0xe6,0xe6, +0xe6,0xe6,0,0,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0,0xca,0xe6,0xe6,0xe6,0xe6, +0xe6,0xe6,0xe6,0xe6,0,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0,0,0xe6,0xe6,0xe6, +0xe6,0xe6,0,0xe6,0xe6,0xe6,0xe6,0xe6,0xca,0xca,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6, +0xe6,0xe6,0xe6,0xe6,0,0,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xca,0xca,0xe6,0xe6, +0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xca,0xca,0xe6,0xe6,0,0,0xe6,0xe6,0xe6,0xe6, +0xe6,0xe6,0xca,0xca,0xe6,0,0,0,0xe6,0xe6,0xca,0xca,0,0xe6,0xe6,0xca, +0xca,0xe6,0xe6,0,0,0,0,0xe6,0xe6,0xca,0xca,0xe6,0xe6,0,0,0, +0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0,0,0xe6,0xe6,0xca,0xca,0xe6,0xe6,0xe6,0xe6, +0xe6,0xe6,0xca,0xca,0xe6,0xe6,0,0,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6, +0xe6,0xe6,0xca,0xca,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0, +0xd8,0xd8,0,0,0,0,0,0,0,0,0,0,0,0,0,0xd8, +0xd8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6, +0xe6,0,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0,0,0xe6,0xe6,0xe6,0xe6,0xca,0xca, +0xe6,0xe6,0xe6,0xe6,0xe6,0,0,0,0xe6,0xe6,0,0,0xe6,0xe6,0xe6,0xe6, +0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6, +0xe6,0xe6,0xe6,0xe6,0xdc,0xdc,0xdc,0xdc,0,0,0xe6,0xe6,0,0,0,0, +0,0,0xe6,0xe6,0xca,0xca,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xe6,0xe6, +0xe6,0xe6,0xe6,0xe6,0xe6,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0xe6,0xe6,0,0xe6,0,0,0,0xe6,0,0,0,0, +0xe6,0xe6,0xe6,0,0,0,0,0,0,0,0,0,0,0xe6,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0xe6,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0xe6,0xe6,0,0xe6, +0,0,0,0xe6,0,0,0,0,0xe6,0xe6,0xe6,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0xe6,0xe6,0,0,0,0,0,0,0,0,0,0xe6,0xe6,0, +0,0,0,0,0,0,0,0,0,0,0,0,0xe6,0xe6,0xe6,0xe6, +0,0,0xe6,0xe6,0,0,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0,0,0xe6,0xe6, +0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0,0,0xe6,0xe6,0,0, +0,0,0,0,0,0,0xe6,0xe6,0xe6,0xdc,0xe6,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0xe6,0,0xe6,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0xe6,0,0,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6, +0xe6e6,0,0,0xe6e6,0,0,0,0,0,0,0,0,0,7,0,0, +0,0,0,0,0,7,0,0,7,0,0,0,0,0,0,0, +0x707,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0x909,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0x5b,0,0,0,0,0x909,0,0,0,0,0,0, +0,0x5454,0x5b5b,0,0,0,0,0,0,0,0,0,0,0,0x909,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,9,0, +0,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0x707,0,0x909,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0x707,0,0,0,0,0,0,0,0,0,0,0, +0x909,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0xdc,0xdc,0xe6,0xe6, +0xdc,0xdc,0xdc,0xdc,0xe6,0xe6,0xe6,0xe6,0xdc,0xdc,0xdc,0xdc,0xca,0xca,0xdc,0xdc, +0xe6,0xe6,0xe6,0xe6,0xdc,0xdc,0xdc,0xdc,0xe6,0xe6,0xe6,0xe6,0xdc,0xdc,0xe6,0xe6, +0xca,0xca,0xdc,0xdc,0xdc,0xdc,0xe6,0xe6,0xe6,0xe6,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc, +0xe6,0xe6,0xdc,0xdc,0xdc,0xdc,0xe6,0xe6,0xe6,0xe6,0xdc,0xdc,0xe6,0xe6,0xdc,0xdc, +0xdc,0xdc,0xdc,0xdc,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6, +0xe6,0xe6,0xdc,0xdc,0xe6,0xe6,0xdc,0xdc,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6, +0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,0xe6,0xe6,0xe6,0xe6, +0xe6,0xe6,0xdc,0xdc,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xdc,0xdc,0xe6,0xe6, +0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xdc,0xdc,0xdc,0xdc,0xdc,0xe6,0xe6,0xe6,0,0xe6, +0,0,0,0,0xdc,0xdc,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6, +0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xdc,0xdc,0xe6,0xe6, +0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xdc,0xdc,0xdc,0xdc,0xe6,0xe6, +0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6, +0xe6,0xe6,0xdc,0xdc,0xdc,0xdc,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6, +0xdc,0xdc,0xe6,0xe6,0xdc,0xdc,0xe6,0xe6,0xe6,0xe6,0,0,0,0,0,0, +0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6, +0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0,0,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0,0, +0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6, +0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6, +0xe6,0xe6,0,0,0xe6,0xe6,0xe6,0xe6,0xe6,0xe6,0,0,0xe6,0xe6,0xe6,0xe6, +0xe6,0xe6,0xe6,0xe6,0,0xe6,0,0xe6,0,0xe6,0,0xe6,0xf0,0xf0,0xf0,0xf0, +0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0, +0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0, +0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,1,0,0,1,0,0,1,0,1,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0, +0,1,1,1,1,1,0,0,1,1,0,0,1,1,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,1,1,1,1,0,0,0,0, +0,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,8,0,8,0,8,0,8,0, +8,0,8,0,8,0,8,0,8,0,8,0,0,8,0,8, +0,8,0,0,0,0,0,0,8,8,0,8,8,0,8,8, +0,8,8,0,8,8,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,8,0,0,0, +0,0x808,0x808,0,0,0,8,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,8,0,0,8, +8,8,8,0,0,0,8,0,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6, +0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe8e8,0xdcdc,0xdcdc, +0xdcdc,0xdcdc,0xe8e8,0xd8d8,0xdcdc,0xdcdc,0xdcdc,0xdcdc,0xdcdc,0xcaca,0xcaca,0xdcdc,0xdcdc,0xdcdc,0xdcdc,0xcaca, +0xcaca,0xdcdc,0xdcdc,0xdcdc,0xdcdc,0xdcdc,0xdcdc,0xdcdc,0xdcdc,0xdcdc,0xdcdc,0xdcdc,0x101,0x101,0x101,0x101, +0x101,0xdcdc,0xdcdc,0xdcdc,0xdcdc,0xe6e6,0xe6e6,0xe6e6,0,0,0,0,0,0,0,0, +0,0,0,0x1b1b,0x1c1c,0x1d1d,0x1e1e,0x1f1f,0x2020,0x2121,0x2222,0xe6e6,0xe6e6,0xdcdc,0xdcdc,0xe6e6, +0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xdcdc,0xe6e6,0xe6e6,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0x707,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0xdcdc,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xdcdc,0xe6e6, +0xe6e6,0xe6e6,0xdede,0xdcdc,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0x2323,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0x6767,0x6767,0x909,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0x7676,0x7676,0,0,0,0,0,0,0,0,0,0,0x7a7a,0x7a7a,0x7a7a,0x7a7a, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0xdcdc,0xdcdc,0,0,0,0,0,0, +0,0,0,0,0,0,0xdada,0xe4e4,0xe8e8,0xdede,0xe0e0,0xe0e0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0xe6e6,0xe6e6,0xe6e6,0xe6e6, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xe6e6, +0xe6e6,0xe6e6,0xe6e6,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0xe6e6,0xe6e6,0xdcdc,0xdcdc, +0xdcdc,0xdcdc,0xdcdc,0xdcdc,0xe6e6,0xe6e6,0xdcdc,0xe6e6,0xe6e6,0xdede,0xe4e4,0xe6e6,0xa0a,0xb0b,0xc0c,0xd0d, +0xe0e,0xf0f,0x1010,0x1111,0x1212,0x1313,0x1313,0x1414,0x1515,0x1616,0,0x1717,0,0x1818,0x1919,0, +0xe6e6,0xdcdc,0,0x1212,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0xe6e6,0xe6e6,0xe6e6,0xe6e6, +0xe6e6,0xe6e6,0,0,0,0,0,0,0,0,0,0,0xe6e6,0xe6e6,0xe6e6,0xdcdc, +0xe6e6,0,0,0xe6e6,0xe6e6,0,0xdcdc,0xe6e6,0xe6e6,0xdcdc,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0x2424,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0xe6e6,0xdcdc,0xe6e6,0xe6e6,0xdcdc,0xe6e6,0xe6e6,0xdcdc,0xdcdc,0xdcdc,0xe6e6,0xdcdc,0xdcdc,0xe6e6,0xdcdc,0xe6e6, +0xe6e6,0xe6e6,0xdcdc,0xe6e6,0xdcdc,0xe6e6,0xdcdc,0xe6e6,0xdcdc,0xe6e6,0xe6e6,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xdcdc,0xe6e6,0,0,0,0, +0,0,0,0,0,0,0,0,0x6b6b,0x6b6b,0x6b6b,0x6b6b,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0xdcdc,0,0xdcdc,0,0xd8d8,0,0,0,0,0,0,0,0,0xdcdc,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0x909,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0x909,0, +0,0,0,0,0,0,0,0,0,0xe6e6,0,0,0,0,0,0, +0,0,0,0,0,0xe4e4,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0xdede,0xe6e6,0xdcdc,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0xe6e6,0xdcdc,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0xe6e6,0xdcdc,0xe6e6,0xe6e6,0xe6e6, +0xe6e6,0xe6e6,0xe6e6,0xe6e6,0,0,0,0,0,0,0,0,0,0,0,0, +0xe6e6,0xe6e6,0xdcdc,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xdcdc,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0xe6e6,0xdcdc,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0xe6e6,0xe6e6,0x101,0x101, +0xe6e6,0xe6e6,0xe6e6,0xe6e6,0x101,0x101,0x101,0xe6e6,0xe6e6,0,0,0,0,0xe6e6,0,0, +0,0x101,0x101,0xe6e6,0xdcdc,0xe6e6,0x101,0x101,0xdcdc,0xdcdc,0xdcdc,0xdcdc,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0x909,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0xdcdc,0,0xe6e6,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0xe6e6,0x101,0xdcdc,0,0,0,0,0x909,0xdcdc,0xdcdc,0xdcdc,0, +0,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xe6e6,0xdcdc,0xdcdc,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xe6e6,0xe6e6, +0xe6e6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0x820,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0x840,0,0,0, +0,0,0,0,0,0,0,0 +}; + +static const UTrie fcdTrie={ + fcdTrie_index, + NULL, + utrie_defaultGetFoldingOffset, + 2144, + 3352, + 0, + FALSE +}; + +static const uint16_t auxTrie_index[5748]={ +0x228,0x228,0x228,0x228,0x228,0x228,0x3ea,0x3f2,0x3fa,0x402,0x40a,0x412,0x228,0x228,0x41a,0x422, +0x42a,0x432,0x228,0x228,0x228,0x228,0x228,0x228,0x4be,0x4be,0x315,0x230,0x31d,0x228,0x238,0x23e, +0x228,0x228,0x228,0x228,0x513,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x4ec,0x51b,0x523,0x228, +0x527,0x43a,0x4c6,0x448,0x228,0x228,0x43d,0x52f,0x533,0x22c,0x4cb,0x228,0x228,0x228,0x228,0x539, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x445,0x322,0x228,0x228,0x4ce,0x32a,0x228, +0x228,0x332,0x33a,0x228,0x228,0x445,0x53e,0x228,0x228,0x4ce,0x342,0x228,0x228,0x4d6,0x44d,0x228, +0x228,0x228,0x453,0x228,0x228,0x445,0x45b,0x228,0x228,0x4d6,0x44d,0x228,0x228,0x228,0x461,0x228, +0x228,0x4f4,0x502,0x228,0x228,0x4fb,0x502,0x228,0x4fb,0x542,0x34a,0x352,0x35a,0x362,0x549,0x228, +0x228,0x469,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x4de,0x228,0x4e4,0x4cd,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x50c,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x447,0x447,0x228,0x228,0x228,0x228,0x54d,0x228, +0x228,0x228,0x228,0x228,0x228,0x53f,0x228,0x228,0x228,0x555,0x228,0x228,0x228,0x228,0x228,0x228, +0x55c,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x470,0x477,0x539,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x245,0x24d,0x228,0x228,0x228,0x4cb,0x43b, +0x47f,0x487,0x48c,0x492,0x49a,0x4a2,0x4a5,0x4a9,0x228,0x228,0x228,0x369,0x4af,0x371,0x379,0x37f, +0x387,0x228,0x228,0x228,0x228,0x253,0x563,0x56b,0x25b,0x263,0x26b,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x38d,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x390,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x508,0x228,0x228,0x4b6,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x26f,0x228,0x228,0x228,0x274,0x228,0x228,0x228,0x228,0x278,0x280,0x286,0x28e,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x549,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x58c,0x58f,0x228,0x595,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x398,0x398,0x398,0x398,0x398,0x398,0x398,0x398, +0x39d,0x3a5,0x398,0x3ab,0x398,0x398,0x3af,0x228,0x3b6,0x3be,0x3c6,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x50b,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x570,0x574,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x3ca,0x3d2,0x57c,0x3da,0x3e2,0x228, +0x228,0x228,0x584,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0x296,0x29d,0x299,0x2a0,0x2a8,0x2b0,0x29e,0x29a,0x2b7,0x2bf,0x2c7,0x29f,0x2a7,0x296,0x29d,0x299, +0x2a0,0x2cf,0x297,0x29e,0x29a,0x2d7,0x2df,0x2e7,0x2ee,0x2f6,0x2e2,0x2fe,0x2f1,0x306,0x30d,0x228, +0x398,0x398,0x398,0x398,0x398,0x398,0x398,0x398,0x398,0x398,0x398,0x398,0x398,0x398,0x398,0x398, +0x399,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228,0x228, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800, +0,0,0,0,0x400,0,0,0,0,0,1,0,0,0,0x400,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,4,6,8,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0xa,0,0,0,0,0, +0,0xb,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0xc,0xe,0x10,0,0x12,0x14,0x16,0x18,0x1a,0x1c,0x1e,0x20,0x22,0x24,0x26,0, +0x28,0x2a,0x2c,0x2e,0x30,0x32,0x34,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0x36,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0x39,0x3b, +0,0,0,0x3e,0,0x40,0,0x1b,0x1b,0x1b,0,0,0x1d,0x1d,0x23,0, +0,0x27,0x43,0,0,0x2d,0x46,0x2f,0x2f,0x2f,0,0,0x48,0x4b,0x4f,0, +0x52,0,0x400,0,0x53,0,0x400,0x400,0x11,0x3a,0,0,0x15,0x42,0,0x25, +0,0,0,0,0,0,0,0x54,0,0,0x58,0x5a,0,0,0,0, +0,0x13,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0x5c,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0x60,0,0x63,0x66, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0x6a,0,0x6e,0,0x71,0,0,0,0,0x74,0,0,0,0,0, +0x77,0x7a,0x7d,0x80,0x83,0x86,0x89,0x8c,0,0,0x8f,0x92,0x95,0,0,0, +0x98,0x9b,0x9f,0xa3,0xa7,0,0,0,0,0,0,0,0,0,0,0, +0,0xab,0xae,0xb2,0xb6,0,0,0,0,0,0,0,0xba,0xbd,0xc0,0xc3, +0xc6,0xc9,0xcc,0xcf,0xd2,0xd5,0xd8,0xdb,0xde,0xe1,0,0xe4,0,0,0xe7,0xec, +0xf0,0xf3,0,0xf6,0,0xf9,0xfc,0,0,0,0,0,0,0,0,0xff, +0,0x102,0x106,0,0x109,0x10c,0x10f,0x113,0xd,0x11,0x3a,0x13,0x15,0x42,0x19,0x1b, +0x1d,0x1f,0x21,0x23,0x25,0x27,0x29,0x2d,0x47,0x2f,0x38,0x31,0x33,0x65,0x35,0x57, +0xf5,0x53,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0xd,0x11,0x3a,0x13,0x15,0x42,0x19,0x1b, +0x1d,0x1f,0x21,0x23,0x25,0x27,0x29,0x2d,0x47,0x2f,0x38,0x31,0x33,0x65,0x35,0x57, +0xf5,0x53,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0xd,0,0x3a,0x13, +0,0,0x19,0,0,0x1f,0x21,0,0,0x27,0x29,0x2d,0x47,0,0x38,0x31, +0x33,0x65,0x35,0x57,0xf5,0x53,0,0,0,0,0,0,0,0,0,0, +0xd,0x11,0,0x13,0x15,0x42,0x19,0,0,0x1f,0x21,0x23,0x25,0x27,0x29,0x2d, +0x47,0,0x38,0x31,0x33,0x65,0x35,0x57,0xf5,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0xd,0x11,0,0x13,0x15,0x42,0x19,0,0x1d,0x1f,0x21,0x23, +0x25,0,0x29,0,0,0,0x38,0x31,0x33,0x65,0x35,0x57,0xf5,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0xf5,0x53,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0xd,0x11,0x3a,0x13,0,0,0,0, +0,0,0,0,0x117,0x119,0x59,0x11b,0x11d,0x11f,0x121,0x123,3,0x125,0x127,0x7e, +0x129,0x12b,0x12d,0x5b,0x12f,0x124,0xb,0x131,5,0x133,0x135,0x137,0xe0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xb, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0x118,0x11a, +0x59,0x11c,0x11e,0x120,0x122,0x124,3,0x126,0x128,0x7e,0x12a,0x12c,0x12e,0x5b,0x130,0x124, +0xb,0x132,5,0x134,0x136,0x138,0xe0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0xb,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0x118,0x11a,0x59,0x11c,0x11e,0x120,0x122,0x124,3,0x126,0x128,0x7e, +0x12a,0x12c,0x12e,0x5b,0x130,0x124,0xb,0x132,5,0x134,0x136,0x138,0xe0,0,0,0, +0,0,0,0,0,0,0,0,0x128,0x7e,0x12a,0x12c,0x12e,0x5b,0x130,0x124, +0xb,0x132,5,0x134,0x136,0x138,0xe0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0x130,0x124,0xb,0x132,5,0x134,0x136,0x138, +0xe0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0xb,0,0,0,0,0,0,0,0,0,0,0x139,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0xc00,0xc00,0x800,0xc00,0xc00,0x800,0x800,0x800,0x800,0x800,0x800,0x800, +0x800,0x800,0x800,0,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800, +0x800,0x800,0x800,0x800,0,0,0,0,0,0,0,0x400,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0x800,0,0,0,0x800,0x800,0x800,0x800,0,0,0, +0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0,0,0,0,0,0,0,0, +0,0,0,0,0,0x800,0,0,0,0,0,0,0,0,0,0x800, +0,0,0,0,0x400,0x400,0,0x400,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0x400,0,0,0x400,0, +0,0,0,0,0x800,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0x800,0,0,0,0,0,0,0,0,0,0, +0,0x400,0x400,0x400,0,0,0x400,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0x800,0,0,0,0,0,0,0,0,0x800,0x800, +0,0,0,0,0x400,0x400,0,0,0,0,0,0x400,0,0,0,0, +0,0,0,0,0,0x400,0,0,0,0,0x400,0,0,0,0,0x400, +0,0,0,0,0x400,0,0,0,0,0,0,0,0,0,0,0, +0,0x400,0,0,0,0,0,0,0,0x800,0x800,0x400,0x800,0x400,0x400,0, +0x400,0,0x800,0x800,0x800,0x800,0,0,0x800,0x400,0x800,0x800,0x800,0,0x800,0x800, +0,0,0,0,0,0,0,0,0,0,0,0x400,0,0,0,0, +0,0,0,0,0,0x400,0,0,0,0,0x400,0,0,0,0,0x400, +0,0,0,0,0x400,0,0,0,0,0,0,0,0,0x800,0,0x800, +0,0x400,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0x400,0,0x400,0,0x400,0,0x400,0,0x400,0,0x400, +0,0x400,0,0,0x1000,0x1000,0,0,0,0,0,0,0x1000,0x1000,0,0, +0,0,0,0,0,0,0,0x1000,0,0,0,0,0,0,0,0x400, +0x1000,0,0x400,0,0,0,0,0x1000,0,0,0,0,0,0x400,0,0x400, +0x1000,0,0,0,0,0,0,0x400,0,0,0,0,0,0,0,0x400, +0,0,0,0,0,0,0,0x400,0,0,0x400,0x400,0,0,0,0x1000, +0,0,0,0,0,0x400,0,0x400,0x1000,0x400,0,0,0x400,0x400,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0x400,0x400,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0x400,0,0,0, +0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400, +0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400, +0x400,0x400,0,0,0x400,0,0x400,0,0,0x400,0x400,0x400,0x400,0x400,0x400,0x400, +0x400,0x400,0x400,0,0x400,0,0x400,0,0,0x400,0x400,0,0,0,0x400,0x400, +0x400,0x400,0,0,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400, +0x400,0x400,0x400,0x400,0x400,0x400,0x400,0,0,0,0,0,0x400,0x400,0x400,0x400, +0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400, +0x400,0x400,0x400,0x400,0x400,0x400,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0x400,0x800,0x400,0,0,0,0,0,0,0,0, +0,0,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0, +0x400,0x400,0x400,0x400,0x400,0,0x400,0,0x400,0x400,0,0x400,0x400,0,0x400,0x400, +0x400,0x400,0x400,0x400,0x400,0x400,0x400,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0x400,0x400,0x400,0x400,0x400,0x400,0x400,0x800,0x800,0x800, +0x800,0x800,0,0,0,0x800,0x800,0x800,0x800,0x800,0x800,0,0,0,0,0, +0,0,0,0x800,0x800,0x800,0x800,0x800,0,0,0,0,0,0,0,0, +0,0,0x800,0x800,0x800,0x800,0,0,0,0,0,0,0,0,0,0, +0,0,0,0x400,0x400,0x400,0x400,0x400,0x400,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0x1000,0x1000,0,0x1000,0,0,0,0, +0x1000,0x1000,0,0x1000,0x1000,0x1000,0x1000,0,0,0x1000,0x1000,0x1000,0,0,0,0, +0,0x1000,0x1000,0x1000,0,0x1000,0,0,0x1000,0x1000,0,0x1000,0,0,0,0, +0x1000,0x1000,0,0x1000,0x1000,0x1000,0x1000,0,0,0x1000,0x1000,0x1000,0,0,0,0, +0,0x1000,0x1000,0x1000,0,0x1000,0,0x1000,0x1000,0x1000,0,0,0,0,0x1000,0x1000, +0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0,0,0,0,0x1000,0x1000,0x1000,0x1000, +0,0,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0,0,0x1000,0x1000,0,0, +0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0,0,0x1000,0,0,0,0,0,0,0, +0,0x1000,0x1000,0,0,0x1000,0x1000,0,0,0,0,0x1000,0x1000,0,0,0x1000, +0x1000,0,0,0,0,0,0x1000,0x1000,0x1000,0x1000,0,0,0x1000,0x1000,0,0, +0x1000,0x1000,0,0,0x1000,0x1000,0,0,0,0,0,0,0x1000,0x1000,0,0, +0,0,0,0,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0,0,0x1000,0x1000,0x1000,0x1000, +0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000, +0x1000,0x1000,0x1000,0x1000,0x1000,0,0x1000,0x1000,0x1000,0x1000,0,0,0,0,0x1000,0x1000, +0x1000,0x1000,0,0,0,0,0,0,0,0,0,0,0x1000,0x1000,0,0, +0x1000,0x1000,0x1000,0x1000,0,0,0,0,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000, +0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000, +0x1000,0x1000,0x1000,0x1000,0,0,0x1000,0x1000,0,0,0,0,0,0,0,0, +0,0,0x1000,0x1000,0x1000,0x1000,0,0,0x1000,0x1000,0x1000,0x1000,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0x1000,0x1000,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0x800,0x800,0x800,0x800,0x800,0x800, +0x800,0,0,0x800,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0x800,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0x800,0,0,0,0,0,0,0,0,0,0x800,0,0,0,0, +0,0,0,0,0,0,0,0,0,0x800,0,0,0,0,0,0, +0,0x800,0x800,0,0,0,0,0,0,0,0,0,0,0,0x800,0, +0,0,0,0,0,0,0,0,0,0x800,0,0,0,0,0,0, +0,0x800,0x800,0,0,0,0,0,0,0,0,0,0,0,0x800,0, +0,0,0,0x800,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0x800,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0x800,0,0,0,0,0,0,0,0,0x800,0,0x800,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0x800,0x800,0,0,0,0,0,0,0,0,0,0, +0x800,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0x1000,0x1000,0x1000,0x1000, +0,0,0,0,0,0,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0,0,0x1000,0x1000, +0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0,0,0,0,0x1000,0x1000,0x1000,0x1000, +0x1000,0x1000,0x1000,0x1000,0,0,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000, +0x1000,0x1000,0,0,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000, +0x1000,0x1000,0x1000,0x1000,0,0,0,0,0x1000,0x1000,0,0,0x1000,0x1000,0x1000,0x1000, +0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000, +0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0,0,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000, +0,0,0,0,0,0,0x1000,0x1000,0x1000,0x1000,0,0,0,0,0x1000,0x1000, +0x1000,0x1000,0,0,0,0,0,0,0,0,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000, +0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000, +0,0,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000, +0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0x1000,0,0,0x1000,0x1000,0x1000,0x1000,0,0, +0,0,0,0,0x1000,0x1000,0,0,0,0,0,0,0x1000,0x1000,0,0, +0,0,0,0,0x1000,0x1000,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0x800,0x800,0,0,0,0,0,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800, +0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800, +0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0,0,0,0,0,0,0,0, +0,0,0,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800, +0x800,0x800,0x800,0x800,0x800,0x800,0x800,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0x800,0,0x800,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0x800,0,0,0x800,0x800,0x800,0x800,0x800,0x800,0x800, +0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0,0, +0,0,0,0,0,0,0,0,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800, +0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0x800,0x800,0x800,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0x800,0x800,0,0,0,0,0,0,0,0,0,0, +0x800,0x800,0x800,0x800,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0x800,0x800,0x800,0x800,0x800,0x800, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0x800, +0x800,0x800,0x800,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0x800,0x800,0x800,0x800, +0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800, +0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0,0x800,0,0x800,0x800,0, +0x800,0x800,0,0x800,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0x800,0x800,0x800,0x800, +0x800,0x800,0,0,0,0,0,0,0,0,0,0,0x800,0x800,0x800,0x800, +0x800,0,0,0x800,0x800,0,0x800,0x800,0x800,0x800,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0x800,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0x800, +0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0,0,0,0,0,0,0,0, +0,0,0,0,0,0x800,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0x800,0,0x800, +0,0x800,0,0,0,0,0,0,0,0,0x800,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0x800,0,0,0,0,0,0,0,0,0, +0,0x800,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0x800,0x800,0x800, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0x800,0x800,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0x800,0x800,0x800,0x800, +0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0,0,0,0,0x800,0,0, +0,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0x800,0,0x800, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0x800,0x800,0x800,0,0,0,0,0x800, +0x800,0x800,0x800,0,0,0x800,0x800,0x800,0x800,0x800,0x800,0x800,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0x800,0x800,0x800,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0x841,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0xc42,0x43,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0x444,0 +}; + +static const UTrie auxTrie={ + auxTrie_index, + NULL, + getFoldingAuxOffset, + 2208, + 3540, + 0, + FALSE +}; + +static const uint16_t canonStartSets[7724]={ +0x109b,0x8ae,0x153,0x1a9c,0x1ab4,0x1e2c,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0x32,0xc0,0xc6,0x100,0x101,0x102,0x103,0x104,0x105,0x1cd,0x1ce,0x1de,0x1df,0x1e0,0x1e1,0x1fa, +0x1fb,0x200,0x201,0x202,0x203,0x226,0x227,0x1e00,0x1e01,0x1ea0,0x1ea1,0x1ea2,0x1ea3,0x1ea4,0x1ea5,0x1ea6, +0x1ea7,0x1ea8,0x1ea9,0x1eaa,0x1eab,0x1eac,0x1ead,0x1eae,0x1eaf,0x1eb0,0x1eb1,0x1eb2,0x1eb3,0x1eb4,0x1eb5,0x1eb6, +0x1eb7,0x212b,0x212c,6,0x1e02,0x1e03,0x1e04,0x1e05,0x1e06,0x1e07,0xc,0xc7,0xc8,0x106,0x107,0x108, +0x109,0x10a,0x10b,0x10c,0x10d,0x1e08,0x1e09,0xc,0x10e,0x10f,0x1e0a,0x1e0b,0x1e0c,0x1e0d,0x1e0e,0x1e0f, +0x1e10,0x1e11,0x1e12,0x1e13,0x2c,0xc8,0xcc,0x112,0x113,0x114,0x115,0x116,0x117,0x118,0x119,0x11a, +0x11b,0x204,0x205,0x206,0x207,0x228,0x229,0x1e14,0x1e15,0x1e16,0x1e17,0x1e18,0x1e19,0x1e1a,0x1e1b,0x1e1c, +0x1e1d,0x1eb8,0x1eb9,0x1eba,0x1ebb,0x1ebc,0x1ebd,0x1ebe,0x1ebf,0x1ec0,0x1ec1,0x1ec2,0x1ec3,0x1ec4,0x1ec5,0x1ec6, +0x1ec7,0xe,0x11c,0x11d,0x11e,0x11f,0x120,0x121,0x122,0x123,0x1e6,0x1e7,0x1f4,0x1f5,0x1e20,0x1e21, +0xe,0x124,0x125,0x21e,0x21f,0x1e22,0x1e23,0x1e24,0x1e25,0x1e26,0x1e27,0x1e28,0x1e29,0x1e2a,0x1e2b,0x1a, +0xcc,0xd0,0x128,0x129,0x12a,0x12b,0x12c,0x12d,0x12e,0x12f,0x130,0x131,0x1cf,0x1d0,0x208,0x209, +0x20a,0x20b,0x1e2c,0x1e2d,0x1e2e,0x1e2f,0x1ec8,0x1ec9,0x1eca,0x1ecb,0xc,0x136,0x137,0x1e8,0x1e9,0x1e30, +0x1e31,0x1e32,0x1e33,0x1e34,0x1e35,0x212a,0x212b,0xe,0x139,0x13a,0x13b,0x13c,0x13d,0x13e,0x1e36,0x1e37, +0x1e38,0x1e39,0x1e3a,0x1e3b,0x1e3c,0x1e3d,6,0x1e3e,0x1e3f,0x1e40,0x1e41,0x1e42,0x1e43,0x12,0xd1,0xd2, +0x143,0x144,0x145,0x146,0x147,0x148,0x1f8,0x1f9,0x1e44,0x1e45,0x1e46,0x1e47,0x1e48,0x1e49,0x1e4a,0x1e4b, +0x3c,0xd2,0xd7,0x14c,0x14d,0x14e,0x14f,0x150,0x151,0x1a0,0x1a1,0x1d1,0x1d2,0x1ea,0x1eb,0x1ec, +0x1ed,0x20c,0x20d,0x20e,0x20f,0x22a,0x22b,0x22c,0x22d,0x22e,0x22f,0x230,0x231,0x1e4c,0x1e4d,0x1e4e, +0x1e4f,0x1e50,0x1e51,0x1e52,0x1e53,0x1ecc,0x1ecd,0x1ece,0x1ecf,0x1ed0,0x1ed1,0x1ed2,0x1ed3,0x1ed4,0x1ed5,0x1ed6, +0x1ed7,0x1ed8,0x1ed9,0x1eda,0x1edb,0x1edc,0x1edd,0x1ede,0x1edf,0x1ee0,0x1ee1,0x1ee2,0x1ee3,4,0x1e54,0x1e55, +0x1e56,0x1e57,0x12,0x154,0x155,0x156,0x157,0x158,0x159,0x210,0x211,0x212,0x213,0x1e58,0x1e59,0x1e5a, +0x1e5b,0x1e5c,0x1e5d,0x1e5e,0x1e5f,0x14,0x15a,0x15b,0x15c,0x15d,0x15e,0x15f,0x160,0x161,0x218,0x219, +0x1e60,0x1e61,0x1e62,0x1e63,0x1e64,0x1e65,0x1e66,0x1e67,0x1e68,0x1e69,0xe,0x162,0x163,0x164,0x165,0x21a, +0x21b,0x1e6a,0x1e6b,0x1e6c,0x1e6d,0x1e6e,0x1e6f,0x1e70,0x1e71,0x36,0xd9,0xdd,0x168,0x169,0x16a,0x16b, +0x16c,0x16d,0x16e,0x16f,0x170,0x171,0x172,0x173,0x1af,0x1b0,0x1d3,0x1d4,0x1d5,0x1d6,0x1d7,0x1d8, +0x1d9,0x1da,0x1db,0x1dc,0x214,0x215,0x216,0x217,0x1e72,0x1e73,0x1e74,0x1e75,0x1e76,0x1e77,0x1e78,0x1e79, +0x1e7a,0x1e7b,0x1ee4,0x1ee5,0x1ee6,0x1ee7,0x1ee8,0x1ee9,0x1eea,0x1eeb,0x1eec,0x1eed,0x1eee,0x1eef,0x1ef0,0x1ef1, +4,0x1e7c,0x1e7d,0x1e7e,0x1e7f,0xc,0x174,0x175,0x1e80,0x1e81,0x1e82,0x1e83,0x1e84,0x1e85,0x1e86,0x1e87, +0x1e88,0x1e89,4,0x1e8a,0x1e8b,0x1e8c,0x1e8d,0x12,0xdd,0xde,0x176,0x177,0x178,0x179,0x232,0x233, +0x1e8e,0x1e8f,0x1ef2,0x1ef3,0x1ef4,0x1ef5,0x1ef6,0x1ef7,0x1ef8,0x1ef9,0xc,0x179,0x17a,0x17b,0x17c,0x17d, +0x17e,0x1e90,0x1e91,0x1e92,0x1e93,0x1e94,0x1e95,0x30,0xe0,0xe6,0x101,0x102,0x103,0x104,0x105,0x106, +0x1ce,0x1cf,0x1df,0x1e0,0x1e1,0x1e2,0x1fb,0x1fc,0x201,0x202,0x203,0x204,0x227,0x228,0x1e01,0x1e02, +0x1ea1,0x1ea2,0x1ea3,0x1ea4,0x1ea5,0x1ea6,0x1ea7,0x1ea8,0x1ea9,0x1eaa,0x1eab,0x1eac,0x1ead,0x1eae,0x1eaf,0x1eb0, +0x1eb1,0x1eb2,0x1eb3,0x1eb4,0x1eb5,0x1eb6,0x1eb7,0x1eb8,6,0x1e03,0x1e04,0x1e05,0x1e06,0x1e07,0x1e08,0xc, +0xe7,0xe8,0x107,0x108,0x109,0x10a,0x10b,0x10c,0x10d,0x10e,0x1e09,0x1e0a,0xc,0x10f,0x110,0x1e0b, +0x1e0c,0x1e0d,0x1e0e,0x1e0f,0x1e10,0x1e11,0x1e12,0x1e13,0x1e14,0x2c,0xe8,0xec,0x113,0x114,0x115,0x116, +0x117,0x118,0x119,0x11a,0x11b,0x11c,0x205,0x206,0x207,0x208,0x229,0x22a,0x1e15,0x1e16,0x1e17,0x1e18, +0x1e19,0x1e1a,0x1e1b,0x1e1c,0x1e1d,0x1e1e,0x1eb9,0x1eba,0x1ebb,0x1ebc,0x1ebd,0x1ebe,0x1ebf,0x1ec0,0x1ec1,0x1ec2, +0x1ec3,0x1ec4,0x1ec5,0x1ec6,0x1ec7,0x1ec8,0xe,0x11d,0x11e,0x11f,0x120,0x121,0x122,0x123,0x124,0x1e7, +0x1e8,0x1f5,0x1f6,0x1e21,0x1e22,0x10,0x125,0x126,0x21f,0x220,0x1e23,0x1e24,0x1e25,0x1e26,0x1e27,0x1e28, +0x1e29,0x1e2a,0x1e2b,0x1e2c,0x1e96,0x1e97,0x18,0xec,0xf0,0x129,0x12a,0x12b,0x12c,0x12d,0x12e,0x12f, +0x130,0x1d0,0x1d1,0x209,0x20a,0x20b,0x20c,0x1e2d,0x1e2e,0x1e2f,0x1e30,0x1ec9,0x1eca,0x1ecb,0x1ecc,4, +0x135,0x136,0x1f0,0x1f1,0xa,0x137,0x138,0x1e9,0x1ea,0x1e31,0x1e32,0x1e33,0x1e34,0x1e35,0x1e36,0xe, +0x13a,0x13b,0x13c,0x13d,0x13e,0x13f,0x1e37,0x1e38,0x1e39,0x1e3a,0x1e3b,0x1e3c,0x1e3d,0x1e3e,6,0x1e3f, +0x1e40,0x1e41,0x1e42,0x1e43,0x1e44,0x12,0xf1,0xf2,0x144,0x145,0x146,0x147,0x148,0x149,0x1f9,0x1fa, +0x1e45,0x1e46,0x1e47,0x1e48,0x1e49,0x1e4a,0x1e4b,0x1e4c,0x3c,0xf2,0xf7,0x14d,0x14e,0x14f,0x150,0x151, +0x152,0x1a1,0x1a2,0x1d2,0x1d3,0x1eb,0x1ec,0x1ed,0x1ee,0x20d,0x20e,0x20f,0x210,0x22b,0x22c,0x22d, +0x22e,0x22f,0x230,0x231,0x232,0x1e4d,0x1e4e,0x1e4f,0x1e50,0x1e51,0x1e52,0x1e53,0x1e54,0x1ecd,0x1ece,0x1ecf, +0x1ed0,0x1ed1,0x1ed2,0x1ed3,0x1ed4,0x1ed5,0x1ed6,0x1ed7,0x1ed8,0x1ed9,0x1eda,0x1edb,0x1edc,0x1edd,0x1ede,0x1edf, +0x1ee0,0x1ee1,0x1ee2,0x1ee3,0x1ee4,4,0x1e55,0x1e56,0x1e57,0x1e58,0x12,0x155,0x156,0x157,0x158,0x159, +0x15a,0x211,0x212,0x213,0x214,0x1e59,0x1e5a,0x1e5b,0x1e5c,0x1e5d,0x1e5e,0x1e5f,0x1e60,0x14,0x15b,0x15c, +0x15d,0x15e,0x15f,0x160,0x161,0x162,0x219,0x21a,0x1e61,0x1e62,0x1e63,0x1e64,0x1e65,0x1e66,0x1e67,0x1e68, +0x1e69,0x1e6a,0x10,0x163,0x164,0x165,0x166,0x21b,0x21c,0x1e6b,0x1e6c,0x1e6d,0x1e6e,0x1e6f,0x1e70,0x1e71, +0x1e72,0x1e97,0x1e98,0x36,0xf9,0xfd,0x169,0x16a,0x16b,0x16c,0x16d,0x16e,0x16f,0x170,0x171,0x172, +0x173,0x174,0x1b0,0x1b1,0x1d4,0x1d5,0x1d6,0x1d7,0x1d8,0x1d9,0x1da,0x1db,0x1dc,0x1dd,0x215,0x216, +0x217,0x218,0x1e73,0x1e74,0x1e75,0x1e76,0x1e77,0x1e78,0x1e79,0x1e7a,0x1e7b,0x1e7c,0x1ee5,0x1ee6,0x1ee7,0x1ee8, +0x1ee9,0x1eea,0x1eeb,0x1eec,0x1eed,0x1eee,0x1eef,0x1ef0,0x1ef1,0x1ef2,4,0x1e7d,0x1e7e,0x1e7f,0x1e80,0xe, +0x175,0x176,0x1e81,0x1e82,0x1e83,0x1e84,0x1e85,0x1e86,0x1e87,0x1e88,0x1e89,0x1e8a,0x1e98,0x1e99,4,0x1e8b, +0x1e8c,0x1e8d,0x1e8e,0x14,0xfd,0xfe,0xff,0x100,0x177,0x178,0x233,0x234,0x1e8f,0x1e90,0x1e99,0x1e9a, +0x1ef3,0x1ef4,0x1ef5,0x1ef6,0x1ef7,0x1ef8,0x1ef9,0x1efa,0xc,0x17a,0x17b,0x17c,0x17d,0x17e,0x17f,0x1e91, +0x1e92,0x1e93,0x1e94,0x1e95,0x1e96,6,0x385,0x386,0x1fc1,0x1fc2,0x1fed,0x1fef,4,0x1e2,0x1e3,0x1fc, +0x1fd,4,0x1e3,0x1e4,0x1fd,0x1fe,8,0x386,0x387,0x1f08,0x1f10,0x1f88,0x1f90,0x1fb8,0x1fbd,6, +0x388,0x389,0x1f18,0x1f1e,0x1fc8,0x1fca,8,0x389,0x38a,0x1f28,0x1f30,0x1f98,0x1fa0,0x1fca,0x1fcd,8, +0x38a,0x38b,0x3aa,0x3ab,0x1f38,0x1f40,0x1fd8,0x1fdc,6,0x38c,0x38d,0x1f48,0x1f4e,0x1ff8,0x1ffa,0xe, +0x38e,0x38f,0x3ab,0x3ac,0x1f59,0x1f5a,0x1f5b,0x1f5c,0x1f5d,0x1f5e,0x1f5f,0x1f60,0x1fe8,0x1fec,0xa,0x38f, +0x390,0x1f68,0x1f70,0x1fa8,0x1fb0,0x1ffa,0x1ffd,0x2126,0x2127,0xc,0x3ac,0x3ad,0x1f00,0x1f08,0x1f70,0x1f72, +0x1f80,0x1f88,0x1fb0,0x1fb5,0x1fb6,0x1fb8,6,0x3ad,0x3ae,0x1f10,0x1f16,0x1f72,0x1f74,0xc,0x3ae,0x3af, +0x1f20,0x1f28,0x1f74,0x1f76,0x1f90,0x1f98,0x1fc2,0x1fc5,0x1fc6,0x1fc8,0x10,0x390,0x391,0x3af,0x3b0,0x3ca, +0x3cb,0x1f30,0x1f38,0x1f76,0x1f78,0x1fbe,0x1fbf,0x1fd0,0x1fd4,0x1fd6,0x1fd8,6,0x3cc,0x3cd,0x1f40,0x1f46, +0x1f78,0x1f7a,2,0x1fe4,0x1fe6,0xe,0x3b0,0x3b1,0x3cb,0x3cc,0x3cd,0x3ce,0x1f50,0x1f58,0x1f7a,0x1f7c, +0x1fe0,0x1fe4,0x1fe6,0x1fe8,0xc,0x3ce,0x3cf,0x1f60,0x1f68,0x1f7c,0x1f7e,0x1fa0,0x1fa8,0x1ff2,0x1ff5,0x1ff6, +0x1ff8,2,0x3d3,0x3d5,4,0x4d0,0x4d1,0x4d2,0x4d3,4,0x400,0x402,0x4d6,0x4d7,4,0x4c1, +0x4c2,0x4dc,0x4dd,8,0x40d,0x40e,0x419,0x41a,0x4e2,0x4e3,0x4e4,0x4e5,8,0x40e,0x40f,0x4ee, +0x4ef,0x4f0,0x4f1,0x4f2,0x4f3,4,0x4d1,0x4d2,0x4d3,0x4d4,4,0x450,0x452,0x4d7,0x4d8,4, +0x4c2,0x4c3,0x4dd,0x4de,8,0x439,0x43a,0x45d,0x45e,0x4e3,0x4e4,0x4e5,0x4e6,8,0x45e,0x45f, +0x4ef,0x4f0,0x4f1,0x4f2,0x4f3,0x4f4,2,0xfb2e,0xfb31,4,0xfb31,0xfb32,0xfb4c,0xfb4d,4,0xfb35, +0xfb36,0xfb4b,0xfb4c,4,0xfb1d,0xfb1e,0xfb39,0xfb3a,4,0xfb3b,0xfb3c,0xfb4d,0xfb4e,4,0xfb44,0xfb45, +0xfb4e,0xfb4f,4,0xfb2a,0xfb2e,0xfb49,0xfb4a,4,0x622,0x624,0x625,0x626,2,0x9cb,0x9cd,4, +0xb48,0xb49,0xb4b,0xb4d,4,0xbca,0xbcb,0xbcc,0xbcd,4,0xcc7,0xcc9,0xcca,0xccc,4,0xd4a, +0xd4b,0xd4c,0xd4d,4,0xdda,0xddb,0xddc,0xddf,6,0xf73,0xf74,0xf75,0xf76,0xf81,0xf82,2, +0xac00,0xae4c,2,0xae4c,0xb098,2,0xb098,0xb2e4,2,0xb2e4,0xb530,2,0xb530,0xb77c,2,0xb77c, +0xb9c8,2,0xb9c8,0xbc14,2,0xbc14,0xbe60,2,0xbe60,0xc0ac,2,0xc0ac,0xc2f8,2,0xc2f8,0xc544, +2,0xc544,0xc790,2,0xc790,0xc9dc,2,0xc9dc,0xcc28,2,0xcc28,0xce74,2,0xce74,0xd0c0,2, +0xd0c0,0xd30c,2,0xd30c,0xd558,2,0xd558,0xd7a4,2,0x1fcd,0x1fd0,2,0x1fdd,0x1fe0,2,0x3070, +0x3072,2,0x3073,0x3075,2,0x3076,0x3078,2,0x3079,0x307b,2,0x307c,0x307e,2,0x30d0,0x30d2, +2,0x30d3,0x30d5,2,0x30d6,0x30d8,2,0x30d9,0x30db,2,0x30dc,0x30de,0x8004,0,2,0xf80c, +2,0xf80d,0x8004,0,2,0xf813,2,0xf814,0x8004,0,2,0xf9ca,2,0xf9cb,0x8004,0, +2,0xf81f,2,0xf820,0x8004,0,2,0xf824,2,0xf825,0x8004,0,2,0xf867,2,0xf868, +0x8004,0,2,0xf868,2,0xf869,0x8004,0,2,0xf876,2,0xf877,0x8004,0,2,0xf883, +2,0xf884,0x8004,0,2,0xf888,2,0xf889,0x8004,0,2,0xf88a,2,0xf88b,0x8004,0, +2,0xf896,2,0xf897,0x8004,0,2,0xf89b,2,0xf89c,0x8004,0,2,0xf8a2,2,0xf8a3, +0x8004,0,2,0xf8a1,2,0xf8a2,0x8004,0,2,0xf8c2,2,0xf8c3,0x8004,0,2,0xf8c7, +2,0xf8c8,0x8004,0,2,0xf8d1,2,0xf8d2,0x8004,0,2,0xf8d0,2,0xf8d1,0x8004,0, +2,0xf8ce,2,0xf8cf,0x8004,0,2,0xf8de,2,0xf8df,0x8006,2,0xfad2,0xfad3,2,0xf8e7, +2,0xf8e8,0x8004,0,2,0xf8ee,2,0xf8ef,0x8004,0,2,0xf8f2,2,0xf8f3,0x8004,0, +2,0xf90a,2,0xf90b,0x8004,0,2,0xf916,2,0xf917,0x8004,0,2,0xf92a,2,0xf92b, +0x8004,0,2,0xf92c,2,0xf92e,0x8004,0,2,0xf933,2,0xf934,0x8004,0,2,0xf93e, +2,0xf93f,0x8004,0,2,0xf93f,2,0xf940,0x8006,2,0xfad4,0xfad5,2,0xf949,2,0xf94a, +0x8004,0,2,0xf94b,2,0xf94c,0x8004,0,2,0xf94c,2,0xf94d,0x8004,0,2,0xf951, +2,0xf952,0x8004,0,2,0xf958,2,0xf959,0x8004,0,2,0xf960,2,0xf961,0x8004,0, +2,0xf964,2,0xf965,0x8004,0,2,0xf967,2,0xf968,0x8004,0,2,0xf96d,2,0xf96e, +0x8004,0,2,0xf971,2,0xf972,0x8004,0,2,0xf974,2,0xf975,0x8004,0,2,0xf981, +2,0xf982,0x8004,0,2,0xf8d7,2,0xf8d8,0x8004,0,2,0xf984,2,0xf985,0x8004,0, +2,0xf98e,2,0xf98f,0x8004,0,2,0xf9a7,2,0xf9a8,0x8004,0,2,0xf9ae,2,0xf9af, +0x8004,0,2,0xf9af,2,0xf9b0,0x8004,0,2,0xf9b2,2,0xf9b3,0x8004,0,2,0xf9bf, +2,0xf9c0,0x8004,0,2,0xf9c2,2,0xf9c3,0x8004,0,2,0xf9c8,2,0xf9c9,0x8004,0, +2,0xf9cd,2,0xf9ce,0x8004,0,2,0xf9ce,2,0xf9cf,0x8004,0,2,0xf9ef,2,0xf9f0, +0x8004,0,2,0xf9f2,2,0xf9f3,0x8004,0,2,0xf9f8,2,0xf9f9,0x8004,0,2,0xf9f9, +2,0xf9fa,0x8004,0,2,0xf9fc,2,0xf9fd,0x8004,0,2,0xfa03,2,0xfa04,0x8004,0, +2,0xfa08,2,0xfa09,0x8004,0,2,0xfa0d,2,0xfa0e,0x8004,0,2,0xfa0e,2,0xfa0f, +0x8004,0,2,0xfa11,2,0xfa12,0x8004,0,2,0xfa16,2,0xfa17,0x8004,0,2,0xf801, +2,0xf802,0x8004,0,2,0xf800,2,0xf801,0x8004,0,2,0xf802,2,0xf803,0x8004,0, +2,0xf819,2,0xf81a,0x8004,0,2,0xf804,2,0xf805,0x8006,2,0xfa30,0xfa31,2,0xf805, +2,0xf806,0x8004,0,2,0xf806,2,0xf807,0x8004,0,2,0xf807,2,0xf808,0x8004,0, +2,0xf808,2,0xf809,0x8004,0,2,0xf809,2,0xf80a,0x8004,0,2,0xf80b,2,0xf80c, +0x8006,2,0xfa31,0xfa32,2,0xf80a,2,0xf80b,0x8006,2,0xfa32,0xfa33,2,0xf80e,2,0xf80f, +0x8004,0,2,0xf80f,2,0xf810,0x8004,0,2,0xf810,2,0xf811,0x8004,0,2,0xf814, +2,0xf815,0x8004,0,2,0xf811,2,0xf812,0x8004,0,2,0xf815,2,0xf816,0x8004,0, +2,0xf8d2,2,0xf8d3,0x8004,0,2,0xf8d3,2,0xf8d4,0x8004,0,2,0xf817,2,0xf818, +0x8004,0,2,0xf818,2,0xf819,0x8004,0,2,0xf81a,2,0xf81b,0x8006,2,0xfa71,0xfa72, +2,0xf81b,2,0xf81c,0x8004,0,2,0xf81d,2,0xf81e,0x8004,0,2,0xf81e,2,0xf81f, +0x8006,2,0xfa00,0xfa01,2,0xf850,2,0xf851,0x8004,0,2,0xf820,2,0xf821,0x8004,0, +2,0xf821,2,0xf822,0x8004,0,2,0xf822,2,0xf823,0x8004,0,2,0xf823,2,0xf824, +0x8004,0,2,0xf992,2,0xf993,0x8006,2,0xfa76,0xfa77,2,0xf825,2,0xf826,0x8006,2, +0xfa33,0xfa34,2,0xf826,2,0xf827,0x8006,2,0xfa34,0xfa35,2,0xf827,2,0xf828,0x8006,2, +0xfa77,0xfa78,2,0xf828,2,0xf829,0x8004,0,2,0xf829,2,0xf82a,0x8004,0,2,0xf82a, +2,0xf82b,0x8006,2,0xf963,0xf964,2,0xf82b,2,0xf82c,0x8004,0,2,0xf82c,2,0xf82d, +0x8006,2,0xfa35,0xfa36,2,0xf82d,2,0xf82e,0x8004,0,2,0xf82e,2,0xf82f,0x8004,0, +2,0xf82f,2,0xf830,0x8004,0,2,0xf830,2,0xf831,0x8004,0,2,0xf831,2,0xf834, +0x8004,0,2,0xf836,2,0xf837,0x8004,0,2,0xf837,2,0xf838,0x8004,0,2,0xf839, +2,0xf83a,0x8004,0,2,0xf83a,2,0xf83b,0x8004,0,2,0xf83b,2,0xf83c,0x8004,0, +2,0xf83d,2,0xf83e,0x8004,0,2,0xf83e,2,0xf83f,0x8004,0,2,0xf83f,2,0xf840, +0x8004,0,2,0xf83c,2,0xf83d,0x8004,0,2,0xf840,2,0xf841,0x8004,0,2,0xf841, +2,0xf842,0x8004,0,2,0xf842,2,0xf843,0x8004,0,2,0xf843,2,0xf844,0x8004,0, +2,0xf844,2,0xf845,0x8004,0,2,0xf845,2,0xf847,0x8006,2,0xfa7a,0xfa7b,2,0xf847, +2,0xf848,4,0xfa36,0xfa37,0xfa78,0xfa79,0x8004,0,2,0xf848,2,0xf849,0x8004,0,2, +0xf849,2,0xf84a,0x8004,0,2,0xf84a,2,0xf84b,0x8006,2,0xfa37,0xfa38,2,0xf84c,2, +0xf84d,0x8004,0,2,0xf84e,2,0xf84f,0x8004,0,2,0xf84f,2,0xf850,0x8004,0,2, +0xf84b,2,0xf84c,0x8004,0,2,0xf84d,2,0xf84e,0x8004,0,2,0xf855,2,0xf856,0x8004, +0,2,0xf852,2,0xf853,0x8004,0,2,0xf853,2,0xf854,0x8004,0,2,0xf854,2, +0xf855,0x8004,0,2,0xf857,2,0xf858,0x8004,0,2,0xf856,2,0xf857,4,0xfa10,0xfa11, +0xfa7c,0xfa7d,0x8004,0,2,0xf858,2,0xf859,0x8004,0,2,0xf851,2,0xf852,0x8004,0, +2,0xf85a,2,0xf85b,0x8004,0,2,0xf85b,2,0xf85c,0x8004,0,2,0xf85c,2,0xf85d, +0x8004,0,2,0xf85d,2,0xf85e,0x8004,0,2,0xf85e,2,0xf85f,0x8004,0,2,0xf85f, +2,0xf860,0x8004,0,2,0xf865,2,0xf866,0x8004,0,2,0xf862,2,0xf863,0x8004,0, +2,0xf863,2,0xf864,0x8004,0,2,0xf864,2,0xf865,0x8004,0,2,0xf866,2,0xf867, +0x8004,0,2,0xf986,2,0xf987,0x8004,0,2,0xf869,2,0xf86a,0x8004,0,2,0xf86a, +2,0xf86c,0x8004,0,2,0xf86d,2,0xf86e,0x8004,0,2,0xf86e,2,0xf86f,0x8008,4, +0xf95f,0xf960,0xf9aa,0xf9ab,2,0xf86f,2,0xf870,0x8004,0,2,0xf870,2,0xf871,0x8004,0, +2,0xf872,2,0xf873,0x8004,0,2,0xf873,2,0xf874,0x8004,0,2,0xf875,2,0xf876, +0x8004,0,2,0xf877,2,0xf878,0x8006,2,0xfa3c,0xfa3d,2,0xf878,2,0xf879,0x8004,0, +2,0xf87a,2,0xf87b,0x8004,0,2,0xf879,2,0xf87a,0x8004,0,2,0xf87c,2,0xf87d, +0x8004,0,2,0xf87f,2,0xf880,0x8004,0,2,0xf87e,2,0xf87f,0x8004,0,2,0xf880, +2,0xf881,0x8004,0,2,0xf9f4,2,0xf9f5,0x8004,0,2,0xf881,2,0xf882,0x8004,0, +2,0xf882,2,0xf883,0x8004,0,2,0xf884,2,0xf885,0x8004,0,2,0xf885,2,0xf886, +0x8004,0,2,0xf886,2,0xf887,0x8004,0,2,0xf887,2,0xf888,0x8004,0,2,0xf88b, +2,0xf88c,0x8004,0,2,0xf88c,2,0xf88d,0x8004,0,2,0xf88d,2,0xf88e,0x8006,2, +0xf928,0xf929,2,0xf88e,2,0xf88f,0x8004,0,2,0xf890,2,0xf891,0x8004,0,2,0xf894, +2,0xf896,0x8004,0,2,0xf874,2,0xf875,0x8004,0,2,0xf899,2,0xf89a,0x8004,0, +2,0xf89a,2,0xf89b,0x8004,0,2,0xf89c,2,0xf89d,0x8004,0,2,0xf89d,2,0xf89e, +0x8004,0,2,0xf89e,2,0xf89f,0x8004,0,2,0xf89f,2,0xf8a0,0x8004,0,2,0xf8a0, +2,0xf8a1,0x8006,2,0xfa3d,0xfa3e,2,0xf8a3,2,0xf8a4,0x8004,0,2,0xf8a5,2,0xf8a6, +0x8004,0,2,0xf8a6,2,0xf8a7,0x8008,0,2,0xf8a7,2,0xf8a8,2,0xf8a9,2,0xf8aa, +0x8006,2,0xfa87,0xfa88,2,0xf8a8,2,0xf8a9,0x8004,0,2,0xf8aa,2,0xf8ab,0x8008,4, +0xfa3f,0xfa40,0xfa89,0xfa8a,2,0xf8ab,2,0xf8ac,0x8004,0,2,0xf8ad,2,0xf8ae,0x8004,0, +2,0xf8ae,2,0xf8af,0x8004,0,2,0xf8ac,2,0xf8ad,0x8004,0,2,0xf8af,2,0xf8b0, +0x8008,4,0xfa40,0xfa41,0xfa8b,0xfa8c,2,0xf8b0,2,0xf8b1,0x8006,2,0xf90d,0xf90e,2,0xf8b1, +2,0xf8b2,0x8004,0,2,0xf8b2,2,0xf8b3,0x8004,0,2,0xf8b3,2,0xf8b4,0x8004,0, +2,0xf8b4,2,0xf8b5,0x8004,0,2,0xf8b5,2,0xf8b6,0x8004,0,2,0xf8b6,2,0xf8b7, +0x8004,0,2,0xf8ba,2,0xf8bb,0x8004,0,2,0xf8b9,2,0xf8ba,0x8004,0,2,0xf8b7, +2,0xf8b8,0x8004,0,2,0xf8bb,2,0xf8bc,0x8004,0,2,0xf8bc,2,0xf8bd,0x8004,0, +2,0xf8c1,2,0xf8c2,0x8004,0,2,0xf8c0,2,0xf8c1,0x8004,0,2,0xf8bd,2,0xf8be, +0x8004,0,2,0xf8bf,2,0xf8c0,0x8004,0,2,0xf8c3,2,0xf8c4,0x8004,0,2,0xf8c6, +2,0xf8c7,0x8004,0,2,0xf8c4,2,0xf8c5,0x8004,0,2,0xf8c5,2,0xf8c6,0x8006,2, +0xfa41,0xfa42,2,0xf8c8,2,0xf8c9,0x8004,0,2,0xf8c9,2,0xf8ca,0x8004,0,2,0xf8cb, +2,0xf8cc,0x8004,0,2,0xf8cd,2,0xf8ce,4,0xfa12,0xfa13,0xfa91,0xfa92,0x8006,2,0xfa43, +0xfa44,2,0xf8cf,2,0xf8d0,0x8004,0,2,0xf8d5,2,0xf8d6,0x8004,0,2,0xf8cc,2, +0xf8cd,0x8004,0,2,0xf8d4,2,0xf8d5,0x8008,4,0xf929,0xf92a,0xfa92,0xfa93,2,0xf8d8,2, +0xf8d9,0x8006,2,0xfa93,0xfa94,2,0xf8d9,2,0xf8da,0x8004,0,2,0xf8da,2,0xf8db,0x8004, +0,2,0xf8dc,2,0xf8dd,0x8004,0,2,0xf8db,2,0xf8dc,0x8004,0,2,0xf8e0,2, +0xf8e1,0x8004,0,2,0xf8df,2,0xf8e0,0x8004,0,2,0xf8e5,2,0xf8e6,0x8004,0,2, +0xf8e1,2,0xf8e2,0x8006,2,0xfa44,0xfa45,2,0xf8e2,2,0xf8e3,0x8004,0,2,0xf8e4,2, +0xf8e5,0x8004,0,2,0xf8e6,2,0xf8e7,0x8004,0,2,0xf8e8,2,0xf8e9,0x8004,0,2, +0xf8e9,2,0xf8ea,0x8004,0,2,0xf8ea,2,0xf8eb,6,0xf914,0xf915,0xf95c,0xf95d,0xf9bf,0xf9c0, +0x8004,0,2,0xf8eb,2,0xf8ec,0x8004,0,2,0xf8ed,2,0xf8ee,0x8004,0,2,0xf8ef, +2,0xf8f0,0x8004,0,2,0xf8f1,2,0xf8f2,0x8004,0,2,0xf8f3,2,0xf8f4,0x8004,0, +2,0xf8f4,2,0xf8f5,0x8008,4,0xf970,0xf971,0xfa96,0xfa97,2,0xf8f5,2,0xf8f6,0x8004,0, +2,0xf8f6,2,0xf8f7,0x8004,0,2,0xf8fa,2,0xf8fb,0x8004,0,2,0xf8fe,2,0xf8ff, +0x8004,0,2,0xf8fc,2,0xf8fd,0x8004,0,2,0xf8fd,2,0xf8fe,0x8004,0,2,0xf8ff, +2,0xf900,0x8004,0,2,0xf907,2,0xf908,0x8004,0,2,0xf900,2,0xf901,0x8008,4, +0xf9ca,0xf9cb,0xfa97,0xfa98,2,0xf902,2,0xf903,0x8004,0,2,0xf903,2,0xf904,0x8006,2, +0xfa45,0xfa46,2,0xf901,2,0xf902,0x8004,0,2,0xf904,2,0xf905,0x8004,0,2,0xf905, +2,0xf906,0x8004,0,2,0xf90e,2,0xf90f,0x8004,0,2,0xf908,2,0xf909,0x8004,0, +2,0xf909,2,0xf90a,0x8004,0,2,0xf90c,2,0xf90d,0x8006,2,0xfa99,0xfa9a,2,0xf90b, +2,0xf90c,4,0xfa47,0xfa48,0xfa9a,0xfa9b,0x8004,0,2,0xf90f,2,0xf910,0x8004,0,2, +0xf912,2,0xf913,0x8004,0,2,0xf915,2,0xf916,0x8006,2,0xfa9b,0xfa9c,2,0xf914,2, +0xf915,0x8004,0,2,0xf913,2,0xf914,0x8004,0,2,0xf917,2,0xf918,0x8004,0,2, +0xf835,2,0xf836,0x8004,0,2,0xf919,2,0xf91a,0x8004,0,2,0xf918,2,0xf919,0x8004, +0,2,0xf91a,2,0xf91b,0x8004,0,2,0xf91c,2,0xf91d,4,0xfa48,0xfa49,0xfa9c,0xfa9d, +0x8004,0,2,0xf91e,2,0xf91f,0x8004,0,2,0xf920,2,0xf921,0x8006,2,0xfa9e,0xfa9f, +2,0xf921,2,0xf922,0x8004,0,2,0xf922,2,0xf923,0x8004,0,2,0xf924,2,0xf925, +0x8004,0,2,0xf925,2,0xf926,4,0xfa16,0xfa17,0xfaa0,0xfaa1,0x8004,0,2,0xf928,2, +0xf929,4,0xf961,0xf962,0xf9db,0xf9dc,0x8004,0,2,0xf929,2,0xf92a,0x8004,0,2,0xf92b, +2,0xf92c,0x8004,0,2,0xf92e,2,0xf92f,0x8004,0,2,0xf92f,2,0xf930,0x8006,2, +0xfaa1,0xfaa2,2,0xf930,2,0xf931,0x8004,0,2,0xf931,2,0xf932,0x8004,0,2,0xf932, +2,0xf933,0x8004,0,2,0xf934,2,0xf935,0x8004,0,2,0xf936,2,0xf937,0x8006,2, +0xf962,0xf963,2,0xf938,2,0xf939,0x8004,0,2,0xf93a,2,0xf93b,4,0xfa17,0xfa18,0xfaa6, +0xfaa7,0x8006,2,0xfaa8,0xfaa9,2,0xf940,2,0xf941,0x8004,0,2,0xf945,2,0xf946,0x8004, +0,2,0xf946,2,0xf948,0x8006,2,0xfaa9,0xfaaa,2,0xf948,2,0xf949,0x8004,0,2, +0xf94a,2,0xf94b,0x8004,0,2,0xf94e,2,0xf94f,0x8006,2,0xf93b,0xf93c,2,0xf94f,2, +0xf950,0x8006,2,0xfaab,0xfaac,2,0xf950,2,0xf951,0x8006,2,0xfa50,0xfa51,2,0xf953,2, +0xf954,0x8006,2,0xfa1b,0xfa1c,2,0xf956,2,0xf957,0x8004,0,2,0xf957,2,0xf958,0x8006, +2,0xfa54,0xfa55,2,0xf959,2,0xf95a,0x8004,0,2,0xf95a,2,0xf95b,0x8004,0,2, +0xf95b,2,0xf95c,0x8004,0,2,0xf95f,2,0xf960,4,0xfa56,0xfa57,0xfaad,0xfaae,0x8004,0, +2,0xf962,2,0xf963,0x8004,0,2,0xf963,2,0xf964,0x8004,0,2,0xf966,2,0xf967, +0x8004,0,2,0xf969,2,0xf96a,0x8004,0,2,0xf968,2,0xf969,0x8004,0,2,0xf96a, +2,0xf96b,0x8004,0,2,0xf96c,2,0xf96d,0x8004,0,2,0xf96e,2,0xf96f,6,0xf996, +0xf997,0xfa57,0xfa58,0xfab0,0xfab1,0x8004,0,2,0xf96f,2,0xf970,0x8004,0,2,0xf970,2, +0xf971,0x8004,0,2,0xf976,2,0xf977,0x8004,0,2,0xf978,2,0xf979,0x8004,0,2, +0xf979,2,0xf97a,0x8008,4,0xfa5b,0xfa5c,0xfab2,0xfab3,2,0xf97a,2,0xf97b,0x8004,0,2, +0xf97d,2,0xf97e,0x8004,0,2,0xf97f,2,0xf980,0x8004,0,2,0xf8d6,2,0xf8d7,0x8004, +0,2,0xf982,2,0xf983,0x8004,0,2,0xf983,2,0xf984,0x8004,0,2,0xf985,2, +0xf986,0x8008,0,2,0xf893,2,0xf894,2,0xf98b,2,0xf98c,0x8004,0,2,0xf98c,2, +0xf98d,2,0xfa5d,0xfa5f,0x8004,0,2,0xf990,2,0xf991,0x8004,0,2,0xf98f,2,0xf990, +0x8004,0,2,0xf991,2,0xf992,0x8004,0,2,0xf993,2,0xf994,0x8004,0,2,0xf994, +2,0xf995,0x8004,0,2,0xf995,2,0xf996,0x8006,2,0xf974,0xf975,2,0xf998,2,0xf999, +0x8004,0,2,0xf996,2,0xf997,0x8004,0,2,0xf999,2,0xf99a,0x8004,0,2,0xf99c, +2,0xf99d,0x8004,0,2,0xf9a0,2,0xf9a1,0x8004,0,2,0xf99a,2,0xf99b,0x8004,0, +2,0xf99b,2,0xf99c,0x8004,0,2,0xf99d,2,0xf99e,0x8004,0,2,0xf9a1,2,0xf9a2, +0x8004,0,2,0xf9a2,2,0xf9a3,0x8004,0,2,0xf9a3,2,0xf9a4,0x8004,0,2,0xf99e, +2,0xf99f,0x8006,2,0xfa5f,0xfa60,2,0xf99f,2,0xf9a0,0x8004,0,2,0xf9a8,2,0xf9a9, +0x8004,0,2,0xf9a9,2,0xf9aa,0x8004,0,2,0xf9aa,2,0xf9ab,0x8004,0,2,0xf9ac, +2,0xf9ad,0x8004,0,2,0xf9b3,2,0xf9b4,0x8006,2,0xf936,0xf937,2,0xf9b4,2,0xf9b5, +0x8004,0,2,0xf9b5,2,0xf9b6,0x8004,0,2,0xf9b6,2,0xf9b7,0x8004,0,2,0xf9b8, +2,0xf9b9,0x8004,0,2,0xf9b7,2,0xf9b8,0x8004,0,2,0xf9ba,2,0xf9bb,0x8004,0, +2,0xf9b9,2,0xf9ba,0x8004,0,2,0xf9bc,2,0xf9bd,0x8004,0,2,0xf9bd,2,0xf9be, +0x8006,2,0xfab5,0xfab6,2,0xf9bb,2,0xf9bc,0x8004,0,2,0xf9be,2,0xf9bf,0x8004,0, +2,0xf9c0,2,0xf9c1,0x8004,0,2,0xf9c1,2,0xf9c2,0x8004,0,2,0xf9c3,2,0xf9c4, +0x8004,0,2,0xf9c4,2,0xf9c5,0x8004,0,2,0xf9c6,2,0xf9c7,0x8004,0,2,0xf9c7, +2,0xf9c8,0x8004,0,2,0xf9c9,2,0xf9ca,4,0xfa61,0xfa62,0xfab8,0xfab9,0x8004,0,2, +0xf9cf,2,0xf9d0,4,0xf96f,0xf970,0xf9a1,0xf9a2,0x8006,2,0xfabe,0xfabf,2,0xf9d0,2,0xf9d1, +4,0xfa22,0xfa23,0xfaba,0xfabb,4,0xf95d,0xf95e,0xfabd,0xfabe,4,0xfa62,0xfa63,0xfabc,0xfabd,4, +0xfa63,0xfa64,0xfabf,0xfac0,0x8006,2,0xfac0,0xfac1,2,0xf9d1,2,0xf9d2,0x8004,0,2,0xf9d2, +2,0xf9d3,0x8004,0,2,0xf9d4,2,0xf9d5,0x8004,0,2,0xf9d5,2,0xf9d6,4,0xfa65, +0xfa66,0xfac1,0xfac2,0x8004,0,2,0xf9d6,2,0xf9d7,0x8004,0,2,0xf9d7,2,0xf9d8,0x8004, +0,2,0xf9db,2,0xf9dc,0x8004,0,2,0xf9da,2,0xf9db,0x8004,0,2,0xf9dc,2, +0xf9dd,0x8004,0,2,0xf9de,2,0xf9df,0x8006,2,0xfac2,0xfac3,2,0xf9df,2,0xf9e0,0x8004, +0,2,0xf98d,2,0xf98e,4,0xfa25,0xfa26,0xfa67,0xfa68,0x8004,0,2,0xf9e2,2,0xf9e3, +0x8004,0,2,0xf9e3,2,0xf9e4,0x8004,0,2,0xf9e4,2,0xf9e5,0x8004,0,2,0xf9e6, +2,0xf9e7,0x8004,0,2,0xf9e7,2,0xf9e8,0x8004,0,2,0xf9ea,2,0xf9eb,0x8004,0, +2,0xf9e8,2,0xf9e9,0x8004,0,2,0xf9e9,2,0xf9ea,0x8004,0,2,0xf9eb,2,0xf9ec, +0x8004,0,2,0xf9ec,2,0xf9ed,0x8004,0,2,0xf9ee,2,0xf9ef,0x8004,0,2,0xf9f0, +2,0xf9f1,0x8004,0,2,0xf9f3,2,0xf9f4,4,0xfa68,0xfa69,0xfac7,0xfac8,0x8004,0,2, +0xf9f5,2,0xf9f6,4,0xfa1c,0xfa1d,0xfac8,0xfac9,0x8004,0,2,0xf9fa,2,0xf9fb,4,0xfa69, +0xfa6a,0xfaca,0xfacb,0x8006,2,0xfacb,0xfacc,2,0xf9fe,2,0xfa00,0x8004,0,2,0xfa00,2, +0xfa01,4,0xfa6a,0xfa6b,0xfacc,0xfacd,0x8004,0,2,0xfa02,2,0xfa03,0x8004,0,2,0xfa04, +2,0xfa05,0x8004,0,2,0xfa05,2,0xfa06,0x8004,0,2,0xfa06,2,0xfa07,0x8004,0, +2,0xfa07,2,0xfa08,0x8006,2,0xfacd,0xface,2,0xfa0a,2,0xfa0b,0x8004,0,2,0xfa0b, +2,0xfa0c,0x8004,0,2,0xfa0c,2,0xfa0d,0x8004,0,2,0xfa0f,2,0xfa10,0x8004,0, +2,0xfa15,2,0xfa16,0x8004,0,2,0xfa17,2,0xfa18,0x8004,0,2,0xfa18,2,0xfa19, +0x8004,0,2,0xfa19,2,0xfa1a,0x8004,0,2,0xfa1a,2,0xfa1b,0x8004,0,2,0xfa1b, +2,0xfa1c,0x8004,0,2,0xfa1c,2,0xfa1d,4,0xf907,0xf909,0xface,0xfacf,0x8004,0,1, +0xd15f,1,0xd165,0x800c,0,1,0xd1bb,1,0xd1bc,1,0xd1bd,1,0xd1be,1,0xd1bf,1, +0xd1c0,0x800c,0,1,0xd1bc,1,0xd1bd,1,0xd1be,1,0xd1bf,1,0xd1c0,1,0xd1c1,0x8004, +0,2,0xf891,2,0xf893,0x8004,0,2,0xf95d,2,0xf95f,0x3b,0x37e,0x3c,0x226e,0x3d, +0x2260,0x3e,0x226f,0x41,0x4020,0x42,0x4053,0x43,0x405a,0x44,0x4067,0x45,0x4074,0x46,0x1e1e,0x47, +0x40a1,0x48,0x40b0,0x49,0x40bf,0x4a,0x134,0x4b,0x40da,0x4c,0x40e7,0x4d,0x40f6,0x4e,0x40fd,0x4f, +0x4110,0x50,0x414d,0x52,0x4152,0x53,0x4165,0x54,0x417a,0x55,0x4189,0x56,0x41c0,0x57,0x41c5,0x58, +0x41d2,0x59,0x41d7,0x5a,0x41ea,0x60,0x1fef,0x61,0x41f7,0x62,0x4228,0x63,0x422f,0x64,0x423c,0x65, +0x4249,0x66,0x1e1f,0x67,0x4276,0x68,0x4285,0x69,0x4296,0x6a,0x42af,0x6b,0x42b4,0x6c,0x42bf,0x6d, +0x42ce,0x6e,0x42d5,0x6f,0x42e8,0x70,0x4325,0x72,0x432a,0x73,0x433d,0x74,0x4352,0x75,0x4363,0x76, +0x439a,0x77,0x439f,0x78,0x43ae,0x79,0x43b3,0x7a,0x43c8,0xa8,0x43d5,0xb4,0x1ffd,0xb7,0x387,0xc6, +0x43dc,0xd8,0x1fe,0xe6,0x43e1,0xf8,0x1ff,0x17f,0x1e9b,0x1b7,0x1ee,0x292,0x1ef,0x2b9,0x374,0x300, +0x340,0x301,0x341,0x308,0x344,0x313,0x343,0x391,0x43e6,0x395,0x43ef,0x397,0x43f6,0x399,0x43ff,0x39f, +0x4408,0x3a1,0x1fec,0x3a5,0x440f,0x3a9,0x441e,0x3b1,0x4429,0x3b5,0x4436,0x3b7,0x443d,0x3b9,0x444a,0x3bf, +0x445b,0x3c1,0x4462,0x3c5,0x4465,0x3c9,0x4474,0x3d2,0x4481,0x406,0x407,0x410,0x4484,0x413,0x403,0x415, +0x4489,0x416,0x448e,0x417,0x4de,0x418,0x4493,0x41a,0x40c,0x41e,0x4e6,0x423,0x449c,0x427,0x4f4,0x42b, +0x4f8,0x42d,0x4ec,0x430,0x44a5,0x433,0x453,0x435,0x44aa,0x436,0x44af,0x437,0x4df,0x438,0x44b4,0x43a, +0x45c,0x43e,0x4e7,0x443,0x44bd,0x447,0x4f5,0x44b,0x4f9,0x44d,0x4ed,0x456,0x457,0x474,0x476,0x475, +0x477,0x4d8,0x4da,0x4d9,0x4db,0x4e8,0x4ea,0x4e9,0x4eb,0x5d0,0x44c6,0x5d1,0x44c9,0x5d2,0xfb32,0x5d3, +0xfb33,0x5d4,0xfb34,0x5d5,0x44ce,0x5d6,0xfb36,0x5d8,0xfb38,0x5d9,0x44d3,0x5da,0xfb3a,0x5db,0x44d8,0x5dc, +0xfb3c,0x5de,0xfb3e,0x5e0,0xfb40,0x5e1,0xfb41,0x5e3,0xfb43,0x5e4,0x44dd,0x5e6,0xfb46,0x5e7,0xfb47,0x5e8, +0xfb48,0x5e9,0x44e2,0x5ea,0xfb4a,0x5f2,0xfb1f,0x627,0x44e7,0x648,0x624,0x64a,0x626,0x6c1,0x6c2,0x6d2, +0x6d3,0x6d5,0x6c0,0x915,0x958,0x916,0x959,0x917,0x95a,0x91c,0x95b,0x921,0x95c,0x922,0x95d,0x928, +0x929,0x92b,0x95e,0x92f,0x95f,0x930,0x931,0x933,0x934,0x9a1,0x9dc,0x9a2,0x9dd,0x9af,0x9df,0x9c7, +0x44ec,0xa16,0xa59,0xa17,0xa5a,0xa1c,0xa5b,0xa2b,0xa5e,0xa32,0xa33,0xa38,0xa36,0xb21,0xb5c,0xb22, +0xb5d,0xb47,0x44ef,0xb92,0xb94,0xbc6,0x44f4,0xbc7,0xbcb,0xc46,0xc48,0xcbf,0xcc0,0xcc6,0x44f9,0xd46, +0x44fe,0xd47,0xd4b,0xdd9,0x4503,0xf40,0xf69,0xf42,0xf43,0xf4c,0xf4d,0xf51,0xf52,0xf56,0xf57,0xf5b, +0xf5c,0xf71,0x4508,0xf90,0xfb9,0xf92,0xf93,0xf9c,0xf9d,0xfa1,0xfa2,0xfa6,0xfa7,0xfab,0xfac,0xfb2, +0xf76,0xfb3,0xf78,0x1025,0x1026,0x1100,0x450f,0x1101,0x4512,0x1102,0x4515,0x1103,0x4518,0x1104,0x451b,0x1105, +0x451e,0x1106,0x4521,0x1107,0x4524,0x1108,0x4527,0x1109,0x452a,0x110a,0x452d,0x110b,0x4530,0x110c,0x4533,0x110d, +0x4536,0x110e,0x4539,0x110f,0x453c,0x1110,0x453f,0x1111,0x4542,0x1112,0x4545,0x1b05,0x1b06,0x1b07,0x1b08,0x1b09, +0x1b0a,0x1b0b,0x1b0c,0x1b0d,0x1b0e,0x1b11,0x1b12,0x1b3a,0x1b3b,0x1b3c,0x1b3d,0x1b3e,0x1b40,0x1b3f,0x1b41,0x1b42, +0x1b43,0x1fbf,0x4548,0x1ffe,0x454b,0x2002,0x2000,0x2003,0x2001,0x2190,0x219a,0x2192,0x219b,0x2194,0x21ae,0x21d0, +0x21cd,0x21d2,0x21cf,0x21d4,0x21ce,0x2203,0x2204,0x2208,0x2209,0x220b,0x220c,0x2223,0x2224,0x2225,0x2226,0x223c, +0x2241,0x2243,0x2244,0x2245,0x2247,0x2248,0x2249,0x224d,0x226d,0x2261,0x2262,0x2264,0x2270,0x2265,0x2271,0x2272, +0x2274,0x2273,0x2275,0x2276,0x2278,0x2277,0x2279,0x227a,0x2280,0x227b,0x2281,0x227c,0x22e0,0x227d,0x22e1,0x2282, +0x2284,0x2283,0x2285,0x2286,0x2288,0x2287,0x2289,0x2291,0x22e2,0x2292,0x22e3,0x22a2,0x22ac,0x22a8,0x22ad,0x22a9, +0x22ae,0x22ab,0x22af,0x22b2,0x22ea,0x22b3,0x22eb,0x22b4,0x22ec,0x22b5,0x22ed,0x2add,0x2adc,0x3008,0x2329,0x3009, +0x232a,0x3046,0x3094,0x304b,0x304c,0x304d,0x304e,0x304f,0x3050,0x3051,0x3052,0x3053,0x3054,0x3055,0x3056,0x3057, +0x3058,0x3059,0x305a,0x305b,0x305c,0x305d,0x305e,0x305f,0x3060,0x3061,0x3062,0x3064,0x3065,0x3066,0x3067,0x3068, +0x3069,0x306f,0x454e,0x3072,0x4551,0x3075,0x4554,0x3078,0x4557,0x307b,0x455a,0x309d,0x309e,0x30a6,0x30f4,0x30ab, +0x30ac,0x30ad,0x30ae,0x30af,0x30b0,0x30b1,0x30b2,0x30b3,0x30b4,0x30b5,0x30b6,0x30b7,0x30b8,0x30b9,0x30ba,0x30bb, +0x30bc,0x30bd,0x30be,0x30bf,0x30c0,0x30c1,0x30c2,0x30c4,0x30c5,0x30c6,0x30c7,0x30c8,0x30c9,0x30cf,0x455d,0x30d2, +0x4560,0x30d5,0x4563,0x30d8,0x4566,0x30db,0x4569,0x30ef,0x30f7,0x30f0,0x30f8,0x30f1,0x30f9,0x30f2,0x30fa,0x30fd, +0x30fe,0x349e,0x456c,0x34b9,0x4572,0x34bb,0x4578,0x34df,0x457e,0x3515,0x4584,0x36ee,0x458a,0x36fc,0x4590,0x3781, +0x4596,0x382f,0x459c,0x3862,0x45a2,0x387c,0x45a8,0x38c7,0x45ae,0x38e3,0x45b4,0x391c,0x45ba,0x393a,0x45c0,0x3a2e, +0x45c6,0x3a6c,0x45cc,0x3ae4,0x45d2,0x3b08,0x45d8,0x3b19,0x45de,0x3b49,0x45e4,0x3b9d,0x45ea,0x3c18,0x45f2,0x3c4e, +0x45f8,0x3d33,0x45fe,0x3d96,0x4604,0x3eac,0x460a,0x3eb8,0x4610,0x3f1b,0x4616,0x3ffc,0x461c,0x4008,0x4622,0x4018, +0xfad3,0x4039,0x4628,0x4046,0x4630,0x4096,0x4636,0x40e3,0x463c,0x412f,0x4642,0x4202,0x4648,0x4227,0x464e,0x42a0, +0x4654,0x4301,0x465a,0x4334,0x4660,0x4359,0x4666,0x43d5,0x466c,0x43d9,0x4672,0x440b,0x4678,0x446b,0x467e,0x452b, +0x4684,0x455d,0x468a,0x4561,0x4690,0x456b,0x4696,0x45d7,0x469c,0x45f9,0x46a2,0x4635,0x46a8,0x46be,0x46ae,0x46c7, +0x46b4,0x4995,0x46ba,0x49e6,0x46c0,0x4a6e,0x46c6,0x4a76,0x46cc,0x4ab2,0x46d2,0x4b33,0x46d8,0x4bce,0x46de,0x4cce, +0x46e4,0x4ced,0x46ea,0x4cf8,0x46f0,0x4d56,0x46f6,0x4e0d,0xf967,0x4e26,0xfa70,0x4e32,0xf905,0x4e38,0x46fc,0x4e39, +0xf95e,0x4e3d,0x4702,0x4e41,0x4708,0x4e82,0xf91b,0x4e86,0xf9ba,0x4eae,0xf977,0x4ec0,0xf9fd,0x4ecc,0x470e,0x4ee4, +0xf9a8,0x4f60,0x4714,0x4f80,0xfa73,0x4f86,0xf92d,0x4f8b,0xf9b5,0x4fae,0x471a,0x4fbb,0x4722,0x4fbf,0xf965,0x5002, +0x4728,0x502b,0xf9d4,0x507a,0x472e,0x5099,0x4734,0x50cf,0x473a,0x50da,0xf9bb,0x50e7,0x4740,0x5140,0xfa0c,0x5145, +0xfa74,0x514d,0x4748,0x5154,0x4750,0x5164,0x4756,0x5167,0x475c,0x5168,0xfa72,0x5169,0xf978,0x516d,0xf9d1,0x5177, +0x4762,0x5180,0xfa75,0x518d,0x4768,0x5192,0x476e,0x5195,0x4774,0x5197,0x477a,0x51a4,0x4780,0x51ac,0x4786,0x51b5, +0x478c,0x51b7,0xf92e,0x51c9,0xf979,0x51cc,0xf955,0x51dc,0xf954,0x51de,0xfa15,0x51f5,0x4794,0x5203,0x479a,0x5207, +0x47a0,0x5217,0xf99c,0x5229,0xf9dd,0x523a,0xf9ff,0x523b,0x47a8,0x5246,0x47ae,0x5272,0x47b4,0x5277,0x47ba,0x5289, +0xf9c7,0x529b,0xf98a,0x52a3,0xf99d,0x52b3,0x47c0,0x52c7,0x47c6,0x52c9,0x47ce,0x52d2,0xf952,0x52de,0xf92f,0x52e4, +0x47d6,0x52f5,0xf97f,0x52fa,0x47de,0x5305,0x47e6,0x5306,0x47ec,0x5317,0x47f2,0x533f,0xf9eb,0x5349,0x47fa,0x5351, +0x4800,0x535a,0x4808,0x5373,0x480e,0x5375,0xf91c,0x537d,0x4814,0x537f,0x481a,0x53c3,0xf96b,0x53ca,0x4820,0x53df, +0x4826,0x53e5,0xf906,0x53eb,0x482c,0x53f1,0x4832,0x5406,0x4838,0x540f,0xf9de,0x541d,0xf9ed,0x5438,0x483e,0x5442, +0xf980,0x5448,0x4844,0x5468,0x484a,0x549e,0x4850,0x54a2,0x4856,0x54bd,0xf99e,0x54f6,0x485c,0x5510,0x4862,0x5553, +0x4868,0x5555,0xfa79,0x5563,0x486e,0x5584,0x4874,0x5587,0xf90b,0x5599,0x487a,0x559d,0x4882,0x55ab,0x4887,0x55b3, +0x488d,0x55c0,0xfa0d,0x55c2,0x4893,0x55e2,0xfa7b,0x5606,0x4899,0x5651,0x48a1,0x5668,0xfa38,0x5674,0x48a7,0x56f9, +0xf9a9,0x5716,0x48ad,0x5717,0x48b3,0x578b,0x48b9,0x57ce,0x48bf,0x57f4,0x48c5,0x580d,0x48cb,0x5831,0x48d1,0x5832, +0x48d7,0x5840,0xfa39,0x585a,0x48dd,0x585e,0xf96c,0x58a8,0xfa3a,0x58ac,0x48e2,0x58b3,0xfa7d,0x58d8,0xf94a,0x58df, +0xf942,0x58ee,0x48e8,0x58f2,0x48ee,0x58f7,0x48f4,0x5906,0x48fa,0x591a,0x4900,0x5922,0x4906,0x5944,0xfa7e,0x5948, +0xf90c,0x5951,0xf909,0x5954,0xfa7f,0x5962,0x490c,0x5973,0xf981,0x59d8,0x4912,0x59ec,0x4918,0x5a1b,0x491e,0x5a27, +0x4924,0x5a62,0xfa80,0x5a66,0x492a,0x5ab5,0x4930,0x5b08,0x4936,0x5b28,0xfa81,0x5b3e,0x493c,0x5b85,0xfa04,0x5bc3, +0x4942,0x5bd8,0x4948,0x5be7,0x494e,0x5bee,0xf9bc,0x5bf3,0x4958,0x5bff,0x495e,0x5c06,0x4964,0x5c22,0x496a,0x5c3f, +0xf9bd,0x5c60,0x4970,0x5c62,0xf94b,0x5c64,0xfa3b,0x5c65,0xf9df,0x5c6e,0x4976,0x5c8d,0x497e,0x5cc0,0x4984,0x5d19, +0xf9d5,0x5d43,0x498a,0x5d50,0xf921,0x5d6b,0x4990,0x5d6e,0x4996,0x5d7c,0x499c,0x5db2,0x49a2,0x5dba,0xf9ab,0x5de1, +0x49a8,0x5de2,0x49ae,0x5dfd,0x49b4,0x5e28,0x49ba,0x5e3d,0x49c0,0x5e69,0x49c6,0x5e74,0xf98e,0x5ea6,0xfa01,0x5eb0, +0x49cc,0x5eb3,0x49d2,0x5eb6,0x49d8,0x5ec9,0xf9a2,0x5eca,0x49de,0x5ed2,0xfa82,0x5ed3,0xfa0b,0x5ed9,0xfa83,0x5eec, +0xf982,0x5efe,0x49e6,0x5f04,0xf943,0x5f22,0x49ec,0x5f53,0x49f2,0x5f62,0x49f8,0x5f69,0xfa84,0x5f6b,0x49fe,0x5f8b, +0xf9d8,0x5f9a,0x4a04,0x5fa9,0xf966,0x5fad,0xfa85,0x5fcd,0x4a0a,0x5fd7,0x4a10,0x5ff5,0xf9a3,0x5ff9,0x4a16,0x6012, +0xf960,0x601c,0xf9ac,0x6081,0x4a1c,0x6094,0x4a22,0x60c7,0x4a2a,0x60d8,0xfa86,0x60e1,0xf9b9,0x6108,0xfa88,0x6144, +0xf9d9,0x6148,0x4a30,0x614c,0x4a36,0x614e,0x4a40,0x6160,0xfa8a,0x6168,0xfa3e,0x617a,0x4a48,0x618e,0x4a4e,0x6190, +0xf98f,0x61a4,0x4a58,0x61af,0x4a5e,0x61b2,0x4a64,0x61de,0x4a6a,0x61f2,0x4a70,0x61f6,0x4a7a,0x6200,0xf990,0x6210, +0x4a82,0x621b,0x4a88,0x622e,0xf9d2,0x6234,0xfa8c,0x625d,0x4a8e,0x62b1,0x4a94,0x62c9,0xf925,0x62cf,0xf95b,0x62d3, +0xfa02,0x62d4,0x4a9a,0x62fc,0x4aa0,0x62fe,0xf973,0x633d,0x4aa6,0x6350,0x4aac,0x6368,0x4ab2,0x637b,0xf9a4,0x6383, +0x4ab8,0x63a0,0xf975,0x63a9,0x4abe,0x63c4,0xfa8d,0x63c5,0x4ac4,0x63e4,0x4aca,0x641c,0xfa8e,0x6422,0x4ad0,0x6452, +0xfa8f,0x6469,0x4ad6,0x6477,0x4adc,0x647e,0x4ae2,0x649a,0xf991,0x649d,0x4ae8,0x64c4,0xf930,0x654f,0x4aee,0x6556, +0xfa90,0x656c,0x4af6,0x6578,0xf969,0x6599,0xf9be,0x65c5,0xf983,0x65e2,0xfa42,0x65e3,0x4afc,0x6613,0xf9e0,0x6649, +0x4b02,0x6674,0x4b08,0x6688,0xf9c5,0x6691,0x4b0d,0x669c,0x4b15,0x66b4,0xfa06,0x66c6,0xf98b,0x66f4,0xf901,0x66f8, +0x4b1b,0x6700,0x4b21,0x6717,0x4b27,0x671b,0x4b31,0x6721,0x4b39,0x674e,0xf9e1,0x6753,0x4b3f,0x6756,0xfa94,0x675e, +0x4b45,0x677b,0xf9c8,0x6785,0x4b4b,0x6797,0xf9f4,0x67f3,0xf9c9,0x67fa,0x4b51,0x6817,0xf9da,0x681f,0x4b57,0x6852, +0x4b5d,0x6881,0xf97a,0x6885,0x4b63,0x688e,0x4b6b,0x68a8,0xf9e2,0x6914,0x4b71,0x6942,0x4b77,0x69a3,0x4b7d,0x69ea, +0x4b83,0x6a02,0x4b89,0x6a13,0xf94c,0x6aa8,0x4b90,0x6ad3,0xf931,0x6adb,0x4b96,0x6b04,0xf91d,0x6b21,0x4b9c,0x6b54, +0x4ba2,0x6b72,0x4ba8,0x6b77,0xf98c,0x6b79,0xfa95,0x6b9f,0x4bae,0x6bae,0xf9a5,0x6bba,0x4bb4,0x6bbb,0x4bbe,0x6c4e, +0x4bc4,0x6c67,0x4bca,0x6c88,0xf972,0x6cbf,0x4bd0,0x6ccc,0xf968,0x6ccd,0x4bd6,0x6ce5,0xf9e3,0x6d16,0x4bdc,0x6d1b, +0xf915,0x6d1e,0xfa05,0x6d34,0x4be2,0x6d3e,0x4be8,0x6d41,0x4bee,0x6d69,0x4bf8,0x6d6a,0xf92a,0x6d77,0x4bfe,0x6d78, +0x4c06,0x6d85,0x4c0c,0x6dcb,0xf9f5,0x6dda,0xf94d,0x6dea,0xf9d6,0x6df9,0x4c12,0x6e1a,0xfa46,0x6e2f,0x4c18,0x6e6e, +0x4c1e,0x6e9c,0xf9cb,0x6eba,0xf9ec,0x6ec7,0x4c24,0x6ecb,0x4c2a,0x6ed1,0xf904,0x6edb,0xfa98,0x6f0f,0xf94e,0x6f22, +0x4c32,0x6f23,0xf992,0x6f6e,0x4c37,0x6fc6,0x4c3d,0x6feb,0xf922,0x6ffe,0xf984,0x701b,0x4c43,0x701e,0x4c49,0x7039, +0x4c51,0x704a,0x4c57,0x7070,0x4c5d,0x7077,0x4c63,0x707d,0x4c69,0x7099,0xf9fb,0x70ad,0x4c6f,0x70c8,0xf99f,0x70d9, +0xf916,0x7145,0x4c75,0x7149,0xf993,0x716e,0x4c7b,0x719c,0x4c80,0x71ce,0xf9c0,0x71d0,0xf9ee,0x7210,0xf932,0x721b, +0xf91e,0x7228,0x4c86,0x722b,0xfa49,0x7235,0x4c8c,0x7250,0x4c94,0x7262,0xf946,0x7280,0x4c9a,0x7295,0x4ca0,0x72af, +0xfa9f,0x72c0,0xf9fa,0x72fc,0xf92b,0x732a,0x4ca6,0x7375,0xf9a7,0x737a,0x4cab,0x7387,0x4cb1,0x738b,0x4cb6,0x73a5, +0x4cbc,0x73b2,0xf9ad,0x73de,0xf917,0x7406,0xf9e4,0x7409,0xf9cc,0x7422,0xfa4a,0x7447,0x4cc2,0x745c,0x4cc8,0x7469, +0xf9ae,0x7471,0x4cce,0x7485,0x4cd6,0x7489,0xf994,0x7498,0xf9ef,0x74ca,0x4cdc,0x7506,0xfaa2,0x7524,0x4ce2,0x753b, +0xfaa3,0x753e,0x4ce8,0x7559,0xf9cd,0x7565,0xf976,0x7570,0x4cee,0x75e2,0xf9e5,0x7610,0x4cf6,0x761d,0xfaa4,0x761f, +0xfaa5,0x7642,0xf9c1,0x7669,0xf90e,0x76ca,0x4cfc,0x76db,0xfaa7,0x76e7,0xf933,0x76f4,0x4d01,0x7701,0xf96d,0x771e, +0x4d09,0x771f,0x4d0f,0x7740,0xfaaa,0x774a,0x4d15,0x778b,0x4d1d,0x77a7,0xfa9d,0x784e,0x4d23,0x786b,0xf9ce,0x788c, +0x4d29,0x7891,0xfa4b,0x78ca,0xf947,0x78cc,0x4d31,0x78fb,0xf964,0x792a,0xf985,0x793c,0xfa18,0x793e,0xfa4c,0x7948, +0xfa4e,0x7949,0xfa4d,0x7950,0xfa4f,0x7956,0x4d39,0x795d,0xfa51,0x795e,0xfa19,0x7965,0xfa1a,0x797f,0xf93c,0x798d, +0xfa52,0x798e,0xfa53,0x798f,0x4d41,0x79ae,0xf9b6,0x79ca,0xf995,0x79eb,0x4d49,0x7a1c,0xf956,0x7a40,0x4d4f,0x7a4a, +0x4d57,0x7a4f,0x4d5d,0x7a81,0xfa55,0x7ab1,0xfaac,0x7acb,0xf9f7,0x7aee,0x4d63,0x7b20,0xf9f8,0x7bc0,0x4d69,0x7bc6, +0x4d6e,0x7bc9,0x4d74,0x7c3e,0xf9a6,0x7c60,0xf944,0x7c7b,0xfaae,0x7c92,0xf9f9,0x7cbe,0xfa1d,0x7cd2,0x4d7a,0x7cd6, +0xfa03,0x7ce3,0x4d80,0x7ce7,0xf97b,0x7ce8,0x4d86,0x7d00,0x4d8c,0x7d10,0xf9cf,0x7d22,0xf96a,0x7d2f,0xf94f,0x7d5b, +0xfaaf,0x7d63,0x4d92,0x7da0,0xf93d,0x7dbe,0xf957,0x7dc7,0x4d98,0x7df4,0x4d9e,0x7e02,0x4da5,0x7e09,0xfa58,0x7e37, +0xf950,0x7e41,0xfa59,0x7e45,0x4dab,0x7f3e,0xfab1,0x7f72,0xfa5a,0x7f79,0xf9e6,0x7f7a,0x4db1,0x7f85,0xf90f,0x7f95, +0x4db7,0x7f9a,0xf9af,0x7fbd,0xfa1e,0x7ffa,0x4dbd,0x8001,0xf934,0x8005,0x4dc3,0x8046,0xf9b0,0x8060,0x4dcd,0x806f, +0xf997,0x8070,0x4dd3,0x807e,0xf945,0x808b,0xf953,0x80ad,0x4dd9,0x80b2,0x4ddf,0x8103,0x4de5,0x813e,0x4deb,0x81d8, +0xf926,0x81e8,0xf9f6,0x81ed,0xfa5c,0x8201,0x4df1,0x8204,0x4dfb,0x826f,0xf97c,0x8279,0x4e01,0x828b,0x4e04,0x8291, +0x4e0a,0x829d,0x4e10,0x82b1,0x4e16,0x82b3,0x4e1c,0x82bd,0x4e22,0x82e5,0x4e28,0x82e6,0x4e30,0x831d,0x4e36,0x8323, +0x4e3c,0x8336,0xf9fe,0x8352,0xfab3,0x8353,0x4e42,0x8363,0x4e48,0x83ad,0x4e4e,0x83bd,0x4e54,0x83c9,0xf93e,0x83ca, +0x4e5a,0x83cc,0x4e60,0x83dc,0x4e66,0x83e7,0x4e6c,0x83ef,0xfab4,0x83f1,0xf958,0x843d,0xf918,0x8449,0xf96e,0x8457, +0x4e72,0x84ee,0xf999,0x84f1,0x4e7a,0x84f3,0x4e80,0x84fc,0xf9c2,0x8516,0x4e86,0x8564,0x4e8c,0x85cd,0xf923,0x85fa, +0xf9f0,0x8606,0xf935,0x8612,0xfa20,0x862d,0xf91f,0x863f,0xf910,0x8650,0x4e92,0x865c,0x4e98,0x8667,0x4ea0,0x8669, +0x4ea6,0x8688,0x4eac,0x86a9,0x4eb2,0x86e2,0x4eb8,0x870e,0x4ebe,0x8728,0x4ec4,0x876b,0x4eca,0x8779,0x4ed0,0x8786, +0x4ed8,0x87ba,0xf911,0x87e1,0x4ede,0x8801,0x4ee4,0x881f,0xf927,0x884c,0xfa08,0x8860,0x4eea,0x8863,0x4ef0,0x88c2, +0xf9a0,0x88cf,0xf9e7,0x88d7,0x4ef6,0x88de,0x4efc,0x88e1,0xf9e8,0x88f8,0xf912,0x88fa,0x4f02,0x8910,0xfa60,0x8941, +0xfab6,0x8964,0xf924,0x8986,0xfab7,0x898b,0xfa0a,0x8996,0x4f08,0x8aa0,0x4f0d,0x8aaa,0x4f13,0x8abf,0xfab9,0x8acb, +0xfabb,0x8ad2,0xf97d,0x8ad6,0xf941,0x8aed,0x4f18,0x8af8,0x4f20,0x8afe,0x4f25,0x8b01,0x4f2a,0x8b39,0x4f2f,0x8b58, +0xf9fc,0x8b80,0xf95a,0x8b8a,0x4f34,0x8c48,0xf900,0x8c55,0x4f3c,0x8cab,0x4f42,0x8cc1,0x4f48,0x8cc2,0xf948,0x8cc8, +0xf903,0x8cd3,0xfa64,0x8d08,0x4f4e,0x8d1b,0x4f53,0x8d77,0x4f59,0x8dbc,0x4f5f,0x8dcb,0x4f65,0x8def,0xf937,0x8df0, +0x4f6b,0x8eca,0xf902,0x8ed4,0x4f71,0x8f26,0xf998,0x8f2a,0xf9d7,0x8f38,0x4f77,0x8f3b,0xfa07,0x8f62,0xf98d,0x8f9e, +0x4f7f,0x8fb0,0xf971,0x8fb6,0xfa66,0x9023,0xf99a,0x9038,0x4f85,0x9072,0xfac3,0x907c,0xf9c3,0x908f,0xf913,0x9094, +0x4f8a,0x90ce,0xf92c,0x90f1,0x4f90,0x90fd,0xfa26,0x9111,0x4f96,0x911b,0x4f9c,0x916a,0xf919,0x9199,0xfac4,0x91b4, +0xf9b7,0x91cc,0xf9e9,0x91cf,0xf97e,0x91d1,0xf90a,0x9234,0xf9b1,0x9238,0x4fa2,0x9276,0xfac5,0x927c,0x4fa8,0x92d7, +0x4fae,0x92d8,0x4fb4,0x9304,0xf93f,0x934a,0xf99b,0x93f9,0x4fba,0x9415,0x4fc0,0x958b,0x4fc6,0x95ad,0xf986,0x95b7, +0x4fcc,0x962e,0xf9c6,0x964b,0xf951,0x964d,0xfa09,0x9675,0xf959,0x9678,0xf9d3,0x967c,0xfac6,0x9686,0xf9dc,0x96a3, +0xf9f1,0x96b8,0xf9b8,0x96c3,0x4fd2,0x96e2,0xf9ea,0x96e3,0x4fd8,0x96f6,0xf9b2,0x96f7,0xf949,0x9723,0x4fdd,0x9732, +0xf938,0x9748,0xf9b3,0x9756,0x4fe3,0x97db,0xfac9,0x97e0,0x4fe8,0x97ff,0x4fee,0x980b,0x4ff3,0x9818,0xf9b4,0x9829, +0x4ffb,0x983b,0x5001,0x985e,0xf9d0,0x98e2,0x5006,0x98ef,0xfa2a,0x98fc,0xfa2b,0x9928,0xfa2c,0x9929,0x500c,0x99a7, +0x5012,0x99c2,0x5018,0x99f1,0xf91a,0x99fe,0x501e,0x9a6a,0xf987,0x9b12,0x5024,0x9b6f,0xf939,0x9c40,0x502c,0x9c57, +0xf9f2,0x9cfd,0x5032,0x9d67,0x5038,0x9db4,0xfa2d,0x9dfa,0xf93a,0x9e1e,0xf920,0x9e7f,0xf940,0x9e97,0xf988,0x9e9f, +0xf9f3,0x9ebb,0x503e,0x9ece,0xf989,0x9ef9,0x5044,0x9efe,0x504a,0x9f05,0x5050,0x9f0f,0x5056,0x9f16,0x505c,0x9f3b, +0x5062,0x9f43,0xfad8,0x9f8d,0xf9c4,0x9f8e,0xfad9,0x9f9c,0x5068,0x8101,0xd157,0xd15e,1,0xd158,0x106d,1, +0xd1b9,0x1073,1,0xd1ba,0x1081,0x8202,0x122,0xf803,0x8202,0x51c,0xf812,0x8202,0x525,0xf91b,0x8202,0x54b, +0xf816,0x8202,0x63a,0xf80d,0x8202,0x804,0xf9d9,0x8202,0x8de,0xf9dd,0x8202,0xa2c,0xf834,0x8202,0xb63,0xf838, +0x8202,0x14e4,0xf859,0x8202,0x16a8,0xf860,0x8202,0x16ea,0xf861,0x8202,0x19c8,0xf86c,0x8202,0x1b18,0xf871,0x8202, +0x1d0b,0xf8f8,0x8202,0x1de4,0xf87b,0x8202,0x1de6,0xf87d,0x8202,0x2183,0xf889,0x8202,0x219f,0xf939,2,0x2331, +0x108f,0x8202,0x26d4,0xf8a4,0x8002,0x2844,0xfad0,0x8002,0x284a,0xfacf,0x8202,0x2b0c,0xf8b8,0x8202,0x2bf1,0xf8be, +0x8202,0x300a,0xf8ca,0x8202,0x32b8,0xf897,0x8202,0x335f,0xf980,0x8202,0x3393,0xf989,0x8202,0x339c,0xf98a,0x8202, +0x33c3,0xf8dd,0x8002,0x33d5,0xfad1,0x8202,0x346d,0xf8e3,0x8202,0x36a3,0xf8ec,0x8202,0x38a7,0xf8f0,0x8202,0x3a8d, +0xf8f7,0x8202,0x3afa,0xf8f9,0x8202,0x3cbc,0xf8fb,0x8202,0x3d1e,0xf906,0x8202,0x3ed1,0xf90d,0x8202,0x3f5e,0xf910, +0x8202,0x3f8e,0xf911,0x8202,0x4263,0xf91d,0x8202,0x43ab,0xf91f,0x8202,0x4608,0xf923,0x8202,0x4735,0xf926,0x8202, +0x4814,0xf927,0x8202,0x4c36,0xf935,0x8202,0x4c92,0xf937,0x8202,0x4fa1,0xf93b,0x8202,0x4fb8,0xf93c,0x8202,0x5044, +0xf93d,0x8202,0x50f2,0xf942,0x8202,0x50f3,0xf941,0x8202,0x5119,0xf943,0x8202,0x5133,0xf944,0x8002,0x5249,0xfad5, +0x8202,0x541d,0xf94d,0x8202,0x5626,0xf952,0x8202,0x569a,0xf954,0x8202,0x56c5,0xf955,0x8202,0x597c,0xf95c,2, +0x5aa7,0x1095,0x8202,0x5bab,0xf961,0x8202,0x5c80,0xf965,0x8002,0x5cd0,0xfad6,0x8202,0x5f86,0xf96b,0x8202,0x61da, +0xf898,0x8202,0x6228,0xf972,0x8202,0x6247,0xf973,0x8202,0x62d9,0xf975,0x8202,0x633e,0xf977,0x8202,0x64da,0xf97b, +0x8202,0x6523,0xf97c,0x8202,0x65a8,0xf97e,0x8202,0x67a7,0xf987,0x8202,0x67b5,0xf988,0x8202,0x6b3c,0xf997,0x8202, +0x6c36,0xf9a4,0x8202,0x6cd5,0xf9a6,0x8202,0x6d6b,0xf9a5,0x8202,0x6f2c,0xf9ad,0x8202,0x6fb1,0xf9b0,0x8202,0x70d2, +0xf9b1,0x8202,0x73ca,0xf9ab,0x8202,0x7667,0xf9c5,0x8202,0x78ae,0xf9cb,0x8202,0x7966,0xf9cc,0x8202,0x7ca8,0xf9d3, +0x8002,0x7ed3,0xfad7,0x8202,0x7f2f,0xf9d8,0x8202,0x85d2,0xf9e0,0x8202,0x85ed,0xf9e1,0x8202,0x872e,0xf9e5,0x8202, +0x8bfa,0xf9ed,0x8202,0x8d77,0xf9f1,0x8202,0x9145,0xf9f6,0x8202,0x91df,0xf81c,0x8202,0x921a,0xf9f7,0x8202,0x940a, +0xf9fb,0x8202,0x9496,0xf9fd,0x8202,0x95b6,0xfa01,0x8202,0x9b30,0xfa09,0x8202,0xa0ce,0xfa10,0x8202,0xa105,0xfa12, +0x8202,0xa20e,0xfa13,0x8202,0xa291,0xfa14,0x8202,0xa392,0xf88f,0x8202,0xa600,0xfa1d,0x8016,0x12,0xf900,0xfa0e, +0xfa10,0xfa11,0xfa12,0xfa13,0xfa15,0xfa1f,0xfa20,0xfa21,0xfa22,0xfa23,0xfa25,0xfa27,0xfa2a,0xfa2e,0xfa70,0xfada, +2,0xf800,2,0xfa1e,0x8376,0x2ba,0x221,0x222,0x234,0x250,0x2ae,0x2b0,0x2ef,0x300,0x350,0x360, +0x370,0x374,0x376,0x37a,0x37b,0x37e,0x37f,0x384,0x38b,0x38c,0x38d,0x38e,0x3a2,0x3a3,0x3cf,0x3d0, +0x3f7,0x400,0x487,0x488,0x4cf,0x4d0,0x4f6,0x4f8,0x4fa,0x500,0x510,0x531,0x557,0x559,0x560,0x561, +0x588,0x589,0x58b,0x591,0x5a2,0x5a3,0x5ba,0x5bb,0x5c5,0x5d0,0x5eb,0x5f0,0x5f5,0x60c,0x60d,0x61b, +0x61c,0x61f,0x620,0x621,0x63b,0x640,0x656,0x660,0x6ee,0x6f0,0x6ff,0x700,0x70e,0x70f,0x72d,0x730, +0x74b,0x780,0x7b2,0x901,0x904,0x905,0x93a,0x93c,0x94e,0x950,0x955,0x958,0x971,0x981,0x984,0x985, +0x98d,0x98f,0x991,0x993,0x9a9,0x9aa,0x9b1,0x9b2,0x9b3,0x9b6,0x9ba,0x9bc,0x9bd,0x9be,0x9c5,0x9c7, +0x9c9,0x9cb,0x9ce,0x9d7,0x9d8,0x9dc,0x9de,0x9df,0x9e4,0x9e6,0x9fb,0xa02,0xa03,0xa05,0xa0b,0xa0f, +0xa11,0xa13,0xa29,0xa2a,0xa31,0xa32,0xa34,0xa35,0xa37,0xa38,0xa3a,0xa3c,0xa3d,0xa3e,0xa43,0xa47, +0xa49,0xa4b,0xa4e,0xa59,0xa5d,0xa5e,0xa5f,0xa66,0xa75,0xa81,0xa84,0xa85,0xa8c,0xa8d,0xa8e,0xa8f, +0xa92,0xa93,0xaa9,0xaaa,0xab1,0xab2,0xab4,0xab5,0xaba,0xabc,0xac6,0xac7,0xaca,0xacb,0xace,0xad0, +0xad1,0xae0,0xae1,0xae6,0xaf0,0xb01,0xb04,0xb05,0xb0d,0xb0f,0xb11,0xb13,0xb29,0xb2a,0xb31,0xb32, +0xb34,0xb36,0xb3a,0xb3c,0xb44,0xb47,0xb49,0xb4b,0xb4e,0xb56,0xb58,0xb5c,0xb5e,0xb5f,0xb62,0xb66, +0xb71,0xb82,0xb84,0xb85,0xb8b,0xb8e,0xb91,0xb92,0xb96,0xb99,0xb9b,0xb9c,0xb9d,0xb9e,0xba0,0xba3, +0xba5,0xba8,0xbab,0xbae,0xbb6,0xbb7,0xbba,0xbbe,0xbc3,0xbc6,0xbc9,0xbca,0xbce,0xbd7,0xbd8,0xbe7, +0xbf3,0xc01,0xc04,0xc05,0xc0d,0xc0e,0xc11,0xc12,0xc29,0xc2a,0xc34,0xc35,0xc3a,0xc3e,0xc45,0xc46, +0xc49,0xc4a,0xc4e,0xc55,0xc57,0xc60,0xc62,0xc66,0xc70,0xc82,0xc84,0xc85,0xc8d,0xc8e,0xc91,0xc92, +0xca9,0xcaa,0xcb4,0xcb5,0xcba,0xcbe,0xcc5,0xcc6,0xcc9,0xcca,0xcce,0xcd5,0xcd7,0xcde,0xcdf,0xce0, +0xce2,0xce6,0xcf0,0xd02,0xd04,0xd05,0xd0d,0xd0e,0xd11,0xd12,0xd29,0xd2a,0xd3a,0xd3e,0xd44,0xd46, +0xd49,0xd4a,0xd4e,0xd57,0xd58,0xd60,0xd62,0xd66,0xd70,0xd82,0xd84,0xd85,0xd97,0xd9a,0xdb2,0xdb3, +0xdbc,0xdbd,0xdbe,0xdc0,0xdc7,0xdca,0xdcb,0xdcf,0xdd5,0xdd6,0xdd7,0xdd8,0xde0,0xdf2,0xdf5,0xe01, +0xe3b,0xe3f,0xe5c,0xe81,0xe83,0xe84,0xe85,0xe87,0xe89,0xe8a,0xe8b,0xe8d,0xe8e,0xe94,0xe98,0xe99, +0xea0,0xea1,0xea4,0xea5,0xea6,0xea7,0xea8,0xeaa,0xeac,0xead,0xeba,0xebb,0xebe,0xec0,0xec5,0xec6, +0xec7,0xec8,0xece,0xed0,0xeda,0xedc,0xede,0xf00,0xf48,0xf49,0xf6b,0xf71,0xf8c,0xf90,0xf98,0xf99, +0xfbd,0xfbe,0xfcd,0xfcf,0xfd0,0x1000,0x1022,0x1023,0x1028,0x1029,0x102b,0x102c,0x1033,0x1036,0x103a,0x1040, +0x105a,0x10a0,0x10c6,0x10d0,0x10f9,0x10fb,0x10fc,0x1100,0x115a,0x115f,0x11a3,0x11a8,0x11fa,0x1200,0x1207,0x1208, +0x1247,0x1248,0x1249,0x124a,0x124e,0x1250,0x1257,0x1258,0x1259,0x125a,0x125e,0x1260,0x1287,0x1288,0x1289,0x128a, +0x128e,0x1290,0x12af,0x12b0,0x12b1,0x12b2,0x12b6,0x12b8,0x12bf,0x12c0,0x12c1,0x12c2,0x12c6,0x12c8,0x12cf,0x12d0, +0x12d7,0x12d8,0x12ef,0x12f0,0x130f,0x1310,0x1311,0x1312,0x1316,0x1318,0x131f,0x1320,0x1347,0x1348,0x135b,0x1361, +0x137d,0x13a0,0x13f5,0x1401,0x1677,0x1680,0x169d,0x16a0,0x16f1,0x1700,0x170d,0x170e,0x1715,0x1720,0x1737,0x1740, +0x1754,0x1760,0x176d,0x176e,0x1771,0x1772,0x1774,0x1780,0x17dd,0x17e0,0x17ea,0x1800,0x180f,0x1810,0x181a,0x1820, +0x1878,0x1880,0x18aa,0x1e00,0x1e9c,0x1ea0,0x1efa,0x1f00,0x1f16,0x1f18,0x1f1e,0x1f20,0x1f46,0x1f48,0x1f4e,0x1f50, +0x1f58,0x1f59,0x1f5a,0x1f5b,0x1f5c,0x1f5d,0x1f5e,0x1f5f,0x1f7e,0x1f80,0x1fb5,0x1fb6,0x1fc5,0x1fc6,0x1fd4,0x1fd6, +0x1fdc,0x1fdd,0x1ff0,0x1ff2,0x1ff5,0x1ff6,0x1fff,0x2000,0x2053,0x2057,0x2058,0x205f,0x2064,0x206a,0x2072,0x2074, +0x208f,0x20a0,0x20b2,0x20d0,0x20eb,0x2100,0x213b,0x213d,0x214c,0x2153,0x2184,0x2190,0x23cf,0x2400,0x2427,0x2440, +0x244b,0x2460,0x24ff,0x2500,0x2614,0x2616,0x2618,0x2619,0x267e,0x2680,0x268a,0x2701,0x2705,0x2706,0x270a,0x270c, +0x2728,0x2729,0x274c,0x274d,0x274e,0x274f,0x2753,0x2756,0x2757,0x2758,0x275f,0x2761,0x2795,0x2798,0x27b0,0x27b1, +0x27bf,0x27d0,0x27ec,0x27f0,0x2b00,0x2e80,0x2e9a,0x2e9b,0x2ef4,0x2f00,0x2fd6,0x2ff0,0x2ffc,0x3000,0x3040,0x3041, +0x3097,0x3099,0x3100,0x3105,0x312d,0x3131,0x318f,0x3190,0x31b8,0x31f0,0x321d,0x3220,0x3244,0x3251,0x327c,0x327f, +0x32cc,0x32d0,0x32ff,0x3300,0x3377,0x337b,0x33de,0x33e0,0x33ff,0x3400,0x4db6,0x4e00,0x9fa6,0xa000,0xa48d,0xa490, +0xa4c7,0xac00,0xd7a4,0xd800,0xfa2e,0xfa30,0xfa6b,0xfb00,0xfb07,0xfb13,0xfb18,0xfb1d,0xfb37,0xfb38,0xfb3d,0xfb3e, +0xfb3f,0xfb40,0xfb42,0xfb43,0xfb45,0xfb46,0xfbb2,0xfbd3,0xfd40,0xfd50,0xfd90,0xfd92,0xfdc8,0xfdd0,0xfdfd,0xfe00, +0xfe10,0xfe20,0xfe24,0xfe30,0xfe47,0xfe49,0xfe53,0xfe54,0xfe67,0xfe68,0xfe6c,0xfe70,0xfe75,0xfe76,0xfefd,0xfeff, +0xff00,0xff01,0xffbf,0xffc2,0xffc8,0xffca,0xffd0,0xffd2,0xffd8,0xffda,0xffdd,0xffe0,0xffe7,0xffe8,0xffef,0xfff9, +1,0,1,0x300,1,0x31f,1,0x320,1,0x324,1,0x330,1,0x34b,1,0x400, +1,0x426,1,0x428,1,0x44e,1,0xd000,1,0xd0f6,1,0xd100,1,0xd127,1,0xd12a, +1,0xd1de,1,0xd400,1,0xd455,1,0xd456,1,0xd49d,1,0xd49e,1,0xd4a0,1,0xd4a2, +1,0xd4a3,1,0xd4a5,1,0xd4a7,1,0xd4a9,1,0xd4ad,1,0xd4ae,1,0xd4ba,1,0xd4bb, +1,0xd4bc,1,0xd4bd,1,0xd4c1,1,0xd4c2,1,0xd4c4,1,0xd4c5,1,0xd506,1,0xd507, +1,0xd50b,1,0xd50d,1,0xd515,1,0xd516,1,0xd51d,1,0xd51e,1,0xd53a,1,0xd53b, +1,0xd53f,1,0xd540,1,0xd545,1,0xd546,1,0xd547,1,0xd54a,1,0xd551,1,0xd552, +1,0xd6a4,1,0xd6a8,1,0xd7ca,1,0xd7ce,1,0xd800,1,0xfffe,2,0xa6d7,2,0xf800, +2,0xfa1e,2,0xfffe,3,0,3,0xfffe,4,0,4,0xfffe,5,0,5,0xfffe, +6,0,6,0xfffe,7,0,7,0xfffe,8,0,8,0xfffe,9,0,9,0xfffe, +0xa,0,0xa,0xfffe,0xb,0,0xb,0xfffe,0xc,0,0xc,0xfffe,0xd,0,0xd,0xfffe, +0xe,0,0xe,1,0xe,2,0xe,0x20,0xe,0x80,0xe,0xfffe +}; + diff --git a/Build/source/libs/icu/icu-xetex/common/unormcmp.cpp b/Build/source/libs/icu/icu-xetex/common/unormcmp.cpp new file mode 100644 index 00000000000..7b3903d898c --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unormcmp.cpp @@ -0,0 +1,690 @@ +/* +******************************************************************************* +* +* Copyright (C) 2001-2006, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: unormcmp.cpp +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2004sep13 +* created by: Markus W. Scherer +* +* unorm_compare() function moved here from unorm.cpp for better modularization. +* Depends on both normalization and case folding. +* Allows unorm.cpp to not depend on any character properties code. +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_NORMALIZATION + +#include "unicode/ustring.h" +#include "unicode/unorm.h" +#include "unicode/uniset.h" +#include "unormimp.h" +#include "ucase.h" +#include "cmemory.h" + +U_NAMESPACE_USE + +#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) + +/* compare canonically equivalent ------------------------------------------- */ + +/* + * Compare two strings for canonical equivalence. + * Further options include case-insensitive comparison and + * code point order (as opposed to code unit order). + * + * In this function, canonical equivalence is optional as well. + * If canonical equivalence is tested, then both strings must fulfill + * the FCD check. + * + * Semantically, this is equivalent to + * strcmp[CodePointOrder](NFD(foldCase(s1)), NFD(foldCase(s2))) + * where code point order, NFD and foldCase are all optional. + * + * String comparisons almost always yield results before processing both strings + * completely. + * They are generally more efficient working incrementally instead of + * performing the sub-processing (strlen, normalization, case-folding) + * on the entire strings first. + * + * It is also unnecessary to not normalize identical characters. + * + * This function works in principle as follows: + * + * loop { + * get one code unit c1 from s1 (-1 if end of source) + * get one code unit c2 from s2 (-1 if end of source) + * + * if(either string finished) { + * return result; + * } + * if(c1==c2) { + * continue; + * } + * + * // c1!=c2 + * try to decompose/case-fold c1/c2, and continue if one does; + * + * // still c1!=c2 and neither decomposes/case-folds, return result + * return c1-c2; + * } + * + * When a character decomposes, then the pointer for that source changes to + * the decomposition, pushing the previous pointer onto a stack. + * When the end of the decomposition is reached, then the code unit reader + * pops the previous source from the stack. + * (Same for case-folding.) + * + * This is complicated further by operating on variable-width UTF-16. + * The top part of the loop works on code units, while lookups for decomposition + * and case-folding need code points. + * Code points are assembled after the equality/end-of-source part. + * The source pointer is only advanced beyond all code units when the code point + * actually decomposes/case-folds. + * + * If we were on a trail surrogate unit when assembling a code point, + * and the code point decomposes/case-folds, then the decomposition/folding + * result must be compared with the part of the other string that corresponds to + * this string's lead surrogate. + * Since we only assemble a code point when hitting a trail unit when the + * preceding lead units were identical, we back up the other string by one unit + * in such a case. + * + * The optional code point order comparison at the end works with + * the same fix-up as the other code point order comparison functions. + * See ustring.c and the comment near the end of this function. + * + * Assumption: A decomposition or case-folding result string never contains + * a single surrogate. This is a safe assumption in the Unicode Standard. + * Therefore, we do not need to check for surrogate pairs across + * decomposition/case-folding boundaries. + * + * Further assumptions (see verifications tstnorm.cpp): + * The API function checks for FCD first, while the core function + * first case-folds and then decomposes. This requires that case-folding does not + * un-FCD any strings. + * + * The API function may also NFD the input and turn off decomposition. + * This requires that case-folding does not un-NFD strings either. + * + * TODO If any of the above two assumptions is violated, + * then this entire code must be re-thought. + * If this happens, then a simple solution is to case-fold both strings up front + * and to turn off UNORM_INPUT_IS_FCD. + * We already do this when not both strings are in FCD because makeFCD + * would be a partial NFD before the case folding, which does not work. + * Note that all of this is only a problem when case-folding _and_ + * canonical equivalence come together. + * (Comments in unorm_compare() are more up to date than this TODO.) + * + * This function could be moved to a different source file, at increased cost + * for calling the decomposition access function. + */ + +/* stack element for previous-level source/decomposition pointers */ +struct CmpEquivLevel { + const UChar *start, *s, *limit; +}; +typedef struct CmpEquivLevel CmpEquivLevel; + +/* internal function */ +static int32_t +unorm_cmpEquivFold(const UChar *s1, int32_t length1, + const UChar *s2, int32_t length2, + uint32_t options, + UErrorCode *pErrorCode) { + const UCaseProps *csp; + + /* current-level start/limit - s1/s2 as current */ + const UChar *start1, *start2, *limit1, *limit2; + + /* decomposition and case folding variables */ + const UChar *p; + int32_t length; + + /* stacks of previous-level start/current/limit */ + CmpEquivLevel stack1[2], stack2[2]; + + /* decomposition buffers for Hangul */ + UChar decomp1[4], decomp2[4]; + + /* case folding buffers, only use current-level start/limit */ + UChar fold1[UCASE_MAX_STRING_LENGTH+1], fold2[UCASE_MAX_STRING_LENGTH+1]; + + /* track which is the current level per string */ + int32_t level1, level2; + + /* current code units, and code points for lookups */ + UChar32 c1, c2, cp1, cp2; + + /* no argument error checking because this itself is not an API */ + + /* + * assume that at least one of the options _COMPARE_EQUIV and U_COMPARE_IGNORE_CASE is set + * otherwise this function must behave exactly as uprv_strCompare() + * not checking for that here makes testing this function easier + */ + + /* normalization/properties data loaded? */ + if( ((options&_COMPARE_EQUIV)!=0 && !unorm_haveData(pErrorCode)) || + U_FAILURE(*pErrorCode) + ) { + return 0; + } + if((options&U_COMPARE_IGNORE_CASE)!=0) { + csp=ucase_getSingleton(pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return 0; + } + } else { + csp=NULL; + } + + /* initialize */ + start1=s1; + if(length1==-1) { + limit1=NULL; + } else { + limit1=s1+length1; + } + + start2=s2; + if(length2==-1) { + limit2=NULL; + } else { + limit2=s2+length2; + } + + level1=level2=0; + c1=c2=-1; + + /* comparison loop */ + for(;;) { + /* + * here a code unit value of -1 means "get another code unit" + * below it will mean "this source is finished" + */ + + if(c1<0) { + /* get next code unit from string 1, post-increment */ + for(;;) { + if(s1==limit1 || ((c1=*s1)==0 && (limit1==NULL || (options&_STRNCMP_STYLE)))) { + if(level1==0) { + c1=-1; + break; + } + } else { + ++s1; + break; + } + + /* reached end of level buffer, pop one level */ + do { + --level1; + start1=stack1[level1].start; + } while(start1==NULL); + s1=stack1[level1].s; + limit1=stack1[level1].limit; + } + } + + if(c2<0) { + /* get next code unit from string 2, post-increment */ + for(;;) { + if(s2==limit2 || ((c2=*s2)==0 && (limit2==NULL || (options&_STRNCMP_STYLE)))) { + if(level2==0) { + c2=-1; + break; + } + } else { + ++s2; + break; + } + + /* reached end of level buffer, pop one level */ + do { + --level2; + start2=stack2[level2].start; + } while(start2==NULL); + s2=stack2[level2].s; + limit2=stack2[level2].limit; + } + } + + /* + * compare c1 and c2 + * either variable c1, c2 is -1 only if the corresponding string is finished + */ + if(c1==c2) { + if(c1<0) { + return 0; /* c1==c2==-1 indicating end of strings */ + } + c1=c2=-1; /* make us fetch new code units */ + continue; + } else if(c1<0) { + return -1; /* string 1 ends before string 2 */ + } else if(c2<0) { + return 1; /* string 2 ends before string 1 */ + } + /* c1!=c2 && c1>=0 && c2>=0 */ + + /* get complete code points for c1, c2 for lookups if either is a surrogate */ + cp1=c1; + if(U_IS_SURROGATE(c1)) { + UChar c; + + if(U_IS_SURROGATE_LEAD(c1)) { + if(s1!=limit1 && U16_IS_TRAIL(c=*s1)) { + /* advance ++s1; only below if cp1 decomposes/case-folds */ + cp1=U16_GET_SUPPLEMENTARY(c1, c); + } + } else /* isTrail(c1) */ { + if(start1<=(s1-2) && U16_IS_LEAD(c=*(s1-2))) { + cp1=U16_GET_SUPPLEMENTARY(c, c1); + } + } + } + + cp2=c2; + if(U_IS_SURROGATE(c2)) { + UChar c; + + if(U_IS_SURROGATE_LEAD(c2)) { + if(s2!=limit2 && U16_IS_TRAIL(c=*s2)) { + /* advance ++s2; only below if cp2 decomposes/case-folds */ + cp2=U16_GET_SUPPLEMENTARY(c2, c); + } + } else /* isTrail(c2) */ { + if(start2<=(s2-2) && U16_IS_LEAD(c=*(s2-2))) { + cp2=U16_GET_SUPPLEMENTARY(c, c2); + } + } + } + + /* + * go down one level for each string + * continue with the main loop as soon as there is a real change + */ + + if( level1==0 && (options&U_COMPARE_IGNORE_CASE) && + (length=ucase_toFullFolding(csp, (UChar32)cp1, &p, options))>=0 + ) { + /* cp1 case-folds to the code point "length" or to p[length] */ + if(U_IS_SURROGATE(c1)) { + if(U_IS_SURROGATE_LEAD(c1)) { + /* advance beyond source surrogate pair if it case-folds */ + ++s1; + } else /* isTrail(c1) */ { + /* + * we got a supplementary code point when hitting its trail surrogate, + * therefore the lead surrogate must have been the same as in the other string; + * compare this decomposition with the lead surrogate in the other string + * remember that this simulates bulk text replacement: + * the decomposition would replace the entire code point + */ + --s2; + c2=*(s2-1); + } + } + + /* push current level pointers */ + stack1[0].start=start1; + stack1[0].s=s1; + stack1[0].limit=limit1; + ++level1; + + /* copy the folding result to fold1[] */ + if(length<=UCASE_MAX_STRING_LENGTH) { + u_memcpy(fold1, p, length); + } else { + int32_t i=0; + U16_APPEND_UNSAFE(fold1, i, length); + length=i; + } + + /* set next level pointers to case folding */ + start1=s1=fold1; + limit1=fold1+length; + + /* get ready to read from decomposition, continue with loop */ + c1=-1; + continue; + } + + if( level2==0 && (options&U_COMPARE_IGNORE_CASE) && + (length=ucase_toFullFolding(csp, (UChar32)cp2, &p, options))>=0 + ) { + /* cp2 case-folds to the code point "length" or to p[length] */ + if(U_IS_SURROGATE(c2)) { + if(U_IS_SURROGATE_LEAD(c2)) { + /* advance beyond source surrogate pair if it case-folds */ + ++s2; + } else /* isTrail(c2) */ { + /* + * we got a supplementary code point when hitting its trail surrogate, + * therefore the lead surrogate must have been the same as in the other string; + * compare this decomposition with the lead surrogate in the other string + * remember that this simulates bulk text replacement: + * the decomposition would replace the entire code point + */ + --s1; + c1=*(s1-1); + } + } + + /* push current level pointers */ + stack2[0].start=start2; + stack2[0].s=s2; + stack2[0].limit=limit2; + ++level2; + + /* copy the folding result to fold2[] */ + if(length<=UCASE_MAX_STRING_LENGTH) { + u_memcpy(fold2, p, length); + } else { + int32_t i=0; + U16_APPEND_UNSAFE(fold2, i, length); + length=i; + } + + /* set next level pointers to case folding */ + start2=s2=fold2; + limit2=fold2+length; + + /* get ready to read from decomposition, continue with loop */ + c2=-1; + continue; + } + + if( level1<2 && (options&_COMPARE_EQUIV) && + 0!=(p=unorm_getCanonicalDecomposition((UChar32)cp1, decomp1, &length)) + ) { + /* cp1 decomposes into p[length] */ + if(U_IS_SURROGATE(c1)) { + if(U_IS_SURROGATE_LEAD(c1)) { + /* advance beyond source surrogate pair if it decomposes */ + ++s1; + } else /* isTrail(c1) */ { + /* + * we got a supplementary code point when hitting its trail surrogate, + * therefore the lead surrogate must have been the same as in the other string; + * compare this decomposition with the lead surrogate in the other string + * remember that this simulates bulk text replacement: + * the decomposition would replace the entire code point + */ + --s2; + c2=*(s2-1); + } + } + + /* push current level pointers */ + stack1[level1].start=start1; + stack1[level1].s=s1; + stack1[level1].limit=limit1; + ++level1; + + /* set empty intermediate level if skipped */ + if(level1<2) { + stack1[level1++].start=NULL; + } + + /* set next level pointers to decomposition */ + start1=s1=p; + limit1=p+length; + + /* get ready to read from decomposition, continue with loop */ + c1=-1; + continue; + } + + if( level2<2 && (options&_COMPARE_EQUIV) && + 0!=(p=unorm_getCanonicalDecomposition((UChar32)cp2, decomp2, &length)) + ) { + /* cp2 decomposes into p[length] */ + if(U_IS_SURROGATE(c2)) { + if(U_IS_SURROGATE_LEAD(c2)) { + /* advance beyond source surrogate pair if it decomposes */ + ++s2; + } else /* isTrail(c2) */ { + /* + * we got a supplementary code point when hitting its trail surrogate, + * therefore the lead surrogate must have been the same as in the other string; + * compare this decomposition with the lead surrogate in the other string + * remember that this simulates bulk text replacement: + * the decomposition would replace the entire code point + */ + --s1; + c1=*(s1-1); + } + } + + /* push current level pointers */ + stack2[level2].start=start2; + stack2[level2].s=s2; + stack2[level2].limit=limit2; + ++level2; + + /* set empty intermediate level if skipped */ + if(level2<2) { + stack2[level2++].start=NULL; + } + + /* set next level pointers to decomposition */ + start2=s2=p; + limit2=p+length; + + /* get ready to read from decomposition, continue with loop */ + c2=-1; + continue; + } + + /* + * no decomposition/case folding, max level for both sides: + * return difference result + * + * code point order comparison must not just return cp1-cp2 + * because when single surrogates are present then the surrogate pairs + * that formed cp1 and cp2 may be from different string indexes + * + * example: { d800 d800 dc01 } vs. { d800 dc00 }, compare at second code units + * c1=d800 cp1=10001 c2=dc00 cp2=10000 + * cp1-cp2>0 but c1-c2<0 and in fact in UTF-32 it is { d800 10001 } < { 10000 } + * + * therefore, use same fix-up as in ustring.c/uprv_strCompare() + * except: uprv_strCompare() fetches c=*s while this functions fetches c=*s++ + * so we have slightly different pointer/start/limit comparisons here + */ + + if(c1>=0xd800 && c2>=0xd800 && (options&U_COMPARE_CODE_POINT_ORDER)) { + /* subtract 0x2800 from BMP code points to make them smaller than supplementary ones */ + if( + (c1<=0xdbff && s1!=limit1 && U16_IS_TRAIL(*s1)) || + (U16_IS_TRAIL(c1) && start1!=(s1-1) && U16_IS_LEAD(*(s1-2))) + ) { + /* part of a surrogate pair, leave >=d800 */ + } else { + /* BMP code point - may be surrogate code point - make =d800 */ + } else { + /* BMP code point - may be surrogate code point - make >UNORM_COMPARE_NORM_OPTIONS_SHIFT); + nx=unorm_getNX(normOptions, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return 0; + } + + d1=d2=0; + options|=_COMPARE_EQUIV; + result=0; + + /* + * UAX #21 Case Mappings, as fixed for Unicode version 4 + * (see Jitterbug 2021), defines a canonical caseless match as + * + * A string X is a canonical caseless match + * for a string Y if and only if + * NFD(toCasefold(NFD(X))) = NFD(toCasefold(NFD(Y))) + * + * For better performance, we check for FCD (or let the caller tell us that + * both strings are in FCD) for the inner normalization. + * BasicNormalizerTest::FindFoldFCDExceptions() makes sure that + * case-folding preserves the FCD-ness of a string. + * The outer normalization is then only performed by unorm_cmpEquivFold() + * when there is a difference. + * + * Exception: When using the Turkic case-folding option, we do perform + * full NFD first. This is because in the Turkic case precomposed characters + * with 0049 capital I or 0069 small i fold differently whether they + * are first decomposed or not, so an FCD check - a check only for + * canonical order - is not sufficient. + */ + if(options&U_FOLD_CASE_EXCLUDE_SPECIAL_I) { + mode=UNORM_NFD; + options&=~UNORM_INPUT_IS_FCD; + } else { + mode=UNORM_FCD; + } + + if(!(options&UNORM_INPUT_IS_FCD)) { + int32_t _len1, _len2; + UBool isFCD1, isFCD2; + + // check if s1 and/or s2 fulfill the FCD conditions + isFCD1= UNORM_YES==unorm_internalQuickCheck(s1, length1, mode, TRUE, nx, pErrorCode); + isFCD2= UNORM_YES==unorm_internalQuickCheck(s2, length2, mode, TRUE, nx, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return 0; + } + + /* + * ICU 2.4 had a further optimization: + * If both strings were not in FCD, then they were both NFD'ed, + * and the _COMPARE_EQUIV option was turned off. + * It is not entirely clear that this is valid with the current + * definition of the canonical caseless match. + * Therefore, ICU 2.6 removes that optimization. + */ + + if(!isFCD1) { + _len1=unorm_internalNormalizeWithNX(fcd1, LENGTHOF(fcd1), + s1, length1, + mode, normOptions, nx, + pErrorCode); + if(*pErrorCode!=U_BUFFER_OVERFLOW_ERROR) { + s1=fcd1; + } else { + d1=(UChar *)uprv_malloc(_len1*U_SIZEOF_UCHAR); + if(d1==0) { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + goto cleanup; + } + + *pErrorCode=U_ZERO_ERROR; + _len1=unorm_internalNormalizeWithNX(d1, _len1, + s1, length1, + mode, normOptions, nx, + pErrorCode); + if(U_FAILURE(*pErrorCode)) { + goto cleanup; + } + + s1=d1; + } + length1=_len1; + } + + if(!isFCD2) { + _len2=unorm_internalNormalizeWithNX(fcd2, LENGTHOF(fcd2), + s2, length2, + mode, normOptions, nx, + pErrorCode); + if(*pErrorCode!=U_BUFFER_OVERFLOW_ERROR) { + s2=fcd2; + } else { + d2=(UChar *)uprv_malloc(_len2*U_SIZEOF_UCHAR); + if(d2==0) { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + goto cleanup; + } + + *pErrorCode=U_ZERO_ERROR; + _len2=unorm_internalNormalizeWithNX(d2, _len2, + s2, length2, + mode, normOptions, nx, + pErrorCode); + if(U_FAILURE(*pErrorCode)) { + goto cleanup; + } + + s2=d2; + } + length2=_len2; + } + } + + if(U_SUCCESS(*pErrorCode)) { + result=unorm_cmpEquivFold(s1, length1, s2, length2, options, pErrorCode); + } + +cleanup: + if(d1!=0) { + uprv_free(d1); + } + if(d2!=0) { + uprv_free(d2); + } + + return result; +} + +#endif /* #if !UCONFIG_NO_NORMALIZATION */ diff --git a/Build/source/libs/icu/icu-xetex/common/unormimp.h b/Build/source/libs/icu/icu-xetex/common/unormimp.h new file mode 100644 index 00000000000..498bf0a8b5d --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unormimp.h @@ -0,0 +1,792 @@ +/* +******************************************************************************* +* +* Copyright (C) 2001-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: unormimp.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2001may25 +* created by: Markus W. Scherer +*/ + +#ifndef __UNORMIMP_H__ +#define __UNORMIMP_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_NORMALIZATION + +#ifdef XP_CPLUSPLUS +#include "unicode/uniset.h" +#endif + +#include "unicode/uiter.h" +#include "unicode/unorm.h" +#include "unicode/uset.h" +#include "utrie.h" +#include "ustr_imp.h" +#include "udataswp.h" + +/* + * This new implementation of the normalization code loads its data from + * unorm.icu, which is generated with the gennorm tool. + * The format of that file is described at the end of this file. + */ + +/* norm32 value constants */ +enum { + /* quick check flags 0..3 set mean "no" for their forms */ + _NORM_QC_NFC=0x11, /* no|maybe */ + _NORM_QC_NFKC=0x22, /* no|maybe */ + _NORM_QC_NFD=4, /* no */ + _NORM_QC_NFKD=8, /* no */ + + _NORM_QC_ANY_NO=0xf, + + /* quick check flags 4..5 mean "maybe" for their forms; test flags>=_NORM_QC_MAYBE */ + _NORM_QC_MAYBE=0x10, + _NORM_QC_ANY_MAYBE=0x30, + + _NORM_QC_MASK=0x3f, + + _NORM_COMBINES_FWD=0x40, + _NORM_COMBINES_BACK=0x80, + _NORM_COMBINES_ANY=0xc0, + + _NORM_CC_SHIFT=8, /* UnicodeData.txt combining class in bits 15..8 */ + _NORM_CC_MASK=0xff00, + + _NORM_EXTRA_SHIFT=16, /* 16 bits for the index to UChars and other extra data */ + _NORM_EXTRA_INDEX_TOP=0xfc00, /* start of surrogate specials after shift */ + + _NORM_EXTRA_SURROGATE_MASK=0x3ff, + _NORM_EXTRA_SURROGATE_TOP=0x3f0, /* hangul etc. */ + + _NORM_EXTRA_HANGUL=_NORM_EXTRA_SURROGATE_TOP, + _NORM_EXTRA_JAMO_L, + _NORM_EXTRA_JAMO_V, + _NORM_EXTRA_JAMO_T +}; + +/* norm32 value constants using >16 bits */ +#define _NORM_MIN_SPECIAL 0xfc000000 +#define _NORM_SURROGATES_TOP 0xfff00000 +#define _NORM_MIN_HANGUL 0xfff00000 +#define _NORM_MIN_JAMO_V 0xfff20000 +#define _NORM_JAMO_V_TOP 0xfff30000 + +/* value constants for auxTrie */ +enum { + _NORM_AUX_COMP_EX_SHIFT=10, + _NORM_AUX_UNSAFE_SHIFT=11, + _NORM_AUX_NFC_SKIPPABLE_F_SHIFT=12 +}; + +#define _NORM_AUX_MAX_FNC ((int32_t)1<<_NORM_AUX_COMP_EX_SHIFT) + +#define _NORM_AUX_FNC_MASK (uint32_t)(_NORM_AUX_MAX_FNC-1) +#define _NORM_AUX_COMP_EX_MASK ((uint32_t)1<<_NORM_AUX_COMP_EX_SHIFT) +#define _NORM_AUX_UNSAFE_MASK ((uint32_t)1<<_NORM_AUX_UNSAFE_SHIFT) +#define _NORM_AUX_NFC_SKIP_F_MASK ((uint32_t)1<<_NORM_AUX_NFC_SKIPPABLE_F_SHIFT) + +/* canonStartSets[0..31] contains indexes for what is in the array */ +enum { + _NORM_SET_INDEX_CANON_SETS_LENGTH, /* number of uint16_t in canonical starter sets */ + _NORM_SET_INDEX_CANON_BMP_TABLE_LENGTH, /* number of uint16_t in the BMP search table (contains pairs) */ + _NORM_SET_INDEX_CANON_SUPP_TABLE_LENGTH,/* number of uint16_t in the supplementary search table (contains triplets) */ + + /* from formatVersion 2.3: */ + _NORM_SET_INDEX_NX_CJK_COMPAT_OFFSET, /* uint16_t offset from canonStartSets[0] to the + exclusion set for CJK compatibility characters */ + _NORM_SET_INDEX_NX_UNICODE32_OFFSET, /* uint16_t offset from canonStartSets[0] to the + exclusion set for Unicode 3.2 characters */ + _NORM_SET_INDEX_NX_RESERVED_OFFSET, /* uint16_t offset from canonStartSets[0] to the + end of the previous exclusion set */ + + _NORM_SET_INDEX_TOP=32 /* changing this requires a new formatVersion */ +}; + +/* more constants for canonical starter sets */ + +/* 14 bit indexes to canonical USerializedSets */ +#define _NORM_MAX_CANON_SETS 0x4000 + +/* single-code point BMP sets are encoded directly in the search table except if result=0x4000..0x7fff */ +#define _NORM_CANON_SET_BMP_MASK 0xc000 +#define _NORM_CANON_SET_BMP_IS_INDEX 0x4000 + +/* indexes[] value names */ +enum { + _NORM_INDEX_TRIE_SIZE, /* number of bytes in normalization trie */ + _NORM_INDEX_UCHAR_COUNT, /* number of UChars in extra data */ + + _NORM_INDEX_COMBINE_DATA_COUNT, /* number of uint16_t words for combining data */ + _NORM_INDEX_COMBINE_FWD_COUNT, /* number of code points that combine forward */ + _NORM_INDEX_COMBINE_BOTH_COUNT, /* number of code points that combine forward and backward */ + _NORM_INDEX_COMBINE_BACK_COUNT, /* number of code points that combine backward */ + + _NORM_INDEX_MIN_NFC_NO_MAYBE, /* first code point with quick check NFC NO/MAYBE */ + _NORM_INDEX_MIN_NFKC_NO_MAYBE, /* first code point with quick check NFKC NO/MAYBE */ + _NORM_INDEX_MIN_NFD_NO_MAYBE, /* first code point with quick check NFD NO/MAYBE */ + _NORM_INDEX_MIN_NFKD_NO_MAYBE, /* first code point with quick check NFKD NO/MAYBE */ + + _NORM_INDEX_FCD_TRIE_SIZE, /* number of bytes in FCD trie */ + + _NORM_INDEX_AUX_TRIE_SIZE, /* number of bytes in the auxiliary trie */ + _NORM_INDEX_CANON_SET_COUNT, /* number of uint16_t in the array of serialized USet */ + + _NORM_INDEX_TOP=32 /* changing this requires a new formatVersion */ +}; + +enum { + /* FCD check: everything below this code point is known to have a 0 lead combining class */ + _NORM_MIN_WITH_LEAD_CC=0x300 +}; + +enum { + /** + * Bit 7 of the length byte for a decomposition string in extra data is + * a flag indicating whether the decomposition string is + * preceded by a 16-bit word with the leading and trailing cc + * of the decomposition (like for A-umlaut); + * if not, then both cc's are zero (like for compatibility ideographs). + */ + _NORM_DECOMP_FLAG_LENGTH_HAS_CC=0x80, + /** + * Bits 6..0 of the length byte contain the actual length. + */ + _NORM_DECOMP_LENGTH_MASK=0x7f +}; + +#endif /* #if !UCONFIG_NO_NORMALIZATION */ + +/* Korean Hangul and Jamo constants */ +enum { + JAMO_L_BASE=0x1100, /* "lead" jamo */ + JAMO_V_BASE=0x1161, /* "vowel" jamo */ + JAMO_T_BASE=0x11a7, /* "trail" jamo */ + + HANGUL_BASE=0xac00, + + JAMO_L_COUNT=19, + JAMO_V_COUNT=21, + JAMO_T_COUNT=28, + + HANGUL_COUNT=JAMO_L_COUNT*JAMO_V_COUNT*JAMO_T_COUNT +}; + +#if !UCONFIG_NO_NORMALIZATION + +/* Constants for options flags for normalization. @draft ICU 2.6 */ +enum { + /** Options bit 0, do not decompose Hangul syllables. @draft ICU 2.6 */ + UNORM_NX_HANGUL=1, + /** Options bit 1, do not decompose CJK compatibility characters. @draft ICU 2.6 */ + UNORM_NX_CJK_COMPAT=2, + /** + * Options bit 8, use buggy recomposition described in + * Unicode Public Review Issue #29 + * at http://www.unicode.org/review/resolved-pri.html#pri29 + * + * Used in IDNA implementation according to strict interpretation + * of IDNA definition based on Unicode 3.2 which predates PRI #29. + */ + UNORM_BEFORE_PRI_29=0x100 +}; + +/** + * Is the normalizer data loaded? + * This is used internally before other internal normalizer functions + * are called. + * It saves this check in each of many normalization calls that + * are made for, e.g., collation. + * + * @param pErrorCode as usual + * @return boolean value for whether the normalization data is loaded + * + * @internal + */ +U_CAPI UBool U_EXPORT2 +unorm_haveData(UErrorCode *pErrorCode); + +/** + * Internal API for normalizing. + * Does not check for bad input. + * @internal + */ +U_CAPI int32_t U_EXPORT2 +unorm_internalNormalize(UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + UNormalizationMode mode, int32_t options, + UErrorCode *pErrorCode); + +#ifdef XP_CPLUSPLUS + +/** + * Internal API for normalizing. + * Does not check for bad input. + * Requires _haveData() to be true. + * @internal + */ +U_CFUNC int32_t +unorm_internalNormalizeWithNX(UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + UNormalizationMode mode, int32_t options, const U_NAMESPACE_QUALIFIER UnicodeSet *nx, + UErrorCode *pErrorCode); + +#endif + +/** + * internal API, used by normlzr.cpp + * @internal + */ +U_CAPI int32_t U_EXPORT2 +unorm_decompose(UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + UBool compat, int32_t options, + UErrorCode *pErrorCode); + +/** + * internal API, used by normlzr.cpp + * @internal + */ +U_CAPI int32_t U_EXPORT2 +unorm_compose(UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + UBool compat, int32_t options, + UErrorCode *pErrorCode); + +#ifdef XP_CPLUSPLUS + +/** + * internal API, used by unormcmp.cpp + * @internal + */ +U_CFUNC UNormalizationCheckResult +unorm_internalQuickCheck(const UChar *src, + int32_t srcLength, + UNormalizationMode mode, + UBool allowMaybe, + const U_NAMESPACE_QUALIFIER UnicodeSet *nx, + UErrorCode *pErrorCode); + +#endif + +#endif /* #if !UCONFIG_NO_NORMALIZATION */ + +/** + * Internal option for unorm_cmpEquivFold() for decomposing. + * If not set, just do strcasecmp(). + * @internal + */ +#define _COMPARE_EQUIV 0x80000 + +#ifndef U_COMPARE_IGNORE_CASE +/* see also unorm.h */ +/** + * Option bit for unorm_compare: + * Perform case-insensitive comparison. + * @draft ICU 2.2 + */ +#define U_COMPARE_IGNORE_CASE 0x10000 +#endif + +/** + * Internal option for unorm_cmpEquivFold() for strncmp style. + * If set, checks for both string length and terminating NUL. + * @internal + */ +#define _STRNCMP_STYLE 0x1000 + +#if !UCONFIG_NO_NORMALIZATION + +/** + * Internal API to get the 16-bit FCD value (lccc + tccc) for c, + * for u_getIntPropertyValue(). + * @internal + */ +U_CFUNC uint16_t U_EXPORT2 +unorm_getFCD16FromCodePoint(UChar32 c); + +/** + * Internal API, used by collation code. + * Get access to the internal FCD trie table to be able to perform + * incremental, per-code unit, FCD checks in collation. + * One pointer is sufficient because the trie index values are offset + * by the index size, so that the same pointer is used to access the trie data. + * @internal + */ +U_CAPI const uint16_t * U_EXPORT2 +unorm_getFCDTrie(UErrorCode *pErrorCode); + +#ifdef XP_CPLUSPLUS + +/** + * Internal API, used by collation code. + * Get the FCD value for a code unit, with + * bits 15..8 lead combining class + * bits 7..0 trail combining class + * + * If c is a lead surrogate and the value is not 0, + * then instead of combining classes the value + * is used in unorm_getFCD16FromSurrogatePair() to get the real value + * of the supplementary code point. + * + * @internal + */ +static inline uint16_t +unorm_getFCD16(const uint16_t *fcdTrieIndex, UChar c) { + return + fcdTrieIndex[ + (fcdTrieIndex[ + c>>UTRIE_SHIFT + ]<>UTRIE_SHIFT) + ]<-skippable code point? See unormimp.h. + * @internal + */ +U_CAPI UBool U_EXPORT2 +unorm_isNFSkippable(UChar32 c, UNormalizationMode mode); + +#ifdef XP_CPLUSPLUS + +/** + * Get normalization exclusion set for the options. + * Requires unorm_haveData(). + * @internal + */ +U_CFUNC const U_NAMESPACE_QUALIFIER UnicodeSet * +unorm_getNX(int32_t options, UErrorCode *pErrorCode); + +#endif + +/** + * Enumerate each normalization data trie and add the + * start of each range of same properties to the set. + * @internal + */ +U_CAPI void U_EXPORT2 +unorm_addPropertyStarts(const USetAdder *sa, UErrorCode *pErrorCode); + +/** + * Swap unorm.icu. See udataswp.h. + * @internal + */ +U_CAPI int32_t U_EXPORT2 +unorm_swap(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode); + +/** + * Get the NF*_QC property for a code point, for u_getIntPropertyValue(). + * @internal + */ +U_CFUNC UNormalizationCheckResult U_EXPORT2 +unorm_getQuickCheck(UChar32 c, UNormalizationMode mode); + +/** + * Description of the format of unorm.icu version 2.3. + * + * Main change from version 1 to version 2: + * Use of new, common UTrie instead of normalization-specific tries. + * Change to version 2.1: add third/auxiliary trie with associated data. + * Change to version 2.2: add skippable (f) flag data (_NORM_AUX_NFC_SKIP_F_MASK). + * Change to version 2.3: add serialized sets for normalization exclusions + * stored inside canonStartSets[] + * + * For more details of how to use the data structures see the code + * in unorm.cpp (runtime normalization code) and + * in gennorm.c and gennorm/store.c (build-time data generation). + * + * For the serialized format of UTrie see utrie.c/UTrieHeader. + * + * - Overall partition + * + * unorm.dat customarily begins with a UDataInfo structure, see udata.h and .c. + * After that there are the following structures: + * + * int32_t indexes[_NORM_INDEX_TOP]; -- _NORM_INDEX_TOP=32, see enum in this file + * + * UTrie normTrie; -- size in bytes=indexes[_NORM_INDEX_TRIE_SIZE] + * + * uint16_t extraData[extraDataTop]; -- extraDataTop=indexes[_NORM_INDEX_UCHAR_COUNT] + * extraData[0] contains the number of units for + * FC_NFKC_Closure (formatVersion>=2.1) + * + * uint16_t combiningTable[combiningTableTop]; -- combiningTableTop=indexes[_NORM_INDEX_COMBINE_DATA_COUNT] + * combiningTableTop may include one 16-bit padding unit + * to make sure that fcdTrie is 32-bit-aligned + * + * UTrie fcdTrie; -- size in bytes=indexes[_NORM_INDEX_FCD_TRIE_SIZE] + * + * UTrie auxTrie; -- size in bytes=indexes[_NORM_INDEX_AUX_TRIE_SIZE] + * + * uint16_t canonStartSets[canonStartSetsTop] -- canonStartSetsTop=indexes[_NORM_INDEX_CANON_SET_COUNT] + * serialized USets and binary search tables, see below + * + * + * The indexes array contains lengths and sizes of the following arrays and structures + * as well as the following values: + * indexes[_NORM_INDEX_COMBINE_FWD_COUNT]=combineFwdTop + * -- one more than the highest combining index computed for forward-only-combining characters + * indexes[_NORM_INDEX_COMBINE_BOTH_COUNT]=combineBothTop-combineFwdTop + * -- number of combining indexes computed for both-ways-combining characters + * indexes[_NORM_INDEX_COMBINE_BACK_COUNT]=combineBackTop-combineBothTop + * -- number of combining indexes computed for backward-only-combining characters + * + * indexes[_NORM_INDEX_MIN_NF*_NO_MAYBE] (where *={ C, D, KC, KD }) + * -- first code point with a quick check NF* value of NO/MAYBE + * + * + * - Tries + * + * The main structures are two UTrie tables ("compact arrays"), + * each with one index array and one data array. + * See utrie.h and utrie.c. + * + * + * - Tries in unorm.dat + * + * The first trie (normTrie above) + * provides data for the NF* quick checks and normalization. + * The second trie (fcdTrie above) provides data just for FCD checks. + * + * + * - norm32 data words from the first trie + * + * The norm32Table contains one 32-bit word "norm32" per code point. + * It contains the following bit fields: + * 31..16 extra data index, _NORM_EXTRA_SHIFT is used to shift this field down + * if this index is <_NORM_EXTRA_INDEX_TOP then it is an index into + * extraData[] where variable-length normalization data for this + * code point is found + * if this index is <_NORM_EXTRA_INDEX_TOP+_NORM_EXTRA_SURROGATE_TOP + * then this is a norm32 for a leading surrogate, and the index + * value is used together with the following trailing surrogate + * code unit in the second trie access + * if this index is >=_NORM_EXTRA_INDEX_TOP+_NORM_EXTRA_SURROGATE_TOP + * then this is a norm32 for a "special" character, + * i.e., the character is a Hangul syllable or a Jamo + * see _NORM_EXTRA_HANGUL etc. + * generally, instead of extracting this index from the norm32 and + * comparing it with the above constants, + * the normalization code compares the entire norm32 value + * with _NORM_MIN_SPECIAL, _NORM_SURROGATES_TOP, _NORM_MIN_HANGUL etc. + * + * 15..8 combining class (cc) according to UnicodeData.txt + * + * 7..6 _NORM_COMBINES_ANY flags, used in composition to see if a character + * combines with any following or preceding character(s) + * at all + * 7 _NORM_COMBINES_BACK + * 6 _NORM_COMBINES_FWD + * + * 5..0 quick check flags, set for "no" or "maybe", with separate flags for + * each normalization form + * the higher bits are "maybe" flags; for NF*D there are no such flags + * the lower bits are "no" flags for all forms, in the same order + * as the "maybe" flags, + * which is (MSB to LSB): NFKD NFD NFKC NFC + * 5..4 _NORM_QC_ANY_MAYBE + * 3..0 _NORM_QC_ANY_NO + * see further related constants + * + * + * - Extra data per code point + * + * "Extra data" is referenced by the index in norm32. + * It is variable-length data. It is only present, and only those parts + * of it are, as needed for a given character. + * The norm32 extra data index is added to the beginning of extraData[] + * to get to a vector of 16-bit words with data at the following offsets: + * + * [-1] Combining index for composition. + * Stored only if norm32&_NORM_COMBINES_ANY . + * [0] Lengths of the canonical and compatibility decomposition strings. + * Stored only if there are decompositions, i.e., + * if norm32&(_NORM_QC_NFD|_NORM_QC_NFKD) + * High byte: length of NFKD, or 0 if none + * Low byte: length of NFD, or 0 if none + * Each length byte also has another flag: + * Bit 7 of a length byte is set if there are non-zero + * combining classes (cc's) associated with the respective + * decomposition. If this flag is set, then the decomposition + * is preceded by a 16-bit word that contains the + * leading and trailing cc's. + * Bits 6..0 of a length byte are the length of the + * decomposition string, not counting the cc word. + * [1..n] NFD + * [n+1..] NFKD + * + * Each of the two decompositions consists of up to two parts: + * - The 16-bit words with the leading and trailing cc's. + * This is only stored if bit 7 of the corresponding length byte + * is set. In this case, at least one of the cc's is not zero. + * High byte: leading cc==cc of the first code point in the decomposition string + * Low byte: trailing cc==cc of the last code point in the decomposition string + * - The decomposition string in UTF-16, with length code units. + * + * + * - Combining indexes and combiningTable[] + * + * Combining indexes are stored at the [-1] offset of the extra data + * if the character combines forward or backward with any other characters. + * They are used for (re)composition in NF*C. + * Values of combining indexes are arranged according to whether a character + * combines forward, backward, or both ways: + * forward-only < both ways < backward-only + * + * The index values for forward-only and both-ways combining characters + * are indexes into the combiningTable[]. + * The index values for backward-only combining characters are simply + * incremented from the preceding index values to be unique. + * + * In the combiningTable[], a variable-length list + * of variable-length (back-index, code point) pair entries is stored + * for each forward-combining character. + * + * These back-indexes are the combining indexes of both-ways or backward-only + * combining characters that the forward-combining character combines with. + * + * Each list is sorted in ascending order of back-indexes. + * Each list is terminated with the last back-index having bit 15 set. + * + * Each pair (back-index, code point) takes up either 2 or 3 + * 16-bit words. + * The first word of a list entry is the back-index, with its bit 15 set if + * this is the last pair in the list. + * + * The second word contains flags in bits 15..13 that determine + * if there is a third word and how the combined character is encoded: + * 15 set if there is a third word in this list entry + * 14 set if the result is a supplementary character + * 13 set if the result itself combines forward + * + * According to these bits 15..14 of the second word, + * the result character is encoded as follows: + * 00 or 01 The result is <=0x1fff and stored in bits 12..0 of + * the second word. + * 10 The result is 0x2000..0xffff and stored in the third word. + * Bits 12..0 of the second word are not used. + * 11 The result is a supplementary character. + * Bits 9..0 of the leading surrogate are in bits 9..0 of + * the second word. + * Add 0xd800 to these bits to get the complete surrogate. + * Bits 12..10 of the second word are not used. + * The trailing surrogate is stored in the third word. + * + * + * - FCD trie + * + * The FCD trie is very simple. + * It is a folded trie with 16-bit data words. + * In each word, the high byte contains the leading cc of the character, + * and the low byte contains the trailing cc of the character. + * These cc's are the cc's of the first and last code points in the + * canonical decomposition of the character. + * + * Since all 16 bits are used for cc's, lead surrogates must be tested + * by checking the code unit instead of the trie data. + * This is done only if the 16-bit data word is not zero. + * If the code unit is a leading surrogate and the data word is not zero, + * then instead of cc's it contains the offset for the second trie lookup. + * + * + * - Auxiliary trie and data + * + * The auxiliary 16-bit trie contains data for additional properties. + * Bits + * 15..13 reserved + * 12 not NFC_Skippable (f) (formatVersion>=2.2) + * 11 flag: not a safe starter for canonical closure + * 10 composition exclusion + * 9.. 0 index into extraData[] to FC_NFKC_Closure string + * (not for lead surrogate), + * or lead surrogate offset (for lead surrogate, if 9..0 not zero) + * + * - FC_NFKC_Closure strings in extraData[] + * + * Strings are either stored as a single code unit or as the length + * followed by that many units. + * const UChar *s=extraData+(index from auxTrie data bits 9..0); + * int32_t length; + * if(*s<0xff00) { + * // s points to the single-unit string + * length=1; + * } else { + * length=*s&0xff; + * ++s; + * } + * + * Conditions for "NF* Skippable" from Mark Davis' com.ibm.text.UCD.NFSkippable: + * (used in NormalizerTransliterator) + * + * A skippable character is + * a) unassigned, or ALL of the following: + * b) of combining class 0. + * c) not decomposed by this normalization form. + * AND if NFC or NFKC, + * d) can never compose with a previous character. + * e) can never compose with a following character. + * f) can never change if another character is added. + * Example: a-breve might satisfy all but f, but if you + * add an ogonek it changes to a-ogonek + breve + * + * a)..e) must be tested from norm32. + * Since f) is more complicated, the (not-)NFC_Skippable flag (f) is built + * into the auxiliary trie. + * The same bit is used for NFC and NFKC; (c) differs for them. + * As usual, we build the "not skippable" flags so that unassigned + * code points get a 0 bit. + * This bit is only valid after (a)..(e) test FALSE; test NFD_NO before (f) as well. + * Test Hangul LV syllables entirely in code. + * + * + * - structure inside canonStartSets[] + * + * This array maps from code points c to sets of code points (USerializedSet). + * The result sets are the code points whose canonical decompositions start + * with c. + * + * canonStartSets[] contains the following sub-arrays: + * + * indexes[_NORM_SET_INDEX_TOP] + * - contains lengths of sub-arrays etc. + * + * startSets[indexes[_NORM_SET_INDEX_CANON_SETS_LENGTH]-_NORM_SET_INDEX_TOP] + * - contains serialized sets (USerializedSet) of canonical starters for + * enumerating canonically equivalent strings + * indexes[_NORM_SET_INDEX_CANON_SETS_LENGTH] includes _NORM_SET_INDEX_TOP + * for details about the structure see uset.c + * + * bmpTable[indexes[_NORM_SET_INDEX_CANON_BMP_TABLE_LENGTH]] + * - a sorted search table for BMP code points whose results are + * either indexes to USerializedSets or single code points for + * single-code point sets; + * each entry is a pair of { code point, result } with result=(binary) yy xxxxxx xxxxxxxx + * if yy==01 then there is a USerializedSet at canonStartSets+x + * else build a USerializedSet with result as the single code point + * + * suppTable[indexes[_NORM_SET_INDEX_CANON_SUPP_TABLE_LENGTH]] + * - a sorted search table for supplementary code points whose results are + * either indexes to USerializedSets or single code points for + * single-code point sets; + * each entry is a triplet of { high16(cp), low16(cp), result } + * each code point's high-word may contain extra data in bits 15..5: + * if the high word has bit 15 set, then build a set with a single code point + * which is (((high16(cp)&0x1f00)<<8)|result; + * else there is a USerializedSet at canonStartSets+result + * + * FormatVersion 2.3 adds 2 serialized sets for normalization exclusions. + * They are stored in the data file so that the runtime normalization code need + * not depend on other properties and their data and implementation files. + * The _NORM_SET_INDEX_NX_..._OFFSET offsets in the canonStartSets index table + * give the location for each set. + * There is no set stored for UNORM_NX_HANGUL because it's trivial to create + * without using properties. + * + * Set contents: + * + * _NORM_SET_INDEX_NX_CJK_COMPAT_OFFSET (for UNORM_NX_CJK_COMPAT) + * [[:Ideographic:]&[:NFD_QC=No:]] + * =[CJK Ideographs]&[has canonical decomposition] + * + * _NORM_SET_INDEX_NX_UNICODE32_OFFSET (for UNORM_UNICODE_3_2) + * [:^Age=3.2:] + * =set with all code points that were not designated by the specified Unicode version + * + * _NORM_SET_INDEX_NX_RESERVED_OFFSET + * This is an offset that points to where the next, future set would start. + * Currently it indicates where the previous set ends, and thus its length. + * The name for this enum constant may in the future be applied to different + * index slots. In order to get the limit of a set, use its index slot and + * the immediately following one regardless of that one's enum name. + */ + +#endif /* #if !UCONFIG_NO_NORMALIZATION */ + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/uobject.cpp b/Build/source/libs/icu/icu-xetex/common/uobject.cpp new file mode 100644 index 00000000000..5fc5423880e --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uobject.cpp @@ -0,0 +1,114 @@ +/* +****************************************************************************** +* +* Copyright (C) 2002-2005, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: uobject.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2002jun26 +* created by: Markus W. Scherer +*/ + +#include "unicode/uobject.h" +#include "cmemory.h" + +U_NAMESPACE_BEGIN + +#if U_OVERRIDE_CXX_ALLOCATION + +/* + * Default implementation of UMemory::new/delete + * using uprv_malloc() and uprv_free(). + * + * For testing, this is used together with a list of imported symbols to verify + * that ICU is not using the global ::new and ::delete operators. + * + * These operators can be implemented like this or any other appropriate way + * when customizing ICU for certain environments. + * Whenever ICU is customized in binary incompatible ways please be sure + * to use library name suffixes to distinguish such libraries from + * the standard build. + * + * Instead of just modifying these C++ new/delete operators, it is usually best + * to modify the uprv_malloc()/uprv_free()/uprv_realloc() functions in cmemory.c. + * + * Memory test on Windows/MSVC 6: + * The global operators new and delete look as follows: + * 04F 00000000 UNDEF notype () External | ??2@YAPAXI@Z (void * __cdecl operator new(unsigned int)) + * 03F 00000000 UNDEF notype () External | ??3@YAXPAX@Z (void __cdecl operator delete(void *)) + * + * These lines are from output generated by the MSVC 6 tool dumpbin with + * dumpbin /symbols *.obj + * + * ??2@YAPAXI@Z and ??3@YAXPAX@Z are the linker symbols in the .obj + * files and are imported from msvcrtd.dll (in a debug build). + * + * Make sure that with the UMemory operators new and delete defined these two symbols + * do not appear in the dumpbin /symbols output for the ICU libraries! + * + * If such a symbol appears in the output then look in the preceding lines in the output + * for which file and function calls the global new or delete operator, + * and replace with uprv_malloc/uprv_free. + */ + +void * U_EXPORT2 UMemory::operator new(size_t size) { + return uprv_malloc(size); +} + +void U_EXPORT2 UMemory::operator delete(void *p) { + if(p!=NULL) { + uprv_free(p); + } +} + +void * U_EXPORT2 UMemory::operator new[](size_t size) { + return uprv_malloc(size); +} + +void U_EXPORT2 UMemory::operator delete[](void *p) { + if(p!=NULL) { + uprv_free(p); + } +} + +#if U_HAVE_DEBUG_LOCATION_NEW +void * U_EXPORT2 UMemory::operator new(size_t size, const char* /*file*/, int /*line*/) { + return UMemory::operator new(size); +} + +void U_EXPORT2 UMemory::operator delete(void* p, const char* /*file*/, int /*line*/) { + UMemory::operator delete(p); +} +#endif /* U_HAVE_DEBUG_LOCATION_NEW */ + + +#endif + +UObject::~UObject() {} + +// Future implementation for RTTI that support subtyping. [alan] +// +// UClassID UObject::getStaticClassID() { +// return (UClassID) NULL; +// } +// +// UBool UObject::instanceOf(UClassID type) const { +// UClassID c = getDynamicClassID(); +// for (;;) { +// if (c == type) { +// return TRUE; +// } else if (c == (UClassID) NULL) { +// return FALSE; +// } +// c = * (UClassID*) c; +// } +// } + +U_NAMESPACE_END + + diff --git a/Build/source/libs/icu/icu-xetex/common/uprops.c b/Build/source/libs/icu/icu-xetex/common/uprops.c new file mode 100644 index 00000000000..2fb4e603f88 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uprops.c @@ -0,0 +1,545 @@ +/* +******************************************************************************* +* +* Copyright (C) 2002-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: uprops.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2002feb24 +* created by: Markus W. Scherer +* +* Implementations for mostly non-core Unicode character properties +* stored in uprops.icu. +* +* With the APIs implemented here, almost all properties files and +* their associated implementation files are used from this file, +* including those for normalization and case mappings. +*/ + +#include "unicode/utypes.h" +#include "unicode/uchar.h" +#include "unicode/uscript.h" +#include "cstring.h" +#include "ucln_cmn.h" +#include "umutex.h" +#include "unormimp.h" +#include "ubidi_props.h" +#include "uprops.h" +#include "ucase.h" + +#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) + +/* cleanup ------------------------------------------------------------------ */ + +static const UBiDiProps *gBdp=NULL; + +static UBool U_CALLCONV uprops_cleanup(void) { + gBdp=NULL; + return TRUE; +} + +/* bidi/shaping properties API ---------------------------------------------- */ + +/* get the UBiDiProps singleton, or else its dummy, once and for all */ +static const UBiDiProps * +getBiDiProps() { + /* + * This lazy intialization with double-checked locking (without mutex protection for + * the initial check) is transiently unsafe under certain circumstances. + * Check the readme and use u_init() if necessary. + */ + + /* the initial check is performed by the GET_BIDI_PROPS() macro */ + const UBiDiProps *bdp; + UErrorCode errorCode=U_ZERO_ERROR; + + bdp=ubidi_getSingleton(&errorCode); + if(U_FAILURE(errorCode)) { + errorCode=U_ZERO_ERROR; + bdp=ubidi_getDummy(&errorCode); + if(U_FAILURE(errorCode)) { + return NULL; + } + } + + umtx_lock(NULL); + if(gBdp==NULL) { + gBdp=bdp; + bdp=NULL; + ucln_common_registerCleanup(UCLN_COMMON_UPROPS, uprops_cleanup); + } + umtx_unlock(NULL); + + return gBdp; +} + +/* see comment for GET_CASE_PROPS() */ +#define GET_BIDI_PROPS() (gBdp!=NULL ? gBdp : getBiDiProps()) + +/* general properties API functions ----------------------------------------- */ + +static const struct { + int32_t column; + uint32_t mask; +} binProps[UCHAR_BINARY_LIMIT]={ + /* + * column and mask values for binary properties from u_getUnicodeProperties(). + * Must be in order of corresponding UProperty, + * and there must be exacly one entry per binary UProperty. + * + * Properties with mask 0 are handled in code. + * For them, column is the UPropertySource value. + */ + { 1, U_MASK(UPROPS_ALPHABETIC) }, + { 1, U_MASK(UPROPS_ASCII_HEX_DIGIT) }, + { UPROPS_SRC_BIDI, 0 }, /* UCHAR_BIDI_CONTROL */ + { UPROPS_SRC_BIDI, 0 }, /* UCHAR_BIDI_MIRRORED */ + { 1, U_MASK(UPROPS_DASH) }, + { 1, U_MASK(UPROPS_DEFAULT_IGNORABLE_CODE_POINT) }, + { 1, U_MASK(UPROPS_DEPRECATED) }, + { 1, U_MASK(UPROPS_DIACRITIC) }, + { 1, U_MASK(UPROPS_EXTENDER) }, + { UPROPS_SRC_NORM, 0 }, /* UCHAR_FULL_COMPOSITION_EXCLUSION */ + { 1, U_MASK(UPROPS_GRAPHEME_BASE) }, + { 1, U_MASK(UPROPS_GRAPHEME_EXTEND) }, + { 1, U_MASK(UPROPS_GRAPHEME_LINK) }, + { 1, U_MASK(UPROPS_HEX_DIGIT) }, + { 1, U_MASK(UPROPS_HYPHEN) }, + { 1, U_MASK(UPROPS_ID_CONTINUE) }, + { 1, U_MASK(UPROPS_ID_START) }, + { 1, U_MASK(UPROPS_IDEOGRAPHIC) }, + { 1, U_MASK(UPROPS_IDS_BINARY_OPERATOR) }, + { 1, U_MASK(UPROPS_IDS_TRINARY_OPERATOR) }, + { UPROPS_SRC_BIDI, 0 }, /* UCHAR_JOIN_CONTROL */ + { 1, U_MASK(UPROPS_LOGICAL_ORDER_EXCEPTION) }, + { UPROPS_SRC_CASE, 0 }, /* UCHAR_LOWERCASE */ + { 1, U_MASK(UPROPS_MATH) }, + { 1, U_MASK(UPROPS_NONCHARACTER_CODE_POINT) }, + { 1, U_MASK(UPROPS_QUOTATION_MARK) }, + { 1, U_MASK(UPROPS_RADICAL) }, + { UPROPS_SRC_CASE, 0 }, /* UCHAR_SOFT_DOTTED */ + { 1, U_MASK(UPROPS_TERMINAL_PUNCTUATION) }, + { 1, U_MASK(UPROPS_UNIFIED_IDEOGRAPH) }, + { UPROPS_SRC_CASE, 0 }, /* UCHAR_UPPERCASE */ + { 1, U_MASK(UPROPS_WHITE_SPACE) }, + { 1, U_MASK(UPROPS_XID_CONTINUE) }, + { 1, U_MASK(UPROPS_XID_START) }, + { UPROPS_SRC_CASE, 0 }, /* UCHAR_CASE_SENSITIVE */ + { 2, U_MASK(UPROPS_V2_S_TERM) }, + { 2, U_MASK(UPROPS_V2_VARIATION_SELECTOR) }, + { UPROPS_SRC_NORM, 0 }, /* UCHAR_NFD_INERT */ + { UPROPS_SRC_NORM, 0 }, /* UCHAR_NFKD_INERT */ + { UPROPS_SRC_NORM, 0 }, /* UCHAR_NFC_INERT */ + { UPROPS_SRC_NORM, 0 }, /* UCHAR_NFKC_INERT */ + { UPROPS_SRC_NORM, 0 }, /* UCHAR_SEGMENT_STARTER */ + { 2, U_MASK(UPROPS_V2_PATTERN_SYNTAX) }, + { 2, U_MASK(UPROPS_V2_PATTERN_WHITE_SPACE) }, + { UPROPS_SRC_CHAR_AND_PROPSVEC, 0 }, /* UCHAR_POSIX_ALNUM */ + { UPROPS_SRC_CHAR, 0 }, /* UCHAR_POSIX_BLANK */ + { UPROPS_SRC_CHAR, 0 }, /* UCHAR_POSIX_GRAPH */ + { UPROPS_SRC_CHAR, 0 }, /* UCHAR_POSIX_PRINT */ + { UPROPS_SRC_CHAR, 0 } /* UCHAR_POSIX_XDIGIT */ +}; + +U_CAPI UBool U_EXPORT2 +u_hasBinaryProperty(UChar32 c, UProperty which) { + /* c is range-checked in the functions that are called from here */ + if(which>UPROPS_EA_SHIFT; + case UCHAR_GENERAL_CATEGORY: + return (int32_t)u_charType(c); + case UCHAR_JOINING_GROUP: + return ubidi_getJoiningGroup(GET_BIDI_PROPS(), c); + case UCHAR_JOINING_TYPE: + return ubidi_getJoiningType(GET_BIDI_PROPS(), c); + case UCHAR_LINE_BREAK: + return (int32_t)(u_getUnicodeProperties(c, 0)&UPROPS_LB_MASK)>>UPROPS_LB_SHIFT; + case UCHAR_NUMERIC_TYPE: + type=(int32_t)GET_NUMERIC_TYPE(u_getUnicodeProperties(c, -1)); + if(type>U_NT_NUMERIC) { + /* keep internal variants of U_NT_NUMERIC from becoming visible */ + type=U_NT_NUMERIC; + } + return type; + case UCHAR_SCRIPT: + errorCode=U_ZERO_ERROR; + return (int32_t)uscript_getScript(c, &errorCode); + case UCHAR_HANGUL_SYLLABLE_TYPE: + return uchar_getHST(c); +#if !UCONFIG_NO_NORMALIZATION + case UCHAR_NFD_QUICK_CHECK: + case UCHAR_NFKD_QUICK_CHECK: + case UCHAR_NFC_QUICK_CHECK: + case UCHAR_NFKC_QUICK_CHECK: + return (int32_t)unorm_getQuickCheck(c, (UNormalizationMode)(which-UCHAR_NFD_QUICK_CHECK+UNORM_NFD)); + case UCHAR_LEAD_CANONICAL_COMBINING_CLASS: + return unorm_getFCD16FromCodePoint(c)>>8; + case UCHAR_TRAIL_CANONICAL_COMBINING_CLASS: + return unorm_getFCD16FromCodePoint(c)&0xff; +#endif + case UCHAR_GRAPHEME_CLUSTER_BREAK: + return (int32_t)(u_getUnicodeProperties(c, 2)&UPROPS_GCB_MASK)>>UPROPS_GCB_SHIFT; + case UCHAR_SENTENCE_BREAK: + return (int32_t)(u_getUnicodeProperties(c, 2)&UPROPS_SB_MASK)>>UPROPS_SB_SHIFT; + case UCHAR_WORD_BREAK: + return (int32_t)(u_getUnicodeProperties(c, 2)&UPROPS_WB_MASK)>>UPROPS_WB_SHIFT; + default: + return 0; /* undefined */ + } + } else if(which==UCHAR_GENERAL_CATEGORY_MASK) { + return U_MASK(u_charType(c)); + } else { + return 0; /* undefined */ + } +} + +U_CAPI int32_t U_EXPORT2 +u_getIntPropertyMinValue(UProperty which) { + return 0; /* all binary/enum/int properties have a minimum value of 0 */ +} + +U_CAPI int32_t U_EXPORT2 +u_getIntPropertyMaxValue(UProperty which) { + if(which>UPROPS_BLOCK_SHIFT; + case UCHAR_CANONICAL_COMBINING_CLASS: + case UCHAR_LEAD_CANONICAL_COMBINING_CLASS: + case UCHAR_TRAIL_CANONICAL_COMBINING_CLASS: + return 0xff; /* TODO do we need to be more precise, getting the actual maximum? */ + case UCHAR_DECOMPOSITION_TYPE: + return uprv_getMaxValues(2)&UPROPS_DT_MASK; + case UCHAR_EAST_ASIAN_WIDTH: + return (uprv_getMaxValues(0)&UPROPS_EA_MASK)>>UPROPS_EA_SHIFT; + case UCHAR_GENERAL_CATEGORY: + return (int32_t)U_CHAR_CATEGORY_COUNT-1; + case UCHAR_LINE_BREAK: + return (uprv_getMaxValues(0)&UPROPS_LB_MASK)>>UPROPS_LB_SHIFT; + case UCHAR_NUMERIC_TYPE: + return (int32_t)U_NT_COUNT-1; + case UCHAR_SCRIPT: + return uprv_getMaxValues(0)&UPROPS_SCRIPT_MASK; + case UCHAR_HANGUL_SYLLABLE_TYPE: + return (int32_t)U_HST_COUNT-1; +#if !UCONFIG_NO_NORMALIZATION + case UCHAR_NFD_QUICK_CHECK: + case UCHAR_NFKD_QUICK_CHECK: + return (int32_t)UNORM_YES; /* these are never "maybe", only "no" or "yes" */ + case UCHAR_NFC_QUICK_CHECK: + case UCHAR_NFKC_QUICK_CHECK: + return (int32_t)UNORM_MAYBE; +#endif + case UCHAR_GRAPHEME_CLUSTER_BREAK: + return (uprv_getMaxValues(2)&UPROPS_GCB_MASK)>>UPROPS_GCB_SHIFT; + case UCHAR_SENTENCE_BREAK: + return (uprv_getMaxValues(2)&UPROPS_SB_MASK)>>UPROPS_SB_SHIFT; + case UCHAR_WORD_BREAK: + return (uprv_getMaxValues(2)&UPROPS_WB_MASK)>>UPROPS_WB_SHIFT; + default: + return -1; /* undefined */ + } + } else { + return -1; /* undefined */ + } +} + +U_CFUNC UPropertySource U_EXPORT2 +uprops_getSource(UProperty which) { + if(which>UPROPS_NUMERIC_TYPE_SHIFT)&7) +#define GET_NUMERIC_VALUE(props) (((props)>>UPROPS_NUMERIC_VALUE_SHIFT)&0xff) + +/* internal numeric pseudo-types for special encodings of numeric values */ +enum { + UPROPS_NT_FRACTION=4, /* ==U_NT_COUNT, must not change unless binary format version changes */ + UPROPS_NT_LARGE, + UPROPS_NT_COUNT +}; + +/* encoding of fractional and large numbers */ +enum { + UPROPS_MAX_SMALL_NUMBER=0xff, + + UPROPS_FRACTION_NUM_SHIFT=3, /* numerator: bits 7..3 */ + UPROPS_FRACTION_DEN_MASK=7, /* denominator: bits 2..0 */ + + UPROPS_FRACTION_MAX_NUM=31, + UPROPS_FRACTION_DEN_OFFSET=2, /* denominator values are 2..9 */ + + UPROPS_FRACTION_MIN_DEN=UPROPS_FRACTION_DEN_OFFSET, + UPROPS_FRACTION_MAX_DEN=UPROPS_FRACTION_MIN_DEN+UPROPS_FRACTION_DEN_MASK, + + UPROPS_LARGE_MANT_SHIFT=4, /* mantissa: bits 7..4 */ + UPROPS_LARGE_EXP_MASK=0xf, /* exponent: bits 3..0 */ + UPROPS_LARGE_EXP_OFFSET=2, /* regular exponents 2..17 */ + UPROPS_LARGE_EXP_OFFSET_EXTRA=18, /* extra large exponents 18..33 */ + + UPROPS_LARGE_MIN_EXP=UPROPS_LARGE_EXP_OFFSET, + UPROPS_LARGE_MAX_EXP=UPROPS_LARGE_MIN_EXP+UPROPS_LARGE_EXP_MASK, + UPROPS_LARGE_MAX_EXP_EXTRA=UPROPS_LARGE_EXP_OFFSET_EXTRA+UPROPS_LARGE_EXP_MASK +}; + +/* number of properties vector words */ +#define UPROPS_VECTOR_WORDS 3 + +/* + * Properties in vector word 0 + * Bits + * 31..24 DerivedAge version major/minor one nibble each + * 23..18 Line Break + * 17..15 East Asian Width + * 14.. 7 UBlockCode + * 6.. 0 UScriptCode + */ + +/* derived age: one nibble each for major and minor version numbers */ +#define UPROPS_AGE_MASK 0xff000000 +#define UPROPS_AGE_SHIFT 24 + +#define UPROPS_LB_MASK 0x00FC0000 +#define UPROPS_LB_SHIFT 18 + +#define UPROPS_EA_MASK 0x00038000 +#define UPROPS_EA_SHIFT 15 + +#define UPROPS_BLOCK_MASK 0x00007f80 +#define UPROPS_BLOCK_SHIFT 7 + +#define UPROPS_SCRIPT_MASK 0x0000007f + +/* + * Properties in vector word 1 + * Each bit encodes one binary property. + * The following constants represent the bit number, use 1<=sizeof(pathBuffer)) { + *status=U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } else if(uprv_isInvariantUString(myPath, length)) { + /* + * the invariant converter is sufficient for package and tree names + * and is more efficient + */ + u_UCharsToChars(myPath, path, length+1); /* length+1 to include the NUL */ + } else { +#if !UCONFIG_NO_CONVERSION + /* use the default converter to support variant-character paths */ + UConverter *cnv=u_getDefaultConverter(status); + length=ucnv_fromUChars(cnv, path, (int32_t)sizeof(pathBuffer), myPath, length, status); + u_releaseDefaultConverter(cnv); + if(U_FAILURE(*status)) { + return NULL; + } + if(length>=sizeof(pathBuffer)) { + /* not NUL-terminated - path too long */ + *status=U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } +#else + /* the default converter is not available */ + *status=U_UNSUPPORTED_ERROR; + return NULL; +#endif + } + } + + return ures_open(path, localeID, status); +} diff --git a/Build/source/libs/icu/icu-xetex/common/uresbund.c b/Build/source/libs/icu/icu-xetex/common/uresbund.c new file mode 100644 index 00000000000..df66876269a --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uresbund.c @@ -0,0 +1,2629 @@ +/* +****************************************************************************** +* Copyright (C) 1997-2007, International Business Machines Corporation and * +* others. All Rights Reserved. * +****************************************************************************** +* +* File URESBUND.C +* +* Modification History: +* +* Date Name Description +* 04/01/97 aliu Creation. +* 06/14/99 stephen Removed functions taking a filename suffix. +* 07/20/99 stephen Changed for UResourceBundle typedef'd to void* +* 11/09/99 weiv Added ures_getLocale() +* March 2000 weiv Total overhaul - using data in DLLs +* 06/20/2000 helena OS/400 port changes; mostly typecast. +* 06/24/02 weiv Added support for resource sharing +****************************************************************************** +*/ + +#include "unicode/ustring.h" +#include "unicode/ucnv.h" +#include "uresimp.h" +#include "ustr_imp.h" +#include "cwchar.h" +#include "ucln_cmn.h" +#include "cmemory.h" +#include "cstring.h" +#include "uhash.h" +#include "unicode/uenum.h" +#include "uenumimp.h" +#include "ulocimp.h" +#include "umutex.h" +#include "putilimp.h" + + +/* +Static cache for already opened resource bundles - mostly for keeping fallback info +TODO: This cache should probably be removed when the deprecated code is + completely removed. +*/ +static UHashtable *cache = NULL; + +static UMTX resbMutex = NULL; + +/* INTERNAL: hashes an entry */ +static int32_t U_CALLCONV hashEntry(const UHashTok parm) { + UResourceDataEntry *b = (UResourceDataEntry *)parm.pointer; + UHashTok namekey, pathkey; + namekey.pointer = b->fName; + pathkey.pointer = b->fPath; + return uhash_hashChars(namekey)+37*uhash_hashChars(pathkey); +} + +/* INTERNAL: compares two entries */ +static UBool U_CALLCONV compareEntries(const UHashTok p1, const UHashTok p2) { + UResourceDataEntry *b1 = (UResourceDataEntry *)p1.pointer; + UResourceDataEntry *b2 = (UResourceDataEntry *)p2.pointer; + UHashTok name1, name2, path1, path2; + name1.pointer = b1->fName; + name2.pointer = b2->fName; + path1.pointer = b1->fPath; + path2.pointer = b2->fPath; + return (UBool)(uhash_compareChars(name1, name2) & + uhash_compareChars(path1, path2)); +} + + +/** + * Internal function, gets parts of locale name according + * to the position of '_' character + */ +static UBool chopLocale(char *name) { + char *i = uprv_strrchr(name, '_'); + + if(i != NULL) { + *i = '\0'; + return TRUE; + } + + return FALSE; +} + +/** + * Internal function + */ +static void entryIncrease(UResourceDataEntry *entry) { + umtx_lock(&resbMutex); + entry->fCountExisting++; + while(entry->fParent != NULL) { + entry = entry->fParent; + entry->fCountExisting++; + } + umtx_unlock(&resbMutex); +} + +/** + * Internal function. Tries to find a resource in given Resource + * Bundle, as well as in its parents + */ +static const ResourceData *getFallbackData(const UResourceBundle* resBundle, const char* * resTag, UResourceDataEntry* *realData, Resource *res, UErrorCode *status) { + UResourceDataEntry *resB = resBundle->fData; + int32_t indexR = -1; + int32_t i = 0; + *res = RES_BOGUS; + if(resB != NULL) { + if(resB->fBogus == U_ZERO_ERROR) { /* if this resource is real, */ + *res = res_getTableItemByKey(&(resB->fData), resB->fData.rootRes, &indexR, resTag); /* try to get data from there */ + i++; + } + if(resBundle->fHasFallback == TRUE) { + while(*res == RES_BOGUS && resB->fParent != NULL) { /* Otherwise, we'll look in parents */ + resB = resB->fParent; + if(resB->fBogus == U_ZERO_ERROR) { + i++; + *res = res_getTableItemByKey(&(resB->fData), resB->fData.rootRes, &indexR, resTag); + } + } + } + + if(*res != RES_BOGUS) { /* If the resource is found in parents, we need to adjust the error */ + if(i>1) { + if(uprv_strcmp(resB->fName, uloc_getDefault())==0 || uprv_strcmp(resB->fName, kRootLocaleName)==0) { + *status = U_USING_DEFAULT_WARNING; + } else { + *status = U_USING_FALLBACK_WARNING; + } + } + *realData = resB; + return (&(resB->fData)); + } else { /* If resource is not found, we need to give an error */ + *status = U_MISSING_RESOURCE_ERROR; + return NULL; + } + } else { + *status = U_MISSING_RESOURCE_ERROR; + return NULL; + } +} + +/* Works just like ucnv_flushCache() */ +/* TODO: figure out why fCountExisting may not go to zero. Do not make this function public yet. */ +static int32_t ures_flushCache() +{ + UResourceDataEntry *resB = NULL; + int32_t pos = -1; + int32_t rbDeletedNum = 0; + const UHashElement *e; + + /*if shared data hasn't even been lazy evaluated yet + * return 0 + */ + umtx_lock(&resbMutex); + if (cache == NULL) { + umtx_unlock(&resbMutex); + return 0; + } + + /*creates an enumeration to iterate through every element in the table */ + while ((e = uhash_nextElement(cache, &pos)) != NULL) + { + resB = (UResourceDataEntry *) e->value.pointer; + /* Deletes only if reference counter == 0 + * Don't worry about the children of this node. + * Those will eventually get deleted too, if not already. + * Don't worry about the parents of this node. + * Those will eventually get deleted too, if not already. + */ + /* DONE: figure out why fCountExisting may not go to zero. Do not make this function public yet. */ + /* 04/05/2002 [weiv] fCountExisting should now be accurate. If it's not zero, that means that */ + /* some resource bundles are still open somewhere. */ + + /*U_ASSERT(resB->fCountExisting == 0);*/ + if (resB->fCountExisting == 0) { + rbDeletedNum++; + uhash_removeElement(cache, e); + if(resB->fBogus == U_ZERO_ERROR) { + res_unload(&(resB->fData)); + } + if(resB->fName != NULL) { + uprv_free(resB->fName); + } + if(resB->fPath != NULL) { + uprv_free(resB->fPath); + } + uprv_free(resB); + } + } + umtx_unlock(&resbMutex); + + return rbDeletedNum; +} + +static UBool U_CALLCONV ures_cleanup(void) +{ + if (cache != NULL) { + ures_flushCache(); + if (cache != NULL && uhash_count(cache) == 0) { + uhash_close(cache); + cache = NULL; + } + } + if (cache == NULL && resbMutex != NULL) { + umtx_destroy(&resbMutex); + } + return (cache == NULL); +} + +/** INTERNAL: Initializes the cache for resources */ +static void initCache(UErrorCode *status) { + UBool makeCache = FALSE; + UMTX_CHECK(&resbMutex, (cache == NULL), makeCache); + if(makeCache) { + UHashtable *newCache = uhash_open(hashEntry, compareEntries, NULL, status); + if (U_FAILURE(*status)) { + return; + } + umtx_lock(&resbMutex); + if(cache == NULL) { + cache = newCache; + newCache = NULL; + ucln_common_registerCleanup(UCLN_COMMON_URES, ures_cleanup); + } + umtx_unlock(&resbMutex); + if(newCache != NULL) { + uhash_close(newCache); + } + } +} + +/** INTERNAL: sets the name (locale) of the resource bundle to given name */ + +static void setEntryName(UResourceDataEntry *res, char *name, UErrorCode *status) { + if(res->fName != NULL) { + uprv_free(res->fName); + } + res->fName = (char *)uprv_malloc(sizeof(char)*uprv_strlen(name)+1); + if(res->fName == NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + } else { + uprv_strcpy(res->fName, name); + } +} + +/** + * INTERNAL: Inits and opens an entry from a data DLL. + * CAUTION: resbMutex must be locked when calling this function. + */ +static UResourceDataEntry *init_entry(const char *localeID, const char *path, UErrorCode *status) { + UResourceDataEntry *r = NULL; + UResourceDataEntry find; + int32_t hashValue; + char name[96]; + const char *myPath = NULL; + char aliasName[100] = { 0 }; + int32_t aliasLen = 0; + /*UBool isAlias = FALSE;*/ + UHashTok hashkey; + + if(U_FAILURE(*status)) { + return NULL; + } + + /* here we try to deduce the right locale name */ + if(localeID == NULL) { /* if localeID is NULL, we're trying to open default locale */ + uprv_strcpy(name, uloc_getDefault()); + } else if(*localeID == 0) { /* if localeID is "" then we try to open root locale */ + uprv_strcpy(name, kRootLocaleName); + } else { /* otherwise, we'll open what we're given */ + uprv_strcpy(name, localeID); + } + + if(path != NULL) { /* if we actually have path, we'll use it */ + myPath = path; + } + + find.fName = name; + find.fPath = (char *)myPath; + + /* calculate the hash value of the entry */ + hashkey.pointer = (void *)&find; + hashValue = hashEntry(hashkey); + + /* check to see if we already have this entry */ + r = (UResourceDataEntry *)uhash_get(cache, &find); + + if(r != NULL) { /* if the entry is already in the hash table */ + r->fCountExisting++; /* we just increase it's reference count */ + /* if the resource has a warning */ + /* we don't want to overwrite a status with no error */ + if(r->fBogus != U_ZERO_ERROR) { + *status = r->fBogus; /* set the returning status */ + } + } else { /* otherwise, we'll try to construct a new entry */ + UBool result = FALSE; + + r = (UResourceDataEntry *) uprv_malloc(sizeof(UResourceDataEntry)); + + if(r == NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + r->fCountExisting = 1; + + r->fName = NULL; + setEntryName(r, name, status); + + r->fPath = NULL; + if(myPath != NULL && !U_FAILURE(*status)) { + r->fPath = (char *)uprv_malloc(sizeof(char)*uprv_strlen(myPath)+1); + if(r->fPath == NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + } else { + uprv_strcpy(r->fPath, myPath); + } + } + + r->fHashKey = hashValue; + r->fParent = NULL; + uprv_memset(&r->fData, 0, sizeof(ResourceData)); + r->fBogus = U_ZERO_ERROR; + + /* this is the actual loading - returns bool true/false */ + result = res_load(&(r->fData), r->fPath, r->fName, status); + + if (result == FALSE || U_FAILURE(*status)) { + /* we have no such entry in dll, so it will always use fallback */ + *status = U_USING_FALLBACK_WARNING; + r->fBogus = U_USING_FALLBACK_WARNING; + } else { /* if we have a regular entry */ + /* We might be able to do this a wee bit more efficiently (we could check whether the aliased data) */ + /* is already in the cache), but it's good the way it is */ + /* handle the alias by trying to get out the %%Alias tag.*/ + /* We'll try to get alias string from the bundle */ + Resource aliasres = res_getResource(&(r->fData), "%%ALIAS"); + if (aliasres != RES_BOGUS) { + const UChar *alias = res_getString(&(r->fData), aliasres, &aliasLen); + if(alias != NULL && aliasLen > 0) { /* if there is actual alias - unload and load new data */ + u_UCharsToChars(alias, aliasName, aliasLen+1); + /*isAlias = TRUE;*/ + res_unload(&(r->fData)); + result = res_load(&(r->fData), r->fPath, aliasName, status); + if (result == FALSE || U_FAILURE(*status)) { + /* we couldn't load aliased data - so we have no data */ + *status = U_USING_FALLBACK_WARNING; + r->fBogus = U_USING_FALLBACK_WARNING; + } + setEntryName(r, aliasName, status); + } + } + } + + { + UResourceDataEntry *oldR = NULL; + if((oldR = (UResourceDataEntry *)uhash_get(cache, r)) == NULL) { /* if the data is not cached */ + /* just insert it in the cache */ + uhash_put(cache, (void *)r, r, status); + } else { + /* somebody have already inserted it while we were working, discard newly opened data */ + /* Also, we could get here IF we opened an alias */ + uprv_free(r->fName); + if(r->fPath != NULL) { + uprv_free(r->fPath); + } + res_unload(&(r->fData)); + uprv_free(r); + r = oldR; + r->fCountExisting++; + } + } + + } + return r; +} + +/* INTERNAL: */ +/* CAUTION: resbMutex must be locked when calling this function! */ +static UResourceDataEntry *findFirstExisting(const char* path, char* name, UBool *isRoot, UBool *hasChopped, UBool *isDefault, UErrorCode* status) { + UResourceDataEntry *r = NULL; + UBool hasRealData = FALSE; + const char *defaultLoc = uloc_getDefault(); + UErrorCode intStatus = U_ZERO_ERROR; + *hasChopped = TRUE; /* we're starting with a fresh name */ + + while(*hasChopped && !hasRealData) { + r = init_entry(name, path, &intStatus); + *isDefault = (UBool)(uprv_strncmp(name, defaultLoc, uprv_strlen(name)) == 0); + hasRealData = (UBool)(r->fBogus == U_ZERO_ERROR); + if(!hasRealData) { + /* this entry is not real. We will discard it. */ + /* However, the parent line for this entry is */ + /* not to be used - as there might be parent */ + /* lines in cache from previous openings that */ + /* are not updated yet. */ + r->fCountExisting--; + /*entryCloseInt(r);*/ + r = NULL; + *status = U_USING_FALLBACK_WARNING; + } else { + uprv_strcpy(name, r->fName); /* this is needed for supporting aliases */ + } + + *isRoot = (UBool)(uprv_strcmp(name, kRootLocaleName) == 0); + + /*Fallback data stuff*/ + *hasChopped = chopLocale(name); + } + return r; +} + +static void ures_setIsStackObject( UResourceBundle* resB, UBool state) { + if(state) { + resB->fMagic1 = 0; + resB->fMagic2 = 0; + } else { + resB->fMagic1 = MAGIC1; + resB->fMagic2 = MAGIC2; + } +} + +static UBool ures_isStackObject(const UResourceBundle* resB) { + return((resB->fMagic1 == MAGIC1 && resB->fMagic2 == MAGIC2)?FALSE:TRUE); +} + + +U_CFUNC void ures_initStackObject(UResourceBundle* resB) { + uprv_memset(resB, 0, sizeof(UResourceBundle)); + ures_setIsStackObject(resB, TRUE); +} + +static UResourceDataEntry *entryOpen(const char* path, const char* localeID, UErrorCode* status) { + UErrorCode intStatus = U_ZERO_ERROR; + UErrorCode parentStatus = U_ZERO_ERROR; + UResourceDataEntry *r = NULL; + UResourceDataEntry *t1 = NULL; + UResourceDataEntry *t2 = NULL; + UBool isDefault = FALSE; + UBool isRoot = FALSE; + UBool hasRealData = FALSE; + UBool hasChopped = TRUE; + char name[96]; + + if(U_FAILURE(*status)) { + return NULL; + } + + initCache(status); + + uprv_strcpy(name, localeID); + + umtx_lock(&resbMutex); + { /* umtx_lock */ + /* We're going to skip all the locales that do not have any data */ + r = findFirstExisting(path, name, &isRoot, &hasChopped, &isDefault, &intStatus); + + if(r != NULL) { /* if there is one real locale, we can look for parents. */ + t1 = r; + hasRealData = TRUE; + while (hasChopped && !isRoot && t1->fParent == NULL && !t1->fData.noFallback) { + /* insert regular parents */ + t2 = init_entry(name, r->fPath, &parentStatus); + t1->fParent = t2; + t1 = t2; + hasChopped = chopLocale(name); + } + } + + /* we could have reached this point without having any real data */ + /* if that is the case, we need to chain in the default locale */ + if(r==NULL && !isDefault && !isRoot /*&& t1->fParent == NULL*/) { + /* insert default locale */ + uprv_strcpy(name, uloc_getDefault()); + r = findFirstExisting(path, name, &isRoot, &hasChopped, &isDefault, &intStatus); + intStatus = U_USING_DEFAULT_WARNING; + if(r != NULL) { /* the default locale exists */ + t1 = r; + hasRealData = TRUE; + isDefault = TRUE; + while (hasChopped && t1->fParent == NULL) { + /* insert chopped defaults */ + t2 = init_entry(name, r->fPath, &parentStatus); + t1->fParent = t2; + t1 = t2; + hasChopped = chopLocale(name); + } + } + } + + /* we could still have r == NULL at this point - maybe even default locale is not */ + /* present */ + if(r == NULL) { + uprv_strcpy(name, kRootLocaleName); + r = findFirstExisting(path, name, &isRoot, &hasChopped, &isDefault, &intStatus); + if(r != NULL) { + t1 = r; + intStatus = U_USING_DEFAULT_WARNING; + hasRealData = TRUE; + } else { /* we don't even have the root locale */ + *status = U_MISSING_RESOURCE_ERROR; + } + } else if(!isRoot && uprv_strcmp(t1->fName, kRootLocaleName) != 0 && t1->fParent == NULL && !r->fData.noFallback) { + /* insert root locale */ + t2 = init_entry(kRootLocaleName, r->fPath, &parentStatus); + if(!hasRealData) { + r->fBogus = U_USING_DEFAULT_WARNING; + } + hasRealData = (UBool)((t2->fBogus == U_ZERO_ERROR) | hasRealData); + t1->fParent = t2; + t1 = t2; + } + + while(r != NULL && !isRoot && t1->fParent != NULL) { + t1->fParent->fCountExisting++; + t1 = t1->fParent; + hasRealData = (UBool)((t1->fBogus == U_ZERO_ERROR) | hasRealData); + } + } /* umtx_lock */ + umtx_unlock(&resbMutex); + + if(U_SUCCESS(*status)) { + if(U_SUCCESS(parentStatus)) { + if(intStatus != U_ZERO_ERROR) { + *status = intStatus; + } + return r; + } else { + *status = parentStatus; + return NULL; + } + } else { + return NULL; + } +} + + +/** + * Functions to create and destroy resource bundles. + * CAUTION: resbMutex must be locked when calling this function. + */ +/* INTERNAL: */ +static void entryCloseInt(UResourceDataEntry *resB) { + UResourceDataEntry *p = resB; + + while(resB != NULL) { + p = resB->fParent; + resB->fCountExisting--; + + /* Entries are left in the cache. TODO: add ures_cacheFlush() to force a flush + of the cache. */ +/* + if(resB->fCountExisting <= 0) { + uhash_remove(cache, resB); + if(resB->fBogus == U_ZERO_ERROR) { + res_unload(&(resB->fData)); + } + if(resB->fName != NULL) { + uprv_free(resB->fName); + } + if(resB->fPath != NULL) { + uprv_free(resB->fPath); + } + uprv_free(resB); + } +*/ + + resB = p; + } +} + +/** + * API: closes a resource bundle and cleans up. + */ + +static void entryClose(UResourceDataEntry *resB) { + umtx_lock(&resbMutex); + entryCloseInt(resB); + umtx_unlock(&resbMutex); +} + +/* +U_CFUNC void ures_setResPath(UResourceBundle *resB, const char* toAdd) { + if(resB->fResPath == NULL) { + resB->fResPath = resB->fResBuf; + *(resB->fResPath) = 0; + } + resB->fResPathLen = uprv_strlen(toAdd); + if(RES_BUFSIZE <= resB->fResPathLen+1) { + if(resB->fResPath == resB->fResBuf) { + resB->fResPath = (char *)uprv_malloc((resB->fResPathLen+1)*sizeof(char)); + } else { + resB->fResPath = (char *)uprv_realloc(resB->fResPath, (resB->fResPathLen+1)*sizeof(char)); + } + } + uprv_strcpy(resB->fResPath, toAdd); +} +*/ +static void ures_appendResPath(UResourceBundle *resB, const char* toAdd, int32_t lenToAdd) { + int32_t resPathLenOrig = resB->fResPathLen; + if(resB->fResPath == NULL) { + resB->fResPath = resB->fResBuf; + *(resB->fResPath) = 0; + resB->fResPathLen = 0; + } + resB->fResPathLen += lenToAdd; + if(RES_BUFSIZE <= resB->fResPathLen+1) { + if(resB->fResPath == resB->fResBuf) { + resB->fResPath = (char *)uprv_malloc((resB->fResPathLen+1)*sizeof(char)); + uprv_strcpy(resB->fResPath, resB->fResBuf); + } else { + resB->fResPath = (char *)uprv_realloc(resB->fResPath, (resB->fResPathLen+1)*sizeof(char)); + } + } + uprv_strcpy(resB->fResPath + resPathLenOrig, toAdd); +} + +static void ures_freeResPath(UResourceBundle *resB) { + if (resB->fResPath && resB->fResPath != resB->fResBuf) { + uprv_free(resB->fResPath); + } + resB->fResPath = NULL; + resB->fResPathLen = 0; +} + +static void +ures_closeBundle(UResourceBundle* resB, UBool freeBundleObj) +{ + if(resB != NULL) { + if(resB->fData != NULL) { + entryClose(resB->fData); + } + if(resB->fVersion != NULL) { + uprv_free(resB->fVersion); + } + ures_freeResPath(resB); + + if(ures_isStackObject(resB) == FALSE && freeBundleObj) { + uprv_free(resB); + } +#if 0 /*U_DEBUG*/ + else { + /* poison the data */ + uprv_memset(resB, -1, sizeof(UResourceBundle)); + } +#endif + } +} + +U_CAPI void U_EXPORT2 +ures_close(UResourceBundle* resB) +{ + ures_closeBundle(resB, TRUE); +} + +static UResourceBundle *init_resb_result(const ResourceData *rdata, Resource r, + const char *key, int32_t index, UResourceDataEntry *realData, + const UResourceBundle *parent, int32_t noAlias, + UResourceBundle *resB, UErrorCode *status) +{ + if(status == NULL || U_FAILURE(*status)) { + return resB; + } + if(RES_GET_TYPE(r) == URES_ALIAS) { /* This is an alias, need to exchange with real data */ + if(noAlias < URES_MAX_ALIAS_LEVEL) { + int32_t len = 0; + const UChar *alias = res_getAlias(rdata, r, &len); + if(len > 0) { + /* we have an alias, now let's cut it up */ + char stackAlias[200]; + char *chAlias = NULL, *path = NULL, *locale = NULL, *keyPath = NULL; + int32_t capacity; + + /* + * Allocate enough space for both the char * version + * of the alias and parent->fResPath. + * + * We do this so that res_findResource() can modify the path, + * which allows us to remove redundant _res_findResource() variants + * in uresdata.c. + * res_findResource() now NUL-terminates each segment so that table keys + * can always be compared with strcmp() instead of strncmp(). + * Saves code there and simplifies testing and code coverage. + * + * markus 2003oct17 + */ + ++len; /* count the terminating NUL */ + if(parent != NULL && parent->fResPath != NULL) { + capacity = (int32_t)uprv_strlen(parent->fResPath) + 1; + } else { + capacity = 0; + } + if(capacity < len) { + capacity = len; + } + if(capacity <= sizeof(stackAlias)) { + capacity = sizeof(stackAlias); + chAlias = stackAlias; + } else { + chAlias = (char *)uprv_malloc(capacity); + /* test for NULL */ + if(chAlias == NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + } + u_UCharsToChars(alias, chAlias, len); + + if(*chAlias == RES_PATH_SEPARATOR) { + /* there is a path included */ + locale = uprv_strchr(chAlias+1, RES_PATH_SEPARATOR); + if(locale == NULL) { + locale = uprv_strchr(chAlias, 0); /* avoid locale == NULL to make code below work */ + } else { + *locale = 0; + locale++; + } + path = chAlias+1; + if(uprv_strcmp(path, "LOCALE") == 0) { + /* this is an XPath alias, starting with "/LOCALE/" */ + /* it contains the path to a resource which should be looked up */ + /* starting in the requested locale */ + keyPath = locale; + locale = parent->fTopLevelData->fName; /* this is the requested locale's name */ + path = realData->fPath; /* we will be looking in the same package */ + } else { + if(uprv_strcmp(path, "ICUDATA") == 0) { /* want ICU data */ + path = NULL; + } + keyPath = uprv_strchr(locale, RES_PATH_SEPARATOR); + if(keyPath) { + *keyPath = 0; + keyPath++; + } + } + } else { + /* no path, start with a locale */ + locale = chAlias; + keyPath = uprv_strchr(locale, RES_PATH_SEPARATOR); + if(keyPath) { + *keyPath = 0; + keyPath++; + } + path = realData->fPath; + } + + + { + /* got almost everything, let's try to open */ + /* first, open the bundle with real data */ + UResourceBundle *result = resB; + const char* temp = NULL; + UErrorCode intStatus = U_ZERO_ERROR; + UResourceBundle *mainRes = ures_openDirect(path, locale, &intStatus); + if(U_SUCCESS(intStatus)) { + if(keyPath == NULL) { + /* no key path. This means that we are going to + * to use the corresponding resource from + * another bundle + */ + /* first, we are going to get a corresponding parent + * resource to the one we are searching. + */ + char *aKey = parent->fResPath; + if(aKey) { + uprv_strcpy(chAlias, aKey); /* allocated large enough above */ + aKey = chAlias; + r = res_findResource(&(mainRes->fResData), mainRes->fRes, &aKey, &temp); + } else { + r = mainRes->fRes; + } + if(key) { + /* we need to make keyPath from parent's fResPath and + * current key, if there is a key associated + */ + len = (int32_t)(uprv_strlen(key) + 1); + if(len > capacity) { + capacity = len; + if(chAlias == stackAlias) { + chAlias = (char *)uprv_malloc(capacity); + } else { + chAlias = (char *)uprv_realloc(chAlias, capacity); + } + if(chAlias == NULL) { + ures_close(mainRes); + *status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + } + uprv_memcpy(chAlias, key, len); + aKey = chAlias; + r = res_findResource(&(mainRes->fResData), r, &aKey, &temp); + } else if(index != -1) { + /* if there is no key, but there is an index, try to get by the index */ + /* here we have either a table or an array, so get the element */ + if(RES_GET_TYPE(r) == URES_TABLE || RES_GET_TYPE(r) == URES_TABLE32) { + r = res_getTableItemByIndex(&(mainRes->fResData), r, index, (const char **)&aKey); + } else { /* array */ + r = res_getArrayItem(&(mainRes->fResData), r, index); + } + } + if(r != RES_BOGUS) { + result = init_resb_result(&(mainRes->fResData), r, temp, -1, mainRes->fData, mainRes, noAlias+1, resB, status); + } else { + *status = U_MISSING_RESOURCE_ERROR; + result = resB; + } + } else { + /* this one is a bit trickier. + * we start finding keys, but after we resolve one alias, the path might continue. + * Consider: + * aliastest:alias { "testtypes/anotheralias/Sequence" } + * anotheralias:alias { "/ICUDATA/sh/CollationElements" } + * aliastest resource should finally have the sequence, not collation elements. + */ + UResourceDataEntry *dataEntry = mainRes->fData; + char stackPath[URES_MAX_BUFFER_SIZE]; + char *pathBuf = stackPath, *myPath = pathBuf; + if(uprv_strlen(keyPath) > URES_MAX_BUFFER_SIZE) { + pathBuf = (char *)uprv_malloc((uprv_strlen(keyPath)+1)*sizeof(char)); + if(pathBuf == NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + } + uprv_strcpy(pathBuf, keyPath); + result = mainRes; + /* now we have fallback following here */ + do { + r = dataEntry->fData.rootRes; + /* this loop handles 'found' resources over several levels */ + while(*myPath && U_SUCCESS(*status)) { + r = res_findResource(&(dataEntry->fData), r, &myPath, &temp); + if(r != RES_BOGUS) { /* found a resource, but it might be an indirection */ + resB = init_resb_result(&(dataEntry->fData), r, temp, -1, dataEntry, result, noAlias+1, resB, status); + result = resB; + if(result) { + r = result->fRes; /* switch to a new resource, possibly a new tree */ + dataEntry = result->fData; + } + } else { /* no resource found, we don't really want to look anymore on this level */ + break; + } + } + dataEntry = dataEntry->fParent; + uprv_strcpy(pathBuf, keyPath); + myPath = pathBuf; + } while(r == RES_BOGUS && dataEntry != NULL); + if(r == RES_BOGUS) { + *status = U_MISSING_RESOURCE_ERROR; + result = resB; + } + if(pathBuf != stackPath) { + uprv_free(pathBuf); + } + } + } else { /* we failed to open the resource we're aliasing to */ + *status = intStatus; + } + if(chAlias != stackAlias) { + uprv_free(chAlias); + } + if(mainRes != result) { + ures_close(mainRes); + } + return result; + } + } else { + /* bad alias, should be an error */ + *status = U_ILLEGAL_ARGUMENT_ERROR; + return resB; + } + } else { + *status = U_TOO_MANY_ALIASES_ERROR; + return resB; + } + } + if(resB == NULL) { + resB = (UResourceBundle *)uprv_malloc(sizeof(UResourceBundle)); + /* test for NULL */ + if (resB == NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + ures_setIsStackObject(resB, FALSE); + resB->fResPath = NULL; + resB->fResPathLen = 0; + } else { + if(resB->fData != NULL) { + entryClose(resB->fData); + } + if(resB->fVersion != NULL) { + uprv_free(resB->fVersion); + } + /* + weiv: if stack object was passed in, it doesn't really need to be reinited, + since the purpose of initing is to remove stack junk. However, at this point + we would not do anything to an allocated object, so stack object should be + treated the same + */ + /* + if(ures_isStackObject(resB) != FALSE) { + ures_initStackObject(resB); + } + */ + if(parent != resB) { + ures_freeResPath(resB); + } + } + resB->fData = realData; + entryIncrease(resB->fData); + resB->fHasFallback = FALSE; + resB->fIsTopLevel = FALSE; + resB->fIndex = -1; + resB->fKey = key; + resB->fParentRes = parent; + resB->fTopLevelData = parent->fTopLevelData; + if(parent->fResPath && parent != resB) { + ures_appendResPath(resB, parent->fResPath, parent->fResPathLen); + } + if(key != NULL) { + ures_appendResPath(resB, key, (int32_t)uprv_strlen(key)); + if(resB->fResPath[resB->fResPathLen-1] != RES_PATH_SEPARATOR) { + ures_appendResPath(resB, RES_PATH_SEPARATOR_S, 1); + } + } else if(index >= 0) { + char buf[256]; + int32_t len = T_CString_integerToString(buf, index, 10); + ures_appendResPath(resB, buf, len); + if(resB->fResPath[resB->fResPathLen-1] != RES_PATH_SEPARATOR) { + ures_appendResPath(resB, RES_PATH_SEPARATOR_S, 1); + } + } + /* Make sure that Purify doesn't complain about uninitialized memory copies. */ + { + int32_t usedLen = ((resB->fResBuf == resB->fResPath) ? resB->fResPathLen : 0); + uprv_memset(resB->fResBuf + usedLen, 0, sizeof(resB->fResBuf) - usedLen); + } + + resB->fVersion = NULL; + resB->fRes = r; + /*resB->fParent = parent->fRes;*/ + uprv_memcpy(&resB->fResData, rdata, sizeof(ResourceData)); + resB->fSize = res_countArrayItems(&(resB->fResData), resB->fRes); + return resB; +} + +UResourceBundle *ures_copyResb(UResourceBundle *r, const UResourceBundle *original, UErrorCode *status) { + UBool isStackObject; + if(U_FAILURE(*status) || r == original) { + return r; + } + if(original != NULL) { + if(r == NULL) { + isStackObject = FALSE; + r = (UResourceBundle *)uprv_malloc(sizeof(UResourceBundle)); + /* test for NULL */ + if (r == NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + } else { + isStackObject = ures_isStackObject(r); + ures_closeBundle(r, FALSE); + } + uprv_memcpy(r, original, sizeof(UResourceBundle)); + r->fResPath = NULL; + r->fResPathLen = 0; + if(original->fResPath) { + ures_appendResPath(r, original->fResPath, original->fResPathLen); + } + ures_setIsStackObject(r, isStackObject); + if(r->fData != NULL) { + entryIncrease(r->fData); + } + } + return r; +} + +/** + * Functions to retrieve data from resource bundles. + */ + +U_CAPI const UChar* U_EXPORT2 ures_getString(const UResourceBundle* resB, int32_t* len, UErrorCode* status) { + + if (status==NULL || U_FAILURE(*status)) { + return NULL; + } + if(resB == NULL) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + + switch(RES_GET_TYPE(resB->fRes)) { + case URES_STRING: + return res_getString(&(resB->fResData), resB->fRes, len); + case URES_INT: + case URES_INT_VECTOR: + case URES_BINARY: + case URES_ARRAY: + case URES_TABLE: + case URES_TABLE32: + default: + *status = U_RESOURCE_TYPE_MISMATCH; + } + + return NULL; +} + +static const char * +ures_toUTF8String(const UChar *s16, int32_t length16, + char *dest, int32_t *pLength, + UBool forceCopy, + UErrorCode *status) { + int32_t capacity; + + if (U_FAILURE(*status)) { + return NULL; + } + if (pLength != NULL) { + capacity = *pLength; + } else { + capacity = 0; + } + if (capacity < 0 || (capacity > 0 && dest == NULL)) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + + if (length16 == 0) { + /* empty string, return as read-only pointer */ + if (pLength != NULL) { + *pLength = 0; + } + if (forceCopy) { + u_terminateChars(dest, capacity, 0, status); + return dest; + } else { + return ""; + } + } else { + /* We need to transform the string to the destination buffer. */ + if (capacity < length16) { + /* No chance for the string to fit. Pure preflighting. */ + return u_strToUTF8(NULL, 0, pLength, s16, length16, status); + } + if (!forceCopy && (length16 <= 0x2aaaaaaa)) { + /* + * We know the string will fit into dest because each UChar turns + * into at most three UTF-8 bytes. Fill the latter part of dest + * so that callers do not expect to use dest as a string pointer, + * hopefully leading to more robust code for when resource bundles + * may store UTF-8 natively. + * (In which case dest would not be used at all.) + * + * We do not do this if forceCopy=TRUE because then the caller + * expects the string to start exactly at dest. + * + * The test above for <= 0x2aaaaaaa prevents overflows. + * The +1 is for the NUL terminator. + */ + int32_t maxLength = 3 * length16 + 1; + if (capacity > maxLength) { + dest += capacity - maxLength; + capacity = maxLength; + } + } + return u_strToUTF8(dest, capacity, pLength, s16, length16, status); + } +} + +U_CAPI const char * U_EXPORT2 +ures_getUTF8String(const UResourceBundle *resB, + char *dest, int32_t *pLength, + UBool forceCopy, + UErrorCode *status) { + int32_t length16; + const UChar *s16 = ures_getString(resB, &length16, status); + return ures_toUTF8String(s16, length16, dest, pLength, forceCopy, status); +} + +U_CAPI const uint8_t* U_EXPORT2 ures_getBinary(const UResourceBundle* resB, int32_t* len, + UErrorCode* status) { + if (status==NULL || U_FAILURE(*status)) { + return NULL; + } + if(resB == NULL) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + switch(RES_GET_TYPE(resB->fRes)) { + case URES_BINARY: + return res_getBinary(&(resB->fResData), resB->fRes, len); + case URES_INT: + case URES_STRING: + case URES_INT_VECTOR: + case URES_ARRAY: + case URES_TABLE: + case URES_TABLE32: + default: + *status = U_RESOURCE_TYPE_MISMATCH; + } + + return NULL; +} + +U_CAPI const int32_t* U_EXPORT2 ures_getIntVector(const UResourceBundle* resB, int32_t* len, + UErrorCode* status) { + if (status==NULL || U_FAILURE(*status)) { + return NULL; + } + if(resB == NULL) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + switch(RES_GET_TYPE(resB->fRes)) { + case URES_INT_VECTOR: + return res_getIntVector(&(resB->fResData), resB->fRes, len); + case URES_INT: + case URES_STRING: + case URES_ARRAY: + case URES_BINARY: + case URES_TABLE: + case URES_TABLE32: + default: + *status = U_RESOURCE_TYPE_MISMATCH; + } + + return NULL; +} + +/* this function returns a signed integer */ +/* it performs sign extension */ +U_CAPI int32_t U_EXPORT2 ures_getInt(const UResourceBundle* resB, UErrorCode *status) { + if (status==NULL || U_FAILURE(*status)) { + return 0xffffffff; + } + if(resB == NULL) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return 0xffffffff; + } + if(RES_GET_TYPE(resB->fRes) != URES_INT) { + *status = U_RESOURCE_TYPE_MISMATCH; + return 0xffffffff; + } + return RES_GET_INT(resB->fRes); +} + +U_CAPI uint32_t U_EXPORT2 ures_getUInt(const UResourceBundle* resB, UErrorCode *status) { + if (status==NULL || U_FAILURE(*status)) { + return 0xffffffff; + } + if(resB == NULL) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return 0xffffffff; + } + if(RES_GET_TYPE(resB->fRes) != URES_INT) { + *status = U_RESOURCE_TYPE_MISMATCH; + return 0xffffffff; + } + return RES_GET_UINT(resB->fRes); +} + + +U_CAPI UResType U_EXPORT2 ures_getType(const UResourceBundle *resB) { + UResType type; + + if(resB == NULL) { + return URES_NONE; + } + type = (UResType) RES_GET_TYPE(resB->fRes); + return type == URES_TABLE32 ? URES_TABLE : type; +} + +U_CAPI const char * U_EXPORT2 ures_getKey(const UResourceBundle *resB) { + if(resB == NULL) { + return NULL; + } + + return(resB->fKey); +} + +U_CAPI int32_t U_EXPORT2 ures_getSize(const UResourceBundle *resB) { + if(resB == NULL) { + return 0; + } + + return resB->fSize; +} + +static const UChar* ures_getStringWithAlias(const UResourceBundle *resB, Resource r, int32_t sIndex, int32_t *len, UErrorCode *status) { + if(RES_GET_TYPE(r) == URES_ALIAS) { + const UChar* result = 0; + UResourceBundle *tempRes = ures_getByIndex(resB, sIndex, NULL, status); + result = ures_getString(tempRes, len, status); + ures_close(tempRes); + return result; + } else { + return res_getString(&(resB->fResData), r, len); + } +} + +U_CAPI void U_EXPORT2 ures_resetIterator(UResourceBundle *resB){ + if(resB == NULL) { + return; + } + resB->fIndex = -1; +} + +U_CAPI UBool U_EXPORT2 ures_hasNext(const UResourceBundle *resB) { + if(resB == NULL) { + return FALSE; + } + return (UBool)(resB->fIndex < resB->fSize-1); +} + +U_CAPI const UChar* U_EXPORT2 ures_getNextString(UResourceBundle *resB, int32_t* len, const char ** key, UErrorCode *status) { + Resource r = RES_BOGUS; + + if (status==NULL || U_FAILURE(*status)) { + return NULL; + } + if(resB == NULL) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + + if(resB->fIndex == resB->fSize-1) { + *status = U_INDEX_OUTOFBOUNDS_ERROR; + } else { + resB->fIndex++; + switch(RES_GET_TYPE(resB->fRes)) { + case URES_INT: + case URES_BINARY: + case URES_STRING: + return res_getString(&(resB->fResData), resB->fRes, len); + case URES_TABLE: + case URES_TABLE32: + r = res_getTableItemByIndex(&(resB->fResData), resB->fRes, resB->fIndex, key); + if(r == RES_BOGUS && resB->fHasFallback) { + /* TODO: do the fallback */ + } + return ures_getStringWithAlias(resB, r, resB->fIndex, len, status); + case URES_ARRAY: + r = res_getArrayItem(&(resB->fResData), resB->fRes, resB->fIndex); + if(r == RES_BOGUS && resB->fHasFallback) { + /* TODO: do the fallback */ + } + return ures_getStringWithAlias(resB, r, resB->fIndex, len, status); + case URES_ALIAS: + return ures_getStringWithAlias(resB, resB->fRes, resB->fIndex, len, status); + case URES_INT_VECTOR: + default: + return NULL; + } + } + + return NULL; +} + +U_CAPI UResourceBundle* U_EXPORT2 ures_getNextResource(UResourceBundle *resB, UResourceBundle *fillIn, UErrorCode *status) { + const char *key = NULL; + Resource r = RES_BOGUS; + + if (status==NULL || U_FAILURE(*status)) { + /*return NULL;*/ + return fillIn; + } + if(resB == NULL) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + /*return NULL;*/ + return fillIn; + } + + if(resB->fIndex == resB->fSize-1) { + *status = U_INDEX_OUTOFBOUNDS_ERROR; + /*return NULL;*/ + } else { + resB->fIndex++; + switch(RES_GET_TYPE(resB->fRes)) { + case URES_INT: + case URES_BINARY: + case URES_STRING: + return ures_copyResb(fillIn, resB, status); + case URES_TABLE: + case URES_TABLE32: + r = res_getTableItemByIndex(&(resB->fResData), resB->fRes, resB->fIndex, &key); + if(r == RES_BOGUS && resB->fHasFallback) { + /* TODO: do the fallback */ + } + return init_resb_result(&(resB->fResData), r, key, resB->fIndex, resB->fData, resB, 0, fillIn, status); + case URES_ARRAY: + r = res_getArrayItem(&(resB->fResData), resB->fRes, resB->fIndex); + if(r == RES_BOGUS && resB->fHasFallback) { + /* TODO: do the fallback */ + } + return init_resb_result(&(resB->fResData), r, key, resB->fIndex, resB->fData, resB, 0, fillIn, status); + case URES_INT_VECTOR: + default: + /*return NULL;*/ + return fillIn; + } + } + /*return NULL;*/ + return fillIn; +} + +U_CAPI UResourceBundle* U_EXPORT2 ures_getByIndex(const UResourceBundle *resB, int32_t indexR, UResourceBundle *fillIn, UErrorCode *status) { + const char* key = NULL; + Resource r = RES_BOGUS; + + if (status==NULL || U_FAILURE(*status)) { + /*return NULL;*/ + return fillIn; + } + if(resB == NULL) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + /*return NULL;*/ + return fillIn; + } + + if(indexR >= 0 && resB->fSize > indexR) { + switch(RES_GET_TYPE(resB->fRes)) { + case URES_INT: + case URES_BINARY: + case URES_STRING: + return ures_copyResb(fillIn, resB, status); + case URES_TABLE: + case URES_TABLE32: + r = res_getTableItemByIndex(&(resB->fResData), resB->fRes, indexR, &key); + if(r == RES_BOGUS && resB->fHasFallback) { + /* TODO: do the fallback */ + } + return init_resb_result(&(resB->fResData), r, key, indexR, resB->fData, resB, 0, fillIn, status); + case URES_ARRAY: + r = res_getArrayItem(&(resB->fResData), resB->fRes, indexR); + if(r == RES_BOGUS && resB->fHasFallback) { + /* TODO: do the fallback */ + } + return init_resb_result(&(resB->fResData), r, key, indexR, resB->fData, resB, 0, fillIn, status); + case URES_INT_VECTOR: + default: + /*return NULL;*/ + return fillIn; + } + } else { + *status = U_MISSING_RESOURCE_ERROR; + } + /*return NULL;*/ + return fillIn; +} + +U_CAPI const UChar* U_EXPORT2 ures_getStringByIndex(const UResourceBundle *resB, int32_t indexS, int32_t* len, UErrorCode *status) { + const char* key = NULL; + Resource r = RES_BOGUS; + + if (status==NULL || U_FAILURE(*status)) { + return NULL; + } + if(resB == NULL) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + + if(indexS >= 0 && resB->fSize > indexS) { + switch(RES_GET_TYPE(resB->fRes)) { + case URES_INT: + case URES_BINARY: + case URES_STRING: + return res_getString(&(resB->fResData), resB->fRes, len); + case URES_TABLE: + case URES_TABLE32: + r = res_getTableItemByIndex(&(resB->fResData), resB->fRes, indexS, &key); + if(r == RES_BOGUS && resB->fHasFallback) { + /* TODO: do the fallback */ + } + return ures_getStringWithAlias(resB, r, indexS, len, status); + case URES_ARRAY: + r = res_getArrayItem(&(resB->fResData), resB->fRes, indexS); + if(r == RES_BOGUS && resB->fHasFallback) { + /* TODO: do the fallback */ + } + return ures_getStringWithAlias(resB, r, indexS, len, status); + case URES_ALIAS: + return ures_getStringWithAlias(resB, resB->fRes, indexS, len, status); + + /*case URES_INT_VECTOR:*/ + default: + /* must not occur */ + *status = U_INTERNAL_PROGRAM_ERROR; + break; + } + } else { + *status = U_MISSING_RESOURCE_ERROR; + } + return NULL; +} + +U_CAPI const char * U_EXPORT2 +ures_getUTF8StringByIndex(const UResourceBundle *resB, + int32_t index, + char *dest, int32_t *pLength, + UBool forceCopy, + UErrorCode *status) { + int32_t length16; + const UChar *s16 = ures_getStringByIndex(resB, index, &length16, status); + return ures_toUTF8String(s16, length16, dest, pLength, forceCopy, status); +} + +/*U_CAPI const char *ures_getResPath(UResourceBundle *resB) { + return resB->fResPath; +}*/ + +U_CAPI UResourceBundle* U_EXPORT2 +ures_findResource(const char* path, UResourceBundle *fillIn, UErrorCode *status) +{ + UResourceBundle *first = NULL; + UResourceBundle *result = fillIn; + char *packageName = NULL; + char *pathToResource = NULL, *save = NULL; + char *locale = NULL, *localeEnd = NULL; + int32_t length; + + if(status == NULL || U_FAILURE(*status)) { + return result; + } + + length = (int32_t)(uprv_strlen(path)+1); + save = pathToResource = (char *)uprv_malloc(length*sizeof(char)); + /* test for NULL */ + if(pathToResource == NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + return result; + } + uprv_memcpy(pathToResource, path, length); + + locale = pathToResource; + if(*pathToResource == RES_PATH_SEPARATOR) { /* there is a path specification */ + pathToResource++; + packageName = pathToResource; + pathToResource = uprv_strchr(pathToResource, RES_PATH_SEPARATOR); + if(pathToResource == NULL) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + } else { + *pathToResource = 0; + locale = pathToResource+1; + } + } + + localeEnd = uprv_strchr(locale, RES_PATH_SEPARATOR); + if(localeEnd != NULL) { + *localeEnd = 0; + } + + first = ures_open(packageName, locale, status); + + if(U_SUCCESS(*status)) { + if(localeEnd) { + result = ures_findSubResource(first, localeEnd+1, fillIn, status); + } else { + result = ures_copyResb(fillIn, first, status); + } + ures_close(first); + } + uprv_free(save); + return result; +} + +U_CAPI UResourceBundle* U_EXPORT2 +ures_findSubResource(const UResourceBundle *resB, char* path, UResourceBundle *fillIn, UErrorCode *status) +{ + Resource res = RES_BOGUS; + UResourceBundle *result = fillIn; + const char *key; + + if(status == NULL || U_FAILURE(*status)) { + return result; + } + + /* here we do looping and circular alias checking */ + /* this loop is here because aliasing is resolved on this level, not on res level */ + /* so, when we encounter an alias, it is not an aggregate resource, so we return */ + do { + res = res_findResource(&(resB->fResData), resB->fRes, &path, &key); + if(res != RES_BOGUS) { + result = init_resb_result(&(resB->fResData), res, key, -1, resB->fData, resB, 0, fillIn, status); + resB = result; + } else { + *status = U_MISSING_RESOURCE_ERROR; + break; + } + } while(*path); /* there is more stuff in the path */ + + return result; +} +U_INTERNAL const UChar* U_EXPORT2 +ures_getStringByKeyWithFallback(const UResourceBundle *resB, + const char* inKey, + int32_t* len, + UErrorCode *status) { + + UResourceBundle stack; + const UChar* retVal = NULL; + ures_initStackObject(&stack); + ures_getByKeyWithFallback(resB, inKey, &stack, status); + retVal = ures_getString(&stack, len, status); + ures_close(&stack); + return retVal; +} + +U_CAPI UResourceBundle* U_EXPORT2 +ures_getByKeyWithFallback(const UResourceBundle *resB, + const char* inKey, + UResourceBundle *fillIn, + UErrorCode *status) { + Resource res = RES_BOGUS, rootRes = RES_BOGUS; + /*UResourceDataEntry *realData = NULL;*/ + const char *key = inKey; + UResourceBundle *helper = NULL; + + if (status==NULL || U_FAILURE(*status)) { + return fillIn; + } + if(resB == NULL) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return fillIn; + } + + if(RES_GET_TYPE(resB->fRes) == URES_TABLE || RES_GET_TYPE(resB->fRes) == URES_TABLE32) { + int32_t t; + res = res_getTableItemByKey(&(resB->fResData), resB->fRes, &t, &key); + if(res == RES_BOGUS) { + UResourceDataEntry *dataEntry = resB->fData; + char path[256]; + char* myPath = path; + const char* resPath = resB->fResPath; + int32_t len = resB->fResPathLen; + + while(res == RES_BOGUS && dataEntry->fParent != NULL) { /* Otherwise, we'll look in parents */ + dataEntry = dataEntry->fParent; + rootRes = dataEntry->fData.rootRes; + if(dataEntry->fBogus == U_ZERO_ERROR) { + uprv_strncpy(path, resPath, len); + uprv_strcpy(path+len, inKey); + myPath = path; + key = inKey; + do { + res = res_findResource(&(dataEntry->fData), rootRes, &myPath, &key); + if (RES_GET_TYPE(res) == URES_ALIAS && *myPath) { + /* We hit an alias, but we didn't finish following the path. */ + helper = init_resb_result(&(dataEntry->fData), res, NULL, -1, dataEntry, resB, 0, helper, status); + /*helper = init_resb_result(&(dataEntry->fData), res, inKey, -1, dataEntry, resB, 0, helper, status);*/ + if(helper) { + dataEntry = helper->fData; + rootRes = helper->fRes; + resPath = helper->fResPath; + len = helper->fResPathLen; + + } else { + break; + } + } + } while(*myPath); /* Continue until the whole path is consumed */ + } + } + /*const ResourceData *rd = getFallbackData(resB, &key, &realData, &res, status);*/ + if(res != RES_BOGUS) { + /* check if resB->fResPath gives the right name here */ + if(uprv_strcmp(dataEntry->fName, uloc_getDefault())==0 || uprv_strcmp(dataEntry->fName, kRootLocaleName)==0) { + *status = U_USING_DEFAULT_WARNING; + } else { + *status = U_USING_FALLBACK_WARNING; + } + + fillIn = init_resb_result(&(dataEntry->fData), res, inKey, -1, dataEntry, resB, 0, fillIn, status); + } else { + *status = U_MISSING_RESOURCE_ERROR; + } + } else { + fillIn = init_resb_result(&(resB->fResData), res, key, -1, resB->fData, resB, 0, fillIn, status); + } + } + else { + *status = U_RESOURCE_TYPE_MISMATCH; + } + ures_close(helper); + return fillIn; +} + + +U_CAPI UResourceBundle* U_EXPORT2 ures_getByKey(const UResourceBundle *resB, const char* inKey, UResourceBundle *fillIn, UErrorCode *status) { + Resource res = RES_BOGUS; + UResourceDataEntry *realData = NULL; + const char *key = inKey; + + if (status==NULL || U_FAILURE(*status)) { + return fillIn; + } + if(resB == NULL) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return fillIn; + } + + if(RES_GET_TYPE(resB->fRes) == URES_TABLE || RES_GET_TYPE(resB->fRes) == URES_TABLE32) { + int32_t t; + res = res_getTableItemByKey(&(resB->fResData), resB->fRes, &t, &key); + if(res == RES_BOGUS) { + key = inKey; + if(resB->fHasFallback == TRUE) { + const ResourceData *rd = getFallbackData(resB, &key, &realData, &res, status); + if(U_SUCCESS(*status)) { + /* check if resB->fResPath gives the right name here */ + return init_resb_result(rd, res, key, -1, realData, resB, 0, fillIn, status); + } else { + *status = U_MISSING_RESOURCE_ERROR; + } + } else { + *status = U_MISSING_RESOURCE_ERROR; + } + } else { + return init_resb_result(&(resB->fResData), res, key, -1, resB->fData, resB, 0, fillIn, status); + } + } +#if 0 + /* this is a kind of TODO item. If we have an array with an index table, we could do this. */ + /* not currently */ + else if(RES_GET_TYPE(resB->fRes) == URES_ARRAY && resB->fHasFallback == TRUE) { + /* here should go a first attempt to locate the key using index table */ + const ResourceData *rd = getFallbackData(resB, &key, &realData, &res, status); + if(U_SUCCESS(*status)) { + return init_resb_result(rd, res, key, realData, resB, fillIn, status); + } else { + *status = U_MISSING_RESOURCE_ERROR; + } + } +#endif + else { + *status = U_RESOURCE_TYPE_MISMATCH; + } + return fillIn; +} + +U_CAPI const UChar* U_EXPORT2 ures_getStringByKey(const UResourceBundle *resB, const char* inKey, int32_t* len, UErrorCode *status) { + Resource res = RES_BOGUS; + UResourceDataEntry *realData = NULL; + const char* key = inKey; + + if (status==NULL || U_FAILURE(*status)) { + return NULL; + } + if(resB == NULL) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + + if(RES_GET_TYPE(resB->fRes) == URES_TABLE || RES_GET_TYPE(resB->fRes) == URES_TABLE32) { + int32_t t=0; + + res = res_getTableItemByKey(&(resB->fResData), resB->fRes, &t, &key); + + if(res == RES_BOGUS) { + key = inKey; + if(resB->fHasFallback == TRUE) { + const ResourceData *rd = getFallbackData(resB, &key, &realData, &res, status); + if(U_SUCCESS(*status)) { + switch (RES_GET_TYPE(res)) { + case URES_STRING: + case URES_TABLE: + case URES_TABLE32: + case URES_ARRAY: + return res_getString(rd, res, len); + case URES_ALIAS: + { + const UChar* result = 0; + UResourceBundle *tempRes = ures_getByKey(resB, inKey, NULL, status); + result = ures_getString(tempRes, len, status); + ures_close(tempRes); + return result; + } + default: + *status = U_RESOURCE_TYPE_MISMATCH; + } + } else { + *status = U_MISSING_RESOURCE_ERROR; + } + } else { + *status = U_MISSING_RESOURCE_ERROR; + } + } else { + switch (RES_GET_TYPE(res)) { + case URES_STRING: + case URES_TABLE: + case URES_TABLE32: + case URES_ARRAY: + return res_getString(&(resB->fResData), res, len); + case URES_ALIAS: + { + const UChar* result = 0; + UResourceBundle *tempRes = ures_getByKey(resB, inKey, NULL, status); + result = ures_getString(tempRes, len, status); + ures_close(tempRes); + return result; + } + default: + *status = U_RESOURCE_TYPE_MISMATCH; + } + } + } +#if 0 + /* this is a kind of TODO item. If we have an array with an index table, we could do this. */ + /* not currently */ + else if(RES_GET_TYPE(resB->fRes) == URES_ARRAY && resB->fHasFallback == TRUE) { + /* here should go a first attempt to locate the key using index table */ + const ResourceData *rd = getFallbackData(resB, &key, &realData, &res, status); + if(U_SUCCESS(*status)) { + return res_getString(rd, res, len); + } else { + *status = U_MISSING_RESOURCE_ERROR; + } + } +#endif + else { + *status = U_RESOURCE_TYPE_MISMATCH; + } + return NULL; +} + +U_CAPI const char * U_EXPORT2 +ures_getUTF8StringByKey(const UResourceBundle *resB, + const char *key, + char *dest, int32_t *pLength, + UBool forceCopy, + UErrorCode *status) { + int32_t length16; + const UChar *s16 = ures_getStringByKey(resB, key, &length16, status); + return ures_toUTF8String(s16, length16, dest, pLength, forceCopy, status); +} + +/* TODO: clean from here down */ + +/** + * INTERNAL: Get the name of the first real locale (not placeholder) + * that has resource bundle data. + */ +U_CAPI const char* U_EXPORT2 +ures_getLocale(const UResourceBundle* resourceBundle, UErrorCode* status) +{ + if (status==NULL || U_FAILURE(*status)) { + return NULL; + } + if (!resourceBundle) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } else { + return resourceBundle->fData->fName; + } +} + +U_CAPI const char* U_EXPORT2 +ures_getLocaleByType(const UResourceBundle* resourceBundle, + ULocDataLocaleType type, + UErrorCode* status) { + if (status==NULL || U_FAILURE(*status)) { + return NULL; + } + if (!resourceBundle) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } else { + switch(type) { + case ULOC_ACTUAL_LOCALE: + return resourceBundle->fData->fName; + case ULOC_VALID_LOCALE: + return resourceBundle->fTopLevelData->fName; + case ULOC_REQUESTED_LOCALE: + return NULL; + default: + *status = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + } +} + +U_CFUNC const char* ures_getName(const UResourceBundle* resB) { + if(resB == NULL) { + return NULL; + } + + return resB->fData->fName; +} + +#ifdef URES_DEBUG +U_CFUNC const char* ures_getPath(const UResourceBundle* resB) { + if(resB == NULL) { + return NULL; + } + + return resB->fData->fPath; +} +#endif + +/* OLD API implementation */ + +/** + * API: This function is used to open a resource bundle + * proper fallback chaining is executed while initialization. + * The result is stored in cache for later fallback search. + */ +U_CAPI void U_EXPORT2 +ures_openFillIn(UResourceBundle *r, const char* path, + const char* localeID, UErrorCode* status) { + if(r == NULL) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + } else { + UResourceDataEntry *firstData; + UBool isStackObject = ures_isStackObject(r); + + ures_closeBundle(r, FALSE); + uprv_memset(r, 0, sizeof(UResourceBundle)); + ures_setIsStackObject(r, isStackObject); + r->fHasFallback = TRUE; + r->fIsTopLevel = TRUE; + r->fIndex = -1; + r->fData = entryOpen(path, localeID, status); + if(U_FAILURE(*status)) { + return; + } + /* this is a quick fix to get regular data in bundle - until construction is cleaned up */ + firstData = r->fData; + while(firstData->fBogus != U_ZERO_ERROR && firstData->fParent != NULL) { + firstData = firstData->fParent; + } + uprv_memcpy(&r->fResData, &firstData->fData, sizeof(ResourceData)); + r->fHasFallback=(UBool)!r->fResData.noFallback; + r->fRes = r->fResData.rootRes; + r->fSize = res_countArrayItems(&(r->fResData), r->fRes); + r->fTopLevelData = r->fData; + } +} + +U_CAPI UResourceBundle* U_EXPORT2 +ures_open(const char* path, + const char* localeID, + UErrorCode* status) +{ + char canonLocaleID[100]; + UResourceDataEntry *hasData = NULL; + UResourceBundle *r; + + if(status == NULL || U_FAILURE(*status)) { + return NULL; + } + + /* first "canonicalize" the locale ID */ + uloc_getBaseName(localeID, canonLocaleID, sizeof(canonLocaleID), status); + if(U_FAILURE(*status) || *status == U_STRING_NOT_TERMINATED_WARNING) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + + r = (UResourceBundle *)uprv_malloc(sizeof(UResourceBundle)); + if(r == NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + + uprv_memset(r, 0, sizeof(UResourceBundle)); + r->fHasFallback = TRUE; + r->fIsTopLevel = TRUE; + ures_setIsStackObject(r, FALSE); + r->fIndex = -1; + r->fData = entryOpen(path, canonLocaleID, status); + if(U_FAILURE(*status)) { + uprv_free(r); + return NULL; + } + r->fTopLevelData = r->fData; + + hasData = r->fData; + while(hasData->fBogus != U_ZERO_ERROR) { + hasData = hasData->fParent; + if(hasData == NULL) { + /* This can happen only if fallback chain gets broken by an act of God */ + /* TODO: this unlikely to happen, consider removing it */ + entryClose(r->fData); + uprv_free(r); + *status = U_MISSING_RESOURCE_ERROR; + return NULL; + } + } + + uprv_memcpy(&r->fResData, &hasData->fData, sizeof(ResourceData)); + r->fHasFallback=(UBool)!r->fResData.noFallback; + r->fRes = r->fResData.rootRes; + r->fSize = res_countArrayItems(&(r->fResData), r->fRes); + /* + if(r->fData->fPath != NULL) { + ures_setResPath(r, r->fData->fPath); + ures_appendResPath(r, RES_PATH_PACKAGE_S); + ures_appendResPath(r, r->fData->fName); + } else { + ures_setResPath(r, r->fData->fName); + } + */ + + + return r; +} + +/** + * Opens a resource bundle without "canonicalizing" the locale name. No fallback will be performed + * or sought. However, alias substitution will happen! + */ +U_CAPI UResourceBundle* U_EXPORT2 +ures_openDirect(const char* path, const char* localeID, UErrorCode* status) { + UResourceBundle *r; + UErrorCode subStatus = U_ZERO_ERROR; + + if(status == NULL || U_FAILURE(*status)) { + return NULL; + } + + r = (UResourceBundle *)uprv_malloc(sizeof(UResourceBundle)); + if(r == NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + + r->fHasFallback = FALSE; + r->fIsTopLevel = TRUE; + ures_setIsStackObject(r, FALSE); + r->fIndex = -1; + r->fData = entryOpen(path, localeID, &subStatus); + if(U_FAILURE(subStatus)) { + *status = subStatus; + uprv_free(r); + return NULL; + } + if(subStatus != U_ZERO_ERROR /*r->fData->fBogus != U_ZERO_ERROR*/) { + /* we didn't find one we were looking for - so openDirect */ + /* should fail */ + entryClose(r->fData); + uprv_free(r); + *status = U_MISSING_RESOURCE_ERROR; + return NULL; + } + + r->fKey = NULL; + r->fVersion = NULL; + uprv_memcpy(&r->fResData, &r->fData->fData, sizeof(ResourceData)); + /* r->fHasFallback remains FALSE here in ures_openDirect() */ + r->fRes = r->fResData.rootRes; + /*r->fParent = RES_BOGUS;*/ + r->fSize = res_countArrayItems(&(r->fResData), r->fRes); + r->fResPath = NULL; + r->fResPathLen = 0; + r->fParentRes = NULL; + r->fTopLevelData = r->fData; + + return r; +} + +/** + * API: Counts members. For arrays and tables, returns number of resources. + * For strings, returns 1. + */ +U_CAPI int32_t U_EXPORT2 +ures_countArrayItems(const UResourceBundle* resourceBundle, + const char* resourceKey, + UErrorCode* status) +{ + UResourceBundle resData; + ures_initStackObject(&resData); + if (status==NULL || U_FAILURE(*status)) { + return 0; + } + if(resourceBundle == NULL) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + ures_getByKey(resourceBundle, resourceKey, &resData, status); + + if(resData.fResData.data != NULL) { + int32_t result = res_countArrayItems(&resData.fResData, resData.fRes); + ures_close(&resData); + return result; + } else { + *status = U_MISSING_RESOURCE_ERROR; + ures_close(&resData); + return 0; + } +} + +U_CAPI const char* U_EXPORT2 +ures_getVersionNumber(const UResourceBundle* resourceBundle) +{ + if (!resourceBundle) return NULL; + + if(resourceBundle->fVersion == NULL) { + + /* If the version ID has not been built yet, then do so. Retrieve */ + /* the minor version from the file. */ + UErrorCode status = U_ZERO_ERROR; + int32_t minor_len = 0; + int32_t len; + + const UChar* minor_version = ures_getStringByKey(resourceBundle, kVersionTag, &minor_len, &status); + + /* Determine the length of of the final version string. This is */ + /* the length of the major part + the length of the separator */ + /* (==1) + the length of the minor part (+ 1 for the zero byte at */ + /* the end). */ + + len = (minor_len > 0) ? minor_len : 1; + + /* Allocate the string, and build it up. */ + /* + 1 for zero byte */ + + + ((UResourceBundle *)resourceBundle)->fVersion = (char *)uprv_malloc(1 + len); + + if(minor_len > 0) { + u_UCharsToChars(minor_version, resourceBundle->fVersion , minor_len); + resourceBundle->fVersion[len] = '\0'; + } + else { + uprv_strcpy(resourceBundle->fVersion, kDefaultMinorVersion); + } + } + + return resourceBundle->fVersion; +} + +U_CAPI void U_EXPORT2 ures_getVersion(const UResourceBundle* resB, UVersionInfo versionInfo) { + if (!resB) return; + + u_versionFromString(versionInfo, ures_getVersionNumber(resB)); +} + +/** Tree support functions *******************************/ +#define INDEX_LOCALE_NAME "res_index" +#define INDEX_TAG "InstalledLocales" +#define DEFAULT_TAG "default" + +#if defined(URES_TREE_DEBUG) +#include +#endif + +typedef struct ULocalesContext { + UResourceBundle installed; + UResourceBundle curr; +} ULocalesContext; + +static void U_CALLCONV +ures_loc_closeLocales(UEnumeration *enumerator) { + ULocalesContext *ctx = (ULocalesContext *)enumerator->context; + ures_close(&ctx->curr); + ures_close(&ctx->installed); + uprv_free(ctx); + uprv_free(enumerator); +} + +static int32_t U_CALLCONV +ures_loc_countLocales(UEnumeration *en, UErrorCode *status) { + ULocalesContext *ctx = (ULocalesContext *)en->context; + return ures_getSize(&ctx->installed); +} + +static const char* U_CALLCONV +ures_loc_nextLocale(UEnumeration* en, + int32_t* resultLength, + UErrorCode* status) { + ULocalesContext *ctx = (ULocalesContext *)en->context; + UResourceBundle *res = &(ctx->installed); + UResourceBundle *k = NULL; + const char *result = NULL; + int32_t len = 0; + if(ures_hasNext(res) && (k = ures_getNextResource(res, &ctx->curr, status))) { + result = ures_getKey(k); + len = (int32_t)uprv_strlen(result); + } + if (resultLength) { + *resultLength = len; + } + return result; +} + +static void U_CALLCONV +ures_loc_resetLocales(UEnumeration* en, + UErrorCode* status) { + UResourceBundle *res = &((ULocalesContext *)en->context)->installed; + ures_resetIterator(res); +} + + +static const UEnumeration gLocalesEnum = { + NULL, + NULL, + ures_loc_closeLocales, + ures_loc_countLocales, + uenum_unextDefault, + ures_loc_nextLocale, + ures_loc_resetLocales +}; + + +U_CAPI UEnumeration* U_EXPORT2 +ures_openAvailableLocales(const char *path, UErrorCode *status) +{ + UResourceBundle *index = NULL; + UEnumeration *en = NULL; + ULocalesContext *myContext = NULL; + + if(U_FAILURE(*status)) { + return NULL; + } + myContext = uprv_malloc(sizeof(ULocalesContext)); + en = (UEnumeration *)uprv_malloc(sizeof(UEnumeration)); + if(!en || !myContext) { + *status = U_MEMORY_ALLOCATION_ERROR; + uprv_free(en); + uprv_free(myContext); + return NULL; + } + uprv_memcpy(en, &gLocalesEnum, sizeof(UEnumeration)); + + ures_initStackObject(&myContext->installed); + ures_initStackObject(&myContext->curr); + index = ures_openDirect(path, INDEX_LOCALE_NAME, status); + ures_getByKey(index, INDEX_TAG, &myContext->installed, status); + if(U_SUCCESS(*status)) { +#if defined(URES_TREE_DEBUG) + fprintf(stderr, "Got %s::%s::[%s] : %s\n", + path, INDEX_LOCALE_NAME, INDEX_TAG, ures_getKey(&myContext->installed)); +#endif + en->context = myContext; + } else { +#if defined(URES_TREE_DEBUG) + fprintf(stderr, "%s open failed - %s\n", path, u_errorName(*status)); +#endif + ures_close(&myContext->installed); + uprv_free(myContext); + uprv_free(en); + en = NULL; + } + + ures_close(index); + + return en; +} + +static UBool isLocaleInList(UEnumeration *locEnum, const char *locToSearch, UErrorCode *status) { + const char *loc; + while ((loc = uenum_next(locEnum, NULL, status)) != NULL) { + if (uprv_strcmp(loc, locToSearch) == 0) { + return TRUE; + } + } + return FALSE; +} + +U_CAPI int32_t U_EXPORT2 +ures_getFunctionalEquivalent(char *result, int32_t resultCapacity, + const char *path, const char *resName, const char *keyword, const char *locid, + UBool *isAvailable, UBool omitDefault, UErrorCode *status) +{ + char kwVal[1024] = ""; /* value of keyword 'keyword' */ + char defVal[1024] = ""; /* default value for given locale */ + char defLoc[1024] = ""; /* default value for given locale */ + char base[1024] = ""; /* base locale */ + char found[1024]; + char parent[1024]; + char full[1024] = ""; + UResourceBundle bund1, bund2; + UResourceBundle *res = NULL; + UErrorCode subStatus = U_ZERO_ERROR; + int32_t length = 0; + if(U_FAILURE(*status)) return 0; + uloc_getKeywordValue(locid, keyword, kwVal, 1024-1,&subStatus); + if(!uprv_strcmp(kwVal, DEFAULT_TAG)) { + kwVal[0]=0; + } + uloc_getBaseName(locid, base, 1024-1,&subStatus); +#if defined(URES_TREE_DEBUG) + fprintf(stderr, "getFunctionalEquivalent: \"%s\" [%s=%s] in %s - %s\n", + locid, keyword, kwVal, base, u_errorName(subStatus)); +#endif + ures_initStackObject(&bund1); + ures_initStackObject(&bund2); + + + uprv_strcpy(parent, base); + uprv_strcpy(found, base); + + if(isAvailable) { + UEnumeration *locEnum = ures_openAvailableLocales(path, &subStatus); + *isAvailable = TRUE; + if (U_SUCCESS(subStatus)) { + *isAvailable = isLocaleInList(locEnum, parent, &subStatus); + } + uenum_close(locEnum); + } + + if(U_FAILURE(subStatus)) { + *status = subStatus; + return 0; + } + + do { + subStatus = U_ZERO_ERROR; + res = ures_open(path, parent, &subStatus); + if(((subStatus == U_USING_FALLBACK_WARNING) || + (subStatus == U_USING_DEFAULT_WARNING)) && isAvailable) + { + *isAvailable = FALSE; + } + isAvailable = NULL; /* only want to set this the first time around */ + +#if defined(URES_TREE_DEBUG) + fprintf(stderr, "%s;%s -> %s [%s]\n", path?path:"ICUDATA", parent, u_errorName(subStatus), ures_getLocale(res, &subStatus)); +#endif + if(U_FAILURE(subStatus)) { + *status = subStatus; + } else if(subStatus == U_ZERO_ERROR) { + ures_getByKey(res,resName,&bund1, &subStatus); + if(subStatus == U_ZERO_ERROR) { + const UChar *defUstr; + int32_t defLen; + /* look for default item */ +#if defined(URES_TREE_DEBUG) + fprintf(stderr, "%s;%s : loaded default -> %s\n", + path?path:"ICUDATA", parent, u_errorName(subStatus)); +#endif + defUstr = ures_getStringByKey(&bund1, DEFAULT_TAG, &defLen, &subStatus); + if(U_SUCCESS(subStatus) && defLen) { + u_UCharsToChars(defUstr, defVal, u_strlen(defUstr)); +#if defined(URES_TREE_DEBUG) + fprintf(stderr, "%s;%s -> default %s=%s, %s\n", + path?path:"ICUDATA", parent, keyword, defVal, u_errorName(subStatus)); +#endif + uprv_strcpy(defLoc, parent); + if(kwVal[0]==0) { + uprv_strcpy(kwVal, defVal); +#if defined(URES_TREE_DEBUG) + fprintf(stderr, "%s;%s -> kwVal = %s\n", + path?path:"ICUDATA", parent, keyword, kwVal); +#endif + } + } + } + } + + subStatus = U_ZERO_ERROR; + + if (res != NULL) { + uprv_strcpy(found, ures_getLocaleByType(res, ULOC_VALID_LOCALE, &subStatus)); + } + + uloc_getParent(found,parent,sizeof(parent),&subStatus); + ures_close(res); + } while(!defVal[0] && *found && uprv_strcmp(found, "root") != 0 && U_SUCCESS(*status)); + + /* Now, see if we can find the kwVal collator.. start the search over.. */ + uprv_strcpy(parent, base); + uprv_strcpy(found, base); + + do { + subStatus = U_ZERO_ERROR; + res = ures_open(path, parent, &subStatus); + if((subStatus == U_USING_FALLBACK_WARNING) && isAvailable) { + *isAvailable = FALSE; + } + isAvailable = NULL; /* only want to set this the first time around */ + +#if defined(URES_TREE_DEBUG) + fprintf(stderr, "%s;%s -> %s (looking for %s)\n", + path?path:"ICUDATA", parent, u_errorName(subStatus), kwVal); +#endif + if(U_FAILURE(subStatus)) { + *status = subStatus; + } else if(subStatus == U_ZERO_ERROR) { + ures_getByKey(res,resName,&bund1, &subStatus); +#if defined(URES_TREE_DEBUG) +/**/ fprintf(stderr,"@%d [%s] %s\n", __LINE__, resName, u_errorName(subStatus)); +#endif + if(subStatus == U_ZERO_ERROR) { + ures_getByKey(&bund1, kwVal, &bund2, &subStatus); +#if defined(URES_TREE_DEBUG) +/**/ fprintf(stderr,"@%d [%s] %s\n", __LINE__, kwVal, u_errorName(subStatus)); +#endif + if(subStatus == U_ZERO_ERROR) { +#if defined(URES_TREE_DEBUG) + fprintf(stderr, "%s;%s -> full0 %s=%s, %s\n", + path?path:"ICUDATA", parent, keyword, kwVal, u_errorName(subStatus)); +#endif + uprv_strcpy(full, parent); + if(*full == 0) { + uprv_strcpy(full, "root"); + } + /* now, recalculate default kw if need be */ + if(uprv_strlen(defLoc) > uprv_strlen(full)) { + const UChar *defUstr; + int32_t defLen; + /* look for default item */ +#if defined(URES_TREE_DEBUG) + fprintf(stderr, "%s;%s -> recalculating Default0\n", + path?path:"ICUDATA", full); +#endif + defUstr = ures_getStringByKey(&bund1, DEFAULT_TAG, &defLen, &subStatus); + if(U_SUCCESS(subStatus) && defLen) { + u_UCharsToChars(defUstr, defVal, u_strlen(defUstr)); +#if defined(URES_TREE_DEBUG) + fprintf(stderr, "%s;%s -> default0 %s=%s, %s\n", + path?path:"ICUDATA", full, keyword, defVal, u_errorName(subStatus)); +#endif + uprv_strcpy(defLoc, full); + } + } /* end of recalculate default KW */ +#if defined(URES_TREE_DEBUG) + else { + fprintf(stderr, "No trim0, %s <= %s\n", defLoc, full); + } +#endif + } else { +#if defined(URES_TREE_DEBUG) + fprintf(stderr, "err=%s in %s looking for %s\n", + u_errorName(subStatus), parent, kwVal); +#endif + } + } + } + + subStatus = U_ZERO_ERROR; + + uprv_strcpy(found, parent); + uloc_getParent(found,parent,1023,&subStatus); + ures_close(res); + } while(!full[0] && *found && U_SUCCESS(*status)); + + if((full[0]==0) && uprv_strcmp(kwVal, defVal)) { +#if defined(URES_TREE_DEBUG) + fprintf(stderr, "Failed to locate kw %s - try default %s\n", kwVal, defVal); +#endif + uprv_strcpy(kwVal, defVal); + uprv_strcpy(parent, base); + uprv_strcpy(found, base); + + do { /* search for 'default' named item */ + subStatus = U_ZERO_ERROR; + res = ures_open(path, parent, &subStatus); + if((subStatus == U_USING_FALLBACK_WARNING) && isAvailable) { + *isAvailable = FALSE; + } + isAvailable = NULL; /* only want to set this the first time around */ + +#if defined(URES_TREE_DEBUG) + fprintf(stderr, "%s;%s -> %s (looking for default %s)\n", + path?path:"ICUDATA", parent, u_errorName(subStatus), kwVal); +#endif + if(U_FAILURE(subStatus)) { + *status = subStatus; + } else if(subStatus == U_ZERO_ERROR) { + ures_getByKey(res,resName,&bund1, &subStatus); + if(subStatus == U_ZERO_ERROR) { + ures_getByKey(&bund1, kwVal, &bund2, &subStatus); + if(subStatus == U_ZERO_ERROR) { +#if defined(URES_TREE_DEBUG) + fprintf(stderr, "%s;%s -> full1 %s=%s, %s\n", path?path:"ICUDATA", + parent, keyword, kwVal, u_errorName(subStatus)); +#endif + uprv_strcpy(full, parent); + if(*full == 0) { + uprv_strcpy(full, "root"); + } + + /* now, recalculate default kw if need be */ + if(uprv_strlen(defLoc) > uprv_strlen(full)) { + const UChar *defUstr; + int32_t defLen; + /* look for default item */ +#if defined(URES_TREE_DEBUG) + fprintf(stderr, "%s;%s -> recalculating Default1\n", + path?path:"ICUDATA", full); +#endif + defUstr = ures_getStringByKey(&bund1, DEFAULT_TAG, &defLen, &subStatus); + if(U_SUCCESS(subStatus) && defLen) { + u_UCharsToChars(defUstr, defVal, u_strlen(defUstr)); +#if defined(URES_TREE_DEBUG) + fprintf(stderr, "%s;%s -> default %s=%s, %s\n", + path?path:"ICUDATA", full, keyword, defVal, u_errorName(subStatus)); +#endif + uprv_strcpy(defLoc, full); + } + } /* end of recalculate default KW */ +#if defined(URES_TREE_DEBUG) + else { + fprintf(stderr, "No trim1, %s <= %s\n", defLoc, full); + } +#endif + } + } + } + subStatus = U_ZERO_ERROR; + + uprv_strcpy(found, parent); + uloc_getParent(found,parent,1023,&subStatus); + ures_close(res); + } while(!full[0] && *found && U_SUCCESS(*status)); + } + + if(U_SUCCESS(*status)) { + if(!full[0]) { +#if defined(URES_TREE_DEBUG) + fprintf(stderr, "Still could not load keyword %s=%s\n", keyword, kwVal); +#endif + *status = U_MISSING_RESOURCE_ERROR; + } else if(omitDefault) { +#if defined(URES_TREE_DEBUG) + fprintf(stderr,"Trim? full=%s, defLoc=%s, found=%s\n", full, defLoc, found); +#endif + if(uprv_strlen(defLoc) <= uprv_strlen(full)) { + /* found the keyword in a *child* of where the default tag was present. */ + if(!uprv_strcmp(kwVal, defVal)) { /* if the requested kw is default, */ + /* and the default is in or in an ancestor of the current locale */ +#if defined(URES_TREE_DEBUG) + fprintf(stderr, "Removing unneeded var %s=%s\n", keyword, kwVal); +#endif + kwVal[0]=0; + } + } + } + uprv_strcpy(found, full); + if(kwVal[0]) { + uprv_strcat(found, "@"); + uprv_strcat(found, keyword); + uprv_strcat(found, "="); + uprv_strcat(found, kwVal); + } else if(!omitDefault) { + uprv_strcat(found, "@"); + uprv_strcat(found, keyword); + uprv_strcat(found, "="); + uprv_strcat(found, defVal); + } + } + /* we found the default locale - no need to repeat it.*/ + + ures_close(&bund1); + ures_close(&bund2); + + length = (int32_t)uprv_strlen(found); + + if(U_SUCCESS(*status)) { + int32_t copyLength = uprv_min(length, resultCapacity); + if(copyLength>0) { + uprv_strncpy(result, found, copyLength); + } + if(length == 0) { + *status = U_MISSING_RESOURCE_ERROR; + } + } else { + length = 0; + result[0]=0; + } + return u_terminateChars(result, resultCapacity, length, status); +} + +U_CAPI UEnumeration* U_EXPORT2 +ures_getKeywordValues(const char *path, const char *keyword, UErrorCode *status) +{ +#define VALUES_BUF_SIZE 2048 +#define VALUES_LIST_SIZE 512 + + char valuesBuf[VALUES_BUF_SIZE]; + int32_t valuesIndex = 0; + const char *valuesList[VALUES_LIST_SIZE]; + int32_t valuesCount = 0; + + const char *locale; + int32_t locLen; + + UEnumeration *locs = NULL; + + UResourceBundle item; + UResourceBundle subItem; + + ures_initStackObject(&item); + ures_initStackObject(&subItem); + locs = ures_openAvailableLocales(path, status); + + if(U_FAILURE(*status)) { + ures_close(&item); + ures_close(&subItem); + return NULL; + } + + valuesBuf[0]=0; + valuesBuf[1]=0; + + while((locale = uenum_next(locs, &locLen, status))) { + UResourceBundle *bund = NULL; + UResourceBundle *subPtr = NULL; + UErrorCode subStatus = U_ZERO_ERROR; /* don't fail if a bundle is unopenable */ + bund = ures_openDirect(path, locale, &subStatus); + +#if defined(URES_TREE_DEBUG) + if(!bund || U_FAILURE(subStatus)) { + fprintf(stderr, "%s-%s values: Can't open %s locale - skipping. (%s)\n", + path?path:"", keyword, locale, u_errorName(subStatus)); + } +#endif + + ures_getByKey(bund, keyword, &item, &subStatus); + + if(!bund || U_FAILURE(subStatus)) { +#if defined(URES_TREE_DEBUG) + fprintf(stderr, "%s-%s values: Can't find in %s - skipping. (%s)\n", + path?path:"", keyword, locale, u_errorName(subStatus)); +#endif + ures_close(bund); + bund = NULL; + continue; + } + + while((subPtr = ures_getNextResource(&item,&subItem,&subStatus)) + && U_SUCCESS(subStatus)) { + const char *k; + int32_t i; + k = ures_getKey(subPtr); + +#if defined(URES_TREE_DEBUG) + /* fprintf(stderr, "%s | %s | %s | %s\n", path?path:"", keyword, locale, k); */ +#endif + for(i=0;k&&i= (VALUES_LIST_SIZE-1)) || /* no more space in list .. */ + ((valuesIndex+kLen+1+1) >= VALUES_BUF_SIZE)) { /* no more space in buffer (string + 2 nulls) */ + *status = U_ILLEGAL_ARGUMENT_ERROR; /* out of space.. */ + } else { + uprv_strcpy(valuesBuf+valuesIndex, k); + valuesList[valuesCount++] = valuesBuf+valuesIndex; + valuesIndex += kLen; +#if defined(URES_TREE_DEBUG) + fprintf(stderr, "%s | %s | %s | [%s] (UNIQUE)\n", + path?path:"", keyword, locale, k); +#endif + valuesBuf[valuesIndex++] = 0; /* terminate */ + } + } + } + ures_close(bund); + } + valuesBuf[valuesIndex++] = 0; /* terminate */ + + ures_close(&item); + ures_close(&subItem); + uenum_close(locs); +#if defined(URES_TREE_DEBUG) + fprintf(stderr, "%s: size %d, #%d\n", u_errorName(*status), + valuesIndex, valuesCount); +#endif + return uloc_openKeywordList(valuesBuf, valuesIndex, status); +} +U_INTERNAL UBool U_EXPORT2 +ures_equal(const UResourceBundle* res1, const UResourceBundle* res2){ + if(res1==NULL || res2==NULL){ + return res1==res2; /* pointer comparision */ + } + if(res1->fKey==NULL|| res2->fKey==NULL){ + return (res1->fKey==res2->fKey); + }else{ + if(uprv_strcmp(res1->fKey, res2->fKey)!=0){ + return FALSE; + } + } + if(uprv_strcmp(res1->fData->fName, res2->fData->fName)!=0){ + return FALSE; + } + if(res1->fData->fPath == NULL|| res2->fData->fPath==NULL){ + return (res1->fData->fPath == res2->fData->fPath); + }else{ + if(uprv_strcmp(res1->fData->fPath, res2->fData->fPath)!=0){ + return FALSE; + } + } + if(uprv_strcmp(res1->fData->fParent->fName, res2->fData->fParent->fName)!=0){ + return FALSE; + } + if(uprv_strcmp(res1->fData->fParent->fPath, res2->fData->fParent->fPath)!=0){ + return FALSE; + } + if(uprv_strncmp(res1->fResPath, res2->fResPath, res1->fResPathLen)!=0){ + return FALSE; + } + if(res1->fRes != res2->fRes){ + return FALSE; + } + return TRUE; +} +U_INTERNAL UResourceBundle* U_EXPORT2 +ures_clone(const UResourceBundle* res, UErrorCode* status){ + UResourceBundle* bundle = NULL; + UResourceBundle* ret = NULL; + if(U_FAILURE(*status) || res == NULL){ + return NULL; + } + bundle = ures_open(res->fData->fPath, res->fData->fName, status); + if(res->fResPath!=NULL){ + ret = ures_findSubResource(bundle, res->fResPath, NULL, status); + ures_close(bundle); + }else{ + ret = bundle; + } + return ret; +} +U_INTERNAL const UResourceBundle* U_EXPORT2 +ures_getParentBundle(const UResourceBundle* res){ + if(res==NULL){ + return NULL; + } + return res->fParentRes; +} +/* eof */ diff --git a/Build/source/libs/icu/icu-xetex/common/uresdata.c b/Build/source/libs/icu/icu-xetex/common/uresdata.c new file mode 100644 index 00000000000..4d947f5ba77 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uresdata.c @@ -0,0 +1,1095 @@ +/* +******************************************************************************* +* * +* Copyright (C) 1999-2006, International Business Machines Corporation * +* and others. All Rights Reserved. * +* * +******************************************************************************* +* file name: uresdata.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 1999dec08 +* created by: Markus W. Scherer +* Modification History: +* +* Date Name Description +* 06/20/2000 helena OS/400 port changes; mostly typecast. +* 06/24/02 weiv Added support for resource sharing +*/ + +#include "unicode/utypes.h" +#include "unicode/udata.h" +#include "cmemory.h" +#include "cstring.h" +#include "uarrsort.h" +#include "udataswp.h" +#include "ucol_swp.h" +#include "uresdata.h" +#include "uresimp.h" + +#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) + +/* + * Resource access helpers + */ + +/* get a const char* pointer to the key with the keyOffset byte offset from pRoot */ +#define RES_GET_KEY(pRoot, keyOffset) ((const char *)(pRoot)+(keyOffset)) +#define URESDATA_ITEM_NOT_FOUND -1 + +/* + * All the type-access functions assume that + * the resource is of the expected type. + */ + + +/* + * Array functions + */ +static Resource +_res_getArrayItem(Resource *pRoot, Resource res, int32_t indexR) { + const int32_t *p=(const int32_t *)RES_GET_POINTER(pRoot, res); + if(indexR<*p) { + return ((const Resource *)(p))[1+indexR]; + } else { + return RES_BOGUS; /* indexR>itemCount */ + } +} + +/* + * Table functions + * + * Important: the key offsets are 16-bit byte offsets from pRoot, + * and the itemCount is one more 16-bit, too. + * Thus, there are (count+1) uint16_t values. + * In order to 4-align the Resource item values, there is a padding + * word if count is even, i.e., there is exactly (~count&1) + * 16-bit padding words. + * + * For Table32, both the count and the key offsets are int32_t's + * and need not alignment. + */ +static const char * +_res_getTableKey(const Resource *pRoot, const Resource res, int32_t indexS) { + const uint16_t *p=(const uint16_t *)RES_GET_POINTER(pRoot, res); + if((uint32_t)indexS<(uint32_t)*p) { + return RES_GET_KEY(pRoot, p[indexS+1]); + } else { + return NULL; /* indexS>itemCount */ + } +} + +static const char * +_res_getTable32Key(const Resource *pRoot, const Resource res, int32_t indexS) { + const int32_t *p=(const int32_t *)RES_GET_POINTER(pRoot, res); + if((uint32_t)indexS<(uint32_t)*p) { + return RES_GET_KEY(pRoot, p[indexS+1]); + } else { + return NULL; /* indexS>itemCount */ + } +} + + +static Resource +_res_getTableItem(const Resource *pRoot, const Resource res, int32_t indexR) { + const uint16_t *p=(const uint16_t *)RES_GET_POINTER(pRoot, res); + int32_t count=*p; + if((uint32_t)indexR<(uint32_t)count) { + return ((const Resource *)(p+1+count+(~count&1)))[indexR]; + } else { + return RES_BOGUS; /* indexR>itemCount */ + } +} + +static Resource +_res_getTable32Item(const Resource *pRoot, const Resource res, int32_t indexR) { + const int32_t *p=(const int32_t *)RES_GET_POINTER(pRoot, res); + int32_t count=*p; + if((uint32_t)indexR<(uint32_t)count) { + return ((const Resource *)(p+1+count))[indexR]; + } else { + return RES_BOGUS; /* indexR>itemCount */ + } +} + + +static Resource +_res_findTableItem(const Resource *pRoot, const Resource res, const char *key, + int32_t *index, const char **realKey) { + const uint16_t *p=(const uint16_t *)RES_GET_POINTER(pRoot, res); + uint32_t mid, start, limit; + uint32_t lastMid; + int result; + + limit=*p++; /* number of entries */ + + if(limit != 0) { + /* do a binary search for the key */ + start=0; + lastMid = UINT32_MAX; + for (;;) { + mid = (uint32_t)((start + limit) / 2); + if (lastMid == mid) { /* Have we moved? */ + break; /* We haven't moved, and it wasn't found. */ + } + lastMid = mid; + result = uprv_strcmp(key, RES_GET_KEY(pRoot, p[mid])); + + if (result < 0) { + limit = mid; + } else if (result > 0) { + start = mid; + } else { + /* We found it! */ + *index=mid; + *realKey=RES_GET_KEY(pRoot, p[mid]); + limit=*(p-1); /* itemCount */ + return ((const Resource *)(p+limit+(~limit&1)))[mid]; + } + } + } + + *index=URESDATA_ITEM_NOT_FOUND; + return RES_BOGUS; /* not found or table is empty. */ +} + +static Resource +_res_findTable32Item(const Resource *pRoot, const Resource res, const char *key, + int32_t *index, const char **realKey) { + const int32_t *p=(const int32_t *)RES_GET_POINTER(pRoot, res); + int32_t mid, start, limit; + int32_t lastMid; + int result; + + limit=*p++; /* number of entries */ + + if(limit != 0) { + /* do a binary search for the key */ + start=0; + lastMid = INT32_MAX; + for (;;) { + mid = (uint32_t)((start + limit) / 2); + if (lastMid == mid) { /* Have we moved? */ + break; /* We haven't moved, and it wasn't found. */ + } + lastMid = mid; + result = uprv_strcmp(key, RES_GET_KEY(pRoot, p[mid])); + + if (result < 0) { + limit = mid; + } else if (result > 0) { + start = mid; + } else { + /* We found it! */ + *index=mid; + *realKey=RES_GET_KEY(pRoot, p[mid]); + return ((const Resource *)(p+(*(p-1))))[mid]; + } + } + } + + *index=URESDATA_ITEM_NOT_FOUND; + return RES_BOGUS; /* not found or table is empty. */ +} + +/* helper for res_load() ---------------------------------------------------- */ + +static UBool U_CALLCONV +isAcceptable(void *context, + const char *type, const char *name, + const UDataInfo *pInfo) { + uprv_memcpy(context, pInfo->formatVersion, 4); + return (UBool)( + pInfo->size>=20 && + pInfo->isBigEndian==U_IS_BIG_ENDIAN && + pInfo->charsetFamily==U_CHARSET_FAMILY && + pInfo->sizeofUChar==U_SIZEOF_UCHAR && + pInfo->dataFormat[0]==0x52 && /* dataFormat="ResB" */ + pInfo->dataFormat[1]==0x65 && + pInfo->dataFormat[2]==0x73 && + pInfo->dataFormat[3]==0x42 && + pInfo->formatVersion[0]==1); +} + +/* semi-public functions ---------------------------------------------------- */ + +U_CFUNC UBool +res_load(ResourceData *pResData, + const char *path, const char *name, UErrorCode *errorCode) { + UVersionInfo formatVersion; + UResType rootType; + + /* load the ResourceBundle file */ + pResData->data=udata_openChoice(path, "res", name, isAcceptable, formatVersion, errorCode); + if(U_FAILURE(*errorCode)) { + return FALSE; + } + + /* get its memory and root resource */ + pResData->pRoot=(Resource *)udata_getMemory(pResData->data); + pResData->rootRes=*pResData->pRoot; + pResData->noFallback=FALSE; + + /* currently, we accept only resources that have a Table as their roots */ + rootType=RES_GET_TYPE(pResData->rootRes); + if(rootType!=URES_TABLE && rootType!=URES_TABLE32) { + *errorCode=U_INVALID_FORMAT_ERROR; + udata_close(pResData->data); + pResData->data=NULL; + return FALSE; + } + + if(formatVersion[0]>1 || (formatVersion[0]==1 && formatVersion[1]>=1)) { + /* bundles with formatVersion 1.1 and later contain an indexes[] array */ + const int32_t *indexes=(const int32_t *)pResData->pRoot+1; + if(indexes[URES_INDEX_LENGTH]>URES_INDEX_ATTRIBUTES) { + pResData->noFallback=(UBool)(indexes[URES_INDEX_ATTRIBUTES]&URES_ATT_NO_FALLBACK); + } + } + + return TRUE; +} + +U_CFUNC void +res_unload(ResourceData *pResData) { + if(pResData->data!=NULL) { + udata_close(pResData->data); + pResData->data=NULL; + } +} + +U_CFUNC const UChar * +res_getString(const ResourceData *pResData, const Resource res, int32_t *pLength) { + /* + * The data structure is documented as supporting res==0 for empty strings. + * Return a fixed pointer in such a case. + * This was dropped in uresdata.c 1.17 as part of Jitterbug 1005 work + * on code coverage for ICU 2.0. + * Re-added for consistency with the design and with other code. + */ + static const int32_t emptyString[2]={ 0, 0 }; + if(res!=RES_BOGUS && RES_GET_TYPE(res)==URES_STRING) { + const int32_t *p= res==0 ? emptyString : (const int32_t *)RES_GET_POINTER(pResData->pRoot, res); + if (pLength) { + *pLength=*p; + } + return (const UChar *)++p; + } else { + if (pLength) { + *pLength=0; + } + return NULL; + } +} + +U_CFUNC const UChar * +res_getAlias(const ResourceData *pResData, const Resource res, int32_t *pLength) { + if(res!=RES_BOGUS && RES_GET_TYPE(res)==URES_ALIAS) { + const int32_t *p=(const int32_t *)RES_GET_POINTER(pResData->pRoot, res); + if (pLength) { + *pLength=*p; + } + return (const UChar *)++p; + } else { + if (pLength) { + *pLength=0; + } + return NULL; + } +} + +U_CFUNC const uint8_t * +res_getBinary(const ResourceData *pResData, const Resource res, int32_t *pLength) { + if(res!=RES_BOGUS) { + const int32_t *p=(const int32_t *)RES_GET_POINTER(pResData->pRoot, res); + *pLength=*p++; + if (*pLength == 0) { + p = NULL; + } + return (const uint8_t *)p; + } else { + *pLength=0; + return NULL; + } +} + + +U_CFUNC const int32_t * +res_getIntVector(const ResourceData *pResData, const Resource res, int32_t *pLength) { + if(res!=RES_BOGUS && RES_GET_TYPE(res)==URES_INT_VECTOR) { + const int32_t *p=(const int32_t *)RES_GET_POINTER(pResData->pRoot, res); + *pLength=*p++; + if (*pLength == 0) { + p = NULL; + } + return (const int32_t *)p; + } else { + *pLength=0; + return NULL; + } +} + +U_CFUNC int32_t +res_countArrayItems(const ResourceData *pResData, const Resource res) { + if(res!=RES_BOGUS) { + switch(RES_GET_TYPE(res)) { + case URES_STRING: + case URES_BINARY: + case URES_ALIAS: + case URES_INT: + case URES_INT_VECTOR: + return 1; + case URES_ARRAY: + case URES_TABLE32: { + const int32_t *p=(const int32_t *)RES_GET_POINTER(pResData->pRoot, res); + return *p; + } + case URES_TABLE: { + const uint16_t *p=(const uint16_t *)RES_GET_POINTER(pResData->pRoot, res); + return *p; + } + default: + break; + } + } + return 0; +} + +U_CFUNC Resource +res_getResource(const ResourceData *pResData, const char *key) { + int32_t index; + const char *realKey; + if(RES_GET_TYPE(pResData->rootRes)==URES_TABLE) { + return _res_findTableItem(pResData->pRoot, pResData->rootRes, key, &index, &realKey); + } else { + return _res_findTable32Item(pResData->pRoot, pResData->rootRes, key, &index, &realKey); + } +} + +U_CFUNC Resource +res_getArrayItem(const ResourceData *pResData, Resource array, const int32_t indexR) { + return _res_getArrayItem(pResData->pRoot, array, indexR); +} + +U_CFUNC Resource +res_findResource(const ResourceData *pResData, Resource r, char** path, const char** key) { + /* we pass in a path. CollationElements/Sequence or zoneStrings/3/2 etc. + * iterates over a path and stops when a scalar resource is found. This + * CAN be an alias. Path gets set to the part that has not yet been processed. + */ + + char *pathP = *path, *nextSepP = *path; + char *closeIndex = NULL; + Resource t1 = r; + Resource t2; + int32_t indexR = 0; + UResType type = RES_GET_TYPE(t1); + + /* if you come in with an empty path, you'll be getting back the same resource */ + if(!uprv_strlen(pathP)) { + return r; + } + + /* one needs to have an aggregate resource in order to search in it */ + if(!(type == URES_TABLE || type == URES_TABLE32 || type == URES_ARRAY)) { + return RES_BOGUS; + } + + while(nextSepP && *pathP && t1 != RES_BOGUS && + (type == URES_TABLE || type == URES_TABLE32 || type == URES_ARRAY) + ) { + /* Iteration stops if: the path has been consumed, we found a non-existing + * resource (t1 == RES_BOGUS) or we found a scalar resource (including alias) + */ + nextSepP = uprv_strchr(pathP, RES_PATH_SEPARATOR); + /* if there are more separators, terminate string + * and set path to the remaining part of the string + */ + if(nextSepP != NULL) { + *nextSepP = 0; /* overwrite the separator with a NUL to terminate the key */ + *path = nextSepP+1; + } else { + *path = uprv_strchr(pathP, 0); + } + + /* if the resource is a table */ + /* try the key based access */ + if(type == URES_TABLE) { + t2 = _res_findTableItem(pResData->pRoot, t1, pathP, &indexR, key); + if(t2 == RES_BOGUS) { + /* if we fail to get the resource by key, maybe we got an index */ + indexR = uprv_strtol(pathP, &closeIndex, 10); + if(closeIndex != pathP) { + /* if we indeed have an index, try to get the item by index */ + t2 = res_getTableItemByIndex(pResData, t1, indexR, key); + } + } + } else if(type == URES_TABLE32) { + t2 = _res_findTable32Item(pResData->pRoot, t1, pathP, &indexR, key); + if(t2 == RES_BOGUS) { + /* if we fail to get the resource by key, maybe we got an index */ + indexR = uprv_strtol(pathP, &closeIndex, 10); + if(closeIndex != pathP) { + /* if we indeed have an index, try to get the item by index */ + t2 = res_getTableItemByIndex(pResData, t1, indexR, key); + } + } + } else if(type == URES_ARRAY) { + indexR = uprv_strtol(pathP, &closeIndex, 10); + if(closeIndex != pathP) { + t2 = _res_getArrayItem(pResData->pRoot, t1, indexR); + } else { + t2 = RES_BOGUS; /* have an array, but don't have a valid index */ + } + *key = NULL; + } else { /* can't do much here, except setting t2 to bogus */ + t2 = RES_BOGUS; + } + t1 = t2; + type = RES_GET_TYPE(t1); + /* position pathP to next resource key/index */ + pathP = *path; + } + + return t1; +} + +U_CFUNC Resource +res_getTableItemByKey(const ResourceData *pResData, Resource table, + int32_t *indexR, const char **key ){ + if(key != NULL && *key != NULL) { + if(RES_GET_TYPE(table)==URES_TABLE) { + return _res_findTableItem(pResData->pRoot, table, *key, indexR, key); + } else { + return _res_findTable32Item(pResData->pRoot, table, *key, indexR, key); + } + } else { + return RES_BOGUS; + } +} + +U_CFUNC Resource +res_getTableItemByIndex(const ResourceData *pResData, Resource table, + int32_t indexR, const char **key) { + if(indexR>-1) { + if(RES_GET_TYPE(table)==URES_TABLE) { + if(key != NULL) { + *key = _res_getTableKey(pResData->pRoot, table, indexR); + } + return _res_getTableItem(pResData->pRoot, table, indexR); + } else { + if(key != NULL) { + *key = _res_getTable32Key(pResData->pRoot, table, indexR); + } + return _res_getTable32Item(pResData->pRoot, table, indexR); + } + } else { + return RES_BOGUS; + } +} + +/* resource bundle swapping ------------------------------------------------- */ + +/* + * Need to always enumerate the entire item tree, + * track the lowest address of any item to use as the limit for char keys[], + * track the highest address of any item to return the size of the data. + * + * We should have thought of storing those in the data... + * It is possible to extend the data structure by putting additional values + * in places that are inaccessible by ordinary enumeration of the item tree. + * For example, additional integers could be stored at the beginning or + * end of the key strings; this could be indicated by a minor version number, + * and the data swapping would have to know about these values. + * + * The data structure does not forbid keys to be shared, so we must swap + * all keys once instead of each key when it is referenced. + * + * These swapping functions assume that a resource bundle always has a length + * that is a multiple of 4 bytes. + * Currently, this is trivially true because genrb writes bundle tree leaves + * physically first, before their branches, so that the root table with its + * array of resource items (uint32_t values) is always last. + */ + +/* definitions for table sorting ------------------------ */ + +/* + * row of a temporary array + * + * gets platform-endian key string indexes and sorting indexes; + * after sorting this array by keys, the actual key/value arrays are permutated + * according to the sorting indexes + */ +typedef struct Row { + int32_t keyIndex, sortIndex; +} Row; + +static int32_t +ures_compareRows(const void *context, const void *left, const void *right) { + const char *keyChars=(const char *)context; + return (int32_t)uprv_strcmp(keyChars+((const Row *)left)->keyIndex, + keyChars+((const Row *)right)->keyIndex); +} + +typedef struct TempTable { + const char *keyChars; + Row *rows; + int32_t *resort; +} TempTable; + +enum { + STACK_ROW_CAPACITY=200 +}; + +/* binary data with known formats is swapped too */ +typedef enum UResSpecialType { + URES_NO_SPECIAL_TYPE, + URES_COLLATION_BINARY, + URES_SPECIAL_TYPE_COUNT +} UResSpecialType; + +/* resource table key for collation binaries: "%%CollationBin" */ +static const UChar gCollationBinKey[]={ + 0x25, 0x25, + 0x43, 0x6f, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x42, 0x69, 0x6e, + 0 +}; + +/* + * preflight one resource item and set bottom and top values; + * length, bottom, and top count Resource item offsets (4 bytes each), not bytes + */ +static void +ures_preflightResource(const UDataSwapper *ds, + const Resource *inBundle, int32_t length, + Resource res, + int32_t *pBottom, int32_t *pTop, int32_t *pMaxTableLength, + UErrorCode *pErrorCode) { + const Resource *p; + int32_t offset; + + if(res==0 || RES_GET_TYPE(res)==URES_INT) { + /* empty string or integer, nothing to do */ + return; + } + + /* all other types use an offset to point to their data */ + offset=(int32_t)RES_GET_OFFSET(res); + if(0<=length && length<=offset) { + udata_printError(ds, "ures_preflightResource(res=%08x) resource offset exceeds bundle length %d\n", + res, length); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return; + } else if(offset<*pBottom) { + *pBottom=offset; + } + p=inBundle+offset; + + switch(RES_GET_TYPE(res)) { + case URES_ALIAS: + /* physically same value layout as string, fall through */ + case URES_STRING: + /* top=offset+1+(string length +1)/2 rounded up */ + offset+=1+((udata_readInt32(ds, (int32_t)*p)+1)+1)/2; + break; + case URES_BINARY: + /* top=offset+1+(binary length)/4 rounded up */ + offset+=1+(udata_readInt32(ds, (int32_t)*p)+3)/4; + break; + case URES_TABLE: + case URES_TABLE32: + { + Resource item; + int32_t i, count; + + if(RES_GET_TYPE(res)==URES_TABLE) { + /* get table item count */ + const uint16_t *pKey16=(const uint16_t *)p; + count=ds->readUInt16(*pKey16++); + + /* top=((1+ table item count)/2 rounded up)+(table item count) */ + offset+=((1+count)+1)/2; + } else { + /* get table item count */ + const int32_t *pKey32=(const int32_t *)p; + count=udata_readInt32(ds, *pKey32++); + + /* top=(1+ table item count)+(table item count) */ + offset+=1+count; + } + + if(count>*pMaxTableLength) { + *pMaxTableLength=count; + } + + p=inBundle+offset; /* pointer to table resources */ + offset+=count; + + /* recurse */ + if(offset<=length) { + for(i=0; ireadUInt32(*p++); + ures_preflightResource(ds, inBundle, length, item, + pBottom, pTop, pMaxTableLength, + pErrorCode); + if(U_FAILURE(*pErrorCode)) { + udata_printError(ds, "ures_preflightResource(table res=%08x)[%d].recurse(%08x) failed\n", + res, i, item); + break; + } + } + } + } + break; + case URES_ARRAY: + { + Resource item; + int32_t i, count; + + /* top=offset+1+(array length) */ + count=udata_readInt32(ds, (int32_t)*p++); + offset+=1+count; + + /* recurse */ + if(offset<=length) { + for(i=0; ireadUInt32(*p++); + ures_preflightResource(ds, inBundle, length, item, + pBottom, pTop, pMaxTableLength, + pErrorCode); + if(U_FAILURE(*pErrorCode)) { + udata_printError(ds, "ures_preflightResource(array res=%08x)[%d].recurse(%08x) failed\n", + res, i, item); + break; + } + } + } + } + break; + case URES_INT_VECTOR: + /* top=offset+1+(vector length) */ + offset+=1+udata_readInt32(ds, (int32_t)*p); + break; + default: + /* also catches RES_BOGUS */ + udata_printError(ds, "ures_preflightResource(res=%08x) unknown resource type\n", res); + *pErrorCode=U_UNSUPPORTED_ERROR; + break; + } + + if(U_FAILURE(*pErrorCode)) { + /* nothing to do */ + } else if(0<=length && length*pTop) { + *pTop=offset; + } +} + +/* + * swap one resource item + * since preflighting succeeded, we need not check offsets against length any more + */ +static void +ures_swapResource(const UDataSwapper *ds, + const Resource *inBundle, Resource *outBundle, + Resource res, /* caller swaps res itself */ + UResSpecialType specialType, + TempTable *pTempTable, + UErrorCode *pErrorCode) { + const Resource *p; + Resource *q; + int32_t offset, count; + + if(res==0 || RES_GET_TYPE(res)==URES_INT) { + /* empty string or integer, nothing to do */ + return; + } + + /* all other types use an offset to point to their data */ + offset=(int32_t)RES_GET_OFFSET(res); + p=inBundle+offset; + q=outBundle+offset; + + switch(RES_GET_TYPE(res)) { + case URES_ALIAS: + /* physically same value layout as string, fall through */ + case URES_STRING: + count=udata_readInt32(ds, (int32_t)*p); + /* swap length */ + ds->swapArray32(ds, p, 4, q, pErrorCode); + /* swap each UChar (the terminating NUL would not change) */ + ds->swapArray16(ds, p+1, 2*count, q+1, pErrorCode); + break; + case URES_BINARY: + count=udata_readInt32(ds, (int32_t)*p); + /* swap length */ + ds->swapArray32(ds, p, 4, q, pErrorCode); + /* no need to swap or copy bytes - ures_swap() copied them all */ + + /* swap known formats */ + if(specialType==URES_COLLATION_BINARY) { +#if !UCONFIG_NO_COLLATION + ucol_swapBinary(ds, p+1, count, q+1, pErrorCode); +#endif + } + break; + case URES_TABLE: + case URES_TABLE32: + { + const uint16_t *pKey16; + uint16_t *qKey16; + + const int32_t *pKey32; + int32_t *qKey32; + + Resource item; + int32_t i, oldIndex; + + if(RES_GET_TYPE(res)==URES_TABLE) { + /* get table item count */ + pKey16=(const uint16_t *)p; + qKey16=(uint16_t *)q; + count=ds->readUInt16(*pKey16); + + pKey32=qKey32=NULL; + + /* swap count */ + ds->swapArray16(ds, pKey16++, 2, qKey16++, pErrorCode); + + offset+=((1+count)+1)/2; + } else { + /* get table item count */ + pKey32=(const int32_t *)p; + qKey32=(int32_t *)q; + count=udata_readInt32(ds, *pKey32); + + pKey16=qKey16=NULL; + + /* swap count */ + ds->swapArray32(ds, pKey32++, 4, qKey32++, pErrorCode); + + offset+=1+count; + } + + if(count==0) { + break; + } + + p=inBundle+offset; /* pointer to table resources */ + q=outBundle+offset; + + /* recurse */ + for(i=0; icompareInvChars(ds, outData+readUInt16(pKey[i]), "%%CollationBin") + * etc. + * + * use some UDataSwapFn pointer from somewhere for collation swapping + * because the common library cannot directly call into the i18n library + */ + if(0==ds->compareInvChars(ds, + ((const char *)outBundle)+ + (pKey16!=NULL ? + ds->readUInt16(pKey16[i]) : + udata_readInt32(ds, pKey32[i])), + -1, + gCollationBinKey, LENGTHOF(gCollationBinKey)-1) + ) { + specialType=URES_COLLATION_BINARY; + } else { + specialType=URES_NO_SPECIAL_TYPE; + } + + item=ds->readUInt32(p[i]); + ures_swapResource(ds, inBundle, outBundle, item, specialType, pTempTable, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + udata_printError(ds, "ures_swapResource(table res=%08x)[%d].recurse(%08x) failed\n", + res, i, item); + return; + } + } + + if(ds->inCharset==ds->outCharset) { + /* no need to sort, just swap the offset/value arrays */ + if(pKey16!=NULL) { + ds->swapArray16(ds, pKey16, count*2, qKey16, pErrorCode); + ds->swapArray32(ds, p, count*4, q, pErrorCode); + } else { + /* swap key offsets and items as one array */ + ds->swapArray32(ds, pKey32, count*2*4, qKey32, pErrorCode); + } + break; + } + + /* + * We need to sort tables by outCharset key strings because they + * sort differently for different charset families. + * ures_swap() already set pTempTable->keyChars appropriately. + * First we set up a temporary table with the key indexes and + * sorting indexes and sort that. + * Then we permutate and copy/swap the actual values. + */ + if(pKey16!=NULL) { + for(i=0; irows[i].keyIndex=ds->readUInt16(pKey16[i]); + pTempTable->rows[i].sortIndex=i; + } + } else { + for(i=0; irows[i].keyIndex=udata_readInt32(ds, pKey32[i]); + pTempTable->rows[i].sortIndex=i; + } + } + uprv_sortArray(pTempTable->rows, count, sizeof(Row), + ures_compareRows, pTempTable->keyChars, + FALSE, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + udata_printError(ds, "ures_swapResource(table res=%08x).uprv_sortArray(%d items) failed\n", + res, count); + return; + } + + /* + * copy/swap/permutate items + * + * If we swap in-place, then the permutation must use another + * temporary array (pTempTable->resort) + * before the results are copied to the outBundle. + */ + /* keys */ + if(pKey16!=NULL) { + uint16_t *rKey16; + + if(pKey16!=qKey16) { + rKey16=qKey16; + } else { + rKey16=(uint16_t *)pTempTable->resort; + } + for(i=0; irows[i].sortIndex; + ds->swapArray16(ds, pKey16+oldIndex, 2, rKey16+i, pErrorCode); + } + if(qKey16!=rKey16) { + uprv_memcpy(qKey16, rKey16, 2*count); + } + } else { + int32_t *rKey32; + + if(pKey32!=qKey32) { + rKey32=qKey32; + } else { + rKey32=pTempTable->resort; + } + for(i=0; irows[i].sortIndex; + ds->swapArray32(ds, pKey32+oldIndex, 4, rKey32+i, pErrorCode); + } + if(qKey32!=rKey32) { + uprv_memcpy(qKey32, rKey32, 4*count); + } + } + + /* resources */ + { + Resource *r; + + + if(p!=q) { + r=q; + } else { + r=(Resource *)pTempTable->resort; + } + for(i=0; irows[i].sortIndex; + ds->swapArray32(ds, p+oldIndex, 4, r+i, pErrorCode); + } + if(q!=r) { + uprv_memcpy(q, r, 4*count); + } + } + } + break; + case URES_ARRAY: + { + Resource item; + int32_t i; + + count=udata_readInt32(ds, (int32_t)*p); + /* swap length */ + ds->swapArray32(ds, p++, 4, q++, pErrorCode); + + /* recurse */ + for(i=0; ireadUInt32(p[i]); + ures_swapResource(ds, inBundle, outBundle, item, URES_NO_SPECIAL_TYPE, pTempTable, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + udata_printError(ds, "ures_swapResource(array res=%08x)[%d].recurse(%08x) failed\n", + res, i, item); + return; + } + } + + /* swap items */ + ds->swapArray32(ds, p, 4*count, q, pErrorCode); + } + break; + case URES_INT_VECTOR: + count=udata_readInt32(ds, (int32_t)*p); + /* swap length and each integer */ + ds->swapArray32(ds, p, 4*(1+count), q, pErrorCode); + break; + default: + /* also catches RES_BOGUS */ + *pErrorCode=U_UNSUPPORTED_ERROR; + break; + } +} + +U_CAPI int32_t U_EXPORT2 +ures_swap(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode) { + const UDataInfo *pInfo; + const Resource *inBundle; + Resource rootRes; + int32_t headerSize, maxTableLength; + + Row rows[STACK_ROW_CAPACITY]; + int32_t resort[STACK_ROW_CAPACITY]; + TempTable tempTable; + + /* the following integers count Resource item offsets (4 bytes each), not bytes */ + int32_t bundleLength, stringsBottom, bottom, top; + + /* udata_swapDataHeader checks the arguments */ + headerSize=udata_swapDataHeader(ds, inData, length, outData, pErrorCode); + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + /* check data format and format version */ + pInfo=(const UDataInfo *)((const char *)inData+4); + if(!( + pInfo->dataFormat[0]==0x52 && /* dataFormat="ResB" */ + pInfo->dataFormat[1]==0x65 && + pInfo->dataFormat[2]==0x73 && + pInfo->dataFormat[3]==0x42 && + pInfo->formatVersion[0]==1 + )) { + udata_printError(ds, "ures_swap(): data format %02x.%02x.%02x.%02x (format version %02x) is not a resource bundle\n", + pInfo->dataFormat[0], pInfo->dataFormat[1], + pInfo->dataFormat[2], pInfo->dataFormat[3], + pInfo->formatVersion[0]); + *pErrorCode=U_UNSUPPORTED_ERROR; + return 0; + } + + /* a resource bundle must contain at least one resource item */ + if(length<0) { + bundleLength=-1; + } else { + bundleLength=(length-headerSize)/4; + + /* formatVersion 1.1 must have a root item and at least 5 indexes */ + if( bundleLength< + (pInfo->formatVersion[1]==0 ? 1 : 1+5) + ) { + udata_printError(ds, "ures_swap(): too few bytes (%d after header) for a resource bundle\n", + length-headerSize); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + } + + inBundle=(const Resource *)((const char *)inData+headerSize); + rootRes=ds->readUInt32(*inBundle); + + if(pInfo->formatVersion[1]==0) { + /* preflight to get the bottom, top and maxTableLength values */ + stringsBottom=1; /* just past root */ + bottom=0x7fffffff; + top=maxTableLength=0; + ures_preflightResource(ds, inBundle, bundleLength, rootRes, + &bottom, &top, &maxTableLength, + pErrorCode); + if(U_FAILURE(*pErrorCode)) { + udata_printError(ds, "ures_preflightResource(root res=%08x) failed\n", + rootRes); + return 0; + } + } else { + /* formatVersion 1.1 adds the indexes[] array */ + const int32_t *inIndexes; + + inIndexes=(const int32_t *)(inBundle+1); + + stringsBottom=1+udata_readInt32(ds, inIndexes[URES_INDEX_LENGTH]); + bottom=udata_readInt32(ds, inIndexes[URES_INDEX_STRINGS_TOP]); + top=udata_readInt32(ds, inIndexes[URES_INDEX_BUNDLE_TOP]); + maxTableLength=udata_readInt32(ds, inIndexes[URES_INDEX_MAX_TABLE_LENGTH]); + + if(0<=bundleLength && bundleLength=0) { + Resource *outBundle=(Resource *)((char *)outData+headerSize); + + /* copy the bundle for binary and inaccessible data */ + if(inData!=outData) { + uprv_memcpy(outBundle, inBundle, 4*top); + } + + /* swap the key strings, but not the padding bytes (0xaa) after the last string and its NUL */ + udata_swapInvStringBlock(ds, inBundle+stringsBottom, 4*(bottom-stringsBottom), + outBundle+stringsBottom, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + udata_printError(ds, "ures_swap().udata_swapInvStringBlock(keys[%d]) failed\n", 4*(bottom-1)); + return 0; + } + + /* allocate the temporary table for sorting resource tables */ + tempTable.keyChars=(const char *)outBundle; /* sort by outCharset */ + if(maxTableLength<=STACK_ROW_CAPACITY) { + tempTable.rows=rows; + tempTable.resort=resort; + } else { + tempTable.rows=(Row *)uprv_malloc(maxTableLength*sizeof(Row)+maxTableLength*4); + if(tempTable.rows==NULL) { + udata_printError(ds, "ures_swap(): unable to allocate memory for sorting tables (max length: %d)\n", + maxTableLength); + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return 0; + } + tempTable.resort=(int32_t *)(tempTable.rows+maxTableLength); + } + + /* swap the resources */ + ures_swapResource(ds, inBundle, outBundle, rootRes, URES_NO_SPECIAL_TYPE, &tempTable, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + udata_printError(ds, "ures_swapResource(root res=%08x) failed\n", + rootRes); + } + + if(tempTable.rows!=rows) { + uprv_free(tempTable.rows); + } + + /* swap the root resource and indexes */ + ds->swapArray32(ds, inBundle, stringsBottom*4, outBundle, pErrorCode); + } + + return headerSize+4*top; +} diff --git a/Build/source/libs/icu/icu-xetex/common/uresdata.h b/Build/source/libs/icu/icu-xetex/common/uresdata.h new file mode 100644 index 00000000000..6b8aa8aaede --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uresdata.h @@ -0,0 +1,233 @@ +/* +****************************************************************************** +* * +* Copyright (C) 1999-2006, International Business Machines * +* Corporation and others. All Rights Reserved. * +* * +****************************************************************************** +* file name: uresdata.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 1999dec08 +* created by: Markus W. Scherer +* 06/24/02 weiv Added support for resource sharing +*/ + +#ifndef __RESDATA_H__ +#define __RESDATA_H__ + +#include "unicode/utypes.h" +#include "unicode/udata.h" +#include "udataswp.h" + +/* + * A Resource is a 32-bit value that has 2 bit fields: + * 31..28 4-bit type, see enum below + * 27..0 28-bit four-byte-offset or value according to the type + */ +typedef uint32_t Resource; + +#define RES_BOGUS 0xffffffff + +#define RES_GET_TYPE(res) ((UResType)((res)>>28UL)) +#define RES_GET_OFFSET(res) ((res)&0x0fffffff) +#define RES_GET_POINTER(pRoot, res) ((pRoot)+RES_GET_OFFSET(res)) + +/* get signed and unsigned integer values directly from the Resource handle */ +#define RES_GET_INT(res) (((int32_t)((res)<<4L))>>4L) +#define RES_GET_UINT(res) ((res)&0x0fffffff) + +/* indexes[] value names; indexes are generally 32-bit (Resource) indexes */ +enum { + URES_INDEX_LENGTH, /* [0] contains URES_INDEX_TOP==the length of indexes[] */ + URES_INDEX_STRINGS_TOP, /* [1] contains the top of the strings, */ + /* same as the bottom of resources, rounded up */ + URES_INDEX_RESOURCES_TOP, /* [2] contains the top of all resources */ + URES_INDEX_BUNDLE_TOP, /* [3] contains the top of the bundle, */ + /* in case it were ever different from [2] */ + URES_INDEX_MAX_TABLE_LENGTH,/* [4] max. length of any table */ + URES_INDEX_ATTRIBUTES, /* [5] attributes bit set, see URES_ATT_* (new in formatVersion 1.2) */ + URES_INDEX_TOP +}; + +/* number of bytes at the beginning of the bundle before the strings start */ +enum { + URES_STRINGS_BOTTOM=(1+URES_INDEX_TOP)*4 +}; + +/* + * Nofallback attribute, attribute bit 0 in indexes[URES_INDEX_ATTRIBUTES]. + * New in formatVersion 1.2 (ICU 3.6). + * + * If set, then this resource bundle is a standalone bundle. + * If not set, then the bundle participates in locale fallback, eventually + * all the way to the root bundle. + * If indexes[] is missing or too short, then the attribute cannot be determined + * reliably. Dependency checking should ignore such bundles, and loading should + * use fallbacks. + */ +#define URES_ATT_NO_FALLBACK 1 + +/* + * File format for .res resource bundle files (formatVersion=1.2) + * + * An ICU4C resource bundle file (.res) is a binary, memory-mappable file + * with nested, hierarchical data structures. + * It physically contains the following: + * + * Resource root; -- 32-bit Resource item, root item for this bundle's tree; + * currently, the root item must be a table or table32 resource item + * int32_t indexes[indexes[0]]; -- array of indexes for friendly + * reading and swapping; see URES_INDEX_* above + * new in formatVersion 1.1 (ICU 2.8) + * char keys[]; -- characters for key strings + * (formatVersion 1.0: up to 65k of characters; 1.1: <2G) + * (minus the space for root and indexes[]), + * which consist of invariant characters (ASCII/EBCDIC) and are NUL-terminated; + * padded to multiple of 4 bytes for 4-alignment of the following data + * data; -- data directly and indirectly indexed by the root item; + * the structure is determined by walking the tree + * + * Each resource bundle item has a 32-bit Resource handle (see typedef above) + * which contains the item type number in its upper 4 bits (31..28) and either + * an offset or a direct value in its lower 28 bits (27..0). + * The order of items is undefined and only determined by walking the tree. + * Leaves of the tree may be stored first or last or anywhere in between, + * and it is in theory possible to have unreferenced holes in the file. + * + * Direct values: + * - Empty Unicode strings have an offset value of 0 in the Resource handle itself. + * - Integer values are 28-bit values stored in the Resource handle itself; + * the interpretation of unsigned vs. signed integers is up to the application. + * + * All other types and values use 28-bit offsets to point to the item's data. + * The offset is an index to the first 32-bit word of the value, relative to the + * start of the resource data (i.e., the root item handle is at offset 0). + * To get byte offsets, the offset is multiplied by 4 (or shifted left by 2 bits). + * All resource item values are 4-aligned. + * + * The structures (memory layouts) for the values for each item type are listed + * in the table above. + * + * Nested, hierarchical structures: ------------- + * + * Table items contain key-value pairs where the keys are 16-bit offsets to char * key strings. + * Key string offsets are also relative to the start of the resource data (of the root handle), + * i.e., the first string has an offset of 4 (after the 4-byte root handle). + * + * The values of these pairs are Resource handles. + * + * Array items are simple vectors of Resource handles. + * + * An alias item is special (and new in ICU 2.4): -------------- + * + * Its memory layout is just like for a UnicodeString, but at runtime it resolves to + * another resource bundle's item according to the path in the string. + * This is used to share items across bundles that are in different lookup/fallback + * chains (e.g., large collation data among zh_TW and zh_HK). + * This saves space (for large items) and maintenance effort (less duplication of data). + * + * -------------------------------------------------------------------------- + * + * Resource types: + * + * Most resources have their values stored at four-byte offsets from the start + * of the resource data. These values are at least 4-aligned. + * Some resource values are stored directly in the offset field of the Resource itself. + * See UResType in unicode/ures.h for enumeration constants for Resource types. + * + * Type Name Memory layout of values + * (in parentheses: scalar, non-offset values) + * + * 0 Unicode String: int32_t length, UChar[length], (UChar)0, (padding) + * or (empty string ("") if offset==0) + * 1 Binary: int32_t length, uint8_t[length], (padding) + * - this value should be 32-aligned - + * 2 Table: uint16_t count, uint16_t keyStringOffsets[count], (uint16_t padding), Resource[count] + * 3 Alias: (physically same value layout as string, new in ICU 2.4) + * 4 Table32: int32_t count, int32_t keyStringOffsets[count], Resource[count] + * (new in formatVersion 1.1/ICU 2.8) + * + * 7 Integer: (28-bit offset is integer value) + * 8 Array: int32_t count, Resource[count] + * + * 14 Integer Vector: int32_t length, int32_t[length] + * 15 Reserved: This value denotes special purpose resources and is for internal use. + * + * Note that there are 3 types with data vector values: + * - Vectors of 8-bit bytes stored as type Binary. + * - Vectors of 16-bit words stored as type Unicode String + * (no value restrictions, all values 0..ffff allowed!). + * - Vectors of 32-bit words stored as type Integer Vector. + */ + +/* + * Structure for a single, memory-mapped ResourceBundle. + */ +typedef struct { + UDataMemory *data; + Resource *pRoot; + Resource rootRes; + UBool noFallback; /* see URES_ATT_NO_FALLBACK */ +} ResourceData; + +/* + * Load a resource bundle file. + * The ResourceData structure must be allocated externally. + */ +U_CFUNC UBool +res_load(ResourceData *pResData, + const char *path, const char *name, UErrorCode *errorCode); + +/* + * Release a resource bundle file. + * This does not release the ResourceData structure itself. + */ +U_CFUNC void +res_unload(ResourceData *pResData); + +/* + * Return a pointer to a zero-terminated, const UChar* string + * and set its length in *pLength. + * Returns NULL if not found. + */ +U_CFUNC const UChar * +res_getString(const ResourceData *pResData, const Resource res, int32_t *pLength); + +U_CFUNC const UChar * +res_getAlias(const ResourceData *pResData, const Resource res, int32_t *pLength); + +U_CFUNC const uint8_t * +res_getBinary(const ResourceData *pResData, const Resource res, int32_t *pLength); + +U_CFUNC const int32_t * +res_getIntVector(const ResourceData *pResData, const Resource res, int32_t *pLength); + +U_CFUNC Resource +res_getResource(const ResourceData *pResData, const char *key); + +U_CFUNC int32_t +res_countArrayItems(const ResourceData *pResData, const Resource res); + +U_CFUNC Resource res_getArrayItem(const ResourceData *pResData, Resource array, const int32_t indexS); +U_CFUNC Resource res_getTableItemByIndex(const ResourceData *pResData, Resource table, int32_t indexS, const char ** key); +U_CFUNC Resource res_getTableItemByKey(const ResourceData *pResData, Resource table, int32_t *indexS, const char* * key); + +/* + * Modifies the contents of *path (replacing separators with NULs), + * and also moves *path forward while it finds items. + */ +U_CFUNC Resource res_findResource(const ResourceData *pResData, Resource r, char** path, const char** key); + +/** + * Swap an ICU resource bundle. See udataswp.h. + * @internal + */ +U_CAPI int32_t U_EXPORT2 +ures_swap(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode); + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/uresimp.h b/Build/source/libs/icu/icu-xetex/common/uresimp.h new file mode 100644 index 00000000000..31ad6434d97 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uresimp.h @@ -0,0 +1,243 @@ +/* +********************************************************************** +* Copyright (C) 2000-2006, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +*/ + +#ifndef URESIMP_H +#define URESIMP_H + +#include "unicode/ures.h" + +#include "uresdata.h" + +#define kRootLocaleName "root" + +/* + The default minor version and the version separator must be exactly one + character long. +*/ + +#define kDefaultMinorVersion "0" +#define kVersionSeparator "." +#define kVersionTag "Version" + +#define MAGIC1 19700503 +#define MAGIC2 19641227 + +#define URES_MAX_ALIAS_LEVEL 256 +#define URES_MAX_BUFFER_SIZE 256 + +/* +enum UResEntryType { + ENTRY_OK = 0, + ENTRY_GOTO_ROOT = 1, + ENTRY_GOTO_DEFAULT = 2, + ENTRY_INVALID = 3 +}; + +typedef enum UResEntryType UResEntryType; +*/ + +struct UResourceDataEntry; +typedef struct UResourceDataEntry UResourceDataEntry; + +struct UResourceDataEntry { + char *fName; /* name of the locale for bundle - still to decide whether it is original or fallback */ + char *fPath; /* path to bundle - used for distinguishing between resources with the same name */ + uint32_t fCountExisting; /* how much is this resource used */ + ResourceData fData; /* data for low level access */ + UResourceDataEntry *fParent; /*next resource in fallback chain*/ +/* UResEntryType fStatus;*/ + UErrorCode fBogus; + int32_t fHashKey; /* for faster access in the hashtable */ +}; + +#define RES_BUFSIZE 64 +#define RES_PATH_SEPARATOR '/' +#define RES_PATH_SEPARATOR_S "/" + +struct UResourceBundle { + const char *fKey; /*tag*/ + UResourceDataEntry *fData; /*for low-level access*/ + char *fVersion; + char *fResPath; /* full path to the resource: "zh_TW/CollationElements/Sequence" */ + char fResBuf[RES_BUFSIZE]; + int32_t fResPathLen; + UBool fHasFallback; + UBool fIsTopLevel; + uint32_t fMagic1; /* For determining if it's a stack object */ + uint32_t fMagic2; /* For determining if it's a stack object */ + int32_t fIndex; + int32_t fSize; + ResourceData fResData; + Resource fRes; + + UResourceDataEntry *fTopLevelData; /* for getting the valid locale */ + const UResourceBundle *fParentRes; /* needed to get the actual locale for a child resource */ + +}; + +U_CAPI void U_EXPORT2 ures_initStackObject(UResourceBundle* resB); + +/* Some getters used by the copy constructor */ +U_CFUNC const char* ures_getName(const UResourceBundle* resB); +#ifdef URES_DEBUG +U_CFUNC const char* ures_getPath(const UResourceBundle* resB); +#endif +/*U_CFUNC void ures_appendResPath(UResourceBundle *resB, const char* toAdd, int32_t lenToAdd);*/ +/*U_CFUNC void ures_setResPath(UResourceBundle *resB, const char* toAdd);*/ +/*U_CFUNC void ures_freeResPath(UResourceBundle *resB);*/ + +/* Candidates for export */ +U_CFUNC UResourceBundle *ures_copyResb(UResourceBundle *r, const UResourceBundle *original, UErrorCode *status); + +/** + * Returns a resource that can be located using the pathToResource argument. One needs optional package, locale + * and path inside the locale, for example: "/myData/en/zoneStrings/3". Keys and indexes are supported. Keys + * need to reference data in named structures, while indexes can reference both named and anonymous resources. + * Features a fill-in parameter. + * + * Note, this function does NOT have a syntax for specifying items within a tree. May want to consider a + * syntax that delineates between package/tree and resource. + * + * @param pathToResource a path that will lead to the requested resource + * @param fillIn if NULL a new UResourceBundle struct is allocated and must be deleted by the caller. + * Alternatively, you can supply a struct to be filled by this function. + * @param status fills in the outgoing error code. + * @return a pointer to a UResourceBundle struct. If fill in param was NULL, caller must delete it + * @draft ICU 2.2 + */ +U_CAPI UResourceBundle* U_EXPORT2 +ures_findResource(const char* pathToResource, + UResourceBundle *fillIn, UErrorCode *status); + +/** + * Returns a sub resource that can be located using the pathToResource argument. One needs a path inside + * the supplied resource, for example, if you have "en_US" resource bundle opened, you might ask for + * "zoneStrings/3". Keys and indexes are supported. Keys + * need to reference data in named structures, while indexes can reference both + * named and anonymous resources. + * Features a fill-in parameter. + * + * @param resourceBundle a resource + * @param pathToResource a path that will lead to the requested resource + * @param fillIn if NULL a new UResourceBundle struct is allocated and must be deleted by the caller. + * Alternatively, you can supply a struct to be filled by this function. + * @param status fills in the outgoing error code. + * @return a pointer to a UResourceBundle struct. If fill in param was NULL, caller must delete it + * @draft ICU 2.2 + */ +U_CAPI UResourceBundle* U_EXPORT2 +ures_findSubResource(const UResourceBundle *resB, + char* pathToResource, + UResourceBundle *fillIn, UErrorCode *status); + +/** + * Returns a functionally equivalent locale (considering keywords) for the specified keyword. + * @param result fillin for the equivalent locale + * @param resultCapacity capacity of the fillin buffer + * @param path path to the tree, or NULL for ICU data + * @param resName top level resource. Example: "collations" + * @param keyword locale keyword. Example: "collation" + * @param locid The requested locale + * @param isAvailable If non-null, pointer to fillin parameter that indicates whether the + * requested locale was available. The locale is defined as 'available' if it physically + * exists within the specified tree. + * @param omitDefault if TRUE, omit keyword and value if default. 'de_DE\@collation=standard' -> 'de_DE' + * @param status error code + * @return the actual buffer size needed for the full locale. If it's greater + * than resultCapacity, the returned full name will be truncated and an error code will be returned. + * @internal ICU 3.0 + */ +U_INTERNAL int32_t U_EXPORT2 +ures_getFunctionalEquivalent(char *result, int32_t resultCapacity, + const char *path, const char *resName, const char *keyword, const char *locid, + UBool *isAvailable, UBool omitDefault, UErrorCode *status); + +/** + * Given a tree path and keyword, return a string enumeration of all possible values for that keyword. + * @param path path to the tree, or NULL for ICU data + * @param keyword a particular keyword to consider, must match a top level resource name + * within the tree. + * @param status error code + * @internal ICU 3.0 + */ +U_INTERNAL UEnumeration* U_EXPORT2 +ures_getKeywordValues(const char *path, const char *keyword, UErrorCode *status); + +/** + * Test if 2 resource bundles are equal + * @param res1 + * @param res2 + * @param status error code + * @internal ICU 3.6 + */ +U_INTERNAL UBool U_EXPORT2 +ures_equal(const UResourceBundle* res1, const UResourceBundle* res2); + +/** + * Clones the given resource bundle + * @param res + * @param status error code + * @internal ICU 3.6 + */ +U_INTERNAL UResourceBundle* U_EXPORT2 +ures_clone(const UResourceBundle* res, UErrorCode* status); + +/** + * Returns the parent bundle. Internal. DONOT close the returned bundle!!! + * @param res + * @internal ICU 3.6 + */ +U_INTERNAL const UResourceBundle* U_EXPORT2 +ures_getParentBundle(const UResourceBundle* res); + + +/** + * Get a resource with multi-level fallback. Normally only the top level resources will + * fallback to its parent. This performs fallback on subresources. For example, when a table + * is defined in a resource bundle and a parent resource bundle, normally no fallback occurs + * on the sub-resources because the table is defined in the current resource bundle, but this + * function can perform fallback on the sub-resources of the table. + * @param resB a resource + * @param inKey a key associated with the requested resource + * @param fillIn if NULL a new UResourceBundle struct is allocated and must be deleted by the caller. + * Alternatively, you can supply a struct to be filled by this function. + * @param status: fills in the outgoing error code + * could be U_MISSING_RESOURCE_ERROR if the key is not found + * could be a non-failing error + * e.g.: U_USING_FALLBACK_WARNING,U_USING_DEFAULT_WARNING + * @return a pointer to a UResourceBundle struct. If fill in param was NULL, caller must delete it + * @internal ICU 3.0 + */ +U_INTERNAL UResourceBundle* U_EXPORT2 +ures_getByKeyWithFallback(const UResourceBundle *resB, + const char* inKey, + UResourceBundle *fillIn, + UErrorCode *status); + + +/** + * Get a String with multi-level fallback. Normally only the top level resources will + * fallback to its parent. This performs fallback on subresources. For example, when a table + * is defined in a resource bundle and a parent resource bundle, normally no fallback occurs + * on the sub-resources because the table is defined in the current resource bundle, but this + * function can perform fallback on the sub-resources of the table. + * @param resB a resource + * @param inKey a key associated with the requested resource + * @param status: fills in the outgoing error code + * could be U_MISSING_RESOURCE_ERROR if the key is not found + * could be a non-failing error + * e.g.: U_USING_FALLBACK_WARNING,U_USING_DEFAULT_WARNING + * @return a pointer to a UResourceBundle struct. If fill in param was NULL, caller must delete it + * @internal ICU 3.4 + * @draft ICU 3.4 + */ +U_INTERNAL const UChar* U_EXPORT2 +ures_getStringByKeyWithFallback(const UResourceBundle *resB, + const char* inKey, + int32_t* len, + UErrorCode *status); +#endif /*URESIMP_H*/ diff --git a/Build/source/libs/icu/icu-xetex/common/usc_impl.c b/Build/source/libs/icu/icu-xetex/common/usc_impl.c new file mode 100644 index 00000000000..1807e1bda48 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/usc_impl.c @@ -0,0 +1,361 @@ +/* +********************************************************************** +* Copyright (C) 1999-2004, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* +* File USC_IMPL.C +* +* Modification History: +* +* Date Name Description +* 07/08/2002 Eric Mader Creation. +****************************************************************************** +*/ + +#include "unicode/uscript.h" +#include "usc_impl.h" +#include "cmemory.h" + +#define ARRAY_SIZE(array) (sizeof array / sizeof array[0]) + +#define PAREN_STACK_DEPTH 32 + +#define MOD(sp) ((sp) % PAREN_STACK_DEPTH) +#define LIMIT_INC(sp) (((sp) < PAREN_STACK_DEPTH)? (sp) + 1 : PAREN_STACK_DEPTH) +#define INC(sp,count) (MOD((sp) + (count))) +#define INC1(sp) (INC(sp, 1)) +#define DEC(sp,count) (MOD((sp) + PAREN_STACK_DEPTH - (count))) +#define DEC1(sp) (DEC(sp, 1)) +#define STACK_IS_EMPTY(scriptRun) ((scriptRun)->pushCount <= 0) +#define STACK_IS_NOT_EMPTY(scriptRun) (! STACK_IS_EMPTY(scriptRun)) +#define TOP(scriptRun) ((scriptRun)->parenStack[(scriptRun)->parenSP]) +#define SYNC_FIXUP(scriptRun) ((scriptRun)->fixupCount = 0) + +struct ParenStackEntry +{ + int32_t pairIndex; + UScriptCode scriptCode; +}; + +struct UScriptRun +{ + int32_t textLength; + const UChar *textArray; + + int32_t scriptStart; + int32_t scriptLimit; + UScriptCode scriptCode; + + struct ParenStackEntry parenStack[PAREN_STACK_DEPTH]; + int32_t parenSP; + int32_t pushCount; + int32_t fixupCount; +}; + +static int8_t highBit(int32_t value); + +static const UChar32 pairedChars[] = { + 0x0028, 0x0029, /* ascii paired punctuation */ + 0x003c, 0x003e, + 0x005b, 0x005d, + 0x007b, 0x007d, + 0x00ab, 0x00bb, /* guillemets */ + 0x2018, 0x2019, /* general punctuation */ + 0x201c, 0x201d, + 0x2039, 0x203a, + 0x3008, 0x3009, /* chinese paired punctuation */ + 0x300a, 0x300b, + 0x300c, 0x300d, + 0x300e, 0x300f, + 0x3010, 0x3011, + 0x3014, 0x3015, + 0x3016, 0x3017, + 0x3018, 0x3019, + 0x301a, 0x301b +}; + +static void push(UScriptRun *scriptRun, int32_t pairIndex, UScriptCode scriptCode) +{ + scriptRun->pushCount = LIMIT_INC(scriptRun->pushCount); + scriptRun->fixupCount = LIMIT_INC(scriptRun->fixupCount); + + scriptRun->parenSP = INC1(scriptRun->parenSP); + scriptRun->parenStack[scriptRun->parenSP].pairIndex = pairIndex; + scriptRun->parenStack[scriptRun->parenSP].scriptCode = scriptCode; +} + +static void pop(UScriptRun *scriptRun) +{ + if (STACK_IS_EMPTY(scriptRun)) { + return; + } + + if (scriptRun->fixupCount > 0) { + scriptRun->fixupCount -= 1; + } + + scriptRun->pushCount -= 1; + scriptRun->parenSP = DEC1(scriptRun->parenSP); + + /* If the stack is now empty, reset the stack + pointers to their initial values. + */ + if (STACK_IS_EMPTY(scriptRun)) { + scriptRun->parenSP = -1; + } +} + +static void fixup(UScriptRun *scriptRun, UScriptCode scriptCode) +{ + int32_t fixupSP = DEC(scriptRun->parenSP, scriptRun->fixupCount); + + while (scriptRun->fixupCount-- > 0) { + fixupSP = INC1(fixupSP); + scriptRun->parenStack[fixupSP].scriptCode = scriptCode; + } +} + +static int8_t +highBit(int32_t value) +{ + int8_t bit = 0; + + if (value <= 0) { + return -32; + } + + if (value >= 1 << 16) { + value >>= 16; + bit += 16; + } + + if (value >= 1 << 8) { + value >>= 8; + bit += 8; + } + + if (value >= 1 << 4) { + value >>= 4; + bit += 4; + } + + if (value >= 1 << 2) { + value >>= 2; + bit += 2; + } + + if (value >= 1 << 1) { + value >>= 1; + bit += 1; + } + + return bit; +} + +static int32_t +getPairIndex(UChar32 ch) +{ + int32_t pairedCharCount = ARRAY_SIZE(pairedChars); + int32_t pairedCharPower = 1 << highBit(pairedCharCount); + int32_t pairedCharExtra = pairedCharCount - pairedCharPower; + + int32_t probe = pairedCharPower; + int32_t index = 0; + + if (ch >= pairedChars[pairedCharExtra]) { + index = pairedCharExtra; + } + + while (probe > (1 << 0)) { + probe >>= 1; + + if (ch >= pairedChars[index + probe]) { + index += probe; + } + } + + if (pairedChars[index] != ch) { + index = -1; + } + + return index; +} + +static UBool +sameScript(UScriptCode scriptOne, UScriptCode scriptTwo) +{ + return scriptOne <= USCRIPT_INHERITED || scriptTwo <= USCRIPT_INHERITED || scriptOne == scriptTwo; +} + +U_CAPI UScriptRun * U_EXPORT2 +uscript_openRun(const UChar *src, int32_t length, UErrorCode *pErrorCode) +{ + UScriptRun *result = NULL; + + if (pErrorCode == NULL || U_FAILURE(*pErrorCode)) { + return NULL; + } + + result = uprv_malloc(sizeof (UScriptRun)); + + if (result == NULL) { + *pErrorCode = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + + uscript_setRunText(result, src, length, pErrorCode); + + /* Release the UScriptRun if uscript_setRunText() returns an error */ + if (U_FAILURE(*pErrorCode)) { + uprv_free(result); + result = NULL; + } + + return result; +} + +U_CAPI void U_EXPORT2 +uscript_closeRun(UScriptRun *scriptRun) +{ + if (scriptRun != NULL) { + uprv_free(scriptRun); + } +} + +U_CAPI void U_EXPORT2 +uscript_resetRun(UScriptRun *scriptRun) +{ + if (scriptRun != NULL) { + scriptRun->scriptStart = 0; + scriptRun->scriptLimit = 0; + scriptRun->scriptCode = USCRIPT_INVALID_CODE; + scriptRun->parenSP = -1; + scriptRun->pushCount = 0; + scriptRun->fixupCount = 0; + } +} + +U_CAPI void U_EXPORT2 +uscript_setRunText(UScriptRun *scriptRun, const UChar *src, int32_t length, UErrorCode *pErrorCode) +{ + if (pErrorCode == NULL || U_FAILURE(*pErrorCode)) { + return; + } + + if (scriptRun == NULL || length < 0 || ((src == NULL) != (length == 0))) { + *pErrorCode = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + scriptRun->textArray = src; + scriptRun->textLength = length; + + uscript_resetRun(scriptRun); +} + +U_CAPI UBool U_EXPORT2 +uscript_nextRun(UScriptRun *scriptRun, int32_t *pRunStart, int32_t *pRunLimit, UScriptCode *pRunScript) +{ + UErrorCode error = U_ZERO_ERROR; + + /* if we've fallen off the end of the text, we're done */ + if (scriptRun == NULL || scriptRun->scriptLimit >= scriptRun->textLength) { + return FALSE; + } + + SYNC_FIXUP(scriptRun); + scriptRun->scriptCode = USCRIPT_COMMON; + + for (scriptRun->scriptStart = scriptRun->scriptLimit; scriptRun->scriptLimit < scriptRun->textLength; scriptRun->scriptLimit += 1) { + UChar high = scriptRun->textArray[scriptRun->scriptLimit]; + UChar32 ch = high; + UScriptCode sc; + int32_t pairIndex; + + /* + * if the character is a high surrogate and it's not the last one + * in the text, see if it's followed by a low surrogate + */ + if (high >= 0xD800 && high <= 0xDBFF && scriptRun->scriptLimit < scriptRun->textLength - 1) { + UChar low = scriptRun->textArray[scriptRun->scriptLimit + 1]; + + /* + * if it is followed by a low surrogate, + * consume it and form the full character + */ + if (low >= 0xDC00 && low <= 0xDFFF) { + ch = (high - 0xD800) * 0x0400 + low - 0xDC00 + 0x10000; + scriptRun->scriptLimit += 1; + } + } + + sc = uscript_getScript(ch, &error); + pairIndex = getPairIndex(ch); + + /* + * Paired character handling: + * + * if it's an open character, push it onto the stack. + * if it's a close character, find the matching open on the + * stack, and use that script code. Any non-matching open + * characters above it on the stack will be poped. + */ + if (pairIndex >= 0) { + if ((pairIndex & 1) == 0) { + push(scriptRun, pairIndex, scriptRun->scriptCode); + } else { + int32_t pi = pairIndex & ~1; + + while (STACK_IS_NOT_EMPTY(scriptRun) && TOP(scriptRun).pairIndex != pi) { + pop(scriptRun); + } + + if (STACK_IS_NOT_EMPTY(scriptRun)) { + sc = TOP(scriptRun).scriptCode; + } + } + } + + if (sameScript(scriptRun->scriptCode, sc)) { + if (scriptRun->scriptCode <= USCRIPT_INHERITED && sc > USCRIPT_INHERITED) { + scriptRun->scriptCode = sc; + + fixup(scriptRun, scriptRun->scriptCode); + } + + /* + * if this character is a close paired character, + * pop the matching open character from the stack + */ + if (pairIndex >= 0 && (pairIndex & 1) != 0) { + pop(scriptRun); + } + } else { + /* + * if the run broke on a surrogate pair, + * end it before the high surrogate + */ + if (ch >= 0x10000) { + scriptRun->scriptLimit -= 1; + } + + break; + } + } + + + if (pRunStart != NULL) { + *pRunStart = scriptRun->scriptStart; + } + + if (pRunLimit != NULL) { + *pRunLimit = scriptRun->scriptLimit; + } + + if (pRunScript != NULL) { + *pRunScript = scriptRun->scriptCode; + } + + return TRUE; +} diff --git a/Build/source/libs/icu/icu-xetex/common/usc_impl.h b/Build/source/libs/icu/icu-xetex/common/usc_impl.h new file mode 100644 index 00000000000..790a709145b --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/usc_impl.h @@ -0,0 +1,149 @@ +/* +********************************************************************** +* Copyright (C) 1999-2002, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* +* File USC_IMPL.H +* +* Modification History: +* +* Date Name Description +* 07/08/2002 Eric Mader Creation. +****************************************************************************** +*/ + +#ifndef USC_IMPL_H +#define USC_IMPL_H +#include "unicode/utypes.h" +#include "unicode/uscript.h" + +/** + * UScriptRun is used to find runs of characters in + * the same script. It implements a simple iterator over an array + * of characters. The iterator will resolve script-neutral characters + * like punctuation into the script of the surrounding characters. + * + * The iterator will try to match paired punctuation. If it sees an + * opening punctuation character, it will remember the script that + * was assigned to that character, and assign the same script to the + * matching closing punctuation. + * + * Scripts are chosen based on the UScriptCode enumeration. + * No attempt is made to combine related scripts into a single run. In + * particular, Hiragana, Katakana, and Han characters will appear in seperate + * runs. + + * Here is an example of how to iterate over script runs: + *
+ * \code
+ * void printScriptRuns(const UChar *text, int32_t length)
+ * {
+ *     UErrorCode error = U_ZERO_ERROR;
+ *     UScriptRun *scriptRun = uscript_openRun(text, testLength, &error);
+ *     int32_t start = 0, limit = 0;
+ *     UScriptCode code = USCRIPT_INVALID_CODE;
+ *
+ *     while (uscript_nextRun(&start, &limit, &code)) {
+ *         printf("Script '%s' from %d to %d.\n", uscript_getName(code), start, limit);
+ *     }
+ *
+ *     uscript_closeRun(scriptRun);
+ *  }
+ * 
+ * + * @draft ICU 2.2 + */ +struct UScriptRun; + +typedef struct UScriptRun UScriptRun; + +/** + * Create a UScriptRun object for iterating over the given text. This object must + * be freed using uscript_closeRun(). Note that this object does not copy the source text, + * only the pointer to it. You must make sure that the pointer remains valid until you call + * uscript_closeRun() or uscript_setRunText(). + * + * @param src is the address of the array of characters over which to iterate. + * if src == NULL and length == 0, + * an empty UScriptRun object will be returned. + * + * @param length is the number of characters over which to iterate. + * + * @param pErrorCode is a pointer to a valid UErrorCode value. If this value + * indicates a failure on entry, the function will immediately return. + * On exit the value will indicate the success of the operation. + * + * @return the address of UScriptRun object which will iterate over the text, + * or NULL if the operation failed. + * + * @draft ICU 2.2 + */ +U_CAPI UScriptRun * U_EXPORT2 +uscript_openRun(const UChar *src, int32_t length, UErrorCode *pErrorCode); + +/** + * Frees the given UScriptRun object and any storage associated with it. + * On return, scriptRun no longer points to a valid UScriptRun object. + * + * @param scriptRun is the UScriptRun object which will be freed. + * + * @draft ICU 2.2 + */ +U_CAPI void U_EXPORT2 +uscript_closeRun(UScriptRun *scriptRun); + +/** + * Reset the UScriptRun object so that it will start iterating from + * the beginning. + * + * @param scriptRun is the address of the UScriptRun object to be reset. + * + * @draft ICU 2.2 + */ +U_CAPI void U_EXPORT2 +uscript_resetRun(UScriptRun *scriptRun); + +/** + * Change the text over which the given UScriptRun object iterates. + * + * @param scriptRun is the UScriptRun object which will be changed. + * + * @param src is the address of the new array of characters over which to iterate. + * If src == NULL and length == 0, + * the UScriptRun object will become empty. + * + * @param length is the new number of characters over which to iterate + * + * @param pErrorCode is a pointer to a valid UErrorCode value. If this value + * indicates a failure on entry, the function will immediately return. + * On exit the value will indicate the success of the operation. + * + * @draft ICU 2.2 + */ +U_CAPI void U_EXPORT2 +uscript_setRunText(UScriptRun *scriptRun, const UChar *src, int32_t length, UErrorCode *pErrorCode); + +/** + * Advance the UScriptRun object to the next script run, return the start and limit + * offsets, and the script of the run. + * + * @param scriptRun is the address of the UScriptRun object. + * + * @param pRunStart is a pointer to the variable to receive the starting offset of the next run. + * This pointer can be NULL if the value is not needed. + * + * @param pRunLimit is a pointer to the variable to receive the limit offset of the next run. + * This pointer can be NULL if the value is not needed. + * + * @param pRunScript is a pointer to the variable to receive the UScriptCode for the + * script of the current run. This pointer can be NULL if the value is not needed. + * + * @return true if there was another script run. + * + * @draft ICU 2.2 + */ +U_CAPI UBool U_EXPORT2 +uscript_nextRun(UScriptRun *scriptRun, int32_t *pRunStart, int32_t *pRunLimit, UScriptCode *pRunScript); + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/uscript.c b/Build/source/libs/icu/icu-xetex/common/uscript.c new file mode 100644 index 00000000000..47ae56daf00 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uscript.c @@ -0,0 +1,112 @@ +/* +********************************************************************** +* Copyright (C) 1997-2006, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* +* File USCRIPT.C +* +* Modification History: +* +* Date Name Description +* 07/06/2001 Ram Creation. +****************************************************************************** +*/ + +#include "unicode/uscript.h" +#include "unicode/ures.h" +#include "unicode/uchar.h" +#include "unicode/putil.h" +#include "uprops.h" +#include "cmemory.h" +#include "cstring.h" + +static const char kLocaleScript[] = "LocaleScript"; +static const char kHyphen = '-'; +static const char kUnderscore = '_'; + +/* TODO: this is a bad API should be deprecated */ +U_CAPI int32_t U_EXPORT2 +uscript_getCode(const char* nameOrAbbrOrLocale, + UScriptCode* fillIn, + int32_t capacity, + UErrorCode* err){ + + UScriptCode code = USCRIPT_INVALID_CODE; + int32_t numFilled=0; + int32_t len=0; + /* check arguments */ + if(err==NULL ||U_FAILURE(*err)){ + return numFilled; + } + if(nameOrAbbrOrLocale==NULL || fillIn == NULL || capacity<0){ + *err = U_ILLEGAL_ARGUMENT_ERROR; + return numFilled; + } + + if(uprv_strchr(nameOrAbbrOrLocale, kHyphen)==NULL && uprv_strchr(nameOrAbbrOrLocale, kUnderscore)==NULL ){ + /* try long and abbreviated script names first */ + code = (UScriptCode) u_getPropertyValueEnum(UCHAR_SCRIPT, nameOrAbbrOrLocale); + + } + if(code==(UScriptCode)UCHAR_INVALID_CODE){ + /* Do not propagate error codes from just not finding a locale bundle. */ + UErrorCode localErrorCode = U_ZERO_ERROR; + UResourceBundle* resB = ures_open(NULL,nameOrAbbrOrLocale,&localErrorCode); + if(U_SUCCESS(localErrorCode)&& localErrorCode != U_USING_DEFAULT_WARNING){ + UResourceBundle* resD = ures_getByKey(resB,kLocaleScript,NULL,&localErrorCode); + if(U_SUCCESS(localErrorCode) ){ + len =0; + while(ures_hasNext(resD)){ + const UChar* name = ures_getNextString(resD,&len,NULL,&localErrorCode); + if(U_SUCCESS(localErrorCode)){ + char cName[50] = {'\0'}; + u_UCharsToChars(name,cName,len); + code = (UScriptCode) u_getPropertyValueEnum(UCHAR_SCRIPT, cName); + /* got the script code now fill in the buffer */ + if(numFilled<=capacity){ + *(fillIn)++=code; + numFilled++; + }else{ + ures_close(resD); + ures_close(resB); + *err=U_BUFFER_OVERFLOW_ERROR; + return len; + } + } + } + } + ures_close(resD); + } + ures_close(resB); + code = USCRIPT_INVALID_CODE; + } + if(code==(UScriptCode)UCHAR_INVALID_CODE){ + /* still not found .. try long and abbreviated script names again */ + code = (UScriptCode) u_getPropertyValueEnum(UCHAR_SCRIPT, nameOrAbbrOrLocale); + } + if(code!=(UScriptCode)UCHAR_INVALID_CODE){ + /* we found it */ + if(numFilled<=capacity){ + *(fillIn)++=code; + numFilled++; + }else{ + *err=U_BUFFER_OVERFLOW_ERROR; + return len; + } + } + return numFilled; +} + +U_CAPI const char* U_EXPORT2 +uscript_getName(UScriptCode scriptCode){ + return u_getPropertyValueName(UCHAR_SCRIPT, scriptCode, + U_LONG_PROPERTY_NAME); +} + +U_CAPI const char* U_EXPORT2 +uscript_getShortName(UScriptCode scriptCode){ + return u_getPropertyValueName(UCHAR_SCRIPT, scriptCode, + U_SHORT_PROPERTY_NAME); +} + diff --git a/Build/source/libs/icu/icu-xetex/common/uset.cpp b/Build/source/libs/icu/icu-xetex/common/uset.cpp new file mode 100644 index 00000000000..add0ee0f6a8 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uset.cpp @@ -0,0 +1,634 @@ +/* +******************************************************************************* +* +* Copyright (C) 2002-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: uset.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2002mar07 +* created by: Markus W. Scherer +* +* The serialized structure, the array of range limits, is +* the same as in UnicodeSet, except that the HIGH value is not stored. +* +* There are functions to efficiently serialize a USet into an array of uint16_t +* and functions to use such a serialized form efficiently without +* instantiating a new USet. +*/ + +#include "unicode/utypes.h" +#include "unicode/uobject.h" +#include "unicode/uset.h" +#include "unicode/uniset.h" +#include "cmemory.h" +#include "unicode/ustring.h" +#include "unicode/parsepos.h" + +U_NAMESPACE_USE + +U_CAPI USet* U_EXPORT2 +uset_open(UChar32 start, UChar32 end) { + return (USet*) new UnicodeSet(start, end); +} + +U_CAPI void U_EXPORT2 +uset_close(USet* set) { + delete (UnicodeSet*) set; +} + +U_CAPI USet * U_EXPORT2 +uset_clone(const USet *set) { + return (USet*) (((UnicodeSet*) set)->UnicodeSet::clone()); +} + +U_CAPI UBool U_EXPORT2 +uset_isFrozen(const USet *set) { + return ((UnicodeSet*) set)->UnicodeSet::isFrozen(); +} + +U_CAPI void U_EXPORT2 +uset_freeze(USet *set) { + ((UnicodeSet*) set)->UnicodeSet::freeze(); +} + +U_CAPI USet * U_EXPORT2 +uset_cloneAsThawed(const USet *set) { + return (USet*) (((UnicodeSet*) set)->UnicodeSet::cloneAsThawed()); +} + +U_CAPI void U_EXPORT2 +uset_set(USet* set, + UChar32 start, UChar32 end) { + ((UnicodeSet*) set)->UnicodeSet::set(start, end); +} + +U_CAPI void U_EXPORT2 +uset_addAll(USet* set, const USet *additionalSet) { + ((UnicodeSet*) set)->UnicodeSet::addAll(*((const UnicodeSet*)additionalSet)); +} + +U_CAPI void U_EXPORT2 +uset_add(USet* set, UChar32 c) { + ((UnicodeSet*) set)->UnicodeSet::add(c); +} + +U_CAPI void U_EXPORT2 +uset_addRange(USet* set, UChar32 start, UChar32 end) { + ((UnicodeSet*) set)->UnicodeSet::add(start, end); +} + +U_CAPI void U_EXPORT2 +uset_addString(USet* set, const UChar* str, int32_t strLen) { + // UnicodeString handles -1 for strLen + UnicodeString s(strLen<0, str, strLen); + ((UnicodeSet*) set)->UnicodeSet::add(s); +} + +U_CAPI void U_EXPORT2 +uset_addAllCodePoints(USet* set, const UChar *str, int32_t strLen) { + // UnicodeString handles -1 for strLen + UnicodeString s(str, strLen); + ((UnicodeSet*) set)->UnicodeSet::addAll(s); +} + +U_CAPI void U_EXPORT2 +uset_remove(USet* set, UChar32 c) { + ((UnicodeSet*) set)->UnicodeSet::remove(c); +} + +U_CAPI void U_EXPORT2 +uset_removeRange(USet* set, UChar32 start, UChar32 end) { + ((UnicodeSet*) set)->UnicodeSet::remove(start, end); +} + +U_CAPI void U_EXPORT2 +uset_removeString(USet* set, const UChar* str, int32_t strLen) { + UnicodeString s(strLen==-1, str, strLen); + ((UnicodeSet*) set)->UnicodeSet::remove(s); +} + +U_CAPI void U_EXPORT2 +uset_removeAll(USet* set, const USet* remove) { + ((UnicodeSet*) set)->UnicodeSet::removeAll(*(const UnicodeSet*)remove); +} + +U_CAPI void U_EXPORT2 +uset_retain(USet* set, UChar32 start, UChar32 end) { + ((UnicodeSet*) set)->UnicodeSet::retain(start, end); +} + +U_CAPI void U_EXPORT2 +uset_retainAll(USet* set, const USet* retain) { + ((UnicodeSet*) set)->UnicodeSet::retainAll(*(const UnicodeSet*)retain); +} + +U_CAPI void U_EXPORT2 +uset_compact(USet* set) { + ((UnicodeSet*) set)->UnicodeSet::compact(); +} + +U_CAPI void U_EXPORT2 +uset_complement(USet* set) { + ((UnicodeSet*) set)->UnicodeSet::complement(); +} + +U_CAPI void U_EXPORT2 +uset_complementAll(USet* set, const USet* complement) { + ((UnicodeSet*) set)->UnicodeSet::complementAll(*(const UnicodeSet*)complement); +} + +U_CAPI void U_EXPORT2 +uset_clear(USet* set) { + ((UnicodeSet*) set)->UnicodeSet::clear(); +} + +U_CAPI UBool U_EXPORT2 +uset_isEmpty(const USet* set) { + return ((const UnicodeSet*) set)->UnicodeSet::isEmpty(); +} + +U_CAPI UBool U_EXPORT2 +uset_contains(const USet* set, UChar32 c) { + return ((const UnicodeSet*) set)->UnicodeSet::contains(c); +} + +U_CAPI UBool U_EXPORT2 +uset_containsRange(const USet* set, UChar32 start, UChar32 end) { + return ((const UnicodeSet*) set)->UnicodeSet::contains(start, end); +} + +U_CAPI UBool U_EXPORT2 +uset_containsString(const USet* set, const UChar* str, int32_t strLen) { + UnicodeString s(strLen==-1, str, strLen); + return ((const UnicodeSet*) set)->UnicodeSet::contains(s); +} + +U_CAPI UBool U_EXPORT2 +uset_containsAll(const USet* set1, const USet* set2) { + return ((const UnicodeSet*) set1)->UnicodeSet::containsAll(* (const UnicodeSet*) set2); +} + +U_CAPI UBool U_EXPORT2 +uset_containsAllCodePoints(const USet* set, const UChar *str, int32_t strLen) { + // Create a string alias, since nothing is being added to the set. + UnicodeString s(strLen==-1, str, strLen); + return ((const UnicodeSet*) set)->UnicodeSet::containsAll(s); +} + +U_CAPI UBool U_EXPORT2 +uset_containsNone(const USet* set1, const USet* set2) { + return ((const UnicodeSet*) set1)->UnicodeSet::containsNone(* (const UnicodeSet*) set2); +} + +U_CAPI UBool U_EXPORT2 +uset_containsSome(const USet* set1, const USet* set2) { + return ((const UnicodeSet*) set1)->UnicodeSet::containsSome(* (const UnicodeSet*) set2); +} + +U_CAPI int32_t U_EXPORT2 +uset_span(const USet *set, const UChar *s, int32_t length, USetSpanCondition spanCondition) { + return ((UnicodeSet*) set)->UnicodeSet::span(s, length, spanCondition); +} + +U_CAPI int32_t U_EXPORT2 +uset_spanBack(const USet *set, const UChar *s, int32_t length, USetSpanCondition spanCondition) { + return ((UnicodeSet*) set)->UnicodeSet::spanBack(s, length, spanCondition); +} + +U_CAPI int32_t U_EXPORT2 +uset_spanUTF8(const USet *set, const char *s, int32_t length, USetSpanCondition spanCondition) { + return ((UnicodeSet*) set)->UnicodeSet::spanUTF8(s, length, spanCondition); +} + +U_CAPI int32_t U_EXPORT2 +uset_spanBackUTF8(const USet *set, const char *s, int32_t length, USetSpanCondition spanCondition) { + return ((UnicodeSet*) set)->UnicodeSet::spanBackUTF8(s, length, spanCondition); +} + +U_CAPI UBool U_EXPORT2 +uset_equals(const USet* set1, const USet* set2) { + return *(const UnicodeSet*)set1 == *(const UnicodeSet*)set2; +} + +U_CAPI int32_t U_EXPORT2 +uset_indexOf(const USet* set, UChar32 c) { + return ((UnicodeSet*) set)->UnicodeSet::indexOf(c); +} + +U_CAPI UChar32 U_EXPORT2 +uset_charAt(const USet* set, int32_t index) { + return ((UnicodeSet*) set)->UnicodeSet::charAt(index); +} + +U_CAPI int32_t U_EXPORT2 +uset_size(const USet* set) { + return ((const UnicodeSet*) set)->UnicodeSet::size(); +} + +U_NAMESPACE_BEGIN +/** + * This class only exists to provide access to the UnicodeSet private + * USet support API. Declaring a class a friend is more portable than + * trying to declare extern "C" functions as friends. + */ +class USetAccess /* not : public UObject because all methods are static */ { +public: + /* Try to have the compiler inline these*/ + inline static int32_t getStringCount(const UnicodeSet& set) { + return set.getStringCount(); + } + inline static const UnicodeString* getString(const UnicodeSet& set, + int32_t i) { + return set.getString(i); + } +private: + /* do not instantiate*/ + USetAccess(); +}; +U_NAMESPACE_END + +U_CAPI int32_t U_EXPORT2 +uset_getItemCount(const USet* uset) { + const UnicodeSet& set = *(const UnicodeSet*)uset; + return set.getRangeCount() + USetAccess::getStringCount(set); +} + +U_CAPI int32_t U_EXPORT2 +uset_getItem(const USet* uset, int32_t itemIndex, + UChar32* start, UChar32* end, + UChar* str, int32_t strCapacity, + UErrorCode* ec) { + if (U_FAILURE(*ec)) return 0; + const UnicodeSet& set = *(const UnicodeSet*)uset; + int32_t rangeCount; + + if (itemIndex < 0) { + *ec = U_ILLEGAL_ARGUMENT_ERROR; + return -1; + } else if (itemIndex < (rangeCount = set.getRangeCount())) { + *start = set.getRangeStart(itemIndex); + *end = set.getRangeEnd(itemIndex); + return 0; + } else { + itemIndex -= rangeCount; + if (itemIndex < USetAccess::getStringCount(set)) { + const UnicodeString* s = USetAccess::getString(set, itemIndex); + return s->extract(str, strCapacity, *ec); + } else { + *ec = U_INDEX_OUTOFBOUNDS_ERROR; + return -1; + } + } +} + +//U_CAPI int32_t U_EXPORT2 +//uset_getRangeCount(const USet* set) { +// return ((const UnicodeSet*) set)->getRangeCount(); +//} +// +//U_CAPI UBool U_EXPORT2 +//uset_getRange(const USet* set, int32_t rangeIndex, +// UChar32* pStart, UChar32* pEnd) { +// if ((uint32_t) rangeIndex >= (uint32_t) uset_getRangeCount(set)) { +// return FALSE; +// } +// const UnicodeSet* us = (const UnicodeSet*) set; +// *pStart = us->getRangeStart(rangeIndex); +// *pEnd = us->getRangeEnd(rangeIndex); +// return TRUE; +//} + +/* + * Serialize a USet into 16-bit units. + * Store BMP code points as themselves with one 16-bit unit each. + * + * Important: the code points in the array are in ascending order, + * therefore all BMP code points precede all supplementary code points. + * + * Store each supplementary code point in 2 16-bit units, + * simply with higher-then-lower 16-bit halfs. + * + * Precede the entire list with the length. + * If there are supplementary code points, then set bit 15 in the length + * and add the bmpLength between it and the array. + * + * In other words: + * - all BMP: (length=bmpLength) BMP, .., BMP + * - some supplementary: (length|0x8000) (bmpLengthUnicodeSet::serialize(dest, destCapacity,* ec); +} + +U_CAPI UBool U_EXPORT2 +uset_getSerializedSet(USerializedSet* fillSet, const uint16_t* src, int32_t srcLength) { + int32_t length; + + if(fillSet==NULL) { + return FALSE; + } + if(src==NULL || srcLength<=0) { + fillSet->length=fillSet->bmpLength=0; + return FALSE; + } + + length=*src++; + if(length&0x8000) { + /* there are supplementary values */ + length&=0x7fff; + if(srcLength<(2+length)) { + fillSet->length=fillSet->bmpLength=0; + return FALSE; + } + fillSet->bmpLength=*src++; + } else { + /* only BMP values */ + if(srcLength<(1+length)) { + fillSet->length=fillSet->bmpLength=0; + return FALSE; + } + fillSet->bmpLength=length; + } + fillSet->array=src; + fillSet->length=length; + return TRUE; +} + +U_CAPI void U_EXPORT2 +uset_setSerializedToOne(USerializedSet* fillSet, UChar32 c) { + if(fillSet==NULL || (uint32_t)c>0x10ffff) { + return; + } + + fillSet->array=fillSet->staticArray; + if(c<0xffff) { + fillSet->bmpLength=fillSet->length=2; + fillSet->staticArray[0]=(uint16_t)c; + fillSet->staticArray[1]=(uint16_t)c+1; + } else if(c==0xffff) { + fillSet->bmpLength=1; + fillSet->length=3; + fillSet->staticArray[0]=0xffff; + fillSet->staticArray[1]=1; + fillSet->staticArray[2]=0; + } else if(c<0x10ffff) { + fillSet->bmpLength=0; + fillSet->length=4; + fillSet->staticArray[0]=(uint16_t)(c>>16); + fillSet->staticArray[1]=(uint16_t)c; + ++c; + fillSet->staticArray[2]=(uint16_t)(c>>16); + fillSet->staticArray[3]=(uint16_t)c; + } else /* c==0x10ffff */ { + fillSet->bmpLength=0; + fillSet->length=2; + fillSet->staticArray[0]=0x10; + fillSet->staticArray[1]=0xffff; + } +} + +U_CAPI UBool U_EXPORT2 +uset_serializedContains(const USerializedSet* set, UChar32 c) { + const uint16_t* array; + + if(set==NULL || (uint32_t)c>0x10ffff) { + return FALSE; + } + + array=set->array; + if(c<=0xffff) { + /* find c in the BMP part */ + int32_t lo = 0; + int32_t hi = set->bmpLength-1; + if (c < array[0]) { + hi = 0; + } else if (c < array[hi]) { + for(;;) { + int32_t i = (lo + hi) >> 1; + if (i == lo) { + break; // Done! + } else if (c < array[i]) { + hi = i; + } else { + lo = i; + } + } + } else { + hi += 1; + } + return (UBool)(hi&1); + } else { + /* find c in the supplementary part */ + uint16_t high=(uint16_t)(c>>16), low=(uint16_t)c; + int32_t base = set->bmpLength; + int32_t lo = 0; + int32_t hi = set->length - 2 - base; + if (high < array[base] || (high==array[base] && low> 1) & ~1; // Guarantee even result + int32_t iabs = i + base; + if (i == lo) { + break; // Done! + } else if (high < array[iabs] || (high==array[iabs] && lowbmpLength+(set->length-set->bmpLength)/2+1)/2; +} + +U_CAPI UBool U_EXPORT2 +uset_getSerializedRange(const USerializedSet* set, int32_t rangeIndex, + UChar32* pStart, UChar32* pEnd) { + const uint16_t* array; + int32_t bmpLength, length; + + if(set==NULL || rangeIndex<0 || pStart==NULL || pEnd==NULL) { + return FALSE; + } + + array=set->array; + length=set->length; + bmpLength=set->bmpLength; + + rangeIndex*=2; /* address start/limit pairs */ + if(rangeIndex0) { +// if(c>=array[length-1]) { +// return length; +// } +// +// /* do not check the last range limit again in the loop below */ +// --length; +// } +// +// for(i=0; i=array[i]; ++i) {} +// return i; +// } +// +// static UBool +// addRemove(USet* set, UChar32 c, int32_t doRemove) { +// int32_t i, length, more; +// +// if(set==NULL || (uint32_t)c>0x10ffff) { +// return FALSE; +// } +// +// length=set->length; +// i=findChar(set->array, length, c); +// if((i&1)^doRemove) { +// /* c is already in the set */ +// return TRUE; +// } +// +// /* how many more array items do we need? */ +// if(iarray[i]) { +// /* c is just before the following range, extend that in-place by one */ +// set->array[i]=c; +// if(i>0) { +// --i; +// if(c==set->array[i]) { +// /* the previous range collapsed, remove it */ +// set->length=length-=2; +// if(iarray+i, set->array+i+2, (length-i)*4); +// } +// } +// } +// return TRUE; +// } else if(i>0 && c==set->array[i-1]) { +// /* c is just after the previous range, extend that in-place by one */ +// if(++c<=0x10ffff) { +// set->array[i-1]=c; +// if(iarray[i]) { +// /* the following range collapsed, remove it */ +// --i; +// set->length=length-=2; +// if(iarray+i, set->array+i+2, (length-i)*4); +// } +// } +// } else { +// /* extend the previous range (had limit 0x10ffff) to the end of Unicode */ +// set->length=i-1; +// } +// return TRUE; +// } else if(i==length && c==0x10ffff) { +// /* insert one range limit c */ +// more=1; +// } else { +// /* insert two range limits c, c+1 */ +// more=2; +// } +// +// /* insert range limits */ +// if(length+more>set->capacity) { +// /* reallocate */ +// int32_t newCapacity=set->capacity+set->capacity/2+USET_GROW_DELTA; +// UChar32* newArray=(UChar32* )uprv_malloc(newCapacity*4); +// if(newArray==NULL) { +// return FALSE; +// } +// set->capacity=newCapacity; +// uprv_memcpy(newArray, set->array, length*4); +// +// if(set->array!=set->staticBuffer) { +// uprv_free(set->array); +// } +// set->array=newArray; +// } +// +// if(iarray+i+more, set->array+i, (length-i)*4); +// } +// set->array[i]=c; +// if(more==2) { +// set->array[i+1]=c+1; +// } +// set->length+=more; +// +// return TRUE; +// } +// +// U_CAPI UBool U_EXPORT2 +// uset_add(USet* set, UChar32 c) { +// return addRemove(set, c, 0); +// } +// +// U_CAPI void U_EXPORT2 +// uset_remove(USet* set, UChar32 c) { +// addRemove(set, c, 1); +// } diff --git a/Build/source/libs/icu/icu-xetex/common/uset_imp.h b/Build/source/libs/icu/icu-xetex/common/uset_imp.h new file mode 100644 index 00000000000..d9916f40351 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uset_imp.h @@ -0,0 +1,56 @@ +/* +******************************************************************************* +* +* Copyright (C) 2004-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: uset_imp.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2004sep07 +* created by: Markus W. Scherer +* +* Internal USet definitions. +*/ + +#ifndef __USET_IMP_H__ +#define __USET_IMP_H__ + +#include "unicode/utypes.h" +#include "unicode/uset.h" + +U_CDECL_BEGIN + +typedef void U_CALLCONV +USetAdd(USet *set, UChar32 c); + +typedef void U_CALLCONV +USetAddRange(USet *set, UChar32 start, UChar32 end); + +typedef void U_CALLCONV +USetAddString(USet *set, const UChar *str, int32_t length); + +typedef void U_CALLCONV +USetRemove(USet *set, UChar32 c); + +/** + * Interface for adding items to a USet, to keep low-level code from + * statically depending on the USet implementation. + * Calls will look like sa->add(sa->set, c); + */ +struct USetAdder { + USet *set; + USetAdd *add; + USetAddRange *addRange; + USetAddString *addString; + USetRemove *remove; +}; +typedef struct USetAdder USetAdder; + +U_CDECL_END + +#endif + diff --git a/Build/source/libs/icu/icu-xetex/common/uset_props.cpp b/Build/source/libs/icu/icu-xetex/common/uset_props.cpp new file mode 100644 index 00000000000..fb607390a2d --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uset_props.cpp @@ -0,0 +1,136 @@ +/* +******************************************************************************* +* +* Copyright (C) 2002-2006, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: uset_props.cpp +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2004aug30 +* created by: Markus W. Scherer +* +* C wrappers around UnicodeSet functions that are implemented in +* uniset_props.cpp, split off for modularization. +*/ + +#include "unicode/utypes.h" +#include "unicode/uobject.h" +#include "unicode/uset.h" +#include "unicode/uniset.h" +#include "cmemory.h" +#include "unicode/ustring.h" +#include "unicode/parsepos.h" + +U_NAMESPACE_USE + +U_CAPI USet* U_EXPORT2 +uset_openPattern(const UChar* pattern, int32_t patternLength, + UErrorCode* ec) +{ + UnicodeString pat(patternLength==-1, pattern, patternLength); + UnicodeSet* set = new UnicodeSet(pat, *ec); + /* test for NULL */ + if(set == 0) { + *ec = U_MEMORY_ALLOCATION_ERROR; + return 0; + } + + if (U_FAILURE(*ec)) { + delete set; + set = NULL; + } + return (USet*) set; +} + +U_CAPI USet* U_EXPORT2 +uset_openPatternOptions(const UChar* pattern, int32_t patternLength, + uint32_t options, + UErrorCode* ec) +{ + UnicodeString pat(patternLength==-1, pattern, patternLength); + UnicodeSet* set = new UnicodeSet(pat, options, NULL, *ec); + /* test for NULL */ + if(set == 0) { + *ec = U_MEMORY_ALLOCATION_ERROR; + return 0; + } + + if (U_FAILURE(*ec)) { + delete set; + set = NULL; + } + return (USet*) set; +} + + +U_CAPI int32_t U_EXPORT2 +uset_applyPattern(USet *set, + const UChar *pattern, int32_t patternLength, + uint32_t options, + UErrorCode *status){ + + // status code needs to be checked since we + // dereference it + if(status == NULL || U_FAILURE(*status)){ + return 0; + } + + // check only the set paramenter + // if pattern is NULL or null terminate + // UnicodeString constructor takes care of it + if(set == NULL){ + *status = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + UnicodeString pat(pattern, patternLength); + + ParsePosition pos; + + ((UnicodeSet*) set)->applyPattern(pat, pos, options, NULL, *status); + + return pos.getIndex(); +} + +U_CAPI void U_EXPORT2 +uset_applyIntPropertyValue(USet* set, + UProperty prop, int32_t value, UErrorCode* ec) { + ((UnicodeSet*) set)->applyIntPropertyValue(prop, value, *ec); +} + +U_CAPI void U_EXPORT2 +uset_applyPropertyAlias(USet* set, + const UChar *prop, int32_t propLength, + const UChar *value, int32_t valueLength, + UErrorCode* ec) { + + UnicodeString p(prop, propLength); + UnicodeString v(value, valueLength); + + ((UnicodeSet*) set)->applyPropertyAlias(p, v, *ec); +} + +U_CAPI UBool U_EXPORT2 +uset_resemblesPattern(const UChar *pattern, int32_t patternLength, + int32_t pos) { + + UnicodeString pat(pattern, patternLength); + + return ((pos+1) < pat.length() && + pat.charAt(pos) == (UChar)91/*[*/) || + UnicodeSet::resemblesPattern(pat, pos); +} + +U_CAPI int32_t U_EXPORT2 +uset_toPattern(const USet* set, + UChar* result, int32_t resultCapacity, + UBool escapeUnprintable, + UErrorCode* ec) { + UnicodeString pat; + ((const UnicodeSet*) set)->toPattern(pat, escapeUnprintable); + return pat.extract(result, resultCapacity, *ec); +} diff --git a/Build/source/libs/icu/icu-xetex/common/usetiter.cpp b/Build/source/libs/icu/icu-xetex/common/usetiter.cpp new file mode 100644 index 00000000000..44858ed46bd --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/usetiter.cpp @@ -0,0 +1,150 @@ +/* +********************************************************************** +* Copyright (c) 2002-2006, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +*/ +#include "unicode/usetiter.h" +#include "unicode/uniset.h" +#include "unicode/unistr.h" +#include "uvector.h" + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(UnicodeSetIterator) + +/** + * Create an iterator + * @param set set to iterate over + */ +UnicodeSetIterator::UnicodeSetIterator(const UnicodeSet& uSet) { + cpString = NULL; + reset(uSet); +} + +/** + * Create an iterator. Convenience for when the contents are to be set later. + */ +UnicodeSetIterator::UnicodeSetIterator() { + this->set = NULL; + cpString = NULL; + reset(); +} + +UnicodeSetIterator::~UnicodeSetIterator() { + delete cpString; +} + +/** + * Returns the next element in the set. + * @return true if there was another element in the set. + * if so, if codepoint == IS_STRING, the value is a string in the string field + * else the value is a single code point in the codepoint field. + *
You are guaranteed that the codepoints are in sorted order, and the strings are in sorted order, + * and that all code points are returned before any strings are returned. + *
Note also that the codepointEnd is undefined after calling this method. + */ +UBool UnicodeSetIterator::next() { + if (nextElement <= endElement) { + codepoint = codepointEnd = nextElement++; + string = NULL; + return TRUE; + } + if (range < endRange) { + loadRange(++range); + codepoint = codepointEnd = nextElement++; + string = NULL; + return TRUE; + } + + if (nextString >= stringCount) return FALSE; + codepoint = (UChar32)IS_STRING; // signal that value is actually a string + string = (const UnicodeString*) set->strings->elementAt(nextString++); + return TRUE; +} + +/** + * @return true if there was another element in the set. + * if so, if codepoint == IS_STRING, the value is a string in the string field + * else the value is a range of codepoints in the fields. + *
Note that the codepoints are in sorted order, and the strings are in sorted order, + * and that all code points are returned before any strings are returned. + *
You are guaranteed that the ranges are in sorted order, and the strings are in sorted order, + * and that all ranges are returned before any strings are returned. + *
You are also guaranteed that ranges are disjoint and non-contiguous. + *
Note also that the codepointEnd is undefined after calling this method. + */ +UBool UnicodeSetIterator::nextRange() { + string = NULL; + if (nextElement <= endElement) { + codepointEnd = endElement; + codepoint = nextElement; + nextElement = endElement+1; + return TRUE; + } + if (range < endRange) { + loadRange(++range); + codepointEnd = endElement; + codepoint = nextElement; + nextElement = endElement+1; + return TRUE; + } + + if (nextString >= stringCount) return FALSE; + codepoint = (UChar32)IS_STRING; // signal that value is actually a string + string = (const UnicodeString*) set->strings->elementAt(nextString++); + return TRUE; +} + +/** + *@param set the set to iterate over. This allows reuse of the iterator. + */ +void UnicodeSetIterator::reset(const UnicodeSet& uSet) { + this->set = &uSet; + reset(); +} + +/** + * Resets to the start, to allow the iteration to start over again. + */ +void UnicodeSetIterator::reset() { + if (set == NULL) { + // Set up indices to empty iteration + endRange = -1; + stringCount = 0; + } else { + endRange = set->getRangeCount() - 1; + stringCount = set->strings->size(); + } + range = 0; + endElement = -1; + nextElement = 0; + if (endRange >= 0) { + loadRange(range); + } + nextString = 0; + string = NULL; +} + +void UnicodeSetIterator::loadRange(int32_t iRange) { + nextElement = set->getRangeStart(iRange); + endElement = set->getRangeEnd(iRange); +} + + +const UnicodeString& UnicodeSetIterator::getString() { + if (string==NULL && codepoint!=(UChar32)IS_STRING) { + if (cpString == NULL) { + cpString = new UnicodeString(); + } + if (cpString != NULL) { + cpString->setTo((UChar32)codepoint); + } + string = cpString; + } + return *string; +} + +U_NAMESPACE_END + +//eof diff --git a/Build/source/libs/icu/icu-xetex/common/ushape.c b/Build/source/libs/icu/icu-xetex/common/ushape.c new file mode 100644 index 00000000000..04d0823ce95 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ushape.c @@ -0,0 +1,1198 @@ +/* + ****************************************************************************** + * + * Copyright (C) 2000-2007, International Business Machines + * Corporation and others. All Rights Reserved. + * + ****************************************************************************** + * file name: ushape.c + * encoding: US-ASCII + * tab size: 8 (not used) + * indentation:4 + * + * created on: 2000jun29 + * created by: Markus W. Scherer + * + * Arabic letter shaping implemented by Ayman Roshdy + */ + +#include "unicode/utypes.h" +#include "unicode/uchar.h" +#include "unicode/ustring.h" +#include "unicode/ushape.h" +#include "cmemory.h" +#include "putilimp.h" +#include "ustr_imp.h" +#include "ubidi_props.h" + +#if UTF_SIZE<16 + /* + * This implementation assumes that the internal encoding is UTF-16 + * or UTF-32, not UTF-8. + * The main assumption is that the Arabic characters and their + * presentation forms each fit into a single UChar. + * With UTF-8, they occupy 2 or 3 bytes, and more than the ASCII + * characters. + */ +# error This implementation assumes UTF-16 or UTF-32 (check UTF_SIZE) +#endif + +/* + * ### TODO in general for letter shaping: + * - the letter shaping code is UTF-16-unaware; needs update + * + especially invertBuffer()?! + * - needs to handle the "Arabic Tail" that is used in some legacy codepages + * as a glyph fragment of wide-glyph letters + * + IBM Unicode conversion tables map it to U+200B (ZWSP) + * + IBM Egypt has proposed to encode the tail in Unicode among Arabic Presentation Forms + */ + +/* definitions for Arabic letter shaping ------------------------------------ */ + +#define IRRELEVANT 4 +#define LAMTYPE 16 +#define ALEFTYPE 32 +#define LINKR 1 +#define LINKL 2 +#define APRESENT 8 +#define SHADDA 64 +#define CSHADDA 128 +#define COMBINE (SHADDA+CSHADDA) + + +static const UChar IrrelevantPos[] = { + 0x0, 0x2, 0x4, 0x6, + 0x8, 0xA, 0xC, 0xE, +}; + +static const UChar convertLamAlef[] = +{ +/*FEF5*/ 0x0622, +/*FEF6*/ 0x0622, +/*FEF7*/ 0x0623, +/*FEF8*/ 0x0623, +/*FEF9*/ 0x0625, +/*FEFA*/ 0x0625, +/*FEFB*/ 0x0627, +/*FEFC*/ 0x0627 +}; + +static const UChar araLink[178]= +{ + 1 + 32 + 256 * 0x11,/*0x0622*/ + 1 + 32 + 256 * 0x13,/*0x0623*/ + 1 + 256 * 0x15,/*0x0624*/ + 1 + 32 + 256 * 0x17,/*0x0625*/ + 1 + 2 + 256 * 0x19,/*0x0626*/ + 1 + 32 + 256 * 0x1D,/*0x0627*/ + 1 + 2 + 256 * 0x1F,/*0x0628*/ + 1 + 256 * 0x23,/*0x0629*/ + 1 + 2 + 256 * 0x25,/*0x062A*/ + 1 + 2 + 256 * 0x29,/*0x062B*/ + 1 + 2 + 256 * 0x2D,/*0x062C*/ + 1 + 2 + 256 * 0x31,/*0x062D*/ + 1 + 2 + 256 * 0x35,/*0x062E*/ + 1 + 256 * 0x39,/*0x062F*/ + 1 + 256 * 0x3B,/*0x0630*/ + 1 + 256 * 0x3D,/*0x0631*/ + 1 + 256 * 0x3F,/*0x0632*/ + 1 + 2 + 256 * 0x41,/*0x0633*/ + 1 + 2 + 256 * 0x45,/*0x0634*/ + 1 + 2 + 256 * 0x49,/*0x0635*/ + 1 + 2 + 256 * 0x4D,/*0x0636*/ + 1 + 2 + 256 * 0x51,/*0x0637*/ + 1 + 2 + 256 * 0x55,/*0x0638*/ + 1 + 2 + 256 * 0x59,/*0x0639*/ + 1 + 2 + 256 * 0x5D,/*0x063A*/ + 0, 0, 0, 0, 0, /*0x063B-0x063F*/ + 1 + 2, /*0x0640*/ + 1 + 2 + 256 * 0x61,/*0x0641*/ + 1 + 2 + 256 * 0x65,/*0x0642*/ + 1 + 2 + 256 * 0x69,/*0x0643*/ + 1 + 2 + 16 + 256 * 0x6D,/*0x0644*/ + 1 + 2 + 256 * 0x71,/*0x0645*/ + 1 + 2 + 256 * 0x75,/*0x0646*/ + 1 + 2 + 256 * 0x79,/*0x0647*/ + 1 + 256 * 0x7D,/*0x0648*/ + 1 + 256 * 0x7F,/*0x0649*/ + 1 + 2 + 256 * 0x81,/*0x064A*/ + 4 + 256 * 1, /*0x064B*/ + 4 + 128 + 256 * 1, /*0x064C*/ + 4 + 128 + 256 * 1, /*0x064D*/ + 4 + 128 + 256 * 1, /*0x064E*/ + 4 + 128 + 256 * 1, /*0x064F*/ + 4 + 128 + 256 * 1, /*0x0650*/ + 4 + 64 + 256 * 3, /*0x0651*/ + 4 + 256 * 1, /*0x0652*/ + 4 + 256 * 7, /*0x0653*/ + 4 + 256 * 8, /*0x0654*/ + 4 + 256 * 8, /*0x0655*/ + 4 + 256 * 1, /*0x0656*/ + 0, 0, 0, 0, 0, /*0x0657-0x065B*/ + 1 + 256 * 0x85,/*0x065C*/ + 1 + 256 * 0x87,/*0x065D*/ + 1 + 256 * 0x89,/*0x065E*/ + 1 + 256 * 0x8B,/*0x065F*/ + 0, 0, 0, 0, 0, /*0x0660-0x0664*/ + 0, 0, 0, 0, 0, /*0x0665-0x0669*/ + 0, 0, 0, 0, 0, 0, /*0x066A-0x066F*/ + 4 + 256 * 6, /*0x0670*/ + 1 + 8 + 256 * 0x00,/*0x0671*/ + 1 + 32, /*0x0672*/ + 1 + 32, /*0x0673*/ + 0, /*0x0674*/ + 1 + 32, /*0x0675*/ + 1, 1, /*0x0676-0x0677*/ + 1+2, 1+2, 1+2, 1+2, 1+2, 1+2, /*0x0678-0x067D*/ + 1+2+8+256 * 0x06, 1+2, 1+2, 1+2, 1+2, 1+2, /*0x067E-0x0683*/ + 1+2, 1+2, 1+2+8+256 * 0x2A, 1+2, /*0x0684-0x0687*/ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /*0x0688-0x0691*/ + 1, 1, 1, 1, 1, 1, 1+8+256 * 0x3A, 1, /*0x0692-0x0699*/ + 1+2, 1+2, 1+2, 1+2, 1+2, 1+2, /*0x069A-0x06A3*/ + 1+2, 1+2, 1+2, 1+2, /*0x069A-0x06A3*/ + 1+2, 1+2, 1+2, 1+2, 1+2, 1+2+8+256 * 0x3E, /*0x06A4-0x06AD*/ + 1+2, 1+2, 1+2, 1+2, /*0x06A4-0x06AD*/ + 1+2, 1+2+8+256 * 0x42, 1+2, 1+2, 1+2, 1+2, /*0x06AE-0x06B7*/ + 1+2, 1+2, 1+2, 1+2, /*0x06AE-0x06B7*/ + 1+2, 1+2, 1+2, 1+2, 1+2, 1+2, /*0x06B8-0x06BF*/ + 1+2, 1+2, /*0x06B8-0x06BF*/ + 1, /*0x06C0*/ + 1+2, /*0x06C1*/ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /*0x06C2-0x06CB*/ + 1+2+8+256 * 0xAC, /*0x06CC*/ + 1, /*0x06CD*/ + 1+2, 1+2, 1+2, 1+2, /*0x06CE-0x06D1*/ + 1, 1 /*0x06D2-0x06D3*/ +}; + +static const UChar presALink[] = { +/***********0*****1*****2*****3*****4*****5*****6*****7*****8*****9*****A*****B*****C*****D*****E*****F*/ +/*FB5*/ 0, 1, 0, 0, 0, 0, 0, 1, 2,1 + 2, 0, 0, 0, 0, 0, 0, +/*FB6*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*FB7*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2,1 + 2, 0, 0, +/*FB8*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, +/*FB9*/ 2,1 + 2, 0, 1, 2,1 + 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*FBA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*FBB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*FBC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*FBD*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*FBE*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*FBF*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2,1 + 2, +/*FC0*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*FC1*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*FC2*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*FC3*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*FC4*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*FC5*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, +/*FC6*/ 4, 4, 4 +}; + +static const UChar presBLink[]= +{ +/***********0*****1*****2*****3*****4*****5*****6*****7*****8*****9*****A*****B*****C*****D*****E*****F*/ +/*FE7*/1 + 2,1 + 2,1 + 2, 0,1 + 2, 0,1 + 2,1 + 2,1 + 2,1 + 2,1 + 2,1 + 2,1 + 2,1 + 2,1 + 2,1 + 2, +/*FE8*/ 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2,1 + 2, 0, 1, 0, +/*FE9*/ 1, 2,1 + 2, 0, 1, 0, 1, 2,1 + 2, 0, 1, 2,1 + 2, 0, 1, 2, +/*FEA*/1 + 2, 0, 1, 2,1 + 2, 0, 1, 2,1 + 2, 0, 1, 0, 1, 0, 1, 0, +/*FEB*/ 1, 0, 1, 2,1 + 2, 0, 1, 2,1 + 2, 0, 1, 2,1 + 2, 0, 1, 2, +/*FEC*/1 + 2, 0, 1, 2,1 + 2, 0, 1, 2,1 + 2, 0, 1, 2,1 + 2, 0, 1, 2, +/*FED*/1 + 2, 0, 1, 2,1 + 2, 0, 1, 2,1 + 2, 0, 1, 2,1 + 2, 0, 1, 2, +/*FEE*/1 + 2, 0, 1, 2,1 + 2, 0, 1, 2,1 + 2, 0, 1, 2,1 + 2, 0, 1, 0, +/*FEF*/ 1, 0, 1, 2,1 + 2, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0 +}; + +static const UChar convertFBto06[] = +{ +/***********0******1******2******3******4******5******6******7******8******9******A******B******C******D******E******F***/ +/*FB5*/ 0x671, 0x671, 0, 0, 0, 0, 0x07E, 0x07E, 0x07E, 0x07E, 0, 0, 0, 0, 0, 0, +/*FB6*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*FB7*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x686, 0x686, 0x686, 0x686, 0, 0, +/*FB8*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x698, 0x698, 0, 0, 0x6A9, 0x6A9, +/*FB9*/ 0x6A9, 0x6A9, 0x6AF, 0x6AF, 0x6AF, 0x6AF, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*FBA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*FBB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*FBC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*FBD*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*FBE*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*FBF*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x6CC, 0x6CC, 0x6CC, 0x6CC +}; + +static const UChar convertFEto06[] = +{ +/***********0******1******2******3******4******5******6******7******8******9******A******B******C******D******E******F***/ +/*FE7*/ 0x64B, 0x64B, 0x64C, 0x64C, 0x64D, 0x64D, 0x64E, 0x64E, 0x64F, 0x64F, 0x650, 0x650, 0x651, 0x651, 0x652, 0x652, +/*FE8*/ 0x621, 0x622, 0x622, 0x623, 0x623, 0x624, 0x624, 0x625, 0x625, 0x626, 0x626, 0x626, 0x626, 0x627, 0x627, 0x628, +/*FE9*/ 0x628, 0x628, 0x628, 0x629, 0x629, 0x62A, 0x62A, 0x62A, 0x62A, 0x62B, 0x62B, 0x62B, 0x62B, 0x62C, 0x62C, 0x62C, +/*FEA*/ 0x62C, 0x62D, 0x62D, 0x62D, 0x62D, 0x62E, 0x62E, 0x62E, 0x62E, 0x62F, 0x62F, 0x630, 0x630, 0x631, 0x631, 0x632, +/*FEB*/ 0x632, 0x633, 0x633, 0x633, 0x633, 0x634, 0x634, 0x634, 0x634, 0x635, 0x635, 0x635, 0x635, 0x636, 0x636, 0x636, +/*FEC*/ 0x636, 0x637, 0x637, 0x637, 0x637, 0x638, 0x638, 0x638, 0x638, 0x639, 0x639, 0x639, 0x639, 0x63A, 0x63A, 0x63A, +/*FED*/ 0x63A, 0x641, 0x641, 0x641, 0x641, 0x642, 0x642, 0x642, 0x642, 0x643, 0x643, 0x643, 0x643, 0x644, 0x644, 0x644, +/*FEE*/ 0x644, 0x645, 0x645, 0x645, 0x645, 0x646, 0x646, 0x646, 0x646, 0x647, 0x647, 0x647, 0x647, 0x648, 0x648, 0x649, +/*FEF*/ 0x649, 0x64A, 0x64A, 0x64A, 0x64A, 0x65C, 0x65C, 0x65D, 0x65D, 0x65E, 0x65E, 0x65F, 0x65F +}; + +static const UChar shapeTable[4][4][4]= +{ + { {0,0,0,0}, {0,0,0,0}, {0,1,0,3}, {0,1,0,1} }, + { {0,0,2,2}, {0,0,1,2}, {0,1,1,2}, {0,1,1,3} }, + { {0,0,0,0}, {0,0,0,0}, {0,1,0,3}, {0,1,0,3} }, + { {0,0,1,2}, {0,0,1,2}, {0,1,1,2}, {0,1,1,3} } +}; + +/* + * This function shapes European digits to Arabic-Indic digits + * in-place, writing over the input characters. + * Since we know that we are only looking for BMP code points, + * we can safely just work with code units (again, at least UTF-16). + */ +static void +_shapeToArabicDigitsWithContext(UChar *s, int32_t length, + UChar digitBase, + UBool isLogical, UBool lastStrongWasAL) { + const UBiDiProps *bdp; + UErrorCode errorCode; + + int32_t i; + UChar c; + + errorCode=U_ZERO_ERROR; + bdp=ubidi_getSingleton(&errorCode); + if(U_FAILURE(errorCode)) { + return; + } + + digitBase-=0x30; + + /* the iteration direction depends on the type of input */ + if(isLogical) { + for(i=0; i0; /* pre-decrement in the body */) { + c=s[--i]; + switch(ubidi_getClass(bdp, c)) { + case U_LEFT_TO_RIGHT: /* L */ + case U_RIGHT_TO_LEFT: /* R */ + lastStrongWasAL=FALSE; + break; + case U_RIGHT_TO_LEFT_ARABIC: /* AL */ + lastStrongWasAL=TRUE; + break; + case U_EUROPEAN_NUMBER: /* EN */ + if(lastStrongWasAL && (uint32_t)(c-0x30)<10) { + s[i]=(UChar)(digitBase+c); /* digitBase+(c-0x30) - digitBase was modified above */ + } + break; + default : + break; + } + } + } +} + +/* + *Name : invertBuffer + *Function : This function inverts the buffer, it's used + * in case the user specifies the buffer to be + * U_SHAPE_TEXT_DIRECTION_LOGICAL + */ +static void +invertBuffer(UChar *buffer,int32_t size,uint32_t options,int32_t lowlimit,int32_t highlimit) { + UChar temp; + int32_t i=0,j=0; + for(i=lowlimit,j=size-highlimit-1;i= 0x0622 && ch <= 0x06D3) { + return(araLink[ch-0x0622]); + } else if(ch == 0x200D) { + return(3); + } else if(ch >= 0x206D && ch <= 0x206F) { + return(4); + } else if(ch >= 0xFB50 && ch <= 0xFC62) { + return(presALink[ch-0xFB50]); + } else if(ch >= 0xFE70 && ch <= 0xFEFC) { + return(presBLink[ch-0xFE70]); + } else { + return(0); + } +} + +/* + *Name : countSpaces + *Function : Counts the number of spaces + * at each end of the logical buffer + */ +static void +countSpaces(UChar *dest,int32_t size,uint32_t options,int32_t *spacesCountl,int32_t *spacesCountr) { + int32_t i = 0; + int32_t countl = 0,countr = 0; + while(dest[i] == 0x0020) { + countl++; + i++; + } + while(dest[size-1] == 0x0020) { + countr++; + size--; + } + *spacesCountl = countl; + *spacesCountr = countr; +} + +/* + *Name : isTashkeelChar + *Function : Returns 1 for Tashkeel characters else return 0 + */ +static U_INLINE int32_t +isTashkeelChar(UChar ch) { + return (int32_t)( ch>=0x064B && ch<= 0x0652 ); +} + +/* + *Name : isAlefChar + *Function : Returns 1 for Alef characters else return 0 + */ +static U_INLINE int32_t +isAlefChar(UChar ch) { + return (int32_t)( (ch==0x0622)||(ch==0x0623)||(ch==0x0625)||(ch==0x0627) ); +} + +/* + *Name : isLamAlefChar + *Function : Returns 1 for LamAlef characters else return 0 + */ +static U_INLINE int32_t +isLamAlefChar(UChar ch) { + return (int32_t)( (ch>=0xFEF5)&&(ch<=0xFEFC) ); +} + +/* + *Name : calculateSize + *Function : This function calculates the destSize to be used in preflighting + * when the destSize is equal to 0 + */ +static int32_t +calculateSize(const UChar *source, int32_t sourceLength, + int32_t destSize,uint32_t options) { + int32_t i = 0; + destSize = sourceLength; + switch(options&U_SHAPE_LETTERS_MASK) { + case U_SHAPE_LETTERS_SHAPE : + case U_SHAPE_LETTERS_SHAPE_TASHKEEL_ISOLATED: + if((options&U_SHAPE_TEXT_DIRECTION_MASK)==U_SHAPE_TEXT_DIRECTION_VISUAL_LTR) { + for(i=0;i= 0) { + tempbuffer[i] = 0x0000; + i--; + count--; + } + + uprv_memcpy(dest, tempbuffer, sourceLength*U_SIZEOF_UCHAR); + destSize = u_strlen(dest); + break; + + case U_SHAPE_LENGTH_FIXED_SPACES_NEAR : + /* Lam+Alef is already shaped into LamAlef + FFFF */ + i = 0; + while(i < sourceLength) { + if(dest[i] == 0xFFFF) + dest[i] = 0x0020; + i++; + } + destSize = sourceLength; + break; + + case U_SHAPE_LENGTH_FIXED_SPACES_AT_BEGINNING : + tempbuffer = (UChar *)uprv_malloc((sourceLength+1)*U_SIZEOF_UCHAR); + + /* Test for NULL */ + if(tempbuffer == NULL) { + *pErrorCode = U_MEMORY_ALLOCATION_ERROR; + return 0; + } + + uprv_memset(tempbuffer, 0, (sourceLength+1)*U_SIZEOF_UCHAR); + + i = j = sourceLength; + while(i >= 0) { + if(dest[i] == 0xFFFF) { + j++; + count++; + } + else + tempbuffer[j] = dest[i]; + i--; + j--; + } + for(i=0;i= 0) { + tempbuffer[i] = 0x0020; + i--; + count--; + } + + uprv_memcpy(dest,tempbuffer, sourceLength*U_SIZEOF_UCHAR); + destSize = sourceLength; + break; + + default : + /* will not occur */ + break; + } + + if(tempbuffer) + uprv_free(tempbuffer); + + return destSize; +} + +/* + *Name : expandLamAlef + *Function : LamAlef needs special handling as the LamAlef is + * one character while expanding it will give two + * characters Lam + Alef, so we need to expand the LamAlef + * in near or far spaces according to the options the user + * specifies or increase the buffer size. + * If there are no spaces to expand the LamAlef, an error + * will be set to U_NO_SPACE_AVAILABLE as defined in utypes.h + */ +static int32_t +expandLamAlef(UChar *dest, int32_t sourceLength, + int32_t destSize,uint32_t options, + UErrorCode *pErrorCode) { + + int32_t i = 0,j = 0; + int32_t countl = 0; + int32_t countr = 0; + int32_t inpsize = sourceLength; + UChar lamalefChar; + UChar *tempbuffer=NULL; + + switch(options&U_SHAPE_LENGTH_MASK) { + + case U_SHAPE_LENGTH_GROW_SHRINK : + destSize = calculateSize(dest,sourceLength,destSize,options); + tempbuffer = (UChar *)uprv_malloc((destSize+1)*U_SIZEOF_UCHAR); + + /* Test for NULL */ + if(tempbuffer == NULL) { + *pErrorCode = U_MEMORY_ALLOCATION_ERROR; + return 0; + } + + uprv_memset(tempbuffer, 0, (destSize+1)*U_SIZEOF_UCHAR); + + i = j = 0; + while(i < destSize && j < destSize) { + if( isLamAlefChar(dest[i]) ) { + tempbuffer[j] = convertLamAlef[ dest[i] - 0xFEF5 ]; + tempbuffer[j+1] = 0x0644; + j++; + } + else + tempbuffer[j] = dest[i]; + i++; + j++; + } + + uprv_memcpy(dest, tempbuffer, destSize*U_SIZEOF_UCHAR); + break; + + case U_SHAPE_LENGTH_FIXED_SPACES_NEAR : + for(i = 0;i= 0 && j >= 0) { + if( countl>0 && isLamAlefChar(dest[i]) ) { + tempbuffer[j] = 0x0644; + tempbuffer[j-1] = convertLamAlef[ dest[i] - 0xFEF5 ]; + j--; + countl--; + } + else { + if( countl == 0 && isLamAlefChar(dest[i]) ) + *pErrorCode=U_NO_SPACE_AVAILABLE; + tempbuffer[j] = dest[i]; + } + i--; + j--; + } + + uprv_memcpy(dest, tempbuffer, sourceLength*U_SIZEOF_UCHAR); + destSize = sourceLength; + break; + + case U_SHAPE_LENGTH_FIXED_SPACES_AT_END : + /* LamAlef expansion below is done from right to left to make sure that we consume + * the spaces with the LamAlefs as they appear in the visual buffer from right to left + */ + tempbuffer = (UChar *)uprv_malloc((sourceLength+1)*U_SIZEOF_UCHAR); + + /* Test for NULL */ + if(tempbuffer == NULL) { + *pErrorCode = U_MEMORY_ALLOCATION_ERROR; + return 0; + } + + uprv_memset(tempbuffer, 0, (sourceLength+1)*U_SIZEOF_UCHAR); + + while(dest[inpsize-1] == 0x0020) { + countr++; + inpsize--; + } + + i = sourceLength - countr - 1; + j = sourceLength - 1; + + while(i >= 0 && j >= 0) { + if( countr>0 && isLamAlefChar(dest[i]) ) { + tempbuffer[j] = 0x0644; + tempbuffer[j-1] = convertLamAlef[ dest[i] - 0xFEF5 ]; + j--; + countr--; + } + else { + if( countr == 0 && isLamAlefChar(dest[i]) ) + *pErrorCode=U_NO_SPACE_AVAILABLE; + tempbuffer[j] = dest[i]; + } + i--; + j--; + } + + if(countr > 0) { + uprv_memmove(tempbuffer, tempbuffer+countr, sourceLength*U_SIZEOF_UCHAR); + if(u_strlen(tempbuffer) < sourceLength) { + for(i=sourceLength-1;i>=sourceLength-countr;i--) { + tempbuffer[i] = 0x0020; + } + } + } + + uprv_memcpy(dest, tempbuffer, sourceLength*U_SIZEOF_UCHAR); + + destSize = sourceLength; + break; + + default : + /* will never occur because of validity checks */ + break; + } + + if(tempbuffer) + uprv_free(tempbuffer); + + return destSize; +} + +/* + *Name : shapeUnicode + *Function : Converts an Arabic Unicode buffer in 06xx Range into a shaped + * arabic Unicode buffer in FExx Range + */ +static int32_t +shapeUnicode(UChar *dest, int32_t sourceLength, + int32_t destSize,uint32_t options, + UErrorCode *pErrorCode, + int tashkeelFlag) { + + int32_t i, iend; + int32_t step; + int32_t lastPos,Nx, Nw; + unsigned int Shape; + int32_t lamalef_found = 0; + UChar prevLink = 0, lastLink = 0, currLink, nextLink = 0; + UChar wLamalef; + + /* + * Converts the input buffer from FExx Range into 06xx Range + * to make sure that all characters are in the 06xx range + * even the lamalef is converted to the special region in + * the 06xx range + */ + if ((options & U_SHAPE_PRESERVE_PRESENTATION_MASK) == U_SHAPE_PRESERVE_PRESENTATION_NOOP) { + for (i = 0; i < sourceLength; i++) { + UChar inputChar = dest[i]; + if ( (inputChar >= 0xFB50) && (inputChar <= 0xFBFF)) { + UChar c = convertFBto06 [ (inputChar - 0xFB50) ]; + if (c != 0) + dest[i] = c; + } else if ( (inputChar >= 0xFE70) && (inputChar <= 0xFEFC)) { + dest[i] = convertFEto06 [ (inputChar - 0xFE70) ] ; + } else { + dest[i] = inputChar ; + } + } + } + + /* sets the index to the end of the buffer, together with the step point to -1 */ + i = sourceLength - 1; + iend = -1; + step = -1; + + /* + * This function resolves the link between the characters . + * Arabic characters have four forms : + * Isolated Form, Initial Form, Middle Form and Final Form + */ + currLink = getLink(dest[i]); + + lastPos = i; + Nx = -2, Nw = 0; + + while (i != iend) { + /* If high byte of currLink > 0 then more than one shape */ + if ((currLink & 0xFF00) > 0 || (getLink(dest[i]) & IRRELEVANT) != 0) { + Nw = i + step; + while (Nx < 0) { /* we need to know about next char */ + if(Nw == iend) { + nextLink = 0; + Nx = 3000; + } else { + nextLink = getLink(dest[Nw]); + if((nextLink & IRRELEVANT) == 0) { + Nx = Nw; + } else { + Nw = Nw + step; + } + } + } + + if ( ((currLink & ALEFTYPE) > 0) && ((lastLink & LAMTYPE) > 0) ) { + lamalef_found = 1; + wLamalef = changeLamAlef(dest[i]); /*get from 0x065C-0x065f */ + if ( wLamalef != 0) { + dest[i] = 0xFFFF; /* The default case is to drop the Alef and replace */ + dest[lastPos] =wLamalef; /* it by 0xFFFF which is the last character in the */ + i=lastPos; /* unicode private use area, this is done to make */ + } /* sure that removeLamAlefSpaces() handles only the */ + lastLink = prevLink; /* spaces generated during lamalef generation. */ + currLink = getLink(wLamalef); /* 0xFFFF is added here and is replaced by spaces */ + } /* in removeLamAlefSpaces() */ + /* + * get the proper shape according to link ability of neighbors + * and of character; depends on the order of the shapes + * (isolated, initial, middle, final) in the compatibility area + */ + Shape = shapeTable[nextLink & (LINKR + LINKL)] + [lastLink & (LINKR + LINKL)] + [currLink & (LINKR + LINKL)]; + + if ((currLink & (LINKR+LINKL)) == 1) { + Shape &= 1; + } else if(isTashkeelChar(dest[i])) { + if( (lastLink & LINKL) && (nextLink & LINKR) && (tashkeelFlag == 1) && + dest[i] != 0x064C && dest[i] != 0x064D ) + { + Shape = 1; + if( (nextLink&ALEFTYPE) == ALEFTYPE && (lastLink&LAMTYPE) == LAMTYPE ) { + Shape = 0; + } + } + else { + Shape = 0; + } + } + if ((dest[i] ^ 0x0600) < 0x100) { + if(isTashkeelChar(dest[i])) + dest[i] = 0xFE70 + IrrelevantPos[(dest[i] - 0x064B)] + Shape; + else if ((currLink & APRESENT) > 0) + dest[i] = (UChar)(0xFB50 + (currLink >> 8) + Shape); + else if ((currLink >> 8) > 0 && (currLink & IRRELEVANT) == 0) + dest[i] = (UChar)(0xFE70 + (currLink >> 8) + Shape); + } + } + + /* move one notch forward */ + if ((currLink & IRRELEVANT) == 0) { + prevLink = lastLink; + lastLink = currLink; + lastPos = i; + } + + i = i + step; + if (i == Nx) { + currLink = nextLink; + Nx = -2; + } else if(i != iend) { + currLink = getLink(dest[i]); + } + } + + if(lamalef_found != 0) + destSize = removeLamAlefSpaces(dest,sourceLength,destSize,options,pErrorCode); + else + destSize = sourceLength; + + return destSize; +} + +/* + *Name : deShapeUnicode + *Function : Converts an Arabic Unicode buffer in FExx Range into unshaped + * arabic Unicode buffer in 06xx Range + */ +static int32_t +deShapeUnicode(UChar *dest, int32_t sourceLength, + int32_t destSize,uint32_t options, + UErrorCode *pErrorCode) { + int32_t i = 0; + int32_t lamalef_found = 0; + + /* + *This for loop changes the buffer from the Unicode FE range to + *the Unicode 06 range + */ + for(i = 0; i < sourceLength; i++) { + UChar inputChar = dest[i]; + if ( (inputChar >= 0xFB50) && (inputChar <= 0xFBFF)) { /* FBxx Arabic range */ + UChar c = convertFBto06 [ (inputChar - 0xFB50) ]; + if (c != 0) + dest[i] = c; + } else if (( inputChar >= 0xFE70) && (inputChar <= 0xFEF4 )) { /* FExx Arabic range */ + dest[i] = convertFEto06 [ (inputChar - 0xFE70) ] ; + } else { + dest[i] = inputChar ; + } + if( isLamAlefChar(dest[i]) ) + lamalef_found = 1; + } + + /* If there is lamalef in the buffer call expandLamAlef */ + if(lamalef_found != 0) + destSize = expandLamAlef(dest,sourceLength,destSize,options,pErrorCode); + else + destSize = sourceLength; + + return destSize; +} + +U_CAPI int32_t U_EXPORT2 +u_shapeArabic(const UChar *source, int32_t sourceLength, + UChar *dest, int32_t destCapacity, + uint32_t options, + UErrorCode *pErrorCode) { + + int32_t destLength; + + /* usual error checking */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + /* make sure that no reserved options values are used; allow dest==NULL only for preflighting */ + if( source==NULL || sourceLength<-1 || (dest==NULL && destCapacity!=0) || destCapacity<0 || + (options&U_SHAPE_DIGIT_TYPE_RESERVED)==U_SHAPE_DIGIT_TYPE_RESERVED || + (options&U_SHAPE_DIGITS_MASK)==U_SHAPE_DIGITS_RESERVED || + ((options&U_SHAPE_LENGTH_MASK) != U_SHAPE_LENGTH_GROW_SHRINK && + (options&U_SHAPE_AGGREGATE_TASHKEEL_MASK) != 0) || + ((options&U_SHAPE_AGGREGATE_TASHKEEL_MASK) == U_SHAPE_AGGREGATE_TASHKEEL && + (options&U_SHAPE_LETTERS_SHAPE_TASHKEEL_ISOLATED) != U_SHAPE_LETTERS_SHAPE_TASHKEEL_ISOLATED) + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* determine the source length */ + if(sourceLength==-1) { + sourceLength=u_strlen(source); + } + if(sourceLength<=0) { + return u_terminateUChars(dest, destCapacity, 0, pErrorCode); + } + + /* check that source and destination do not overlap */ + if( dest!=NULL && + ((source<=dest && dest0) { + int32_t logical_order = (options&U_SHAPE_TEXT_DIRECTION_MASK) == U_SHAPE_TEXT_DIRECTION_LOGICAL; + int32_t aggregate_tashkeel = + (options&(U_SHAPE_AGGREGATE_TASHKEEL_MASK+U_SHAPE_LETTERS_SHAPE_TASHKEEL_ISOLATED)) == + (U_SHAPE_AGGREGATE_TASHKEEL+U_SHAPE_LETTERS_SHAPE_TASHKEEL_ISOLATED); + int step=logical_order?1:-1; + int j=logical_order?-1:2*sourceLength; + int i=logical_order?-1:sourceLength; + int end=logical_order?sourceLength:-1; + int aggregation_possible = 1; + UChar prev = 0; + UChar prevLink, currLink = 0; + int newSourceLength = 0; + tempsource = (UChar *)uprv_malloc(2*sourceLength*U_SIZEOF_UCHAR); + if(tempsource == NULL) { + *pErrorCode = U_MEMORY_ALLOCATION_ERROR; + return 0; + } + + while ((i+=step) != end) { + prevLink = currLink; + currLink = getLink(source[i]); + if (aggregate_tashkeel && ((prevLink|currLink)&COMBINE) == COMBINE && aggregation_possible) { + aggregation_possible = 0; + tempsource[j] = (prevdestCapacity) { + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + if (tempsource != NULL) uprv_free(tempsource); + return outputSize; + } + + /* + * need a temporary buffer of size max(outputSize, sourceLength) + * because at first we copy source->temp + */ + if(sourceLength>outputSize) { + outputSize=sourceLength; + } + + /* Start of Arabic letter shaping part */ + if(outputSize<=sizeof(buffer)/U_SIZEOF_UCHAR) { + outputSize=sizeof(buffer)/U_SIZEOF_UCHAR; + tempbuffer=buffer; + } else { + tempbuffer = (UChar *)uprv_malloc(outputSize*U_SIZEOF_UCHAR); + + /*Test for NULL*/ + if(tempbuffer == NULL) { + *pErrorCode = U_MEMORY_ALLOCATION_ERROR; + if (tempsource != NULL) uprv_free(tempsource); + return 0; + } + } + uprv_memcpy(tempbuffer, source, sourceLength*U_SIZEOF_UCHAR); + if (tempsource != NULL) uprv_free(tempsource); + if(sourceLengthdestCapacity) { + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + return destLength; + } + + /* End of Arabic letter shaping part */ + } else { + /* + * No letter shaping: + * just make sure the destination is large enough and copy the string. + */ + if(destCapacitysize>=20 && + pInfo->isBigEndian==U_IS_BIG_ENDIAN && + pInfo->charsetFamily==U_CHARSET_FAMILY && + pInfo->dataFormat[0]==0x53 && /* dataFormat="SPRP" */ + pInfo->dataFormat[1]==0x50 && + pInfo->dataFormat[2]==0x52 && + pInfo->dataFormat[3]==0x50 && + pInfo->formatVersion[0]==3 && + pInfo->formatVersion[2]==UTRIE_SHIFT && + pInfo->formatVersion[3]==UTRIE_INDEX_SHIFT + ) { + uprv_memcpy(formatVersion, pInfo->formatVersion, 4); + uprv_memcpy(dataVersion, pInfo->dataVersion, 4); + return TRUE; + } else { + return FALSE; + } +} + +static int32_t U_CALLCONV +getSPrepFoldingOffset(uint32_t data) { + + return (int32_t)data; + +} + +/* hashes an entry */ +static int32_t U_CALLCONV +hashEntry(const UHashTok parm) { + UStringPrepKey *b = (UStringPrepKey *)parm.pointer; + UHashTok namekey, pathkey; + namekey.pointer = b->name; + pathkey.pointer = b->path; + return uhash_hashChars(namekey)+37*uhash_hashChars(pathkey); +} + +/* compares two entries */ +static UBool U_CALLCONV +compareEntries(const UHashTok p1, const UHashTok p2) { + UStringPrepKey *b1 = (UStringPrepKey *)p1.pointer; + UStringPrepKey *b2 = (UStringPrepKey *)p2.pointer; + UHashTok name1, name2, path1, path2; + name1.pointer = b1->name; + name2.pointer = b2->name; + path1.pointer = b1->path; + path2.pointer = b2->path; + return ((UBool)(uhash_compareChars(name1, name2) & + uhash_compareChars(path1, path2))); +} + +static void +usprep_unload(UStringPrepProfile* data){ + udata_close(data->sprepData); +} + +static int32_t +usprep_internal_flushCache(UBool noRefCount){ + UStringPrepProfile *profile = NULL; + UStringPrepKey *key = NULL; + int32_t pos = -1; + int32_t deletedNum = 0; + const UHashElement *e; + + /* + * if shared data hasn't even been lazy evaluated yet + * return 0 + */ + umtx_lock(&usprepMutex); + if (SHARED_DATA_HASHTABLE == NULL) { + umtx_unlock(&usprepMutex); + return 0; + } + + /*creates an enumeration to iterate through every element in the table */ + while ((e = uhash_nextElement(SHARED_DATA_HASHTABLE, &pos)) != NULL) + { + profile = (UStringPrepProfile *) e->value.pointer; + key = (UStringPrepKey *) e->key.pointer; + + if ((noRefCount== FALSE && profile->refCount == 0) || + noRefCount== TRUE) { + deletedNum++; + uhash_removeElement(SHARED_DATA_HASHTABLE, e); + + /* unload the data */ + usprep_unload(profile); + + if(key->name != NULL) { + uprv_free(key->name); + key->name=NULL; + } + if(key->path != NULL) { + uprv_free(key->path); + key->path=NULL; + } + uprv_free(profile); + uprv_free(key); + } + + } + umtx_unlock(&usprepMutex); + + return deletedNum; +} + +/* Works just like ucnv_flushCache() +static int32_t +usprep_flushCache(){ + return usprep_internal_flushCache(FALSE); +} +*/ + +static UBool U_CALLCONV usprep_cleanup(void){ + if (SHARED_DATA_HASHTABLE != NULL) { + usprep_internal_flushCache(TRUE); + if (SHARED_DATA_HASHTABLE != NULL && uhash_count(SHARED_DATA_HASHTABLE) == 0) { + uhash_close(SHARED_DATA_HASHTABLE); + SHARED_DATA_HASHTABLE = NULL; + } + } + + umtx_destroy(&usprepMutex); /* Don't worry about destroying the mutex even */ + /* if the hash table still exists. The mutex */ + /* will lazily re-init itself if needed. */ + return (SHARED_DATA_HASHTABLE == NULL); +} +U_CDECL_END + +static void +usprep_init() { + umtx_init(&usprepMutex); +} + +/** Initializes the cache for resources */ +static void +initCache(UErrorCode *status) { + UBool makeCache; + UMTX_CHECK(&usprepMutex, (SHARED_DATA_HASHTABLE == NULL), makeCache); + if(makeCache) { + UHashtable *newCache = uhash_open(hashEntry, compareEntries, NULL, status); + if (U_SUCCESS(*status)) { + umtx_lock(&usprepMutex); + if(SHARED_DATA_HASHTABLE == NULL) { + SHARED_DATA_HASHTABLE = newCache; + ucln_common_registerCleanup(UCLN_COMMON_USPREP, usprep_cleanup); + newCache = NULL; + } + umtx_unlock(&usprepMutex); + if(newCache != NULL) { + uhash_close(newCache); + } + } + } +} + +static UBool U_CALLCONV +loadData(UStringPrepProfile* profile, + const char* path, + const char* name, + const char* type, + UErrorCode* errorCode) { + /* load Unicode SPREP data from file */ + UTrie _sprepTrie={ 0,0,0,0,0,0,0 }; + UDataMemory *dataMemory; + const int32_t *p=NULL; + const uint8_t *pb; + UVersionInfo normUnicodeVersion; + int32_t normUniVer, sprepUniVer, normCorrVer; + + if(errorCode==NULL || U_FAILURE(*errorCode)) { + return 0; + } + + /* open the data outside the mutex block */ + //TODO: change the path + dataMemory=udata_openChoice(path, type, name, isSPrepAcceptable, NULL, errorCode); + if(U_FAILURE(*errorCode)) { + return FALSE; + } + + p=(const int32_t *)udata_getMemory(dataMemory); + pb=(const uint8_t *)(p+_SPREP_INDEX_TOP); + utrie_unserialize(&_sprepTrie, pb, p[_SPREP_INDEX_TRIE_SIZE], errorCode); + _sprepTrie.getFoldingOffset=getSPrepFoldingOffset; + + + if(U_FAILURE(*errorCode)) { + udata_close(dataMemory); + return FALSE; + } + + /* in the mutex block, set the data for this process */ + umtx_lock(&usprepMutex); + if(profile->sprepData==NULL) { + profile->sprepData=dataMemory; + dataMemory=NULL; + uprv_memcpy(&profile->indexes, p, sizeof(profile->indexes)); + uprv_memcpy(&profile->sprepTrie, &_sprepTrie, sizeof(UTrie)); + } else { + p=(const int32_t *)udata_getMemory(profile->sprepData); + } + umtx_unlock(&usprepMutex); + /* initialize some variables */ + profile->mappingData=(uint16_t *)((uint8_t *)(p+_SPREP_INDEX_TOP)+profile->indexes[_SPREP_INDEX_TRIE_SIZE]); + + unorm_getUnicodeVersion(&normUnicodeVersion, errorCode); + normUniVer = (normUnicodeVersion[0] << 24) + (normUnicodeVersion[1] << 16) + + (normUnicodeVersion[2] << 8 ) + (normUnicodeVersion[3]); + sprepUniVer = (dataVersion[0] << 24) + (dataVersion[1] << 16) + + (dataVersion[2] << 8 ) + (dataVersion[3]); + normCorrVer = profile->indexes[_SPREP_NORM_CORRECTNS_LAST_UNI_VERSION]; + + if(U_FAILURE(*errorCode)){ + udata_close(dataMemory); + return FALSE; + } + if( normUniVer < sprepUniVer && /* the Unicode version of SPREP file must be less than the Unicode Vesion of the normalization data */ + normUniVer < normCorrVer && /* the Unicode version of the NormalizationCorrections.txt file should be less than the Unicode Vesion of the normalization data */ + ((profile->indexes[_SPREP_OPTIONS] & _SPREP_NORMALIZATION_ON) > 0) /* normalization turned on*/ + ){ + *errorCode = U_INVALID_FORMAT_ERROR; + udata_close(dataMemory); + return FALSE; + } + profile->isDataLoaded = TRUE; + + /* if a different thread set it first, then close the extra data */ + if(dataMemory!=NULL) { + udata_close(dataMemory); /* NULL if it was set correctly */ + } + + + return profile->isDataLoaded; +} + +static UStringPrepProfile* +usprep_getProfile(const char* path, + const char* name, + UErrorCode *status){ + + UStringPrepProfile* profile = NULL; + + initCache(status); + + if(U_FAILURE(*status)){ + return NULL; + } + + UStringPrepKey stackKey; + /* + * const is cast way to save malloc, strcpy and free calls + * we use the passed in pointers for fetching the data from the + * hash table which is safe + */ + stackKey.name = (char*) name; + stackKey.path = (char*) path; + + /* fetch the data from the cache */ + umtx_lock(&usprepMutex); + profile = (UStringPrepProfile*) (uhash_get(SHARED_DATA_HASHTABLE,&stackKey)); + umtx_unlock(&usprepMutex); + + if(profile == NULL){ + UStringPrepKey* key = (UStringPrepKey*) uprv_malloc(sizeof(UStringPrepKey)); + if(key == NULL){ + *status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + /* else load the data and put the data in the cache */ + profile = (UStringPrepProfile*) uprv_malloc(sizeof(UStringPrepProfile)); + if(profile == NULL){ + *status = U_MEMORY_ALLOCATION_ERROR; + uprv_free(key); + return NULL; + } + + /* initialize the data struct members */ + uprv_memset(profile->indexes,0,sizeof(profile->indexes)); + profile->mappingData = NULL; + profile->sprepData = NULL; + profile->refCount = 0; + + /* initialize the key memebers */ + key->name = (char*) uprv_malloc(uprv_strlen(name)+1); + if(key->name == NULL){ + *status = U_MEMORY_ALLOCATION_ERROR; + uprv_free(key); + uprv_free(profile); + return NULL; + } + + uprv_strcpy(key->name, name); + + key->path=NULL; + + if(path != NULL){ + key->path = (char*) uprv_malloc(uprv_strlen(path)+1); + if(key->path == NULL){ + *status = U_MEMORY_ALLOCATION_ERROR; + uprv_free(key->name); + uprv_free(key); + uprv_free(profile); + return NULL; + } + uprv_strcpy(key->path, path); + } + + /* load the data */ + if(!loadData(profile, path, name, _SPREP_DATA_TYPE, status) || U_FAILURE(*status) ){ + uprv_free(key->path); + uprv_free(key->name); + uprv_free(key); + uprv_free(profile); + return NULL; + } + + /* get the options */ + profile->doNFKC = (UBool)((profile->indexes[_SPREP_OPTIONS] & _SPREP_NORMALIZATION_ON) > 0); + profile->checkBiDi = (UBool)((profile->indexes[_SPREP_OPTIONS] & _SPREP_CHECK_BIDI_ON) > 0); + + if(profile->checkBiDi) { + profile->bdp = ubidi_getSingleton(status); + if(U_FAILURE(*status)) { + usprep_unload(profile); + uprv_free(key->path); + uprv_free(key->name); + uprv_free(key); + uprv_free(profile); + return NULL; + } + } else { + profile->bdp = NULL; + } + + umtx_lock(&usprepMutex); + /* add the data object to the cache */ + uhash_put(SHARED_DATA_HASHTABLE, key, profile, status); + umtx_unlock(&usprepMutex); + } + umtx_lock(&usprepMutex); + /* increment the refcount */ + profile->refCount++; + umtx_unlock(&usprepMutex); + + return profile; +} + +U_CAPI UStringPrepProfile* U_EXPORT2 +usprep_open(const char* path, + const char* name, + UErrorCode* status){ + + if(status == NULL || U_FAILURE(*status)){ + return NULL; + } + /* initialize the mutex */ + usprep_init(); + + /* initialize the profile struct members */ + return usprep_getProfile(path,name,status); +} + +U_CAPI void U_EXPORT2 +usprep_close(UStringPrepProfile* profile){ + if(profile==NULL){ + return; + } + + umtx_lock(&usprepMutex); + /* decrement the ref count*/ + if(profile->refCount > 0){ + profile->refCount--; + } + umtx_unlock(&usprepMutex); + +} + +U_CFUNC void +uprv_syntaxError(const UChar* rules, + int32_t pos, + int32_t rulesLen, + UParseError* parseError){ + if(parseError == NULL){ + return; + } + parseError->offset = pos; + parseError->line = 0 ; // we are not using line numbers + + // for pre-context + int32_t start = (pos < U_PARSE_CONTEXT_LEN)? 0 : (pos - (U_PARSE_CONTEXT_LEN-1)); + int32_t limit = pos; + + u_memcpy(parseError->preContext,rules+start,limit-start); + //null terminate the buffer + parseError->preContext[limit-start] = 0; + + // for post-context; include error rules[pos] + start = pos; + limit = start + (U_PARSE_CONTEXT_LEN-1); + if (limit > rulesLen) { + limit = rulesLen; + } + if (start < rulesLen) { + u_memcpy(parseError->postContext,rules+start,limit-start); + } + //null terminate the buffer + parseError->postContext[limit-start]= 0; +} + + +static inline UStringPrepType +getValues(uint16_t trieWord, int16_t& value, UBool& isIndex){ + + UStringPrepType type; + if(trieWord == 0){ + /* + * Initial value stored in the mapping table + * just return USPREP_TYPE_LIMIT .. so that + * the source codepoint is copied to the destination + */ + type = USPREP_TYPE_LIMIT; + isIndex =FALSE; + value = 0; + }else if(trieWord >= _SPREP_TYPE_THRESHOLD){ + type = (UStringPrepType) (trieWord - _SPREP_TYPE_THRESHOLD); + isIndex =FALSE; + value = 0; + }else{ + /* get the type */ + type = USPREP_MAP; + /* ascertain if the value is index or delta */ + if(trieWord & 0x02){ + isIndex = TRUE; + value = trieWord >> 2; //mask off the lower 2 bits and shift + }else{ + isIndex = FALSE; + value = (int16_t)trieWord; + value = (value >> 2); + } + + if((trieWord>>2) == _SPREP_MAX_INDEX_VALUE){ + type = USPREP_DELETE; + isIndex =FALSE; + value = 0; + } + } + return type; +} + + + +static int32_t +usprep_map( const UStringPrepProfile* profile, + const UChar* src, int32_t srcLength, + UChar* dest, int32_t destCapacity, + int32_t options, + UParseError* parseError, + UErrorCode* status ){ + + uint16_t result; + int32_t destIndex=0; + int32_t srcIndex; + UBool allowUnassigned = (UBool) ((options & USPREP_ALLOW_UNASSIGNED)>0); + UStringPrepType type; + int16_t value; + UBool isIndex; + const int32_t* indexes = profile->indexes; + + // no error checking the caller check for error and arguments + // no string length check the caller finds out the string length + + for(srcIndex=0;srcIndexsprepTrie,ch,result); + + type = getValues(result, value, isIndex); + + // check if the source codepoint is unassigned + if(type == USPREP_UNASSIGNED && allowUnassigned == FALSE){ + + uprv_syntaxError(src,srcIndex-U16_LENGTH(ch), srcLength,parseError); + *status = U_STRINGPREP_UNASSIGNED_ERROR; + return 0; + + }else if(type == USPREP_MAP){ + + int32_t index, length; + + if(isIndex){ + index = value; + if(index >= indexes[_SPREP_ONE_UCHAR_MAPPING_INDEX_START] && + index < indexes[_SPREP_TWO_UCHARS_MAPPING_INDEX_START]){ + length = 1; + }else if(index >= indexes[_SPREP_TWO_UCHARS_MAPPING_INDEX_START] && + index < indexes[_SPREP_THREE_UCHARS_MAPPING_INDEX_START]){ + length = 2; + }else if(index >= indexes[_SPREP_THREE_UCHARS_MAPPING_INDEX_START] && + index < indexes[_SPREP_FOUR_UCHARS_MAPPING_INDEX_START]){ + length = 3; + }else{ + length = profile->mappingData[index++]; + + } + + /* copy mapping to destination */ + for(int32_t i=0; i< length; i++){ + if(destIndex < destCapacity ){ + dest[destIndex] = profile->mappingData[index+i]; + } + destIndex++; /* for pre-flighting */ + } + continue; + }else{ + // subtract the delta to arrive at the code point + ch -= value; + } + + }else if(type==USPREP_DELETE){ + // just consume the codepoint and contine + continue; + } + //copy the code point into destination + if(ch <= 0xFFFF){ + if(destIndex < destCapacity ){ + dest[destIndex] = (UChar)ch; + } + destIndex++; + }else{ + if(destIndex+1 < destCapacity ){ + dest[destIndex] = U16_LEAD(ch); + dest[destIndex+1] = U16_TRAIL(ch); + } + destIndex +=2; + } + + } + + return u_terminateUChars(dest, destCapacity, destIndex, status); +} + + +static int32_t +usprep_normalize( const UChar* src, int32_t srcLength, + UChar* dest, int32_t destCapacity, + UErrorCode* status ){ + /* + * Option UNORM_BEFORE_PRI_29: + * + * IDNA as interpreted by IETF members (see unicode mailing list 2004H1) + * requires strict adherence to Unicode 3.2 normalization, + * including buggy composition from before fixing Public Review Issue #29. + * Note that this results in some valid but nonsensical text to be + * either corrupted or rejected, depending on the text. + * See http://www.unicode.org/review/resolved-pri.html#pri29 + * See unorm.cpp and cnormtst.c + */ + return unorm_normalize( + src, srcLength, + UNORM_NFKC, UNORM_UNICODE_3_2|UNORM_BEFORE_PRI_29, + dest, destCapacity, + status); +} + + + /* + 1) Map -- For each character in the input, check if it has a mapping + and, if so, replace it with its mapping. + + 2) Normalize -- Possibly normalize the result of step 1 using Unicode + normalization. + + 3) Prohibit -- Check for any characters that are not allowed in the + output. If any are found, return an error. + + 4) Check bidi -- Possibly check for right-to-left characters, and if + any are found, make sure that the whole string satisfies the + requirements for bidirectional strings. If the string does not + satisfy the requirements for bidirectional strings, return an + error. + [Unicode3.2] defines several bidirectional categories; each character + has one bidirectional category assigned to it. For the purposes of + the requirements below, an "RandALCat character" is a character that + has Unicode bidirectional categories "R" or "AL"; an "LCat character" + is a character that has Unicode bidirectional category "L". Note + + + that there are many characters which fall in neither of the above + definitions; Latin digits ( through ) are examples of + this because they have bidirectional category "EN". + + In any profile that specifies bidirectional character handling, all + three of the following requirements MUST be met: + + 1) The characters in section 5.8 MUST be prohibited. + + 2) If a string contains any RandALCat character, the string MUST NOT + contain any LCat character. + + 3) If a string contains any RandALCat character, a RandALCat + character MUST be the first character of the string, and a + RandALCat character MUST be the last character of the string. +*/ + +#define MAX_STACK_BUFFER_SIZE 300 + + +U_CAPI int32_t U_EXPORT2 +usprep_prepare( const UStringPrepProfile* profile, + const UChar* src, int32_t srcLength, + UChar* dest, int32_t destCapacity, + int32_t options, + UParseError* parseError, + UErrorCode* status ){ + + // check error status + if(status == NULL || U_FAILURE(*status)){ + return 0; + } + + //check arguments + if(profile==NULL || src==NULL || srcLength<-1 || (dest==NULL && destCapacity!=0)) { + *status=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + UChar b1Stack[MAX_STACK_BUFFER_SIZE], b2Stack[MAX_STACK_BUFFER_SIZE]; + UChar *b1 = b1Stack, *b2 = b2Stack; + int32_t b1Len, b2Len=0, + b1Capacity = MAX_STACK_BUFFER_SIZE , + b2Capacity = MAX_STACK_BUFFER_SIZE; + uint16_t result; + int32_t b2Index = 0; + UCharDirection direction=U_CHAR_DIRECTION_COUNT, firstCharDir=U_CHAR_DIRECTION_COUNT; + UBool leftToRight=FALSE, rightToLeft=FALSE; + int32_t rtlPos =-1, ltrPos =-1; + + //get the string length + if(srcLength == -1){ + srcLength = u_strlen(src); + } + // map + b1Len = usprep_map(profile, src, srcLength, b1, b1Capacity, options, parseError, status); + + if(*status == U_BUFFER_OVERFLOW_ERROR){ + // redo processing of string + /* we do not have enough room so grow the buffer*/ + b1 = (UChar*) uprv_malloc(b1Len * U_SIZEOF_UCHAR); + if(b1==NULL){ + *status = U_MEMORY_ALLOCATION_ERROR; + goto CLEANUP; + } + + *status = U_ZERO_ERROR; // reset error + + b1Len = usprep_map(profile, src, srcLength, b1, b1Len, options, parseError, status); + + } + + // normalize + if(profile->doNFKC == TRUE){ + b2Len = usprep_normalize(b1,b1Len, b2,b2Capacity,status); + + if(*status == U_BUFFER_OVERFLOW_ERROR){ + // redo processing of string + /* we do not have enough room so grow the buffer*/ + b2 = (UChar*) uprv_malloc(b2Len * U_SIZEOF_UCHAR); + if(b2==NULL){ + *status = U_MEMORY_ALLOCATION_ERROR; + goto CLEANUP; + } + + *status = U_ZERO_ERROR; // reset error + + b2Len = usprep_normalize(b1,b1Len, b2,b2Len,status); + + } + + }else{ + b2 = b1; + b2Len = b1Len; + } + + + if(U_FAILURE(*status)){ + goto CLEANUP; + } + + UChar32 ch; + UStringPrepType type; + int16_t value; + UBool isIndex; + + // Prohibit and checkBiDi in one pass + for(b2Index=0; b2IndexsprepTrie,ch,result); + + type = getValues(result, value, isIndex); + + if( type == USPREP_PROHIBITED || + ((result < _SPREP_TYPE_THRESHOLD) && (result & 0x01) /* first bit says it the code point is prohibited*/) + ){ + *status = U_STRINGPREP_PROHIBITED_ERROR; + uprv_syntaxError(b1, b2Index-U16_LENGTH(ch), b2Len, parseError); + goto CLEANUP; + } + + if(profile->checkBiDi) { + direction = ubidi_getClass(profile->bdp, ch); + if(firstCharDir == U_CHAR_DIRECTION_COUNT){ + firstCharDir = direction; + } + if(direction == U_LEFT_TO_RIGHT){ + leftToRight = TRUE; + ltrPos = b2Index-1; + } + if(direction == U_RIGHT_TO_LEFT || direction == U_RIGHT_TO_LEFT_ARABIC){ + rightToLeft = TRUE; + rtlPos = b2Index-1; + } + } + } + if(profile->checkBiDi == TRUE){ + // satisfy 2 + if( leftToRight == TRUE && rightToLeft == TRUE){ + *status = U_STRINGPREP_CHECK_BIDI_ERROR; + uprv_syntaxError(b2,(rtlPos>ltrPos) ? rtlPos : ltrPos, b2Len, parseError); + goto CLEANUP; + } + + //satisfy 3 + if( rightToLeft == TRUE && + !((firstCharDir == U_RIGHT_TO_LEFT || firstCharDir == U_RIGHT_TO_LEFT_ARABIC) && + (direction == U_RIGHT_TO_LEFT || direction == U_RIGHT_TO_LEFT_ARABIC)) + ){ + *status = U_STRINGPREP_CHECK_BIDI_ERROR; + uprv_syntaxError(b2, rtlPos, b2Len, parseError); + return FALSE; + } + } + if(b2Len>0 && b2Len <= destCapacity){ + uprv_memmove(dest,b2, b2Len*U_SIZEOF_UCHAR); + } + +CLEANUP: + if(b1!=b1Stack){ + uprv_free(b1); + b1=NULL; + } + + if(b2!=b1Stack && b2!=b2Stack && b2!=b1 /* b1 should not be freed twice */){ + uprv_free(b2); + b2=NULL; + } + return u_terminateUChars(dest, destCapacity, b2Len, status); +} + + +/* data swapping ------------------------------------------------------------ */ + +U_CAPI int32_t U_EXPORT2 +usprep_swap(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode) { + const UDataInfo *pInfo; + int32_t headerSize; + + const uint8_t *inBytes; + uint8_t *outBytes; + + const int32_t *inIndexes; + int32_t indexes[16]; + + int32_t i, offset, count, size; + + /* udata_swapDataHeader checks the arguments */ + headerSize=udata_swapDataHeader(ds, inData, length, outData, pErrorCode); + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + /* check data format and format version */ + pInfo=(const UDataInfo *)((const char *)inData+4); + if(!( + pInfo->dataFormat[0]==0x53 && /* dataFormat="SPRP" */ + pInfo->dataFormat[1]==0x50 && + pInfo->dataFormat[2]==0x52 && + pInfo->dataFormat[3]==0x50 && + pInfo->formatVersion[0]==3 + )) { + udata_printError(ds, "usprep_swap(): data format %02x.%02x.%02x.%02x (format version %02x) is not recognized as StringPrep .spp data\n", + pInfo->dataFormat[0], pInfo->dataFormat[1], + pInfo->dataFormat[2], pInfo->dataFormat[3], + pInfo->formatVersion[0]); + *pErrorCode=U_UNSUPPORTED_ERROR; + return 0; + } + + inBytes=(const uint8_t *)inData+headerSize; + outBytes=(uint8_t *)outData+headerSize; + + inIndexes=(const int32_t *)inBytes; + + if(length>=0) { + length-=headerSize; + if(length<16*4) { + udata_printError(ds, "usprep_swap(): too few bytes (%d after header) for StringPrep .spp data\n", + length); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + } + + /* read the first 16 indexes (ICU 2.8/format version 3: _SPREP_INDEX_TOP==16, might grow) */ + for(i=0; i<16; ++i) { + indexes[i]=udata_readInt32(ds, inIndexes[i]); + } + + /* calculate the total length of the data */ + size= + 16*4+ /* size of indexes[] */ + indexes[_SPREP_INDEX_TRIE_SIZE]+ + indexes[_SPREP_INDEX_MAPPING_DATA_SIZE]; + + if(length>=0) { + if(lengthswapArray32(ds, inBytes, count, outBytes, pErrorCode); + offset+=count; + + /* swap the UTrie */ + count=indexes[_SPREP_INDEX_TRIE_SIZE]; + utrie_swap(ds, inBytes+offset, count, outBytes+offset, pErrorCode); + offset+=count; + + /* swap the uint16_t mappingTable[] */ + count=indexes[_SPREP_INDEX_MAPPING_DATA_SIZE]; + ds->swapArray16(ds, inBytes+offset, count, outBytes+offset, pErrorCode); + offset+=count; + } + + return headerSize+size; +} + +#endif /* #if !UCONFIG_NO_IDNA */ diff --git a/Build/source/libs/icu/icu-xetex/common/ustack.cpp b/Build/source/libs/icu/icu-xetex/common/ustack.cpp new file mode 100644 index 00000000000..76118bce99a --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ustack.cpp @@ -0,0 +1,61 @@ +/* +********************************************************************** +* Copyright (C) 2003-2004, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +*/ + +#include "uvector.h" + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(UStack) + +UStack::UStack(UErrorCode &status) : + UVector(status) +{ +} + +UStack::UStack(int32_t initialCapacity, UErrorCode &status) : + UVector(initialCapacity, status) +{ +} + +UStack::UStack(UObjectDeleter *d, UKeyComparator *c, UErrorCode &status) : + UVector(d, c, status) +{ +} + +UStack::UStack(UObjectDeleter *d, UKeyComparator *c, int32_t initialCapacity, UErrorCode &status) : + UVector(d, c, initialCapacity, status) +{ +} + +UStack::~UStack() {} + +void* UStack::pop(void) { + int32_t n = size() - 1; + void* result = 0; + if (n >= 0) { + result = elementAt(n); + removeElementAt(n); + } + return result; +} + +int32_t UStack::popi(void) { + int32_t n = size() - 1; + int32_t result = 0; + if (n >= 0) { + result = elementAti(n); + removeElementAt(n); + } + return result; +} + +int32_t UStack::search(void* obj) const { + int32_t i = indexOf(obj); + return (i >= 0) ? size() - i : i; +} + +U_NAMESPACE_END diff --git a/Build/source/libs/icu/icu-xetex/common/ustr_cnv.c b/Build/source/libs/icu/icu-xetex/common/ustr_cnv.c new file mode 100644 index 00000000000..05d10258658 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ustr_cnv.c @@ -0,0 +1,230 @@ +/* +******************************************************************************* +* +* Copyright (C) 1998-2004, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: ustr_cnv.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2004aug24 +* created by: Markus W. Scherer +* +* Character conversion functions moved here from ustring.c +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "unicode/ustring.h" +#include "unicode/ucnv.h" +#include "cstring.h" +#include "cmemory.h" +#include "umutex.h" +#include "ustr_cnv.h" + +/* mutexed access to a shared default converter ----------------------------- */ + +static UConverter *gDefaultConverter = NULL; + +U_CAPI UConverter* U_EXPORT2 +u_getDefaultConverter(UErrorCode *status) +{ + UConverter *converter = NULL; + + if (gDefaultConverter != NULL) { + umtx_lock(NULL); + + /* need to check to make sure it wasn't taken out from under us */ + if (gDefaultConverter != NULL) { + converter = gDefaultConverter; + gDefaultConverter = NULL; + } + umtx_unlock(NULL); + } + + /* if the cache was empty, create a converter */ + if(converter == NULL) { + converter = ucnv_open(NULL, status); + if(U_FAILURE(*status)) { + ucnv_close(converter); + converter = NULL; + } + } + + return converter; +} + +U_CAPI void U_EXPORT2 +u_releaseDefaultConverter(UConverter *converter) +{ + if(gDefaultConverter == NULL) { + if (converter != NULL) { + ucnv_reset(converter); + } + umtx_lock(NULL); + + if(gDefaultConverter == NULL) { + gDefaultConverter = converter; + converter = NULL; + } + umtx_unlock(NULL); + } + + if(converter != NULL) { + ucnv_close(converter); + } +} + +/* conversions between char* and UChar* ------------------------------------- */ + +/* maximum string length for u_uastrcpy() and u_austrcpy() implementations */ +#define MAX_STRLEN 0x0FFFFFFF + +/* + returns the minimum of (the length of the null-terminated string) and n. +*/ +static int32_t u_astrnlen(const char *s1, int32_t n) +{ + int32_t len = 0; + + if (s1) + { + while (n-- && *(s1++)) + { + len++; + } + } + return len; +} + +U_CAPI UChar* U_EXPORT2 +u_uastrncpy(UChar *ucs1, + const char *s2, + int32_t n) +{ + UChar *target = ucs1; + UErrorCode err = U_ZERO_ERROR; + UConverter *cnv = u_getDefaultConverter(&err); + if(U_SUCCESS(err) && cnv != NULL) { + ucnv_reset(cnv); + ucnv_toUnicode(cnv, + &target, + ucs1+n, + &s2, + s2+u_astrnlen(s2, n), + NULL, + TRUE, + &err); + ucnv_reset(cnv); /* be good citizens */ + u_releaseDefaultConverter(cnv); + if(U_FAILURE(err) && (err != U_BUFFER_OVERFLOW_ERROR) ) { + *ucs1 = 0; /* failure */ + } + if(target < (ucs1+n)) { /* U_BUFFER_OVERFLOW_ERROR isn't an err, just means no termination will happen. */ + *target = 0; /* terminate */ + } + } else { + *ucs1 = 0; + } + return ucs1; +} + +U_CAPI UChar* U_EXPORT2 +u_uastrcpy(UChar *ucs1, + const char *s2 ) +{ + UErrorCode err = U_ZERO_ERROR; + UConverter *cnv = u_getDefaultConverter(&err); + if(U_SUCCESS(err) && cnv != NULL) { + ucnv_toUChars(cnv, + ucs1, + MAX_STRLEN, + s2, + (int32_t)uprv_strlen(s2), + &err); + u_releaseDefaultConverter(cnv); + if(U_FAILURE(err)) { + *ucs1 = 0; + } + } else { + *ucs1 = 0; + } + return ucs1; +} + +/* + returns the minimum of (the length of the null-terminated string) and n. +*/ +static int32_t u_ustrnlen(const UChar *ucs1, int32_t n) +{ + int32_t len = 0; + + if (ucs1) + { + while (n-- && *(ucs1++)) + { + len++; + } + } + return len; +} + +U_CAPI char* U_EXPORT2 +u_austrncpy(char *s1, + const UChar *ucs2, + int32_t n) +{ + char *target = s1; + UErrorCode err = U_ZERO_ERROR; + UConverter *cnv = u_getDefaultConverter(&err); + if(U_SUCCESS(err) && cnv != NULL) { + ucnv_reset(cnv); + ucnv_fromUnicode(cnv, + &target, + s1+n, + &ucs2, + ucs2+u_ustrnlen(ucs2, n), + NULL, + TRUE, + &err); + ucnv_reset(cnv); /* be good citizens */ + u_releaseDefaultConverter(cnv); + if(U_FAILURE(err) && (err != U_BUFFER_OVERFLOW_ERROR) ) { + *s1 = 0; /* failure */ + } + if(target < (s1+n)) { /* U_BUFFER_OVERFLOW_ERROR isn't an err, just means no termination will happen. */ + *target = 0; /* terminate */ + } + } else { + *s1 = 0; + } + return s1; +} + +U_CAPI char* U_EXPORT2 +u_austrcpy(char *s1, + const UChar *ucs2 ) +{ + UErrorCode err = U_ZERO_ERROR; + UConverter *cnv = u_getDefaultConverter(&err); + if(U_SUCCESS(err) && cnv != NULL) { + int32_t len = ucnv_fromUChars(cnv, + s1, + MAX_STRLEN, + ucs2, + -1, + &err); + u_releaseDefaultConverter(cnv); + s1[len] = 0; + } else { + *s1 = 0; + } + return s1; +} + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/ustr_cnv.h b/Build/source/libs/icu/icu-xetex/common/ustr_cnv.h new file mode 100644 index 00000000000..942b4c64179 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ustr_cnv.h @@ -0,0 +1,42 @@ +/* +********************************************************************** +* Copyright (C) 1999-2004, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* file name: ustr_cnv.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2004Aug27 +* created by: George Rhoten +*/ + +#ifndef USTR_CNV_IMP_H +#define USTR_CNV_IMP_H + +#include "unicode/utypes.h" +#include "unicode/ucnv.h" + +#if !UCONFIG_NO_CONVERSION + +/** + * Get the default converter. This is a commonly used converter + * that is used for the ustring and UnicodeString API. + * Remember to use the u_releaseDefaultConverter when you are done. + * @internal + */ +U_CAPI UConverter* U_EXPORT2 +u_getDefaultConverter(UErrorCode *status); + + +/** + * Release the default converter to the converter cache. + * @internal + */ +U_CAPI void U_EXPORT2 +u_releaseDefaultConverter(UConverter *converter); + +#endif + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/ustr_imp.h b/Build/source/libs/icu/icu-xetex/common/ustr_imp.h new file mode 100644 index 00000000000..7c6681682cc --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ustr_imp.h @@ -0,0 +1,211 @@ +/* +********************************************************************** +* Copyright (C) 1999-2007, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* file name: ustr_imp.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2001jan30 +* created by: Markus W. Scherer +*/ + +#ifndef __USTR_IMP_H__ +#define __USTR_IMP_H__ + +#include "unicode/utypes.h" +#include "unicode/uiter.h" +#include "ucase.h" + +/** Simple declaration for u_strToTitle() to avoid including unicode/ubrk.h. */ +#ifndef UBRK_TYPEDEF_UBREAK_ITERATOR +# define UBRK_TYPEDEF_UBREAK_ITERATOR + typedef void UBreakIterator; +#endif + +/** + * Compare two strings in code point order or code unit order. + * Works in strcmp style (both lengths -1), + * strncmp style (lengths equal and >=0, flag TRUE), + * and memcmp/UnicodeString style (at least one length >=0). + * @internal + */ +U_CAPI int32_t U_EXPORT2 +uprv_strCompare(const UChar *s1, int32_t length1, + const UChar *s2, int32_t length2, + UBool strncmpStyle, UBool codePointOrder); + +/** + * Internal API, used by u_strcasecmp() etc. + * Compare strings case-insensitively, + * in code point order or code unit order. + * @internal + */ +U_CFUNC int32_t +u_strcmpFold(const UChar *s1, int32_t length1, + const UChar *s2, int32_t length2, + uint32_t options, + UErrorCode *pErrorCode); + +/** + * Are the Unicode properties loaded? + * This must be used before internal functions are called that do + * not perform this check. + * Generate a debug assertion failure if data is not loaded, to flag the fact + * that u_init() wasn't called first, before trying to access character properties. + * @internal + */ +U_CFUNC UBool +uprv_haveProperties(UErrorCode *pErrorCode); + +/** + * Load the Unicode property data. + * Intended primarily for use from u_init(). + * Has no effect if property data is already loaded. + * NOT thread safe. + * @internal + */ +/*U_CFUNC int8_t +uprv_loadPropsData(UErrorCode *errorCode);*/ + +/** + * Type of a function that may be passed to the internal case mapping functions + * or similar for growing the destination buffer. + * @internal + */ +typedef UBool U_CALLCONV +UGrowBuffer(void *context, /* opaque pointer for this function */ + UChar **pBuffer, /* in/out destination buffer pointer */ + int32_t *pCapacity, /* in/out buffer capacity in numbers of UChars */ + int32_t reqCapacity,/* requested capacity */ + int32_t length); /* number of UChars to be copied to new buffer */ + +/** + * Default implementation of UGrowBuffer. + * Takes a static buffer as context, allocates a new buffer, + * and releases the old one if it is not the same as the one passed as context. + * @internal + */ +U_CAPI UBool /* U_CALLCONV U_EXPORT2 */ +u_growBufferFromStatic(void *context, + UChar **pBuffer, int32_t *pCapacity, int32_t reqCapacity, + int32_t length); + +/* + * Internal string casing functions implementing + * ustring.h/ustrcase.c and UnicodeString case mapping functions. + */ + +/** + * @internal + */ +struct UCaseMap { + const UCaseProps *csp; +#if !UCONFIG_NO_BREAK_ITERATION + UBreakIterator *iter; /* We adopt the iterator, so we own it. */ +#endif + char locale[32]; + int32_t locCache; + uint32_t options; +}; + +#ifndef __UCASEMAP_H__ +typedef struct UCaseMap UCaseMap; +#endif + +/** + * @internal + */ +enum { + TO_LOWER, + TO_UPPER, + TO_TITLE, + FOLD_CASE +}; + +/** + * @internal + */ +U_CFUNC int32_t +ustr_toLower(const UCaseProps *csp, + UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + const char *locale, + UErrorCode *pErrorCode); + +/** + * @internal + */ +U_CFUNC int32_t +ustr_toUpper(const UCaseProps *csp, + UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + const char *locale, + UErrorCode *pErrorCode); + +#if !UCONFIG_NO_BREAK_ITERATION + +/** + * @internal + */ +U_CFUNC int32_t +ustr_toTitle(const UCaseProps *csp, + UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + UBreakIterator *titleIter, + const char *locale, uint32_t options, + UErrorCode *pErrorCode); + +#endif + +/** + * Internal case folding function. + * @internal + */ +U_CFUNC int32_t +ustr_foldCase(const UCaseProps *csp, + UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + uint32_t options, + UErrorCode *pErrorCode); + +/** + * NUL-terminate a UChar * string if possible. + * If length < destCapacity then NUL-terminate. + * If length == destCapacity then do not terminate but set U_STRING_NOT_TERMINATED_WARNING. + * If length > destCapacity then do not terminate but set U_BUFFER_OVERFLOW_ERROR. + * + * @param dest Destination buffer, can be NULL if destCapacity==0. + * @param destCapacity Number of UChars available at dest. + * @param length Number of UChars that were (to be) written to dest. + * @param pErrorCode ICU error code. + * @return length + * @internal + */ +U_CAPI int32_t U_EXPORT2 +u_terminateUChars(UChar *dest, int32_t destCapacity, int32_t length, UErrorCode *pErrorCode); + +/** + * NUL-terminate a char * string if possible. + * Same as u_terminateUChars() but for a different string type. + */ +U_CAPI int32_t U_EXPORT2 +u_terminateChars(char *dest, int32_t destCapacity, int32_t length, UErrorCode *pErrorCode); + +/** + * NUL-terminate a UChar32 * string if possible. + * Same as u_terminateUChars() but for a different string type. + */ +U_CAPI int32_t U_EXPORT2 +u_terminateUChar32s(UChar32 *dest, int32_t destCapacity, int32_t length, UErrorCode *pErrorCode); + +/** + * NUL-terminate a wchar_t * string if possible. + * Same as u_terminateUChars() but for a different string type. + */ +U_CAPI int32_t U_EXPORT2 +u_terminateWChars(wchar_t *dest, int32_t destCapacity, int32_t length, UErrorCode *pErrorCode); + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/ustr_wcs.c b/Build/source/libs/icu/icu-xetex/common/ustr_wcs.c new file mode 100644 index 00000000000..3b0e4b08452 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ustr_wcs.c @@ -0,0 +1,522 @@ +/* +******************************************************************************* +* +* Copyright (C) 2001-2006, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: ustr_wcs.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2004sep07 +* created by: Markus W. Scherer +* +* u_strToWCS() and u_strFromWCS() functions +* moved here from ustrtrns.c for better modularization. +*/ + +#include "unicode/utypes.h" +#include "unicode/ustring.h" +#include "cstring.h" +#include "cwchar.h" +#include "cmemory.h" +#include "ustr_imp.h" +#include "ustr_cnv.h" + +#if defined(U_WCHAR_IS_UTF16) || defined(U_WCHAR_IS_UTF32) || !UCONFIG_NO_CONVERSION + +#define _STACK_BUFFER_CAPACITY 1000 +#define _BUFFER_CAPACITY_MULTIPLIER 2 + +#if !defined(U_WCHAR_IS_UTF16) && !defined(U_WCHAR_IS_UTF32) +static U_INLINE UBool +u_growAnyBufferFromStatic(void *context, + void **pBuffer, int32_t *pCapacity, int32_t reqCapacity, + int32_t length, int32_t size) { + + void *newBuffer=uprv_malloc(reqCapacity*size); + if(newBuffer!=NULL) { + if(length>0) { + uprv_memcpy(newBuffer, *pBuffer, length*size); + } + *pCapacity=reqCapacity; + } else { + *pCapacity=0; + } + + /* release the old pBuffer if it was not statically allocated */ + if(*pBuffer!=(void *)context) { + uprv_free(*pBuffer); + } + + *pBuffer=newBuffer; + return (UBool)(newBuffer!=NULL); +} + +/* helper function */ +static wchar_t* +_strToWCS(wchar_t *dest, + int32_t destCapacity, + int32_t *pDestLength, + const UChar *src, + int32_t srcLength, + UErrorCode *pErrorCode){ + + char stackBuffer [_STACK_BUFFER_CAPACITY]; + char* tempBuf = stackBuffer; + int32_t tempBufCapacity = _STACK_BUFFER_CAPACITY; + char* tempBufLimit = stackBuffer + tempBufCapacity; + UConverter* conv = NULL; + char* saveBuf = tempBuf; + wchar_t* intTarget=NULL; + int32_t intTargetCapacity=0; + int count=0,retVal=0; + + const UChar *pSrcLimit =NULL; + const UChar *pSrc = src; + + conv = u_getDefaultConverter(pErrorCode); + + if(U_FAILURE(*pErrorCode)){ + return NULL; + } + + if(srcLength == -1){ + srcLength = u_strlen(pSrc); + } + + pSrcLimit = pSrc + srcLength; + + for(;;) { + /* reset the error state */ + *pErrorCode = U_ZERO_ERROR; + + /* convert to chars using default converter */ + ucnv_fromUnicode(conv,&tempBuf,tempBufLimit,&pSrc,pSrcLimit,NULL,(UBool)(pSrc==pSrcLimit),pErrorCode); + count =(tempBuf - saveBuf); + + /* This should rarely occur */ + if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR){ + tempBuf = saveBuf; + + /* we dont have enough room on the stack grow the buffer */ + if(!u_growAnyBufferFromStatic(stackBuffer,(void**) &tempBuf, &tempBufCapacity, + (_BUFFER_CAPACITY_MULTIPLIER * (srcLength)), count,sizeof(char))){ + goto cleanup; + } + + saveBuf = tempBuf; + tempBufLimit = tempBuf + tempBufCapacity; + tempBuf = tempBuf + count; + + } else { + break; + } + } + + if(U_FAILURE(*pErrorCode)){ + goto cleanup; + } + + /* done with conversion null terminate the char buffer */ + if(count>=tempBufCapacity){ + tempBuf = saveBuf; + /* we dont have enough room on the stack grow the buffer */ + if(!u_growAnyBufferFromStatic(stackBuffer,(void**) &tempBuf, &tempBufCapacity, + tempBufCapacity-count+1, count,sizeof(char))){ + goto cleanup; + } + saveBuf = tempBuf; + } + + saveBuf[count]=0; + + + /* allocate more space than required + * here we assume that every char requires + * no more than 2 wchar_ts + */ + intTargetCapacity = (count * _BUFFER_CAPACITY_MULTIPLIER + 1) /*for null termination */; + intTarget = (wchar_t*)uprv_malloc( intTargetCapacity * sizeof(wchar_t) ); + + if(intTarget){ + + int32_t nulLen = 0; + int32_t remaining = intTargetCapacity; + wchar_t* pIntTarget=intTarget; + tempBuf = saveBuf; + + /* now convert the mbs to wcs */ + for(;;){ + + /* we can call the system API since we are sure that + * there is atleast 1 null in the input + */ + retVal = uprv_mbstowcs(pIntTarget,(tempBuf+nulLen),remaining); + + if(retVal==-1){ + *pErrorCode = U_INVALID_CHAR_FOUND; + break; + }else if(retVal== remaining){/* should never occur */ + int numWritten = (pIntTarget-intTarget); + u_growAnyBufferFromStatic(NULL,(void**) &intTarget, + &intTargetCapacity, + intTargetCapacity * _BUFFER_CAPACITY_MULTIPLIER, + numWritten, + sizeof(wchar_t)); + pIntTarget = intTarget; + remaining=intTargetCapacity; + + if(nulLen!=count){ /*there are embedded nulls*/ + pIntTarget+=numWritten; + remaining-=numWritten; + } + + }else{ + int32_t nulVal; + /*scan for nulls */ + /* we donot check for limit since tempBuf is null terminated */ + while(tempBuf[nulLen++] != 0){ + } + nulVal = (nulLen < srcLength) ? 1 : 0; + pIntTarget = pIntTarget + retVal+nulVal; + remaining -=(retVal+nulVal); + + /* check if we have reached the source limit*/ + if(nulLen>=(count)){ + break; + } + } + } + count = (int32_t)(pIntTarget-intTarget); + + if(0 < count && count <= destCapacity){ + uprv_memcpy(dest,intTarget,count*sizeof(wchar_t)); + } + + if(pDestLength){ + *pDestLength = count; + } + + /* free the allocated memory */ + uprv_free(intTarget); + + }else{ + *pErrorCode = U_MEMORY_ALLOCATION_ERROR; + } +cleanup: + /* are we still using stack buffer */ + if(stackBuffer != saveBuf){ + uprv_free(saveBuf); + } + u_terminateWChars(dest,destCapacity,count,pErrorCode); + + u_releaseDefaultConverter(conv); + + return dest; +} +#endif + +U_CAPI wchar_t* U_EXPORT2 +u_strToWCS(wchar_t *dest, + int32_t destCapacity, + int32_t *pDestLength, + const UChar *src, + int32_t srcLength, + UErrorCode *pErrorCode){ + + /* args check */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)){ + return NULL; + } + + if((src==NULL) || (srcLength < -1) || (destCapacity<0) || (!dest && destCapacity > 0)){ + *pErrorCode = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + +#ifdef U_WCHAR_IS_UTF16 + /* wchar_t is UTF-16 just do a memcpy */ + if(srcLength == -1){ + srcLength = u_strlen(src); + } + if(0 < srcLength && srcLength <= destCapacity){ + uprv_memcpy(dest,src,srcLength*U_SIZEOF_UCHAR); + } + if(pDestLength){ + *pDestLength = srcLength; + } + + u_terminateUChars(dest,destCapacity,srcLength,pErrorCode); + + return dest; + +#elif defined U_WCHAR_IS_UTF32 + + return (wchar_t*)u_strToUTF32((UChar32*)dest, destCapacity, pDestLength, + src, srcLength, pErrorCode); + +#else + + return _strToWCS(dest,destCapacity,pDestLength,src,srcLength, pErrorCode); + +#endif + +} + +#if !defined(U_WCHAR_IS_UTF16) && !defined(U_WCHAR_IS_UTF32) +/* helper function */ +static UChar* +_strFromWCS( UChar *dest, + int32_t destCapacity, + int32_t *pDestLength, + const wchar_t *src, + int32_t srcLength, + UErrorCode *pErrorCode) +{ + int32_t retVal =0, count =0 ; + UConverter* conv = NULL; + UChar* pTarget = NULL; + UChar* pTargetLimit = NULL; + UChar* target = NULL; + + UChar uStack [_STACK_BUFFER_CAPACITY]; + + wchar_t wStack[_STACK_BUFFER_CAPACITY]; + wchar_t* pWStack = wStack; + + + char cStack[_STACK_BUFFER_CAPACITY]; + int32_t cStackCap = _STACK_BUFFER_CAPACITY; + char* pCSrc=cStack; + char* pCSave=pCSrc; + char* pCSrcLimit=NULL; + + const wchar_t* pSrc = src; + const wchar_t* pSrcLimit = NULL; + + if(srcLength ==-1){ + /* if the wchar_t source is null terminated we can safely + * assume that there are no embedded nulls, this is a fast + * path for null terminated strings. + */ + for(;;){ + /* convert wchars to chars */ + retVal = uprv_wcstombs(pCSrc,src, cStackCap); + + if(retVal == -1){ + *pErrorCode = U_ILLEGAL_CHAR_FOUND; + goto cleanup; + }else if(retVal >= (cStackCap-1)){ + /* Should rarely occur */ + u_growAnyBufferFromStatic(cStack,(void**)&pCSrc,&cStackCap, + cStackCap * _BUFFER_CAPACITY_MULTIPLIER, 0, sizeof(char)); + pCSave = pCSrc; + }else{ + /* converted every thing */ + pCSrc = pCSrc+retVal; + break; + } + } + + }else{ + /* here the source is not null terminated + * so it may have nulls embeded and we need to + * do some extra processing + */ + int32_t remaining =cStackCap; + + pSrcLimit = src + srcLength; + + for(;;){ + register int32_t nulLen = 0; + + /* find nulls in the string */ + while(nulLen= _STACK_BUFFER_CAPACITY){ + /* Should rarely occcur */ + /* allocate new buffer buffer */ + pWStack =(wchar_t*) uprv_malloc(sizeof(wchar_t) * (nulLen + 1)); + if(pWStack==NULL){ + *pErrorCode = U_MEMORY_ALLOCATION_ERROR; + goto cleanup; + } + } + if(nulLen>0){ + /* copy the contents to tempStack */ + uprv_memcpy(pWStack,pSrc,nulLen*sizeof(wchar_t)); + } + + /* null terminate the tempBuffer */ + pWStack[nulLen] =0 ; + + if(remaining < (nulLen * MB_CUR_MAX)){ + /* Should rarely occur */ + int32_t len = (pCSrc-pCSave); + pCSrc = pCSave; + /* we do not have enough room so grow the buffer*/ + u_growAnyBufferFromStatic(cStack,(void**)&pCSrc,&cStackCap, + cStackCap+(nulLen*MB_CUR_MAX),len,sizeof(char)); + + pCSave = pCSrc; + pCSrc = pCSave+len; + remaining = cStackCap-(pCSrc - pCSave); + } + /* convert to chars */ + retVal = uprv_wcstombs(pCSrc,pWStack,remaining); + + pCSrc += retVal; + pSrc += nulLen; + srcLength-=nulLen; /* decrement the srcLength */ + break; + } + } + } + + /* OK..now we have converted from wchar_ts to chars now + * convert chars to UChars + */ + pCSrcLimit = pCSrc; + pCSrc = pCSave; + pTarget = target= dest; + pTargetLimit = dest + destCapacity; + + conv= u_getDefaultConverter(pErrorCode); + + if(U_FAILURE(*pErrorCode)|| conv==NULL){ + goto cleanup; + } + + for(;;) { + + *pErrorCode = U_ZERO_ERROR; + + /* convert to stack buffer*/ + ucnv_toUnicode(conv,&pTarget,pTargetLimit,(const char**)&pCSrc,pCSrcLimit,NULL,(UBool)(pCSrc==pCSrcLimit),pErrorCode); + + /* increment count to number written to stack */ + count+= pTarget - target; + + if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR){ + target = uStack; + pTarget = uStack; + pTargetLimit = uStack + _STACK_BUFFER_CAPACITY; + } else { + break; + } + + } + + if(pDestLength){ + *pDestLength =count; + } + + u_terminateUChars(dest,destCapacity,count,pErrorCode); + +cleanup: + + if(cStack != pCSave){ + uprv_free(pCSave); + } + + if(wStack != pWStack){ + uprv_free(pWStack); + } + + u_releaseDefaultConverter(conv); + + return dest; +} +#endif + +U_CAPI UChar* U_EXPORT2 +u_strFromWCS(UChar *dest, + int32_t destCapacity, + int32_t *pDestLength, + const wchar_t *src, + int32_t srcLength, + UErrorCode *pErrorCode) +{ + + /* args check */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)){ + return NULL; + } + + if((src==NULL) || (srcLength < -1) || (destCapacity<0) || (!dest && destCapacity > 0)){ + *pErrorCode = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + +#ifdef U_WCHAR_IS_UTF16 + /* wchar_t is UTF-16 just do a memcpy */ + if(srcLength == -1){ + srcLength = u_strlen(src); + } + if(0 < srcLength && srcLength <= destCapacity){ + uprv_memcpy(dest,src,srcLength*U_SIZEOF_UCHAR); + } + if(pDestLength){ + *pDestLength = srcLength; + } + + u_terminateUChars(dest,destCapacity,srcLength,pErrorCode); + + return dest; + +#elif defined U_WCHAR_IS_UTF32 + + return u_strFromUTF32(dest, destCapacity, pDestLength, + (UChar32*)src, srcLength, pErrorCode); + +#else + + return _strFromWCS(dest,destCapacity,pDestLength,src,srcLength,pErrorCode); + +#endif + +} + +#endif /* #if !defined(U_WCHAR_IS_UTF16) && !defined(U_WCHAR_IS_UTF32) && !UCONFIG_NO_CONVERSION */ diff --git a/Build/source/libs/icu/icu-xetex/common/ustrcase.c b/Build/source/libs/icu/icu-xetex/common/ustrcase.c new file mode 100644 index 00000000000..402201d8dc0 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ustrcase.c @@ -0,0 +1,981 @@ +/* +******************************************************************************* +* +* Copyright (C) 2001-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: ustrcase.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2002feb20 +* created by: Markus W. Scherer +* +* Implementation file for string casing C API functions. +* Uses functions from uchar.c for basic functionality that requires access +* to the Unicode Character Database (uprops.dat). +*/ + +#include "unicode/utypes.h" +#include "unicode/uloc.h" +#include "unicode/ustring.h" +#include "unicode/ucasemap.h" +#include "unicode/ubrk.h" +#include "cmemory.h" +#include "ucase.h" +#include "unormimp.h" +#include "ustr_imp.h" + +/* string casing ------------------------------------------------------------ */ + +/* append a full case mapping result, see UCASE_MAX_STRING_LENGTH */ +static U_INLINE int32_t +appendResult(UChar *dest, int32_t destIndex, int32_t destCapacity, + int32_t result, const UChar *s) { + UChar32 c; + int32_t length; + + /* decode the result */ + if(result<0) { + /* (not) original code point */ + c=~result; + length=-1; + } else if(result<=UCASE_MAX_STRING_LENGTH) { + c=U_SENTINEL; + length=result; + } else { + c=result; + length=-1; + } + + if(destIndex0) { + dest[destIndex++]=*s++; + --length; + } + } else { + /* overflow */ + destIndex+=length; + } + } + } else { + /* preflight */ + if(length<0) { + destIndex+=U16_LENGTH(c); + } else { + destIndex+=length; + } + } + return destIndex; +} + +static UChar32 U_CALLCONV +utf16_caseContextIterator(void *context, int8_t dir) { + UCaseContext *csc=(UCaseContext *)context; + UChar32 c; + + if(dir<0) { + /* reset for backward iteration */ + csc->index=csc->cpStart; + csc->dir=dir; + } else if(dir>0) { + /* reset for forward iteration */ + csc->index=csc->cpLimit; + csc->dir=dir; + } else { + /* continue current iteration direction */ + dir=csc->dir; + } + + if(dir<0) { + if(csc->startindex) { + U16_PREV((const UChar *)csc->p, csc->start, csc->index, c); + return c; + } + } else { + if(csc->indexlimit) { + U16_NEXT((const UChar *)csc->p, csc->index, csc->limit, c); + return c; + } + } + return U_SENTINEL; +} + +/* + * Case-maps [srcStart..srcLimit[ but takes + * context [0..srcLength[ into account. + */ +static int32_t +_caseMap(const UCaseMap *csm, UCaseMapFull *map, + UChar *dest, int32_t destCapacity, + const UChar *src, UCaseContext *csc, + int32_t srcStart, int32_t srcLimit, + UErrorCode *pErrorCode) { + const UChar *s; + UChar32 c, c2; + int32_t srcIndex, destIndex; + int32_t locCache; + + locCache=csm->locCache; + + /* case mapping loop */ + srcIndex=srcStart; + destIndex=0; + while(srcIndexcpStart=srcIndex; + U16_NEXT(src, srcIndex, srcLimit, c); + csc->cpLimit=srcIndex; + c=map(csm->csp, c, utf16_caseContextIterator, csc, &s, csm->locale, &locCache); + if((destIndexdestCapacity) { + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + return destIndex; +} + +static void +setTempCaseMapLocale(UCaseMap *csm, const char *locale, UErrorCode *pErrorCode) { + /* + * We could call ucasemap_setLocale(), but here we really only care about + * the initial language subtag, we need not return the real string via + * ucasemap_getLocale(), and we don't care about only getting "x" from + * "x-some-thing" etc. + * + * We ignore locales with a longer-than-3 initial subtag. + * + * We also do not fill in the locCache because it is rarely used, + * and not worth setting unless we reuse it for many case mapping operations. + * (That's why UCaseMap was created.) + */ + int i; + char c; + + /* the internal functions require locale!=NULL */ + if(locale==NULL) { + locale=uloc_getDefault(); + } + for(i=0; i<4 && (c=locale[i])!=0 && c!='-' && c!='_'; ++i) { + csm->locale[i]=c; + } + if(i<=3) { + csm->locale[i]=0; /* Up to 3 non-separator characters. */ + } else { + csm->locale[0]=0; /* Longer-than-3 initial subtag: Ignore. */ + } +} + +/* + * Set parameters on an empty UCaseMap, for UCaseMap-less API functions. + * Do this fast because it is called with every function call. + */ +static U_INLINE void +setTempCaseMap(UCaseMap *csm, const char *locale, UErrorCode *pErrorCode) { + if(csm->csp==NULL) { + csm->csp=ucase_getSingleton(pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return; + } + } + if(locale!=NULL && locale[0]==0) { + csm->locale[0]=0; + } else { + setTempCaseMapLocale(csm, locale, pErrorCode); + } +} + +#if !UCONFIG_NO_BREAK_ITERATION + +/* + * Internal titlecasing function. + */ +static int32_t +_toTitle(UCaseMap *csm, + UChar *dest, int32_t destCapacity, + const UChar *src, UCaseContext *csc, + int32_t srcLength, + UErrorCode *pErrorCode) { + const UChar *s; + UChar32 c; + int32_t prev, titleStart, titleLimit, index, destIndex, length; + UBool isFirstIndex; + + if(csm->iter!=NULL) { + ubrk_setText(csm->iter, src, srcLength, pErrorCode); + } else { + csm->iter=ubrk_open(UBRK_WORD, csm->locale, + src, srcLength, + pErrorCode); + } + if(U_FAILURE(*pErrorCode)) { + return 0; + } + + /* set up local variables */ + destIndex=0; + prev=0; + isFirstIndex=TRUE; + + /* titlecasing loop */ + while(previter); + } else { + index=ubrk_next(csm->iter); + } + if(index==UBRK_DONE || index>srcLength) { + index=srcLength; + } + + /* + * Unicode 4 & 5 section 3.13 Default Case Operations: + * + * R3 toTitlecase(X): Find the word boundaries based on Unicode Standard Annex + * #29, "Text Boundaries." Between each pair of word boundaries, find the first + * cased character F. If F exists, map F to default_title(F); then map each + * subsequent character C to default_lower(C). + * + * In this implementation, segment [prev..index[ into 3 parts: + * a) uncased characters (copy as-is) [prev..titleStart[ + * b) first case letter (titlecase) [titleStart..titleLimit[ + * c) subsequent characters (lowercase) [titleLimit..index[ + */ + if(prevoptions&U_TITLECASE_NO_BREAK_ADJUSTMENT)==0 && UCASE_NONE==ucase_getType(csm->csp, c)) { + /* Adjust the titlecasing index (titleStart) to the next cased character. */ + for(;;) { + titleStart=titleLimit; + if(titleLimit==index) { + /* + * only uncased characters in [prev..index[ + * stop with titleStart==titleLimit==index + */ + break; + } + U16_NEXT(src, titleLimit, index, c); + if(UCASE_NONE!=ucase_getType(csm->csp, c)) { + break; /* cased letter at [titleStart..titleLimit[ */ + } + } + length=titleStart-prev; + if(length>0) { + if((destIndex+length)<=destCapacity) { + uprv_memcpy(dest+destIndex, src+prev, length*U_SIZEOF_UCHAR); + } + destIndex+=length; + } + } + + if(titleStartcpStart=titleStart; + csc->cpLimit=titleLimit; + c=ucase_toFullTitle(csm->csp, c, utf16_caseContextIterator, csc, &s, csm->locale, &csm->locCache); + destIndex=appendResult(dest, destIndex, destCapacity, c, s); + + /* lowercase [titleLimit..index[ */ + if(titleLimitoptions&U_TITLECASE_NO_LOWERCASE)==0) { + /* Normal operation: Lowercase the rest of the word. */ + destIndex+= + _caseMap( + csm, ucase_toFullLower, + dest+destIndex, destCapacity-destIndex, + src, csc, + titleLimit, index, + pErrorCode); + } else { + /* Optionally just copy the rest of the word unchanged. */ + length=index-titleLimit; + if((destIndex+length)<=destCapacity) { + uprv_memcpy(dest+destIndex, src+titleLimit, length*U_SIZEOF_UCHAR); + } + destIndex+=length; + } + } + } + } + + prev=index; + } + + if(destIndex>destCapacity) { + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + return destIndex; +} + +#endif + +/* functions available in the common library (for unistr_case.cpp) */ + +U_CFUNC int32_t +ustr_toLower(const UCaseProps *csp, + UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + const char *locale, + UErrorCode *pErrorCode) { + UCaseMap csm={ NULL }; + UCaseContext csc={ NULL }; + + csm.csp=csp; + setTempCaseMap(&csm, locale, pErrorCode); + csc.p=(void *)src; + csc.limit=srcLength; + + return _caseMap(&csm, ucase_toFullLower, + dest, destCapacity, + src, &csc, 0, srcLength, + pErrorCode); +} + +U_CFUNC int32_t +ustr_toUpper(const UCaseProps *csp, + UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + const char *locale, + UErrorCode *pErrorCode) { + UCaseMap csm={ NULL }; + UCaseContext csc={ NULL }; + + csm.csp=csp; + setTempCaseMap(&csm, locale, pErrorCode); + csc.p=(void *)src; + csc.limit=srcLength; + + return _caseMap(&csm, ucase_toFullUpper, + dest, destCapacity, + src, &csc, 0, srcLength, + pErrorCode); +} + +#if !UCONFIG_NO_BREAK_ITERATION + +U_CFUNC int32_t +ustr_toTitle(const UCaseProps *csp, + UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + UBreakIterator *titleIter, + const char *locale, uint32_t options, + UErrorCode *pErrorCode) { + UCaseMap csm={ NULL }; + UCaseContext csc={ NULL }; + int32_t length; + + csm.csp=csp; + csm.iter=titleIter; + csm.options=options; + setTempCaseMap(&csm, locale, pErrorCode); + csc.p=(void *)src; + csc.limit=srcLength; + + length=_toTitle(&csm, + dest, destCapacity, + src, &csc, srcLength, + pErrorCode); + if(titleIter==NULL && csm.iter!=NULL) { + ubrk_close(csm.iter); + } + return length; +} + +#endif + +U_CFUNC int32_t +ustr_foldCase(const UCaseProps *csp, + UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + uint32_t options, + UErrorCode *pErrorCode) { + int32_t srcIndex, destIndex; + + const UChar *s; + UChar32 c, c2; + + /* case mapping loop */ + srcIndex=destIndex=0; + while(srcIndexdestCapacity) { + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + return destIndex; +} + +/* + * Implement argument checking and buffer handling + * for string case mapping as a common function. + */ + +/* common internal function for public API functions */ + +static int32_t +caseMap(const UCaseMap *csm, + UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + int32_t toWhichCase, + UErrorCode *pErrorCode) { + UChar buffer[300]; + UChar *temp; + + int32_t destLength; + + /* check argument values */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + if( destCapacity<0 || + (dest==NULL && destCapacity>0) || + src==NULL || + srcLength<-1 + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* get the string length */ + if(srcLength==-1) { + srcLength=u_strlen(src); + } + + /* check for overlapping source and destination */ + if( dest!=NULL && + ((src>=dest && src<(dest+destCapacity)) || + (dest>=src && dest<(src+srcLength))) + ) { + /* overlap: provide a temporary destination buffer and later copy the result */ + if(destCapacity<=(sizeof(buffer)/U_SIZEOF_UCHAR)) { + /* the stack buffer is large enough */ + temp=buffer; + } else { + /* allocate a buffer */ + temp=(UChar *)uprv_malloc(destCapacity*U_SIZEOF_UCHAR); + if(temp==NULL) { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return 0; + } + } + } else { + temp=dest; + } + + destLength=0; + + if(toWhichCase==FOLD_CASE) { + destLength=ustr_foldCase(csm->csp, temp, destCapacity, src, srcLength, + csm->options, pErrorCode); + } else { + UCaseContext csc={ NULL }; + + csc.p=(void *)src; + csc.limit=srcLength; + + if(toWhichCase==TO_LOWER) { + destLength=_caseMap(csm, ucase_toFullLower, + temp, destCapacity, + src, &csc, + 0, srcLength, + pErrorCode); + } else if(toWhichCase==TO_UPPER) { + destLength=_caseMap(csm, ucase_toFullUpper, + temp, destCapacity, + src, &csc, + 0, srcLength, + pErrorCode); + } else /* if(toWhichCase==TO_TITLE) */ { +#if UCONFIG_NO_BREAK_ITERATION + *pErrorCode=U_UNSUPPORTED_ERROR; +#else + /* UCaseMap is actually non-const in toTitle() APIs. */ + destLength=_toTitle((UCaseMap *)csm, temp, destCapacity, + src, &csc, srcLength, + pErrorCode); +#endif + } + } + if(temp!=dest) { + /* copy the result string to the destination buffer */ + if(destLength>0) { + int32_t copyLength= destLength<=destCapacity ? destLength : destCapacity; + if(copyLength>0) { + uprv_memmove(dest, temp, copyLength*U_SIZEOF_UCHAR); + } + } + if(temp!=buffer) { + uprv_free(temp); + } + } + + return u_terminateUChars(dest, destCapacity, destLength, pErrorCode); +} + +/* public API functions */ + +U_CAPI int32_t U_EXPORT2 +u_strToLower(UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + const char *locale, + UErrorCode *pErrorCode) { + UCaseMap csm={ NULL }; + setTempCaseMap(&csm, locale, pErrorCode); + return caseMap(&csm, + dest, destCapacity, + src, srcLength, + TO_LOWER, pErrorCode); +} + +U_CAPI int32_t U_EXPORT2 +u_strToUpper(UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + const char *locale, + UErrorCode *pErrorCode) { + UCaseMap csm={ NULL }; + setTempCaseMap(&csm, locale, pErrorCode); + return caseMap(&csm, + dest, destCapacity, + src, srcLength, + TO_UPPER, pErrorCode); +} + +#if !UCONFIG_NO_BREAK_ITERATION + +U_CAPI int32_t U_EXPORT2 +u_strToTitle(UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + UBreakIterator *titleIter, + const char *locale, + UErrorCode *pErrorCode) { + UCaseMap csm={ NULL }; + int32_t length; + + csm.iter=titleIter; + setTempCaseMap(&csm, locale, pErrorCode); + length=caseMap(&csm, + dest, destCapacity, + src, srcLength, + TO_TITLE, pErrorCode); + if(titleIter==NULL && csm.iter!=NULL) { + ubrk_close(csm.iter); + } + return length; +} + +U_CAPI int32_t U_EXPORT2 +ucasemap_toTitle(UCaseMap *csm, + UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + UErrorCode *pErrorCode) { + return caseMap(csm, + dest, destCapacity, + src, srcLength, + TO_TITLE, pErrorCode); +} + +#endif + +U_CAPI int32_t U_EXPORT2 +u_strFoldCase(UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + uint32_t options, + UErrorCode *pErrorCode) { + UCaseMap csm={ NULL }; + csm.csp=ucase_getSingleton(pErrorCode); + csm.options=options; + return caseMap(&csm, + dest, destCapacity, + src, srcLength, + FOLD_CASE, pErrorCode); +} + +/* case-insensitive string comparisons -------------------------------------- */ + +/* + * This function is a copy of unorm_cmpEquivFold() minus the parts for + * canonical equivalence. + * Keep the functions in sync, and see there for how this works. + * The duplication is for modularization: + * It makes caseless (but not canonical caseless) matches independent of + * the normalization code. + */ + +/* stack element for previous-level source/decomposition pointers */ +struct CmpEquivLevel { + const UChar *start, *s, *limit; +}; +typedef struct CmpEquivLevel CmpEquivLevel; + +/* internal function */ +U_CFUNC int32_t +u_strcmpFold(const UChar *s1, int32_t length1, + const UChar *s2, int32_t length2, + uint32_t options, + UErrorCode *pErrorCode) { + const UCaseProps *csp; + + /* current-level start/limit - s1/s2 as current */ + const UChar *start1, *start2, *limit1, *limit2; + + /* case folding variables */ + const UChar *p; + int32_t length; + + /* stacks of previous-level start/current/limit */ + CmpEquivLevel stack1[2], stack2[2]; + + /* case folding buffers, only use current-level start/limit */ + UChar fold1[UCASE_MAX_STRING_LENGTH+1], fold2[UCASE_MAX_STRING_LENGTH+1]; + + /* track which is the current level per string */ + int32_t level1, level2; + + /* current code units, and code points for lookups */ + UChar32 c1, c2, cp1, cp2; + + /* no argument error checking because this itself is not an API */ + + /* + * assume that at least the option U_COMPARE_IGNORE_CASE is set + * otherwise this function would have to behave exactly as uprv_strCompare() + */ + csp=ucase_getSingleton(pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return 0; + } + + /* initialize */ + start1=s1; + if(length1==-1) { + limit1=NULL; + } else { + limit1=s1+length1; + } + + start2=s2; + if(length2==-1) { + limit2=NULL; + } else { + limit2=s2+length2; + } + + level1=level2=0; + c1=c2=-1; + + /* comparison loop */ + for(;;) { + /* + * here a code unit value of -1 means "get another code unit" + * below it will mean "this source is finished" + */ + + if(c1<0) { + /* get next code unit from string 1, post-increment */ + for(;;) { + if(s1==limit1 || ((c1=*s1)==0 && (limit1==NULL || (options&_STRNCMP_STYLE)))) { + if(level1==0) { + c1=-1; + break; + } + } else { + ++s1; + break; + } + + /* reached end of level buffer, pop one level */ + do { + --level1; + start1=stack1[level1].start; + } while(start1==NULL); + s1=stack1[level1].s; + limit1=stack1[level1].limit; + } + } + + if(c2<0) { + /* get next code unit from string 2, post-increment */ + for(;;) { + if(s2==limit2 || ((c2=*s2)==0 && (limit2==NULL || (options&_STRNCMP_STYLE)))) { + if(level2==0) { + c2=-1; + break; + } + } else { + ++s2; + break; + } + + /* reached end of level buffer, pop one level */ + do { + --level2; + start2=stack2[level2].start; + } while(start2==NULL); + s2=stack2[level2].s; + limit2=stack2[level2].limit; + } + } + + /* + * compare c1 and c2 + * either variable c1, c2 is -1 only if the corresponding string is finished + */ + if(c1==c2) { + if(c1<0) { + return 0; /* c1==c2==-1 indicating end of strings */ + } + c1=c2=-1; /* make us fetch new code units */ + continue; + } else if(c1<0) { + return -1; /* string 1 ends before string 2 */ + } else if(c2<0) { + return 1; /* string 2 ends before string 1 */ + } + /* c1!=c2 && c1>=0 && c2>=0 */ + + /* get complete code points for c1, c2 for lookups if either is a surrogate */ + cp1=c1; + if(U_IS_SURROGATE(c1)) { + UChar c; + + if(U_IS_SURROGATE_LEAD(c1)) { + if(s1!=limit1 && U16_IS_TRAIL(c=*s1)) { + /* advance ++s1; only below if cp1 decomposes/case-folds */ + cp1=U16_GET_SUPPLEMENTARY(c1, c); + } + } else /* isTrail(c1) */ { + if(start1<=(s1-2) && U16_IS_LEAD(c=*(s1-2))) { + cp1=U16_GET_SUPPLEMENTARY(c, c1); + } + } + } + + cp2=c2; + if(U_IS_SURROGATE(c2)) { + UChar c; + + if(U_IS_SURROGATE_LEAD(c2)) { + if(s2!=limit2 && U16_IS_TRAIL(c=*s2)) { + /* advance ++s2; only below if cp2 decomposes/case-folds */ + cp2=U16_GET_SUPPLEMENTARY(c2, c); + } + } else /* isTrail(c2) */ { + if(start2<=(s2-2) && U16_IS_LEAD(c=*(s2-2))) { + cp2=U16_GET_SUPPLEMENTARY(c, c2); + } + } + } + + /* + * go down one level for each string + * continue with the main loop as soon as there is a real change + */ + + if( level1==0 && + (length=ucase_toFullFolding(csp, (UChar32)cp1, &p, options))>=0 + ) { + /* cp1 case-folds to the code point "length" or to p[length] */ + if(U_IS_SURROGATE(c1)) { + if(U_IS_SURROGATE_LEAD(c1)) { + /* advance beyond source surrogate pair if it case-folds */ + ++s1; + } else /* isTrail(c1) */ { + /* + * we got a supplementary code point when hitting its trail surrogate, + * therefore the lead surrogate must have been the same as in the other string; + * compare this decomposition with the lead surrogate in the other string + * remember that this simulates bulk text replacement: + * the decomposition would replace the entire code point + */ + --s2; + c2=*(s2-1); + } + } + + /* push current level pointers */ + stack1[0].start=start1; + stack1[0].s=s1; + stack1[0].limit=limit1; + ++level1; + + /* copy the folding result to fold1[] */ + if(length<=UCASE_MAX_STRING_LENGTH) { + u_memcpy(fold1, p, length); + } else { + int32_t i=0; + U16_APPEND_UNSAFE(fold1, i, length); + length=i; + } + + /* set next level pointers to case folding */ + start1=s1=fold1; + limit1=fold1+length; + + /* get ready to read from decomposition, continue with loop */ + c1=-1; + continue; + } + + if( level2==0 && + (length=ucase_toFullFolding(csp, (UChar32)cp2, &p, options))>=0 + ) { + /* cp2 case-folds to the code point "length" or to p[length] */ + if(U_IS_SURROGATE(c2)) { + if(U_IS_SURROGATE_LEAD(c2)) { + /* advance beyond source surrogate pair if it case-folds */ + ++s2; + } else /* isTrail(c2) */ { + /* + * we got a supplementary code point when hitting its trail surrogate, + * therefore the lead surrogate must have been the same as in the other string; + * compare this decomposition with the lead surrogate in the other string + * remember that this simulates bulk text replacement: + * the decomposition would replace the entire code point + */ + --s1; + c1=*(s1-1); + } + } + + /* push current level pointers */ + stack2[0].start=start2; + stack2[0].s=s2; + stack2[0].limit=limit2; + ++level2; + + /* copy the folding result to fold2[] */ + if(length<=UCASE_MAX_STRING_LENGTH) { + u_memcpy(fold2, p, length); + } else { + int32_t i=0; + U16_APPEND_UNSAFE(fold2, i, length); + length=i; + } + + /* set next level pointers to case folding */ + start2=s2=fold2; + limit2=fold2+length; + + /* get ready to read from decomposition, continue with loop */ + c2=-1; + continue; + } + + /* + * no decomposition/case folding, max level for both sides: + * return difference result + * + * code point order comparison must not just return cp1-cp2 + * because when single surrogates are present then the surrogate pairs + * that formed cp1 and cp2 may be from different string indexes + * + * example: { d800 d800 dc01 } vs. { d800 dc00 }, compare at second code units + * c1=d800 cp1=10001 c2=dc00 cp2=10000 + * cp1-cp2>0 but c1-c2<0 and in fact in UTF-32 it is { d800 10001 } < { 10000 } + * + * therefore, use same fix-up as in ustring.c/uprv_strCompare() + * except: uprv_strCompare() fetches c=*s while this functions fetches c=*s++ + * so we have slightly different pointer/start/limit comparisons here + */ + + if(c1>=0xd800 && c2>=0xd800 && (options&U_COMPARE_CODE_POINT_ORDER)) { + /* subtract 0x2800 from BMP code points to make them smaller than supplementary ones */ + if( + (c1<=0xdbff && s1!=limit1 && U16_IS_TRAIL(*s1)) || + (U16_IS_TRAIL(c1) && start1!=(s1-1) && U16_IS_LEAD(*(s1-2))) + ) { + /* part of a surrogate pair, leave >=d800 */ + } else { + /* BMP code point - may be surrogate code point - make =d800 */ + } else { + /* BMP code point - may be surrogate code point - make charsCapacity) { + if(capacity<(charsCapacity+charsCapacity/2)) { + // avoid allocation thrashing + capacity=charsCapacity+charsCapacity/2; + } + if(chars!=charsBuffer) { + uprv_free(chars); + } + chars=(char *)uprv_malloc(capacity); + if(chars==NULL) { + chars=charsBuffer; + charsCapacity=sizeof(charsBuffer); + status=U_MEMORY_ALLOCATION_ERROR; + } else { + charsCapacity=capacity; + } + } +} + +UnicodeString * +StringEnumeration::setChars(const char *s, int32_t length, UErrorCode &status) { + if(U_SUCCESS(status) && s!=NULL) { + if(length<0) { + length=(int32_t)uprv_strlen(s); + } + + UChar *buffer=unistr.getBuffer(length+1); + if(buffer!=NULL) { + u_charsToUChars(s, buffer, length); + buffer[length]=0; + unistr.releaseBuffer(length); + return &unistr; + } else { + status=U_MEMORY_ALLOCATION_ERROR; + } + } + + return NULL; +} +UBool +StringEnumeration::operator==(const StringEnumeration& that)const { + return getDynamicClassID() == that.getDynamicClassID(); +} + +UBool +StringEnumeration::operator!=(const StringEnumeration& that)const { + return !operator==(that); +} + +// UStringEnumeration implementation --------------------------------------- *** + +UStringEnumeration::UStringEnumeration(UEnumeration* _uenum) : + uenum(_uenum) { + U_ASSERT(_uenum != 0); +} + +UStringEnumeration::~UStringEnumeration() { + uenum_close(uenum); +} + +int32_t UStringEnumeration::count(UErrorCode& status) const { + return uenum_count(uenum, &status); +} + +const UnicodeString* UStringEnumeration::snext(UErrorCode& status) { + int32_t length; + const UChar* str = uenum_unext(uenum, &length, &status); + if (str == 0 || U_FAILURE(status)) { + return 0; + } + return &unistr.setTo(str, length); +} + +void UStringEnumeration::reset(UErrorCode& status) { + uenum_reset(uenum, &status); +} + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(UStringEnumeration) +U_NAMESPACE_END + +// C wrapper --------------------------------------------------------------- *** + +#define THIS(en) ((U_NAMESPACE_QUALIFIER StringEnumeration*)(en->context)) + +U_CDECL_BEGIN + +/** + * Wrapper API to make StringEnumeration look like UEnumeration. + */ +static void U_CALLCONV +ustrenum_close(UEnumeration* en) { + delete THIS(en); + uprv_free(en); +} + +/** + * Wrapper API to make StringEnumeration look like UEnumeration. + */ +static int32_t U_CALLCONV +ustrenum_count(UEnumeration* en, + UErrorCode* ec) +{ + return THIS(en)->count(*ec); +} + +/** + * Wrapper API to make StringEnumeration look like UEnumeration. + */ +static const UChar* U_CALLCONV +ustrenum_unext(UEnumeration* en, + int32_t* resultLength, + UErrorCode* ec) +{ + return THIS(en)->unext(resultLength, *ec); +} + +/** + * Wrapper API to make StringEnumeration look like UEnumeration. + */ +static const char* U_CALLCONV +ustrenum_next(UEnumeration* en, + int32_t* resultLength, + UErrorCode* ec) +{ + return THIS(en)->next(resultLength, *ec); +} + +/** + * Wrapper API to make StringEnumeration look like UEnumeration. + */ +static void U_CALLCONV +ustrenum_reset(UEnumeration* en, + UErrorCode* ec) +{ + THIS(en)->reset(*ec); +} + +/** + * Pseudo-vtable for UEnumeration wrapper around StringEnumeration. + * The StringEnumeration pointer will be stored in 'context'. + */ +static const UEnumeration USTRENUM_VT = { + NULL, + NULL, // store StringEnumeration pointer here + ustrenum_close, + ustrenum_count, + ustrenum_unext, + ustrenum_next, + ustrenum_reset +}; + +U_CDECL_END + +/** + * Given a StringEnumeration, wrap it in a UEnumeration. The + * StringEnumeration is adopted; after this call, the caller must not + * delete it (regardless of error status). + */ +U_CAPI UEnumeration* U_EXPORT2 +uenum_openStringEnumeration(U_NAMESPACE_QUALIFIER StringEnumeration* adopted, UErrorCode* ec) { + UEnumeration* result = NULL; + if (U_SUCCESS(*ec) && adopted != NULL) { + result = (UEnumeration*) uprv_malloc(sizeof(UEnumeration)); + if (result == NULL) { + *ec = U_MEMORY_ALLOCATION_ERROR; + } else { + uprv_memcpy(result, &USTRENUM_VT, sizeof(USTRENUM_VT)); + result->context = adopted; + } + } + if (result == NULL) { + delete adopted; + } + return result; +} + +// C wrapper --------------------------------------------------------------- *** + +U_CDECL_BEGIN + +typedef struct UCharStringEnumeration { + UEnumeration uenum; + int32_t index, count; +} UCharStringEnumeration; + +static void U_CALLCONV +ucharstrenum_close(UEnumeration* en) { + uprv_free(en); +} + +static int32_t U_CALLCONV +ucharstrenum_count(UEnumeration* en, + UErrorCode* /*ec*/) { + return ((UCharStringEnumeration*)en)->count; +} + +static const char* U_CALLCONV +ucharstrenum_next(UEnumeration* en, + int32_t* resultLength, + UErrorCode* /*ec*/) { + UCharStringEnumeration *e = (UCharStringEnumeration*) en; + if (e->index >= e->count) { + return NULL; + } + const char* result = ((const char**)e->uenum.context)[e->index++]; + if (resultLength) { + *resultLength = (int32_t)uprv_strlen(result); + } + return result; +} + +static void U_CALLCONV +ucharstrenum_reset(UEnumeration* en, + UErrorCode* /*ec*/) { + ((UCharStringEnumeration*)en)->index = 0; +} + +static const UEnumeration UCHARSTRENUM_VT = { + NULL, + NULL, // store StringEnumeration pointer here + ucharstrenum_close, + ucharstrenum_count, + uenum_unextDefault, + ucharstrenum_next, + ucharstrenum_reset +}; + +U_CDECL_END + +U_CAPI UEnumeration* U_EXPORT2 +uenum_openCharStringsEnumeration(const char** strings, int32_t count, + UErrorCode* ec) { + UCharStringEnumeration* result = NULL; + if (U_SUCCESS(*ec) && count >= 0 && (count == 0 || strings != 0)) { + result = (UCharStringEnumeration*) uprv_malloc(sizeof(UCharStringEnumeration)); + if (result == NULL) { + *ec = U_MEMORY_ALLOCATION_ERROR; + } else { + U_ASSERT((char*)result==(char*)(&result->uenum)); + uprv_memcpy(result, &UCHARSTRENUM_VT, sizeof(UCHARSTRENUM_VT)); + result->uenum.context = strings; + result->index = 0; + result->count = count; + } + } + return (UEnumeration*) result; +} + + diff --git a/Build/source/libs/icu/icu-xetex/common/ustrenum.h b/Build/source/libs/icu/icu-xetex/common/ustrenum.h new file mode 100644 index 00000000000..6580ceaca70 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ustrenum.h @@ -0,0 +1,98 @@ +/* +********************************************************************** +* Copyright (c) 2002-2006, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Author: Alan Liu +* Created: November 11 2002 +* Since: ICU 2.4 +********************************************************************** +*/ +#ifndef _USTRENUM_H_ +#define _USTRENUM_H_ + +#include "unicode/uenum.h" +#include "unicode/strenum.h" + +/** + * Given a StringEnumeration, wrap it in a UEnumeration. The + * StringEnumeration is adopted; after this call, the caller must not + * delete it (regardless of error status). + */ +U_CAPI UEnumeration* U_EXPORT2 +uenum_openStringEnumeration(U_NAMESPACE_QUALIFIER StringEnumeration* adopted, UErrorCode* ec); + +/** + * Given an array of const char* strings (invariant chars only), + * return a UEnumeration. Must have strings[i] != 0 for i in + * 0..count-1. + */ +U_CAPI UEnumeration* U_EXPORT2 +uenum_openCharStringsEnumeration(const char** strings, int32_t count, + UErrorCode* ec); + +//---------------------------------------------------------------------- +U_NAMESPACE_BEGIN + +/** + * A wrapper to make a UEnumeration into a StringEnumeration. The + * wrapper adopts the UEnumeration is wraps. + */ +class U_COMMON_API UStringEnumeration : public StringEnumeration { + +public: + /** + * Constructor. This constructor adopts its UEnumeration + * argument. + * @param uenum a UEnumeration object. This object takes + * ownership of 'uenum' and will close it in its destructor. The + * caller must not call uenum_close on 'uenum' after calling this + * constructor. + */ + UStringEnumeration(UEnumeration* uenum); + + /** + * Destructor. This closes the UEnumeration passed in to the + * constructor. + */ + virtual ~UStringEnumeration(); + + /** + * Return the number of elements that the iterator traverses. + * @param status the error code. + * @return number of elements in the iterator. + */ + virtual int32_t count(UErrorCode& status) const; + + /** + * Returns the next element a UnicodeString*. If there are no + * more elements, returns NULL. + * @param status the error code. + * @return a pointer to the string, or NULL. + */ + virtual const UnicodeString* snext(UErrorCode& status); + + /** + * Resets the iterator. + * @param status the error code. + */ + virtual void reset(UErrorCode& status); + + /** + * ICU4C "poor man's RTTI", returns a UClassID for the actual ICU class. + */ + virtual UClassID getDynamicClassID() const; + + /** + * ICU4C "poor man's RTTI", returns a UClassID for this ICU class. + */ + static UClassID U_EXPORT2 getStaticClassID(); + +private: + UEnumeration *uenum; // owned +}; + +U_NAMESPACE_END + +#endif + diff --git a/Build/source/libs/icu/icu-xetex/common/ustrfmt.c b/Build/source/libs/icu/icu-xetex/common/ustrfmt.c new file mode 100644 index 00000000000..5e9fb924f8d --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ustrfmt.c @@ -0,0 +1,57 @@ +/* +********************************************************************** +* Copyright (C) 2001-2006, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +*/ + +#include "cstring.h" +#include "ustrfmt.h" + + +/*** + * Fills in a UChar* string with the radix-based representation of a + * uint32_t number padded with zeroes to minwidth. The result + * will be null terminated if there is room. + * + * @param buffer UChar buffer to receive result + * @param capacity capacity of buffer + * @param i the unsigned number to be formatted + * @param radix the radix from 2..36 + * @param minwidth the minimum width. If the result is narrower than + * this, '0's will be added on the left. Must be <= + * capacity. + * @return the length of the result, not including any terminating + * null + */ +U_CAPI int32_t U_EXPORT2 +uprv_itou (UChar * buffer, int32_t capacity, + uint32_t i, uint32_t radix, int32_t minwidth) +{ + int32_t length = 0; + int digit; + int32_t j; + UChar temp; + + do{ + digit = (int)(i % radix); + buffer[length++]=(UChar)(digit<=9?(0x0030+digit):(0x0030+digit+7)); + i=i/radix; + } while(i && length= 0) { + return (UChar *)string + index; + } else { + return NULL; + } +} + +/* Search for a codepoint in a string that matches one of the matchSet codepoints. */ +U_CAPI int32_t U_EXPORT2 +u_strcspn(const UChar *string, const UChar *matchSet) +{ + int32_t index = _matchFromSet(string, matchSet, TRUE); + if(index >= 0) { + return index; + } else { + return -index - 1; /* == u_strlen(string) */ + } +} + +/* Search for a codepoint in a string that does not match one of the matchSet codepoints. */ +U_CAPI int32_t U_EXPORT2 +u_strspn(const UChar *string, const UChar *matchSet) +{ + int32_t index = _matchFromSet(string, matchSet, FALSE); + if(index >= 0) { + return index; + } else { + return -index - 1; /* == u_strlen(string) */ + } +} + +/* ----- Text manipulation functions --- */ + +U_CAPI UChar* U_EXPORT2 +u_strtok_r(UChar *src, + const UChar *delim, + UChar **saveState) +{ + UChar *tokSource; + UChar *nextToken; + uint32_t nonDelimIdx; + + /* If saveState is NULL, the user messed up. */ + if (src != NULL) { + tokSource = src; + *saveState = src; /* Set to "src" in case there are no delimiters */ + } + else if (*saveState) { + tokSource = *saveState; + } + else { + /* src == NULL && *saveState == NULL */ + /* This shouldn't happen. We already finished tokenizing. */ + return NULL; + } + + /* Skip initial delimiters */ + nonDelimIdx = u_strspn(tokSource, delim); + tokSource = &tokSource[nonDelimIdx]; + + if (*tokSource) { + nextToken = u_strpbrk(tokSource, delim); + if (nextToken != NULL) { + /* Create a token */ + *(nextToken++) = 0; + *saveState = nextToken; + return tokSource; + } + else if (*saveState) { + /* Return the last token */ + *saveState = NULL; + return tokSource; + } + } + else { + /* No tokens were found. Only delimiters were left. */ + *saveState = NULL; + } + return NULL; +} + +/* Miscellaneous functions -------------------------------------------------- */ + +U_CAPI UChar* U_EXPORT2 +u_strcat(UChar *dst, + const UChar *src) +{ + UChar *anchor = dst; /* save a pointer to start of dst */ + + while(*dst != 0) { /* To end of first string */ + ++dst; + } + while((*(dst++) = *(src++)) != 0) { /* copy string 2 over */ + } + + return anchor; +} + +U_CAPI UChar* U_EXPORT2 +u_strncat(UChar *dst, + const UChar *src, + int32_t n ) +{ + if(n > 0) { + UChar *anchor = dst; /* save a pointer to start of dst */ + + while(*dst != 0) { /* To end of first string */ + ++dst; + } + while((*dst = *src) != 0) { /* copy string 2 over */ + ++dst; + if(--n == 0) { + *dst = 0; + break; + } + ++src; + } + + return anchor; + } else { + return dst; + } +} + +/* ----- Text property functions --- */ + +U_CAPI int32_t U_EXPORT2 +u_strcmp(const UChar *s1, + const UChar *s2) +{ + UChar c1, c2; + + for(;;) { + c1=*s1++; + c2=*s2++; + if (c1 != c2 || c1 == 0) { + break; + } + } + return (int32_t)c1 - (int32_t)c2; +} + +U_CAPI int32_t U_EXPORT2 +uprv_strCompare(const UChar *s1, int32_t length1, + const UChar *s2, int32_t length2, + UBool strncmpStyle, UBool codePointOrder) { + const UChar *start1, *start2, *limit1, *limit2; + UChar c1, c2; + + /* setup for fix-up */ + start1=s1; + start2=s2; + + /* compare identical prefixes - they do not need to be fixed up */ + if(length1<0 && length2<0) { + /* strcmp style, both NUL-terminated */ + if(s1==s2) { + return 0; + } + + for(;;) { + c1=*s1; + c2=*s2; + if(c1!=c2) { + break; + } + if(c1==0) { + return 0; + } + ++s1; + ++s2; + } + + /* setup for fix-up */ + limit1=limit2=NULL; + } else if(strncmpStyle) { + /* special handling for strncmp, assume length1==length2>=0 but also check for NUL */ + if(s1==s2) { + return 0; + } + + limit1=start1+length1; + + for(;;) { + /* both lengths are same, check only one limit */ + if(s1==limit1) { + return 0; + } + + c1=*s1; + c2=*s2; + if(c1!=c2) { + break; + } + if(c1==0) { + return 0; + } + ++s1; + ++s2; + } + + /* setup for fix-up */ + limit2=start2+length1; /* use length1 here, too, to enforce assumption */ + } else { + /* memcmp/UnicodeString style, both length-specified */ + int32_t lengthResult; + + if(length1<0) { + length1=u_strlen(s1); + } + if(length2<0) { + length2=u_strlen(s2); + } + + /* limit1=start1+min(lenght1, length2) */ + if(length1length2 */ { + lengthResult=1; + limit1=start1+length2; + } + + if(s1==s2) { + return lengthResult; + } + + for(;;) { + /* check pseudo-limit */ + if(s1==limit1) { + return lengthResult; + } + + c1=*s1; + c2=*s2; + if(c1!=c2) { + break; + } + ++s1; + ++s2; + } + + /* setup for fix-up */ + limit1=start1+length1; + limit2=start2+length2; + } + + /* if both values are in or above the surrogate range, fix them up */ + if(c1>=0xd800 && c2>=0xd800 && codePointOrder) { + /* subtract 0x2800 from BMP code points to make them smaller than supplementary ones */ + if( + (c1<=0xdbff && (s1+1)!=limit1 && UTF_IS_TRAIL(*(s1+1))) || + (UTF_IS_TRAIL(c1) && start1!=s1 && UTF_IS_LEAD(*(s1-1))) + ) { + /* part of a surrogate pair, leave >=d800 */ + } else { + /* BMP code point - may be surrogate code point - make =d800 */ + } else { + /* BMP code point - may be surrogate code point - make move(iter1, 0, UITER_START); + iter2->move(iter2, 0, UITER_START); + + /* compare identical prefixes - they do not need to be fixed up */ + for(;;) { + c1=iter1->next(iter1); + c2=iter2->next(iter2); + if(c1!=c2) { + break; + } + if(c1==-1) { + return 0; + } + } + + /* if both values are in or above the surrogate range, fix them up */ + if(c1>=0xd800 && c2>=0xd800 && codePointOrder) { + /* subtract 0x2800 from BMP code points to make them smaller than supplementary ones */ + if( + (c1<=0xdbff && UTF_IS_TRAIL(iter1->current(iter1))) || + (UTF_IS_TRAIL(c1) && (iter1->previous(iter1), UTF_IS_LEAD(iter1->previous(iter1)))) + ) { + /* part of a surrogate pair, leave >=d800 */ + } else { + /* BMP code point - may be surrogate code point - make current(iter2))) || + (UTF_IS_TRAIL(c2) && (iter2->previous(iter2), UTF_IS_LEAD(iter2->previous(iter2)))) + ) { + /* part of a surrogate pair, leave >=d800 */ + } else { + /* BMP code point - may be surrogate code point - make =0. + * + * Consistently leaving them _behind_ the different units is not an option + * because the current "unit" is the end of the string if that is reached, + * and in such a case the iterator does not move. + * For example, when comparing "ab" with "abc", both iterators rest _on_ the end + * of their strings. Calling previous() on each does not move them to where + * the comparison fails. + * + * So the simplest semantics is to not define where the iterators end up. + * + * The following fragment is part of what would need to be done for backing up. + */ +void fragment { + /* iff a surrogate is part of a surrogate pair, leave >=d800 */ + if(c1<=0xdbff) { + if(!UTF_IS_TRAIL(iter1->current(iter1))) { + /* lead surrogate code point - make getIndex(iter1, UITER_CURRENT); + iter1->previous(iter1); /* ==c1 */ + if(!UTF_IS_LEAD(iter1->previous(iter1))) { + /* trail surrogate code point - make move(iter1, index, UITER_ZERO); + } else /* 0xe000<=c1<=0xffff */ { + /* BMP code point - make 0) { + int32_t rc; + for(;;) { + rc = (int32_t)*s1 - (int32_t)*s2; + if(rc != 0 || *s1 == 0 || --n == 0) { + return rc; + } + ++s1; + ++s2; + } + } else { + return 0; + } +} + +U_CAPI int32_t U_EXPORT2 +u_strncmpCodePointOrder(const UChar *s1, const UChar *s2, int32_t n) { + return uprv_strCompare(s1, n, s2, n, TRUE, TRUE); +} + +U_CAPI UChar* U_EXPORT2 +u_strcpy(UChar *dst, + const UChar *src) +{ + UChar *anchor = dst; /* save a pointer to start of dst */ + + while((*(dst++) = *(src++)) != 0) { /* copy string 2 over */ + } + + return anchor; +} + +U_CAPI UChar* U_EXPORT2 +u_strncpy(UChar *dst, + const UChar *src, + int32_t n) +{ + UChar *anchor = dst; /* save a pointer to start of dst */ + + /* copy string 2 over */ + while(n > 0 && (*(dst++) = *(src++)) != 0) { + --n; + } + + return anchor; +} + +U_CAPI int32_t U_EXPORT2 +u_strlen(const UChar *s) +{ +#if U_SIZEOF_WCHAR_T == U_SIZEOF_UCHAR + return (int32_t)uprv_wcslen(s); +#else + const UChar *t = s; + while(*t != 0) { + ++t; + } + return t - s; +#endif +} + +U_CAPI int32_t U_EXPORT2 +u_countChar32(const UChar *s, int32_t length) { + int32_t count; + + if(s==NULL || length<-1) { + return 0; + } + + count=0; + if(length>=0) { + while(length>0) { + ++count; + if(UTF_IS_LEAD(*s) && length>=2 && UTF_IS_TRAIL(*(s+1))) { + s+=2; + length-=2; + } else { + ++s; + --length; + } + } + } else /* length==-1 */ { + UChar c; + + for(;;) { + if((c=*s++)==0) { + break; + } + ++count; + + /* + * sufficient to look ahead one because of UTF-16; + * safe to look ahead one because at worst that would be the terminating NUL + */ + if(UTF_IS_LEAD(c) && UTF_IS_TRAIL(*s)) { + ++s; + } + } + } + return count; +} + +U_CAPI UBool U_EXPORT2 +u_strHasMoreChar32Than(const UChar *s, int32_t length, int32_t number) { + + if(number<0) { + return TRUE; + } + if(s==NULL || length<-1) { + return FALSE; + } + + if(length==-1) { + /* s is NUL-terminated */ + UChar c; + + /* count code points until they exceed */ + for(;;) { + if((c=*s++)==0) { + return FALSE; + } + if(number==0) { + return TRUE; + } + if(U16_IS_LEAD(c) && U16_IS_TRAIL(*s)) { + ++s; + } + --number; + } + } else { + /* length>=0 known */ + const UChar *limit; + int32_t maxSupplementary; + + /* s contains at least (length+1)/2 code points: <=2 UChars per cp */ + if(((length+1)/2)>number) { + return TRUE; + } + + /* check if s does not even contain enough UChars */ + maxSupplementary=length-number; + if(maxSupplementary<=0) { + return FALSE; + } + /* there are maxSupplementary=length-number more UChars than asked-for code points */ + + /* + * count code points until they exceed and also check that there are + * no more than maxSupplementary supplementary code points (UChar pairs) + */ + limit=s+length; + for(;;) { + if(s==limit) { + return FALSE; + } + if(number==0) { + return TRUE; + } + if(U16_IS_LEAD(*s++) && s!=limit && U16_IS_TRAIL(*s)) { + ++s; + if(--maxSupplementary<=0) { + /* too many pairs - too few code points */ + return FALSE; + } + } + --number; + } + } +} + +U_CAPI UChar * U_EXPORT2 +u_memcpy(UChar *dest, const UChar *src, int32_t count) { + return (UChar *)uprv_memcpy(dest, src, count*U_SIZEOF_UCHAR); +} + +U_CAPI UChar * U_EXPORT2 +u_memmove(UChar *dest, const UChar *src, int32_t count) { + return (UChar *)uprv_memmove(dest, src, count*U_SIZEOF_UCHAR); +} + +U_CAPI UChar * U_EXPORT2 +u_memset(UChar *dest, UChar c, int32_t count) { + if(count > 0) { + UChar *ptr = dest; + UChar *limit = dest + count; + + while (ptr < limit) { + *(ptr++) = c; + } + } + return dest; +} + +U_CAPI int32_t U_EXPORT2 +u_memcmp(const UChar *buf1, const UChar *buf2, int32_t count) { + if(count > 0) { + const UChar *limit = buf1 + count; + int32_t result; + + while (buf1 < limit) { + result = (int32_t)(uint16_t)*buf1 - (int32_t)(uint16_t)*buf2; + if (result != 0) { + return result; + } + buf1++; + buf2++; + } + } + return 0; +} + +U_CAPI int32_t U_EXPORT2 +u_memcmpCodePointOrder(const UChar *s1, const UChar *s2, int32_t count) { + return uprv_strCompare(s1, count, s2, count, FALSE, TRUE); +} + +/* u_unescape & support fns ------------------------------------------------- */ + +/* This map must be in ASCENDING ORDER OF THE ESCAPE CODE */ +static const UChar UNESCAPE_MAP[] = { + /*" 0x22, 0x22 */ + /*' 0x27, 0x27 */ + /*? 0x3F, 0x3F */ + /*\ 0x5C, 0x5C */ + /*a*/ 0x61, 0x07, + /*b*/ 0x62, 0x08, + /*e*/ 0x65, 0x1b, + /*f*/ 0x66, 0x0c, + /*n*/ 0x6E, 0x0a, + /*r*/ 0x72, 0x0d, + /*t*/ 0x74, 0x09, + /*v*/ 0x76, 0x0b +}; +enum { UNESCAPE_MAP_LENGTH = sizeof(UNESCAPE_MAP) / sizeof(UNESCAPE_MAP[0]) }; + +/* Convert one octal digit to a numeric value 0..7, or -1 on failure */ +static int8_t _digit8(UChar c) { + if (c >= 0x0030 && c <= 0x0037) { + return (int8_t)(c - 0x0030); + } + return -1; +} + +/* Convert one hex digit to a numeric value 0..F, or -1 on failure */ +static int8_t _digit16(UChar c) { + if (c >= 0x0030 && c <= 0x0039) { + return (int8_t)(c - 0x0030); + } + if (c >= 0x0041 && c <= 0x0046) { + return (int8_t)(c - (0x0041 - 10)); + } + if (c >= 0x0061 && c <= 0x0066) { + return (int8_t)(c - (0x0061 - 10)); + } + return -1; +} + +/* Parse a single escape sequence. Although this method deals in + * UChars, it does not use C++ or UnicodeString. This allows it to + * be used from C contexts. */ +U_CAPI UChar32 U_EXPORT2 +u_unescapeAt(UNESCAPE_CHAR_AT charAt, + int32_t *offset, + int32_t length, + void *context) { + + int32_t start = *offset; + UChar c; + UChar32 result = 0; + int8_t n = 0; + int8_t minDig = 0; + int8_t maxDig = 0; + int8_t bitsPerDigit = 4; + int8_t dig; + int32_t i; + UBool braces = FALSE; + + /* Check that offset is in range */ + if (*offset < 0 || *offset >= length) { + goto err; + } + + /* Fetch first UChar after '\\' */ + c = charAt((*offset)++, context); + + /* Convert hexadecimal and octal escapes */ + switch (c) { + case 0x0075 /*'u'*/: + minDig = maxDig = 4; + break; + case 0x0055 /*'U'*/: + minDig = maxDig = 8; + break; + case 0x0078 /*'x'*/: + minDig = 1; + if (*offset < length && charAt(*offset, context) == 0x7B /*{*/) { + ++(*offset); + braces = TRUE; + maxDig = 8; + } else { + maxDig = 2; + } + break; + default: + dig = _digit8(c); + if (dig >= 0) { + minDig = 1; + maxDig = 3; + n = 1; /* Already have first octal digit */ + bitsPerDigit = 3; + result = dig; + } + break; + } + if (minDig != 0) { + while (*offset < length && n < maxDig) { + c = charAt(*offset, context); + dig = (int8_t)((bitsPerDigit == 3) ? _digit8(c) : _digit16(c)); + if (dig < 0) { + break; + } + result = (result << bitsPerDigit) | dig; + ++(*offset); + ++n; + } + if (n < minDig) { + goto err; + } + if (braces) { + if (c != 0x7D /*}*/) { + goto err; + } + ++(*offset); + } + if (result < 0 || result >= 0x110000) { + goto err; + } + /* If an escape sequence specifies a lead surrogate, see if + * there is a trail surrogate after it, either as an escape or + * as a literal. If so, join them up into a supplementary. + */ + if (*offset < length && U16_IS_LEAD(result)) { + int32_t ahead = *offset + 1; + c = charAt(*offset, context); + if (c == 0x5C /*'\\'*/ && ahead < length) { + c = (UChar) u_unescapeAt(charAt, &ahead, length, context); + } + if (U16_IS_TRAIL(c)) { + *offset = ahead; + result = U16_GET_SUPPLEMENTARY(result, c); + } + } + return result; + } + + /* Convert C-style escapes in table */ + for (i=0; i destCapacity) { + srcLen = destCapacity; + } + u_charsToUChars(src, dest, srcLen); +} + +/* Do an invariant conversion of char* -> UChar*, with escape parsing */ +U_CAPI int32_t U_EXPORT2 +u_unescape(const char *src, UChar *dest, int32_t destCapacity) { + const char *segment = src; + int32_t i = 0; + char c; + + while ((c=*src) != 0) { + /* '\\' intentionally written as compiler-specific + * character constant to correspond to compiler-specific + * char* constants. */ + if (c == '\\') { + int32_t lenParsed = 0; + UChar32 c32; + if (src != segment) { + if (dest != NULL) { + _appendUChars(dest + i, destCapacity - i, + segment, src - segment); + } + i += src - segment; + } + ++src; /* advance past '\\' */ + c32 = u_unescapeAt(_charPtr_charAt, &lenParsed, uprv_strlen(src), (void*)src); + if (lenParsed == 0) { + goto err; + } + src += lenParsed; /* advance past escape seq. */ + if (dest != NULL && UTF_CHAR_LENGTH(c32) <= (destCapacity - i)) { + UTF_APPEND_CHAR_UNSAFE(dest, i, c32); + } else { + i += UTF_CHAR_LENGTH(c32); + } + segment = src; + } else { + ++src; + } + } + if (src != segment) { + if (dest != NULL) { + _appendUChars(dest + i, destCapacity - i, + segment, src - segment); + } + i += src - segment; + } + if (dest != NULL && i < destCapacity) { + dest[i] = 0; + } + return i; + + err: + if (dest != NULL && destCapacity > 0) { + *dest = 0; + } + return 0; +} + +/* C UGrowBuffer implementation --------------------------------------------- */ + +U_CAPI UBool /* U_CALLCONV U_EXPORT2 */ +u_growBufferFromStatic(void *context, + UChar **pBuffer, int32_t *pCapacity, int32_t reqCapacity, + int32_t length) { + UChar *newBuffer=(UChar *)uprv_malloc(reqCapacity*U_SIZEOF_UCHAR); + if(newBuffer!=NULL) { + if(length>0) { + uprv_memcpy(newBuffer, *pBuffer, length*U_SIZEOF_UCHAR); + } + *pCapacity=reqCapacity; + } else { + *pCapacity=0; + } + + /* release the old pBuffer if it was not statically allocated */ + if(*pBuffer!=(UChar *)context) { + uprv_free(*pBuffer); + } + + *pBuffer=newBuffer; + return (UBool)(newBuffer!=NULL); +} + +/* NUL-termination of strings ----------------------------------------------- */ + +/** + * NUL-terminate a string no matter what its type. + * Set warning and error codes accordingly. + */ +#define __TERMINATE_STRING(dest, destCapacity, length, pErrorCode) \ + if(pErrorCode!=NULL && U_SUCCESS(*pErrorCode)) { \ + /* not a public function, so no complete argument checking */ \ + \ + if(length<0) { \ + /* assume that the caller handles this */ \ + } else if(lengthdestCapacity */ { \ + /* even the string itself did not fit - set an error code */ \ + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; \ + } \ + } + +U_CAPI int32_t U_EXPORT2 +u_terminateUChars(UChar *dest, int32_t destCapacity, int32_t length, UErrorCode *pErrorCode) { + __TERMINATE_STRING(dest, destCapacity, length, pErrorCode); + return length; +} + +U_CAPI int32_t U_EXPORT2 +u_terminateChars(char *dest, int32_t destCapacity, int32_t length, UErrorCode *pErrorCode) { + __TERMINATE_STRING(dest, destCapacity, length, pErrorCode); + return length; +} + +U_CAPI int32_t U_EXPORT2 +u_terminateUChar32s(UChar32 *dest, int32_t destCapacity, int32_t length, UErrorCode *pErrorCode) { + __TERMINATE_STRING(dest, destCapacity, length, pErrorCode); + return length; +} + +U_CAPI int32_t U_EXPORT2 +u_terminateWChars(wchar_t *dest, int32_t destCapacity, int32_t length, UErrorCode *pErrorCode) { + __TERMINATE_STRING(dest, destCapacity, length, pErrorCode); + return length; +} diff --git a/Build/source/libs/icu/icu-xetex/common/ustrtrns.c b/Build/source/libs/icu/icu-xetex/common/ustrtrns.c new file mode 100644 index 00000000000..5f013c10063 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/ustrtrns.c @@ -0,0 +1,1181 @@ +/* +****************************************************************************** +* +* Copyright (C) 2001-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* +* File ustrtrns.c +* +* Modification History: +* +* Date Name Description +* 9/10/2001 Ram Creation. +****************************************************************************** +*/ + +/******************************************************************************* + * + * u_strTo* and u_strFrom* APIs + * WCS functions moved to ustr_wcs.c for better modularization + * + ******************************************************************************* + */ + + +#include "unicode/putil.h" +#include "unicode/ustring.h" +#include "cstring.h" +#include "cmemory.h" +#include "ustr_imp.h" + +U_CAPI UChar* U_EXPORT2 +u_strFromUTF32(UChar *dest, + int32_t destCapacity, + int32_t *pDestLength, + const UChar32 *src, + int32_t srcLength, + UErrorCode *pErrorCode) +{ + int32_t reqLength = 0; + uint32_t ch =0; + UChar *pDestLimit =dest+destCapacity; + UChar *pDest = dest; + const uint32_t *pSrc = (const uint32_t *)src; + + /* args check */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)){ + return NULL; + } + + if((src==NULL) || (srcLength < -1) || (destCapacity<0) || (!dest && destCapacity > 0)){ + *pErrorCode = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + + /* Check if the source is null terminated */ + if(srcLength == -1 ){ + while(((ch=*pSrc)!=0) && (pDest < pDestLimit)){ + ++pSrc; + if(ch<=0xFFFF){ + *(pDest++)=(UChar)ch; + }else if(ch<=0x10ffff){ + *(pDest++)=UTF16_LEAD(ch); + if(pDest 0)){ + *pErrorCode = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + + if(srcLength==-1) { + while((ch=*pSrc)!=0 && pDest!=pDestLimit) { + ++pSrc; + /*need not check for NUL because NUL fails UTF_IS_TRAIL() anyway*/ + if(UTF_IS_LEAD(ch) && UTF_IS_TRAIL(ch2=*pSrc)) { + ++pSrc; + ch=UTF16_GET_PAIR_VALUE(ch, ch2); + } + *(pDest++)= ch; + } + while((ch=*pSrc++)!=0) { + if(UTF_IS_LEAD(ch) && UTF_IS_TRAIL(ch2=*pSrc)) { + ++pSrc; + } + ++reqLength; + } + } else { + pSrcLimit = pSrc+srcLength; + while(pSrc=4 is always illegal: no more than 3 trail bytes in Unicode's UTF-8 */ + illegal=1; + break; + case 3: + trail=(uint8_t)(*s++ - 0x80); + c=(c<<6)|trail; + if(trail>0x3f || c>=0x110) { + /* not a trail byte, or code point>0x10ffff (outside Unicode) */ + illegal=1; + break; + } + case 2: + trail=(uint8_t)(*s++ - 0x80); + if(trail>0x3f) { + /* not a trail byte */ + illegal=1; + break; + } + c=(c<<6)|trail; + case 1: + trail=(uint8_t)(*s++ - 0x80); + if(trail>0x3f) { + /* not a trail byte */ + illegal=1; + } + c=(c<<6)|trail; + break; + case 0: + return U_SENTINEL; + /* no default branch to optimize switch() - all values are covered */ + } + + /* correct sequence - all trail bytes have (b7..b6)==(10)? */ + /* illegal is also set if count>=4 */ + if(illegal || c0 && UTF8_IS_TRAIL(*s)) { + ++s; + --count; + } + c=U_SENTINEL; + } + *ps=s; + return c; +} + +/* + * Version of utf8_nextCharSafeBody() with the following differences: + * - works with pointers instead of indexes + * - always strict (strict==-1) + * + * *ps points to after the lead byte and will be moved to after the last trail byte. + * c is the lead byte. + * @return the code point, or U_SENTINEL + */ +static UChar32 +utf8_nextCharSafeBodyPointer(const uint8_t **ps, const uint8_t *limit, UChar32 c) { + const uint8_t *s=*ps; + uint8_t trail, illegal=0; + uint8_t count=UTF8_COUNT_TRAIL_BYTES(c); + if((limit-s)>=count) { + UTF8_MASK_LEAD_BYTE((c), count); + /* count==0 for illegally leading trail bytes and the illegal bytes 0xfe and 0xff */ + switch(count) { + /* each branch falls through to the next one */ + case 5: + case 4: + /* count>=4 is always illegal: no more than 3 trail bytes in Unicode's UTF-8 */ + illegal=1; + break; + case 3: + trail=*s++; + c=(c<<6)|(trail&0x3f); + if(c<0x110) { + illegal|=(trail&0xc0)^0x80; + } else { + /* code point>0x10ffff, outside Unicode */ + illegal=1; + break; + } + case 2: + trail=*s++; + c=(c<<6)|(trail&0x3f); + illegal|=(trail&0xc0)^0x80; + case 1: + trail=*s++; + c=(c<<6)|(trail&0x3f); + illegal|=(trail&0xc0)^0x80; + break; + case 0: + return U_SENTINEL; + /* no default branch to optimize switch() - all values are covered */ + } + } else { + illegal=1; /* too few bytes left */ + } + + /* correct sequence - all trail bytes have (b7..b6)==(10)? */ + /* illegal is also set if count>=4 */ + if(illegal || c0 && s 0) || + subchar > 0x10ffff || U_IS_SURROGATE(subchar) + ) { + *pErrorCode = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + + numSubstitutions=0; + + /* + * Inline processing of UTF-8 byte sequences: + * + * Byte sequences for the most common characters are handled inline in + * the conversion loops. In order to reduce the path lengths for those + * characters, the tests are arranged in a kind of binary search. + * ASCII (<=0x7f) is checked first, followed by the dividing point + * between 2- and 3-byte sequences (0xe0). + * The 3-byte branch is tested first to speed up CJK text. + * The compiler should combine the subtractions for the two tests for 0xe0. + * Each branch then tests for the other end of its range. + */ + + if(srcLength < 0){ + /* + * Transform a NUL-terminated string. + * The code explicitly checks for NULs only in the lead byte position. + * A NUL byte in the trail byte position fails the trail byte range check anyway. + */ + while(((ch = *pSrc) != 0) && (pDest < pDestLimit)) { + if(ch <= 0x7f){ + *pDest++=(UChar)ch; + ++pSrc; + } else { + if(ch > 0xe0) { + if( /* handle U+1000..U+CFFF inline */ + ch <= 0xec && + (t1 = (uint8_t)(pSrc[1] - 0x80)) <= 0x3f && + (t2 = (uint8_t)(pSrc[2] - 0x80)) <= 0x3f + ) { + /* no need for (ch & 0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ + *pDest++ = (UChar)((ch << 12) | (t1 << 6) | t2); + pSrc += 3; + continue; + } + } else if(ch < 0xe0) { + if( /* handle U+0080..U+07FF inline */ + ch >= 0xc2 && + (t1 = (uint8_t)(pSrc[1] - 0x80)) <= 0x3f + ) { + *pDest++ = (UChar)(((ch & 0x1f) << 6) | t1); + pSrc += 2; + continue; + } + } + + /* function call for "complicated" and error cases */ + ++pSrc; /* continue after the lead byte */ + ch=utf8_nextCharSafeBodyTerminated(&pSrc, ch); + if(ch<0 && (++numSubstitutions, ch = subchar) < 0) { + *pErrorCode = U_INVALID_CHAR_FOUND; + return NULL; + } else if(ch<=0xFFFF) { + *(pDest++)=(UChar)ch; + } else { + *(pDest++)=UTF16_LEAD(ch); + if(pDest 0xe0) { + if( /* handle U+1000..U+CFFF inline */ + ch <= 0xec && + (uint8_t)(pSrc[1] - 0x80) <= 0x3f && + (uint8_t)(pSrc[2] - 0x80) <= 0x3f + ) { + ++reqLength; + pSrc += 3; + continue; + } + } else if(ch < 0xe0) { + if( /* handle U+0080..U+07FF inline */ + ch >= 0xc2 && + (uint8_t)(pSrc[1] - 0x80) <= 0x3f + ) { + ++reqLength; + pSrc += 2; + continue; + } + } + + /* function call for "complicated" and error cases */ + ++pSrc; /* continue after the lead byte */ + ch=utf8_nextCharSafeBodyTerminated(&pSrc, ch); + if(ch<0 && (++numSubstitutions, ch = subchar) < 0) { + *pErrorCode = U_INVALID_CHAR_FOUND; + return NULL; + } + reqLength += U16_LENGTH(ch); + } + } + } else /* srcLength >= 0 */ { + const uint8_t *pSrcLimit = pSrc + srcLength; + int32_t count; + + /* Faster loop without ongoing checking for pSrcLimit and pDestLimit. */ + for(;;) { + /* + * Each iteration of the inner loop progresses by at most 3 UTF-8 + * bytes and one UChar, for most characters. + * For supplementary code points (4 & 2), which are rare, + * there is an additional adjustment. + */ + count = (int32_t)(pDestLimit - pDest); + srcLength = (int32_t)((pSrcLimit - pSrc) / 3); + if(count > srcLength) { + count = srcLength; /* min(remaining dest, remaining src/3) */ + } + if(count < 3) { + /* + * Too much overhead if we get near the end of the string, + * continue with the next loop. + */ + break; + } + + do { + ch = *pSrc; + if(ch <= 0x7f){ + *pDest++=(UChar)ch; + ++pSrc; + } else { + if(ch > 0xe0) { + if( /* handle U+1000..U+CFFF inline */ + ch <= 0xec && + (t1 = (uint8_t)(pSrc[1] - 0x80)) <= 0x3f && + (t2 = (uint8_t)(pSrc[2] - 0x80)) <= 0x3f + ) { + /* no need for (ch & 0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ + *pDest++ = (UChar)((ch << 12) | (t1 << 6) | t2); + pSrc += 3; + continue; + } + } else if(ch < 0xe0) { + if( /* handle U+0080..U+07FF inline */ + ch >= 0xc2 && + (t1 = (uint8_t)(pSrc[1] - 0x80)) <= 0x3f + ) { + *pDest++ = (UChar)(((ch & 0x1f) << 6) | t1); + pSrc += 2; + continue; + } + } + + if(ch >= 0xf0 || subchar > 0xffff) { + /* + * We may read up to six bytes and write up to two UChars, + * which we didn't account for with computing count, + * so we adjust it here. + */ + if(--count == 0) { + break; + } + } + + /* function call for "complicated" and error cases */ + ++pSrc; /* continue after the lead byte */ + ch=utf8_nextCharSafeBodyPointer(&pSrc, pSrcLimit, ch); + if(ch<0 && (++numSubstitutions, ch = subchar) < 0){ + *pErrorCode = U_INVALID_CHAR_FOUND; + return NULL; + }else if(ch<=0xFFFF){ + *(pDest++)=(UChar)ch; + }else{ + *(pDest++)=UTF16_LEAD(ch); + if(pDest 0); + } + + while((pSrc 0xe0) { + if( /* handle U+1000..U+CFFF inline */ + ch <= 0xec && + ((pSrcLimit - pSrc) >= 3) && + (t1 = (uint8_t)(pSrc[1] - 0x80)) <= 0x3f && + (t2 = (uint8_t)(pSrc[2] - 0x80)) <= 0x3f + ) { + /* no need for (ch & 0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ + *pDest++ = (UChar)((ch << 12) | (t1 << 6) | t2); + pSrc += 3; + continue; + } + } else if(ch < 0xe0) { + if( /* handle U+0080..U+07FF inline */ + ch >= 0xc2 && + ((pSrcLimit - pSrc) >= 2) && + (t1 = (uint8_t)(pSrc[1] - 0x80)) <= 0x3f + ) { + *pDest++ = (UChar)(((ch & 0x1f) << 6) | t1); + pSrc += 2; + continue; + } + } + + /* function call for "complicated" and error cases */ + ++pSrc; /* continue after the lead byte */ + ch=utf8_nextCharSafeBodyPointer(&pSrc, pSrcLimit, ch); + if(ch<0 && (++numSubstitutions, ch = subchar) < 0){ + *pErrorCode = U_INVALID_CHAR_FOUND; + return NULL; + }else if(ch<=0xFFFF){ + *(pDest++)=(UChar)ch; + }else{ + *(pDest++)=UTF16_LEAD(ch); + if(pDest 0xe0) { + if( /* handle U+1000..U+CFFF inline */ + ch <= 0xec && + ((pSrcLimit - pSrc) >= 3) && + (uint8_t)(pSrc[1] - 0x80) <= 0x3f && + (uint8_t)(pSrc[2] - 0x80) <= 0x3f + ) { + reqLength++; + pSrc += 3; + continue; + } + } else if(ch < 0xe0) { + if( /* handle U+0080..U+07FF inline */ + ch >= 0xc2 && + ((pSrcLimit - pSrc) >= 2) && + (uint8_t)(pSrc[1] - 0x80) <= 0x3f + ) { + reqLength++; + pSrc += 2; + continue; + } + } + + /* function call for "complicated" and error cases */ + ++pSrc; /* continue after the lead byte */ + ch=utf8_nextCharSafeBodyPointer(&pSrc, pSrcLimit, ch); + if(ch<0 && (++numSubstitutions, ch = subchar) < 0){ + *pErrorCode = U_INVALID_CHAR_FOUND; + return NULL; + } + reqLength+=UTF_CHAR_LENGTH(ch); + } + } + } + + reqLength+=(int32_t)(pDest - dest); + + if(pNumSubstitutions!=NULL) { + *pNumSubstitutions=numSubstitutions; + } + + if(pDestLength){ + *pDestLength = reqLength; + } + + /* Terminate the buffer */ + u_terminateUChars(dest,destCapacity,reqLength,pErrorCode); + + return dest; +} + +U_CAPI UChar* U_EXPORT2 +u_strFromUTF8(UChar *dest, + int32_t destCapacity, + int32_t *pDestLength, + const char* src, + int32_t srcLength, + UErrorCode *pErrorCode){ + return u_strFromUTF8WithSub( + dest, destCapacity, pDestLength, + src, srcLength, + U_SENTINEL, NULL, + pErrorCode); +} + +U_CAPI UChar * U_EXPORT2 +u_strFromUTF8Lenient(UChar *dest, + int32_t destCapacity, + int32_t *pDestLength, + const char *src, + int32_t srcLength, + UErrorCode *pErrorCode) { + + UChar *pDest = dest; + UChar32 ch; + int32_t reqLength = 0; + uint8_t* pSrc = (uint8_t*) src; + + /* args check */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)){ + return NULL; + } + + if((src==NULL) || (srcLength < -1) || (destCapacity<0) || (!dest && destCapacity > 0)) { + *pErrorCode = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + + if(srcLength < 0) { + /* Transform a NUL-terminated string. */ + UChar *pDestLimit = dest+destCapacity; + uint8_t t1, t2, t3; /* trail bytes */ + + while(((ch = *pSrc) != 0) && (pDest < pDestLimit)) { + if(ch < 0xc0) { + /* + * ASCII, or a trail byte in lead position which is treated like + * a single-byte sequence for better character boundary + * resynchronization after illegal sequences. + */ + *pDest++=(UChar)ch; + ++pSrc; + continue; + } else if(ch < 0xe0) { /* U+0080..U+07FF */ + if((t1 = pSrc[1]) != 0) { + /* 0x3080 = (0xc0 << 6) + 0x80 */ + *pDest++ = (UChar)((ch << 6) + t1 - 0x3080); + pSrc += 2; + continue; + } + } else if(ch < 0xf0) { /* U+0800..U+FFFF */ + if((t1 = pSrc[1]) != 0 && (t2 = pSrc[2]) != 0) { + /* no need for (ch & 0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ + /* 0x2080 = (0x80 << 6) + 0x80 */ + *pDest++ = (UChar)((ch << 12) + (t1 << 6) + t2 - 0x2080); + pSrc += 3; + continue; + } + } else /* f0..f4 */ { /* U+10000..U+10FFFF */ + if((t1 = pSrc[1]) != 0 && (t2 = pSrc[2]) != 0 && (t3 = pSrc[3]) != 0) { + pSrc += 4; + /* 0x3c82080 = (0xf0 << 18) + (0x80 << 12) + (0x80 << 6) + 0x80 */ + ch = (ch << 18) + (t1 << 12) + (t2 << 6) + t3 - 0x3c82080; + *(pDest++) = U16_LEAD(ch); + if(pDest < pDestLimit) { + *(pDest++) = U16_TRAIL(ch); + } else { + reqLength = 1; + break; + } + continue; + } + } + + /* truncated character at the end */ + *pDest++ = 0xfffd; + while(*++pSrc != 0) {} + break; + } + + /* Pre-flight the rest of the string. */ + while((ch = *pSrc) != 0) { + if(ch < 0xc0) { + /* + * ASCII, or a trail byte in lead position which is treated like + * a single-byte sequence for better character boundary + * resynchronization after illegal sequences. + */ + ++reqLength; + ++pSrc; + continue; + } else if(ch < 0xe0) { /* U+0080..U+07FF */ + if(pSrc[1] != 0) { + ++reqLength; + pSrc += 2; + continue; + } + } else if(ch < 0xf0) { /* U+0800..U+FFFF */ + if(pSrc[1] != 0 && pSrc[2] != 0) { + ++reqLength; + pSrc += 3; + continue; + } + } else /* f0..f4 */ { /* U+10000..U+10FFFF */ + if(pSrc[1] != 0 && pSrc[2] != 0 && pSrc[3] != 0) { + reqLength += 2; + pSrc += 4; + continue; + } + } + + /* truncated character at the end */ + ++reqLength; + break; + } + } else /* srcLength >= 0 */ { + const uint8_t *pSrcLimit = pSrc + srcLength; + + /* + * This function requires that if srcLength is given, then it must be + * destCapatity >= srcLength so that we need not check for + * destination buffer overflow in the loop. + */ + if(destCapacity < srcLength) { + if(pDestLength != NULL) { + *pDestLength = srcLength; /* this likely overestimates the true destLength! */ + } + *pErrorCode = U_BUFFER_OVERFLOW_ERROR; + return NULL; + } + + if((pSrcLimit - pSrc) >= 4) { + pSrcLimit -= 3; /* temporarily reduce pSrcLimit */ + + /* in this loop, we can always access at least 4 bytes, up to pSrc+3 */ + do { + ch = *pSrc++; + if(ch < 0xc0) { + /* + * ASCII, or a trail byte in lead position which is treated like + * a single-byte sequence for better character boundary + * resynchronization after illegal sequences. + */ + *pDest++=(UChar)ch; + } else if(ch < 0xe0) { /* U+0080..U+07FF */ + /* 0x3080 = (0xc0 << 6) + 0x80 */ + *pDest++ = (UChar)((ch << 6) + *pSrc++ - 0x3080); + } else if(ch < 0xf0) { /* U+0800..U+FFFF */ + /* no need for (ch & 0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ + /* 0x2080 = (0x80 << 6) + 0x80 */ + ch = (ch << 12) + (*pSrc++ << 6); + *pDest++ = (UChar)(ch + *pSrc++ - 0x2080); + } else /* f0..f4 */ { /* U+10000..U+10FFFF */ + /* 0x3c82080 = (0xf0 << 18) + (0x80 << 12) + (0x80 << 6) + 0x80 */ + ch = (ch << 18) + (*pSrc++ << 12); + ch += *pSrc++ << 6; + ch += *pSrc++ - 0x3c82080; + *(pDest++) = U16_LEAD(ch); + *(pDest++) = U16_TRAIL(ch); + } + } while(pSrc < pSrcLimit); + + pSrcLimit += 3; /* restore original pSrcLimit */ + } + + while(pSrc < pSrcLimit) { + ch = *pSrc++; + if(ch < 0xc0) { + /* + * ASCII, or a trail byte in lead position which is treated like + * a single-byte sequence for better character boundary + * resynchronization after illegal sequences. + */ + *pDest++=(UChar)ch; + continue; + } else if(ch < 0xe0) { /* U+0080..U+07FF */ + if(pSrc < pSrcLimit) { + /* 0x3080 = (0xc0 << 6) + 0x80 */ + *pDest++ = (UChar)((ch << 6) + *pSrc++ - 0x3080); + continue; + } + } else if(ch < 0xf0) { /* U+0800..U+FFFF */ + if((pSrcLimit - pSrc) >= 2) { + /* no need for (ch & 0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ + /* 0x2080 = (0x80 << 6) + 0x80 */ + ch = (ch << 12) + (*pSrc++ << 6); + *pDest++ = (UChar)(ch + *pSrc++ - 0x2080); + pSrc += 3; + continue; + } + } else /* f0..f4 */ { /* U+10000..U+10FFFF */ + if((pSrcLimit - pSrc) >= 3) { + /* 0x3c82080 = (0xf0 << 18) + (0x80 << 12) + (0x80 << 6) + 0x80 */ + ch = (ch << 18) + (*pSrc++ << 12); + ch += *pSrc++ << 6; + ch += *pSrc++ - 0x3c82080; + *(pDest++) = U16_LEAD(ch); + *(pDest++) = U16_TRAIL(ch); + pSrc += 4; + continue; + } + } + + /* truncated character at the end */ + *pDest++ = 0xfffd; + break; + } + } + + reqLength+=(int32_t)(pDest - dest); + + if(pDestLength){ + *pDestLength = reqLength; + } + + /* Terminate the buffer */ + u_terminateUChars(dest,destCapacity,reqLength,pErrorCode); + + return dest; +} + +static U_INLINE uint8_t * +_appendUTF8(uint8_t *pDest, UChar32 c) { + /* it is 0<=c<=0x10ffff and not a surrogate if called by a validating function */ + if((c)<=0x7f) { + *pDest++=(uint8_t)c; + } else if(c<=0x7ff) { + *pDest++=(uint8_t)((c>>6)|0xc0); + *pDest++=(uint8_t)((c&0x3f)|0x80); + } else if(c<=0xffff) { + *pDest++=(uint8_t)((c>>12)|0xe0); + *pDest++=(uint8_t)(((c>>6)&0x3f)|0x80); + *pDest++=(uint8_t)(((c)&0x3f)|0x80); + } else /* if((uint32_t)(c)<=0x10ffff) */ { + *pDest++=(uint8_t)(((c)>>18)|0xf0); + *pDest++=(uint8_t)((((c)>>12)&0x3f)|0x80); + *pDest++=(uint8_t)((((c)>>6)&0x3f)|0x80); + *pDest++=(uint8_t)(((c)&0x3f)|0x80); + } + return pDest; +} + + +U_CAPI char* U_EXPORT2 +u_strToUTF8WithSub(char *dest, + int32_t destCapacity, + int32_t *pDestLength, + const UChar *pSrc, + int32_t srcLength, + UChar32 subchar, int32_t *pNumSubstitutions, + UErrorCode *pErrorCode){ + + int32_t reqLength=0; + uint32_t ch=0,ch2=0; + uint8_t *pDest = (uint8_t *)dest; + uint8_t *pDestLimit = pDest + destCapacity; + int32_t numSubstitutions; + + /* args check */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)){ + return NULL; + } + + if( (pSrc==NULL) || (srcLength < -1) || (destCapacity<0) || (!dest && destCapacity > 0) || + subchar > 0x10ffff || U_IS_SURROGATE(subchar) + ) { + *pErrorCode = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + + numSubstitutions=0; + + if(srcLength==-1) { + while((ch=*pSrc)!=0) { + ++pSrc; + if(ch <= 0x7f) { + if(pDest= 2) { + *pDest++=(uint8_t)((ch>>6)|0xc0); + *pDest++=(uint8_t)((ch&0x3f)|0x80); + } else { + reqLength = 2; + break; + } + } else if(ch <= 0xd7ff || ch >= 0xe000) { + if((pDestLimit - pDest) >= 3) { + *pDest++=(uint8_t)((ch>>12)|0xe0); + *pDest++=(uint8_t)(((ch>>6)&0x3f)|0x80); + *pDest++=(uint8_t)((ch&0x3f)|0x80); + } else { + reqLength = 3; + break; + } + } else /* ch is a surrogate */ { + int32_t length; + + /*need not check for NUL because NUL fails UTF_IS_TRAIL() anyway*/ + if(UTF_IS_SURROGATE_FIRST(ch) && UTF_IS_TRAIL(ch2=*pSrc)) { + ++pSrc; + ch=UTF16_GET_PAIR_VALUE(ch, ch2); + } else if(subchar>=0) { + ch=subchar; + ++numSubstitutions; + } else { + /* Unicode 3.2 forbids surrogate code points in UTF-8 */ + *pErrorCode = U_INVALID_CHAR_FOUND; + return NULL; + } + + length = U8_LENGTH(ch); + if((pDestLimit - pDest) >= length) { + /* convert and append*/ + pDest=_appendUTF8(pDest, ch); + } else { + reqLength = length; + break; + } + } + } + while((ch=*pSrc++)!=0) { + if(ch<=0x7f) { + ++reqLength; + } else if(ch<=0x7ff) { + reqLength+=2; + } else if(!UTF_IS_SURROGATE(ch)) { + reqLength+=3; + } else if(UTF_IS_SURROGATE_FIRST(ch) && UTF_IS_TRAIL(ch2=*pSrc)) { + ++pSrc; + reqLength+=4; + } else if(subchar>=0) { + reqLength+=U8_LENGTH(subchar); + ++numSubstitutions; + } else { + /* Unicode 3.2 forbids surrogate code points in UTF-8 */ + *pErrorCode = U_INVALID_CHAR_FOUND; + return NULL; + } + } + } else { + const UChar *pSrcLimit = pSrc+srcLength; + int32_t count; + + /* Faster loop without ongoing checking for pSrcLimit and pDestLimit. */ + for(;;) { + /* + * Each iteration of the inner loop progresses by at most 3 UTF-8 + * bytes and one UChar, for most characters. + * For supplementary code points (4 & 2), which are rare, + * there is an additional adjustment. + */ + count = (int32_t)((pDestLimit - pDest) / 3); + srcLength = (int32_t)(pSrcLimit - pSrc); + if(count > srcLength) { + count = srcLength; /* min(remaining dest/3, remaining src) */ + } + if(count < 3) { + /* + * Too much overhead if we get near the end of the string, + * continue with the next loop. + */ + break; + } + do { + ch=*pSrc++; + if(ch <= 0x7f) { + *pDest++ = (char)ch; + } else if(ch <= 0x7ff) { + *pDest++=(uint8_t)((ch>>6)|0xc0); + *pDest++=(uint8_t)((ch&0x3f)|0x80); + } else if(ch <= 0xd7ff || ch >= 0xe000) { + *pDest++=(uint8_t)((ch>>12)|0xe0); + *pDest++=(uint8_t)(((ch>>6)&0x3f)|0x80); + *pDest++=(uint8_t)((ch&0x3f)|0x80); + } else /* ch is a surrogate */ { + /* + * We will read two UChars and probably output four bytes, + * which we didn't account for with computing count, + * so we adjust it here. + */ + if(--count == 0) { + --pSrc; /* undo ch=*pSrc++ for the lead surrogate */ + break; /* recompute count */ + } + + if(UTF_IS_SURROGATE_FIRST(ch) && UTF_IS_TRAIL(ch2=*pSrc)) { + ++pSrc; + ch=UTF16_GET_PAIR_VALUE(ch, ch2); + + /* writing 4 bytes per 2 UChars is ok */ + *pDest++=(uint8_t)((ch>>18)|0xf0); + *pDest++=(uint8_t)(((ch>>12)&0x3f)|0x80); + *pDest++=(uint8_t)(((ch>>6)&0x3f)|0x80); + *pDest++=(uint8_t)((ch&0x3f)|0x80); + } else { + /* Unicode 3.2 forbids surrogate code points in UTF-8 */ + if(subchar>=0) { + ch=subchar; + ++numSubstitutions; + } else { + *pErrorCode = U_INVALID_CHAR_FOUND; + return NULL; + } + + /* convert and append*/ + pDest=_appendUTF8(pDest, ch); + } + } + } while(--count > 0); + } + + while(pSrc= 2) { + *pDest++=(uint8_t)((ch>>6)|0xc0); + *pDest++=(uint8_t)((ch&0x3f)|0x80); + } else { + reqLength = 2; + break; + } + } else if(ch <= 0xd7ff || ch >= 0xe000) { + if((pDestLimit - pDest) >= 3) { + *pDest++=(uint8_t)((ch>>12)|0xe0); + *pDest++=(uint8_t)(((ch>>6)&0x3f)|0x80); + *pDest++=(uint8_t)((ch&0x3f)|0x80); + } else { + reqLength = 3; + break; + } + } else /* ch is a surrogate */ { + int32_t length; + + if(UTF_IS_SURROGATE_FIRST(ch) && pSrc=0) { + ch=subchar; + ++numSubstitutions; + } else { + /* Unicode 3.2 forbids surrogate code points in UTF-8 */ + *pErrorCode = U_INVALID_CHAR_FOUND; + return NULL; + } + + length = U8_LENGTH(ch); + if((pDestLimit - pDest) >= length) { + /* convert and append*/ + pDest=_appendUTF8(pDest, ch); + } else { + reqLength = length; + break; + } + } + } + while(pSrc=0) { + reqLength+=U8_LENGTH(subchar); + ++numSubstitutions; + } else { + /* Unicode 3.2 forbids surrogate code points in UTF-8 */ + *pErrorCode = U_INVALID_CHAR_FOUND; + return NULL; + } + } + } + + reqLength+=(int32_t)(pDest - (uint8_t *)dest); + + if(pNumSubstitutions!=NULL) { + *pNumSubstitutions=numSubstitutions; + } + + if(pDestLength){ + *pDestLength = reqLength; + } + + /* Terminate the buffer */ + u_terminateChars((char*)dest,destCapacity,reqLength,pErrorCode); + + return (char*)dest; +} + +U_CAPI char* U_EXPORT2 +u_strToUTF8(char *dest, + int32_t destCapacity, + int32_t *pDestLength, + const UChar *pSrc, + int32_t srcLength, + UErrorCode *pErrorCode){ + return u_strToUTF8WithSub( + dest, destCapacity, pDestLength, + pSrc, srcLength, + U_SENTINEL, NULL, + pErrorCode); +} diff --git a/Build/source/libs/icu/icu-xetex/common/utext.cpp b/Build/source/libs/icu/icu-xetex/common/utext.cpp new file mode 100644 index 00000000000..2c317200372 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/utext.cpp @@ -0,0 +1,2822 @@ +/* +******************************************************************************* +* +* Copyright (C) 2005-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: utext.cpp +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2005apr12 +* created by: Markus W. Scherer +*/ + +#include "unicode/utypes.h" +#include "unicode/ustring.h" +#include "unicode/unistr.h" +#include "unicode/chariter.h" +#include "unicode/utext.h" +#include "ustr_imp.h" +#include "cmemory.h" +#include "cstring.h" +#include "uassert.h" + +U_NAMESPACE_USE + +#define I32_FLAG(bitIndex) ((int32_t)1<<(bitIndex)) + + +static UBool +utext_access(UText *ut, int64_t index, UBool forward) { + return ut->pFuncs->access(ut, index, forward); +} + + + +U_CAPI UBool U_EXPORT2 +utext_moveIndex32(UText *ut, int32_t delta) { + UChar32 c; + if (delta > 0) { + do { + if(ut->chunkOffset>=ut->chunkLength && !utext_access(ut, ut->chunkNativeLimit, TRUE)) { + return FALSE; + } + c = ut->chunkContents[ut->chunkOffset]; + if (U16_IS_SURROGATE(c)) { + c = utext_next32(ut); + if (c == U_SENTINEL) { + return FALSE; + } + } else { + ut->chunkOffset++; + } + } while(--delta>0); + + } else if (delta<0) { + do { + if(ut->chunkOffset<=0 && !utext_access(ut, ut->chunkNativeStart, FALSE)) { + return FALSE; + } + c = ut->chunkContents[ut->chunkOffset-1]; + if (U16_IS_SURROGATE(c)) { + c = utext_previous32(ut); + if (c == U_SENTINEL) { + return FALSE; + } + } else { + ut->chunkOffset--; + } + } while(++delta<0); + } + + return TRUE; +} + + +U_CAPI int64_t U_EXPORT2 +utext_nativeLength(UText *ut) { + return ut->pFuncs->nativeLength(ut); +} + + +U_CAPI UBool U_EXPORT2 +utext_isLengthExpensive(const UText *ut) { + UBool r = (ut->providerProperties & I32_FLAG(UTEXT_PROVIDER_LENGTH_IS_EXPENSIVE)) != 0; + return r; +} + + +U_CAPI int64_t U_EXPORT2 +utext_getNativeIndex(const UText *ut) { + if(ut->chunkOffset <= ut->nativeIndexingLimit) { + return ut->chunkNativeStart+ut->chunkOffset; + } else { + return ut->pFuncs->mapOffsetToNative(ut); + } +} + + +U_CAPI void U_EXPORT2 +utext_setNativeIndex(UText *ut, int64_t index) { + if(indexchunkNativeStart || index>=ut->chunkNativeLimit) { + // The desired position is outside of the current chunk. + // Access the new position. Assume a forward iteration from here, + // which will also be optimimum for a single random access. + // Reverse iterations may suffer slightly. + ut->pFuncs->access(ut, index, TRUE); + } else if((int32_t)(index - ut->chunkNativeStart) <= ut->nativeIndexingLimit) { + // utf-16 indexing. + ut->chunkOffset=(int32_t)(index-ut->chunkNativeStart); + } else { + ut->chunkOffset=ut->pFuncs->mapNativeIndexToUTF16(ut, index); + } + // The convention is that the index must always be on a code point boundary. + // Adjust the index position if it is in the middle of a surrogate pair. + if (ut->chunkOffsetchunkLength) { + UChar c= ut->chunkContents[ut->chunkOffset]; + if (UTF16_IS_TRAIL(c)) { + if (ut->chunkOffset==0) { + ut->pFuncs->access(ut, ut->chunkNativeStart, FALSE); + } + if (ut->chunkOffset>0) { + UChar lead = ut->chunkContents[ut->chunkOffset-1]; + if (UTF16_IS_LEAD(lead)) { + ut->chunkOffset--; + } + } + } + } +} + + + +U_CAPI int64_t U_EXPORT2 +utext_getPreviousNativeIndex(UText *ut) { + // + // Fast-path the common case. + // Common means current position is not at the beginning of a chunk + // and the preceding character is not supplementary. + // + int32_t i = ut->chunkOffset - 1; + int64_t result; + if (i >= 0) { + UChar c = ut->chunkContents[i]; + if (U16_IS_TRAIL(c) == FALSE) { + if (i <= ut->nativeIndexingLimit) { + result = ut->chunkNativeStart + i; + } else { + ut->chunkOffset = i; + result = ut->pFuncs->mapOffsetToNative(ut); + ut->chunkOffset++; + } + return result; + } + } + + // If at the start of text, simply return 0. + if (ut->chunkOffset==0 && ut->chunkNativeStart==0) { + return 0; + } + + // Harder, less common cases. We are at a chunk boundary, or on a surrogate. + // Keep it simple, use other functions to handle the edges. + // + utext_previous32(ut); + result = UTEXT_GETNATIVEINDEX(ut); + utext_next32(ut); + return result; +} + + +// +// utext_current32. Get the UChar32 at the current position. +// UText iteration position is always on a code point boundary, +// never on the trail half of a surrogate pair. +// +U_CAPI UChar32 U_EXPORT2 +utext_current32(UText *ut) { + UChar32 c; + if (ut->chunkOffset==ut->chunkLength) { + // Current position is just off the end of the chunk. + if (ut->pFuncs->access(ut, ut->chunkNativeLimit, TRUE) == FALSE) { + // Off the end of the text. + return U_SENTINEL; + } + } + + c = ut->chunkContents[ut->chunkOffset]; + if (U16_IS_LEAD(c) == FALSE) { + // Normal, non-supplementary case. + return c; + } + + // + // Possible supplementary char. + // + UChar32 trail = 0; + UChar32 supplementaryC = c; + if ((ut->chunkOffset+1) < ut->chunkLength) { + // The trail surrogate is in the same chunk. + trail = ut->chunkContents[ut->chunkOffset+1]; + } else { + // The trail surrogate is in a different chunk. + // Because we must maintain the iteration position, we need to switch forward + // into the new chunk, get the trail surrogate, then revert the chunk back to the + // original one. + // An edge case to be careful of: the entire text may end with an unpaired + // leading surrogate. The attempt to access the trail will fail, but + // the original position before the unpaired lead still needs to be restored. + int64_t nativePosition = ut->chunkNativeLimit; + int32_t originalOffset = ut->chunkOffset; + if (ut->pFuncs->access(ut, nativePosition, TRUE)) { + trail = ut->chunkContents[ut->chunkOffset]; + } + UBool r = ut->pFuncs->access(ut, nativePosition, FALSE); // reverse iteration flag loads preceding chunk + U_ASSERT(r==TRUE); + ut->chunkOffset = originalOffset; + if(!r) { + return U_SENTINEL; + } + } + + if (U16_IS_TRAIL(trail)) { + supplementaryC = U16_GET_SUPPLEMENTARY(c, trail); + } + return supplementaryC; + +} + + +U_CAPI UChar32 U_EXPORT2 +utext_char32At(UText *ut, int64_t nativeIndex) { + UChar32 c = U_SENTINEL; + + // Fast path the common case. + if (nativeIndex>=ut->chunkNativeStart && nativeIndex < ut->chunkNativeStart + ut->nativeIndexingLimit) { + ut->chunkOffset = (int32_t)(nativeIndex - ut->chunkNativeStart); + c = ut->chunkContents[ut->chunkOffset]; + if (U16_IS_SURROGATE(c) == FALSE) { + return c; + } + } + + + utext_setNativeIndex(ut, nativeIndex); + if (nativeIndex>=ut->chunkNativeStart && ut->chunkOffsetchunkLength) { + c = ut->chunkContents[ut->chunkOffset]; + if (U16_IS_SURROGATE(c)) { + // For surrogates, let current32() deal with the complications + // of supplementaries that may span chunk boundaries. + c = utext_current32(ut); + } + } + return c; +} + + +U_CAPI UChar32 U_EXPORT2 +utext_next32(UText *ut) { + UChar32 c; + + if (ut->chunkOffset >= ut->chunkLength) { + if (ut->pFuncs->access(ut, ut->chunkNativeLimit, TRUE) == FALSE) { + return U_SENTINEL; + } + } + + c = ut->chunkContents[ut->chunkOffset++]; + if (U16_IS_LEAD(c) == FALSE) { + // Normal case, not supplementary. + // (A trail surrogate seen here is just returned as is, as a surrogate value. + // It cannot be part of a pair.) + return c; + } + + if (ut->chunkOffset >= ut->chunkLength) { + if (ut->pFuncs->access(ut, ut->chunkNativeLimit, TRUE) == FALSE) { + // c is an unpaired lead surrogate at the end of the text. + // return it as it is. + return c; + } + } + UChar32 trail = ut->chunkContents[ut->chunkOffset]; + if (U16_IS_TRAIL(trail) == FALSE) { + // c was an unpaired lead surrogate, not at the end of the text. + // return it as it is (unpaired). Iteration position is on the + // following character, possibly in the next chunk, where the + // trail surrogate would have been if it had existed. + return c; + } + + UChar32 supplementary = U16_GET_SUPPLEMENTARY(c, trail); + ut->chunkOffset++; // move iteration position over the trail surrogate. + return supplementary; + } + + +U_CAPI UChar32 U_EXPORT2 +utext_previous32(UText *ut) { + UChar32 c; + + if (ut->chunkOffset <= 0) { + if (ut->pFuncs->access(ut, ut->chunkNativeStart, FALSE) == FALSE) { + return U_SENTINEL; + } + } + ut->chunkOffset--; + c = ut->chunkContents[ut->chunkOffset]; + if (U16_IS_TRAIL(c) == FALSE) { + // Normal case, not supplementary. + // (A lead surrogate seen here is just returned as is, as a surrogate value. + // It cannot be part of a pair.) + return c; + } + + if (ut->chunkOffset <= 0) { + if (ut->pFuncs->access(ut, ut->chunkNativeStart, FALSE) == FALSE) { + // c is an unpaired trail surrogate at the start of the text. + // return it as it is. + return c; + } + } + + UChar32 lead = ut->chunkContents[ut->chunkOffset-1]; + if (U16_IS_LEAD(lead) == FALSE) { + // c was an unpaired trail surrogate, not at the end of the text. + // return it as it is (unpaired). Iteration position is at c + return c; + } + + UChar32 supplementary = U16_GET_SUPPLEMENTARY(lead, c); + ut->chunkOffset--; // move iteration position over the lead surrogate. + return supplementary; +} + + + +U_CAPI UChar32 U_EXPORT2 +utext_next32From(UText *ut, int64_t index) { + UChar32 c = U_SENTINEL; + + if(indexchunkNativeStart || index>=ut->chunkNativeLimit) { + // Desired position is outside of the current chunk. + if(!ut->pFuncs->access(ut, index, TRUE)) { + // no chunk available here + return U_SENTINEL; + } + } else if (index - ut->chunkNativeStart <= (int64_t)ut->nativeIndexingLimit) { + // Desired position is in chunk, with direct 1:1 native to UTF16 indexing + ut->chunkOffset = (int32_t)(index - ut->chunkNativeStart); + } else { + // Desired position is in chunk, with non-UTF16 indexing. + ut->chunkOffset = ut->pFuncs->mapNativeIndexToUTF16(ut, index); + } + + c = ut->chunkContents[ut->chunkOffset++]; + if (U16_IS_SURROGATE(c)) { + // Surrogates. Many edge cases. Use other functions that already + // deal with the problems. + utext_setNativeIndex(ut, index); + c = utext_next32(ut); + } + return c; +} + + +U_CAPI UChar32 U_EXPORT2 +utext_previous32From(UText *ut, int64_t index) { + // + // Return the character preceding the specified index. + // Leave the iteration position at the start of the character that was returned. + // + UChar32 cPrev; // The character preceding cCurr, which is what we will return. + + // Address the chunk containg the position preceding the incoming index + // A tricky edge case: + // We try to test the requested native index against the chunkNativeStart to determine + // whether the character preceding the one at the index is in the current chunk. + // BUT, this test can fail with UTF-8 (or any other multibyte encoding), when the + // requested index is on something other than the first position of the first char. + // + if(index<=ut->chunkNativeStart || index>ut->chunkNativeLimit) { + // Requested native index is outside of the current chunk. + if(!ut->pFuncs->access(ut, index, FALSE)) { + // no chunk available here + return U_SENTINEL; + } + } else if(index - ut->chunkNativeStart <= (int64_t)ut->nativeIndexingLimit) { + // Direct UTF-16 indexing. + ut->chunkOffset = (int32_t)(index - ut->chunkNativeStart); + } else { + ut->chunkOffset=ut->pFuncs->mapNativeIndexToUTF16(ut, index); + if (ut->chunkOffset==0 && !ut->pFuncs->access(ut, index, FALSE)) { + // no chunk available here + return U_SENTINEL; + } + } + + // + // Simple case with no surrogates. + // + ut->chunkOffset--; + cPrev = ut->chunkContents[ut->chunkOffset]; + + if (U16_IS_SURROGATE(cPrev)) { + // Possible supplementary. Many edge cases. + // Let other functions do the heavy lifting. + utext_setNativeIndex(ut, index); + cPrev = utext_previous32(ut); + } + return cPrev; +} + + +U_CAPI int32_t U_EXPORT2 +utext_extract(UText *ut, + int64_t start, int64_t limit, + UChar *dest, int32_t destCapacity, + UErrorCode *status) { + return ut->pFuncs->extract(ut, start, limit, dest, destCapacity, status); + } + + + +U_CAPI UBool U_EXPORT2 +utext_equals(const UText *a, const UText *b) { + if (a==NULL || b==NULL || + a->magic != UTEXT_MAGIC || + b->magic != UTEXT_MAGIC) { + // Null or invalid arguments don't compare equal to anything. + return FALSE; + } + + if (a->pFuncs != b->pFuncs) { + // Different types of text providers. + return FALSE; + } + + if (a->context != b->context) { + // Different sources (different strings) + return FALSE; + } + if (utext_getNativeIndex(a) != utext_getNativeIndex(b)) { + // Different current position in the string. + return FALSE; + } + + return TRUE; +} + +U_CAPI UBool U_EXPORT2 +utext_isWritable(const UText *ut) +{ + UBool b = (ut->providerProperties & I32_FLAG(UTEXT_PROVIDER_WRITABLE)) != 0; + return b; +} + + +U_CAPI void U_EXPORT2 +utext_freeze(UText *ut) { + // Zero out the WRITABLE flag. + ut->providerProperties &= ~(I32_FLAG(UTEXT_PROVIDER_WRITABLE)); +} + + +U_CAPI UBool U_EXPORT2 +utext_hasMetaData(const UText *ut) +{ + UBool b = (ut->providerProperties & I32_FLAG(UTEXT_PROVIDER_HAS_META_DATA)) != 0; + return b; +} + + + +U_CAPI int32_t U_EXPORT2 +utext_replace(UText *ut, + int64_t nativeStart, int64_t nativeLimit, + const UChar *replacementText, int32_t replacementLength, + UErrorCode *status) +{ + if (U_FAILURE(*status)) { + return 0; + } + if ((ut->providerProperties & I32_FLAG(UTEXT_PROVIDER_WRITABLE)) == 0) { + *status = U_NO_WRITE_PERMISSION; + return 0; + } + int32_t i = ut->pFuncs->replace(ut, nativeStart, nativeLimit, replacementText, replacementLength, status); + return i; +} + +U_CAPI void U_EXPORT2 +utext_copy(UText *ut, + int64_t nativeStart, int64_t nativeLimit, + int64_t destIndex, + UBool move, + UErrorCode *status) +{ + if (U_FAILURE(*status)) { + return; + } + if ((ut->providerProperties & I32_FLAG(UTEXT_PROVIDER_WRITABLE)) == 0) { + *status = U_NO_WRITE_PERMISSION; + return; + } + ut->pFuncs->copy(ut, nativeStart, nativeLimit, destIndex, move, status); +} + + + +U_CAPI UText * U_EXPORT2 +utext_clone(UText *dest, const UText *src, UBool deep, UBool readOnly, UErrorCode *status) { + UText *result; + result = src->pFuncs->clone(dest, src, deep, status); + if (readOnly) { + utext_freeze(result); + } + return result; +} + + + +//------------------------------------------------------------------------------ +// +// UText common functions implementation +// +//------------------------------------------------------------------------------ + +// +// UText.flags bit definitions +// +enum { + UTEXT_HEAP_ALLOCATED = 1, // 1 if ICU has allocated this UText struct on the heap. + // 0 if caller provided storage for the UText. + + UTEXT_EXTRA_HEAP_ALLOCATED = 2, // 1 if ICU has allocated extra storage as a separate + // heap block. + // 0 if there is no separate allocation. Either no extra + // storage was requested, or it is appended to the end + // of the main UText storage. + + UTEXT_OPEN = 4 // 1 if this UText is currently open + // 0 if this UText is not open. +}; + + +// +// Extended form of a UText. The purpose is to aid in computing the total size required +// when a provider asks for a UText to be allocated with extra storage. + +struct ExtendedUText { + UText ut; + UAlignedMemory extension; +}; + +static const UText emptyText = UTEXT_INITIALIZER; + +U_CAPI UText * U_EXPORT2 +utext_setup(UText *ut, int32_t extraSpace, UErrorCode *status) { + if (U_FAILURE(*status)) { + return ut; + } + + if (ut == NULL) { + // We need to heap-allocate storage for the new UText + int32_t spaceRequired = sizeof(UText); + if (extraSpace > 0) { + spaceRequired = sizeof(ExtendedUText) + extraSpace - sizeof(UAlignedMemory); + } + ut = (UText *)uprv_malloc(spaceRequired); + if (ut == NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + } else { + *ut = emptyText; + ut->flags |= UTEXT_HEAP_ALLOCATED; + if (spaceRequired>0) { + ut->extraSize = extraSpace; + ut->pExtra = &((ExtendedUText *)ut)->extension; + } + } + } else { + // We have been supplied with an already existing UText. + // Verify that it really appears to be a UText. + if (ut->magic != UTEXT_MAGIC) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return ut; + } + // If the ut is already open and there's a provider supplied close + // function, call it. + if ((ut->flags & UTEXT_OPEN) && ut->pFuncs->close != NULL) { + ut->pFuncs->close(ut); + } + ut->flags &= ~UTEXT_OPEN; + + // If extra space was requested by our caller, check whether + // sufficient already exists, and allocate new if needed. + if (extraSpace > ut->extraSize) { + // Need more space. If there is existing separately allocated space, + // delete it first, then allocate new space. + if (ut->flags & UTEXT_EXTRA_HEAP_ALLOCATED) { + uprv_free(ut->pExtra); + ut->extraSize = 0; + } + ut->pExtra = uprv_malloc(extraSpace); + if (ut->pExtra == NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + } else { + ut->extraSize = extraSpace; + ut->flags |= UTEXT_EXTRA_HEAP_ALLOCATED; + } + } + } + if (U_SUCCESS(*status)) { + ut->flags |= UTEXT_OPEN; + + // Initialize all remaining fields of the UText. + // + ut->context = NULL; + ut->chunkContents = NULL; + ut->p = NULL; + ut->q = NULL; + ut->r = NULL; + ut->a = 0; + ut->b = 0; + ut->c = 0; + ut->chunkOffset = 0; + ut->chunkLength = 0; + ut->chunkNativeStart = 0; + ut->chunkNativeLimit = 0; + ut->nativeIndexingLimit = 0; + ut->providerProperties = 0; + ut->privA = 0; + ut->privB = 0; + ut->privC = 0; + ut->privP = NULL; + if (ut->pExtra!=NULL && ut->extraSize>0) + uprv_memset(ut->pExtra, 0, ut->extraSize); + + } + return ut; +} + + +U_CAPI UText * U_EXPORT2 +utext_close(UText *ut) { + if (ut==NULL || + ut->magic != UTEXT_MAGIC || + (ut->flags & UTEXT_OPEN) == 0) + { + // The supplied ut is not an open UText. + // Do nothing. + return ut; + } + + // If the provider gave us a close function, call it now. + // This will clean up anything allocated specifically by the provider. + if (ut->pFuncs->close != NULL) { + ut->pFuncs->close(ut); + } + ut->flags &= ~UTEXT_OPEN; + + // If we (the framework) allocated the UText or subsidiary storage, + // delete it. + if (ut->flags & UTEXT_EXTRA_HEAP_ALLOCATED) { + uprv_free(ut->pExtra); + ut->pExtra = NULL; + ut->flags &= ~UTEXT_EXTRA_HEAP_ALLOCATED; + ut->extraSize = 0; + } + + // Zero out function table of the closed UText. This is a defensive move, + // inteded to cause applications that inadvertantly use a closed + // utext to crash with null pointer errors. + ut->pFuncs = NULL; + + if (ut->flags & UTEXT_HEAP_ALLOCATED) { + // This UText was allocated by UText setup. We need to free it. + // Clear magic, so we can detect if the user messes up and immediately + // tries to reopen another UText using the deleted storage. + ut->magic = 0; + uprv_free(ut); + ut = NULL; + } + return ut; +} + + + + +// +// invalidateChunk Reset a chunk to have no contents, so that the next call +// to access will cause new data to load. +// This is needed when copy/move/replace operate directly on the +// backing text, potentially putting it out of sync with the +// contents in the chunk. +// +static void +invalidateChunk(UText *ut) { + ut->chunkLength = 0; + ut->chunkNativeLimit = 0; + ut->chunkNativeStart = 0; + ut->chunkOffset = 0; + ut->nativeIndexingLimit = 0; +} + +// +// pinIndex Do range pinning on a native index parameter. +// 64 bit pinning is done in place. +// 32 bit truncated result is returned as a convenience for +// use in providers that don't need 64 bits. +static int32_t +pinIndex(int64_t &index, int64_t limit) { + if (index<0) { + index = 0; + } else if (index > limit) { + index = limit; + } + return (int32_t)index; +} + + +U_CDECL_BEGIN + +// +// Pointer relocation function, +// a utility used by shallow clone. +// Adjust a pointer that refers to something within one UText (the source) +// to refer to the same relative offset within a another UText (the target) +// +static void adjustPointer(UText *dest, const void **destPtr, const UText *src) { + // convert all pointers to (char *) so that byte address arithmetic will work. + char *dptr = (char *)*destPtr; + char *dUText = (char *)dest; + char *sUText = (char *)src; + + if (dptr >= (char *)src->pExtra && dptr < ((char*)src->pExtra)+src->extraSize) { + // target ptr was to something within the src UText's pExtra storage. + // relocate it into the target UText's pExtra region. + *destPtr = ((char *)dest->pExtra) + (dptr - (char *)src->pExtra); + } else if (dptr>=sUText && dptr < sUText+src->sizeOfStruct) { + // target ptr was pointing to somewhere within the source UText itself. + // Move it to the same offset within the target UText. + *destPtr = dUText + (dptr-sUText); + } +} + + +// +// Clone. This is a generic copy-the-utext-by-value clone function that can be +// used as-is with some utext types, and as a helper by other clones. +// +static UText * U_CALLCONV +shallowTextClone(UText * dest, const UText * src, UErrorCode * status) { + if (U_FAILURE(*status)) { + return NULL; + } + int32_t srcExtraSize = src->extraSize; + + // + // Use the generic text_setup to allocate storage if required. + // + dest = utext_setup(dest, srcExtraSize, status); + if (U_FAILURE(*status)) { + return dest; + } + + // + // flags (how the UText was allocated) and the pointer to the + // extra storage must retain the values in the cloned utext that + // were set up by utext_setup. Save them separately before + // copying the whole struct. + // + void *destExtra = dest->pExtra; + int32_t flags = dest->flags; + + + // + // Copy the whole UText struct by value. + // Any "Extra" storage is copied also. + // + int sizeToCopy = src->sizeOfStruct; + if (sizeToCopy > dest->sizeOfStruct) { + sizeToCopy = dest->sizeOfStruct; + } + uprv_memcpy(dest, src, sizeToCopy); + dest->pExtra = destExtra; + dest->flags = flags; + if (srcExtraSize > 0) { + uprv_memcpy(dest->pExtra, src->pExtra, srcExtraSize); + } + + // + // Relocate any pointers in the target that refer to the UText itself + // to point to the cloned copy rather than the original source. + // + adjustPointer(dest, &dest->context, src); + adjustPointer(dest, &dest->p, src); + adjustPointer(dest, &dest->q, src); + adjustPointer(dest, &dest->r, src); + adjustPointer(dest, (const void **)&dest->chunkContents, src); + + return dest; +} + + +U_CDECL_END + + + +//------------------------------------------------------------------------------ +// +// UText implementation for UTF-8 char * strings (read-only) +// Limitation: string length must be <= 0x7fffffff in length. +// (length must for in an int32_t variable) +// +// Use of UText data members: +// context pointer to UTF-8 string +// utext.b is the input string length (bytes). +// utext.c Length scanned so far in string +// (for optimizing finding length of zero terminated strings.) +// utext.p pointer to the current buffer +// utext.q pointer to the other buffer. +// +//------------------------------------------------------------------------------ + +// Chunk size. +// Must be less than 85, because of byte mapping from UChar indexes to native indexes. +// Worst case is three native bytes to one UChar. (Supplemenaries are 4 native bytes +// to two UChars.) +// +enum { UTF8_TEXT_CHUNK_SIZE=32 }; + +// +// UTF8Buf Two of these structs will be set up in the UText's extra allocated space. +// Each contains the UChar chunk buffer, the to and from native maps, and +// header info. +// +// because backwards iteration fills the buffers starting at the end and +// working towards the front, the filled part of the buffers may not begin +// at the start of the available storage for the buffers. +// +// Buffer size is one bigger than the specified UTF8_TEXT_CHUNK_SIZE to allow for +// the last character added being a supplementary, and thus requiring a surrogate +// pair. Doing this is simpler than checking for the edge case. +// + +struct UTF8Buf { + int32_t bufNativeStart; // Native index of first char in UChar buf + int32_t bufNativeLimit; // Native index following last char in buf. + int32_t bufStartIdx; // First filled position in buf. + int32_t bufLimitIdx; // Limit of filled range in buf. + int32_t bufNILimit; // Limit of native indexing part of buf + int32_t toUCharsMapStart; // Native index corresponding to + // mapToUChars[0]. + // Set to bufNativeStart when filling forwards. + // Set to computed value when filling backwards. + + UChar buf[UTF8_TEXT_CHUNK_SIZE+4]; // The UChar buffer. Requires one extra position beyond the + // the chunk size, to allow for surrogate at the end. + // Length must be identical to mapToNative array, below, + // because of the way indexing works when the array is + // filled backwards during a reverse iteration. Thus, + // the additional extra size. + uint8_t mapToNative[UTF8_TEXT_CHUNK_SIZE+4]; // map UChar index in buf to + // native offset from bufNativeStart. + // Requires two extra slots, + // one for a supplementary starting in the last normal position, + // and one for an entry for the buffer limit position. + uint8_t mapToUChars[UTF8_TEXT_CHUNK_SIZE*3+6]; // Map native offset from bufNativeStart to + // correspoding offset in filled part of buf. + int32_t align; +}; + +U_CDECL_BEGIN + +// +// utf8TextLength +// +// Get the length of the string. If we don't already know it, +// we'll need to scan for the trailing nul. +// +static int64_t U_CALLCONV +utf8TextLength(UText *ut) { + if (ut->b < 0) { + // Zero terminated string, and we haven't scanned to the end yet. + // Scan it now. + const char *r = (const char *)ut->context + ut->c; + while (*r != 0) { + r++; + } + if ((r - (const char *)ut->context) < 0x7fffffff) { + ut->b = (int32_t)(r - (const char *)ut->context); + } else { + // Actual string was bigger (more than 2 gig) than we + // can handle. Clip it to 2 GB. + ut->b = 0x7fffffff; + } + ut->providerProperties &= ~I32_FLAG(UTEXT_PROVIDER_LENGTH_IS_EXPENSIVE); + } + return ut->b; +} + + + + + + +static UBool U_CALLCONV +utf8TextAccess(UText *ut, int64_t index, UBool forward) { + // + // Apologies to those who are allergic to goto statements. + // Consider each goto to a labelled block to be the equivalent of + // call the named block as if it were a function(); + // return; + // + const uint8_t *s8=(const uint8_t *)ut->context; + UTF8Buf *u8b = NULL; + int32_t length = ut->b; // Length of original utf-8 + int32_t ix= (int32_t)index; // Requested index, trimmed to 32 bits. + int32_t mapIndex = 0; + if (index<0) { + ix=0; + } else if (index > 0x7fffffff) { + // Strings with 64 bit lengths not supported by this UTF-8 provider. + ix = 0x7fffffff; + } + + // Pin requested index to the string length. + if (ix>length) { + if (length>=0) { + ix=length; + } else if (ix>ut->c) { + // Zero terminated string, and requested index is beyond + // the region that has already been scanned. + // Scan up to either the end of the string or to the + // requested position, whichever comes first. + while (ut->cc]!=0) { + ut->c++; + } + // TODO: support for null terminated string length > 32 bits. + if (s8[ut->c] == 0) { + // We just found the actual length of the string. + // Trim the requested index back to that. + ix = ut->c; + ut->b = ut->c; + length = ut->c; + ut->providerProperties &= ~I32_FLAG(UTEXT_PROVIDER_LENGTH_IS_EXPENSIVE); + } + } + } + + // + // Dispatch to the appropriate action for a forward iteration request. + // + if (forward) { + if (ix==ut->chunkNativeLimit) { + // Check for normal sequential iteration cases first. + if (ix==length) { + // Just reached end of string + // Don't swap buffers, but do set the + // current buffer position. + ut->chunkOffset = ut->chunkLength; + return FALSE; + } else { + // End of current buffer. + // check whether other buffer already has what we need. + UTF8Buf *altB = (UTF8Buf *)ut->q; + if (ix>=altB->bufNativeStart && ixbufNativeLimit) { + goto swapBuffers; + } + } + } + + // A random access. Desired index could be in either or niether buf. + // For optimizing the order of testing, first check for the index + // being in the other buffer. This will be the case for uses that + // move back and forth over a fairly limited range + { + u8b = (UTF8Buf *)ut->q; // the alternate buffer + if (ix>=u8b->bufNativeStart && ixbufNativeLimit) { + // Requested index is in the other buffer. + goto swapBuffers; + } + if (ix == length) { + // Requested index is end-of-string. + // (this is the case of randomly seeking to the end. + // The case of iterating off the end is handled earlier.) + if (ix == ut->chunkNativeLimit) { + // Current buffer extends up to the end of the string. + // Leave it as the current buffer. + ut->chunkOffset = ut->chunkLength; + return FALSE; + } + if (ix == u8b->bufNativeLimit) { + // Alternate buffer extends to the end of string. + // Swap it in as the current buffer. + goto swapBuffersAndFail; + } + + // Neither existing buffer extends to the end of the string. + goto makeStubBuffer; + } + + if (ixchunkNativeStart || ix>=ut->chunkNativeLimit) { + // Requested index is in neither buffer. + goto fillForward; + } + + // Requested index is in this buffer. + u8b = (UTF8Buf *)ut->p; // the current buffer + mapIndex = ix - u8b->toUCharsMapStart; + ut->chunkOffset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx; + return TRUE; + + } + } + + + // + // Dispatch to the appropriate action for a + // Backwards Diretion iteration request. + // + if (ix==ut->chunkNativeStart) { + // Check for normal sequential iteration cases first. + if (ix==0) { + // Just reached the start of string + // Don't swap buffers, but do set the + // current buffer position. + ut->chunkOffset = 0; + return FALSE; + } else { + // Start of current buffer. + // check whether other buffer already has what we need. + UTF8Buf *altB = (UTF8Buf *)ut->q; + if (ix>altB->bufNativeStart && ix<=altB->bufNativeLimit) { + goto swapBuffers; + } + } + } + + // A random access. Desired index could be in either or niether buf. + // For optimizing the order of testing, + // Most likely case: in the other buffer. + // Second most likely: in neither buffer. + // Unlikely, but must work: in the current buffer. + u8b = (UTF8Buf *)ut->q; // the alternate buffer + if (ix>u8b->bufNativeStart && ix<=u8b->bufNativeLimit) { + // Requested index is in the other buffer. + goto swapBuffers; + } + // Requested index is start-of-string. + // (this is the case of randomly seeking to the start. + // The case of iterating off the start is handled earlier.) + if (ix==0) { + if (u8b->bufNativeStart==0) { + // Alternate buffer contains the data for the start string. + // Make it be the current buffer. + goto swapBuffersAndFail; + } else { + // Request for data before the start of string, + // neither buffer is usable. + // set up a zero-length buffer. + goto makeStubBuffer; + } + } + + if (ix<=ut->chunkNativeStart || ix>ut->chunkNativeLimit) { + // Requested index is in neither buffer. + goto fillReverse; + } + + // Requested index is in this buffer. + // Set the utf16 buffer index. + u8b = (UTF8Buf *)ut->p; + mapIndex = ix - u8b->toUCharsMapStart; + ut->chunkOffset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx; + if (ut->chunkOffset==0) { + // This occurs when the first character in the text is + // a multi-byte UTF-8 char, and the requested index is to + // one of the trailing bytes. Because there is no preceding , + // character, this access fails. We can't pick up on the + // situation sooner because the requested index is not zero. + return FALSE; + } else { + return TRUE; + } + + + +swapBuffers: + // The alternate buffer (ut->q) has the string data that was requested. + // Swap the primary and alternate buffers, and set the + // chunk index into the new primary buffer. + { + u8b = (UTF8Buf *)ut->q; + ut->q = ut->p; + ut->p = u8b; + ut->chunkContents = &u8b->buf[u8b->bufStartIdx]; + ut->chunkLength = u8b->bufLimitIdx - u8b->bufStartIdx; + ut->chunkNativeStart = u8b->bufNativeStart; + ut->chunkNativeLimit = u8b->bufNativeLimit; + ut->nativeIndexingLimit = u8b->bufNILimit; + + // Index into the (now current) chunk + // Use the map to set the chunk index. It's more trouble than it's worth + // to check whether native indexing can be used. + U_ASSERT(ix>=u8b->bufNativeStart); + U_ASSERT(ix<=u8b->bufNativeLimit); + mapIndex = ix - u8b->toUCharsMapStart; + U_ASSERT(mapIndex>=0); + U_ASSERT(mapIndex<(int32_t)sizeof(u8b->mapToUChars)); + ut->chunkOffset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx; + + return TRUE; + } + + + swapBuffersAndFail: + // We got a request for either the start or end of the string, + // with iteration continuing in the out-of-bounds direction. + // The alternate buffer already contains the data up to the + // start/end. + // Swap the buffers, then return failure, indicating that we couldn't + // make things correct for continuing the iteration in the requested + // direction. The position & buffer are correct should the + // user decide to iterate in the opposite direction. + u8b = (UTF8Buf *)ut->q; + ut->q = ut->p; + ut->p = u8b; + ut->chunkContents = &u8b->buf[u8b->bufStartIdx]; + ut->chunkLength = u8b->bufLimitIdx - u8b->bufStartIdx; + ut->chunkNativeStart = u8b->bufNativeStart; + ut->chunkNativeLimit = u8b->bufNativeLimit; + ut->nativeIndexingLimit = u8b->bufNILimit; + + // Index into the (now current) chunk + // For this function (swapBuffersAndFail), the requested index + // will always be at either the start or end of the chunk. + if (ix==u8b->bufNativeLimit) { + ut->chunkOffset = ut->chunkLength; + } else { + ut->chunkOffset = 0; + U_ASSERT(ix == u8b->bufNativeStart); + } + return FALSE; + +makeStubBuffer: + // The user has done a seek/access past the start or end + // of the string. Rather than loading data that is likely + // to never be used, just set up a zero-length buffer at + // the position. + u8b = (UTF8Buf *)ut->q; + u8b->bufNativeStart = ix; + u8b->bufNativeLimit = ix; + u8b->bufStartIdx = 0; + u8b->bufLimitIdx = 0; + u8b->bufNILimit = 0; + u8b->toUCharsMapStart = ix; + u8b->mapToNative[0] = 0; + u8b->mapToUChars[0] = 0; + goto swapBuffersAndFail; + + + +fillForward: + { + // Move the incoming index to a code point boundary. + U8_SET_CP_START(s8, 0, ix); + + // Swap the UText buffers. + // We want to fill what was previously the alternate buffer, + // and make what was the current buffer be the new alternate. + UTF8Buf *u8b = (UTF8Buf *)ut->q; + ut->q = ut->p; + ut->p = u8b; + + int32_t strLen = ut->b; + UBool nulTerminated = FALSE; + if (strLen < 0) { + strLen = 0x7fffffff; + nulTerminated = TRUE; + } + + UChar *buf = u8b->buf; + uint8_t *mapToNative = u8b->mapToNative; + uint8_t *mapToUChars = u8b->mapToUChars; + int32_t destIx = 0; + int32_t srcIx = ix; + UBool seenNonAscii = FALSE; + UChar32 c; + + // Fill the chunk buffer and mapping arrays. + while (destIx0 && c<0x80) { + // Special case ASCII range for speed. + // zero is excluded to simplify bounds checking. + buf[destIx] = c; + mapToNative[destIx] = srcIx - ix; + mapToUChars[srcIx-ix] = destIx; + srcIx++; + destIx++; + } else { + // General case, handle everything. + if (seenNonAscii == FALSE) { + seenNonAscii = TRUE; + u8b->bufNILimit = destIx; + } + + int32_t cIx = srcIx; + int32_t dIx = destIx; + int32_t dIxSaved = destIx; + U8_NEXT(s8, srcIx, strLen, c); + if (c==0 && nulTerminated) { + srcIx--; + break; + } + if (c<0) { + // Illegal UTF-8. Replace with sub character. + c = 0x0fffd; + } + + U16_APPEND_UNSAFE(buf, destIx, c); + do { + mapToNative[dIx++] = cIx - ix; + } while (dIx < destIx); + + do { + mapToUChars[cIx++ - ix] = dIxSaved; + } while (cIx < srcIx); + } + if (srcIx>=strLen) { + break; + } + + } + + // store Native <--> Chunk Map entries for the end of the buffer. + // There is no actual character here, but the index position is valid. + mapToNative[destIx] = srcIx - ix; + mapToUChars[srcIx - ix] = destIx; + + // fill in Buffer descriptor + u8b->bufNativeStart = ix; + u8b->bufNativeLimit = srcIx; + u8b->bufStartIdx = 0; + u8b->bufLimitIdx = destIx; + if (seenNonAscii == FALSE) { + u8b->bufNILimit = destIx; + } + u8b->toUCharsMapStart = u8b->bufNativeStart; + + // Set UText chunk to refer to this buffer. + ut->chunkContents = buf; + ut->chunkOffset = 0; + ut->chunkLength = u8b->bufLimitIdx; + ut->chunkNativeStart = u8b->bufNativeStart; + ut->chunkNativeLimit = u8b->bufNativeLimit; + ut->nativeIndexingLimit = u8b->bufNILimit; + + // For zero terminated strings, keep track of the maximum point + // scanned so far. + if (nulTerminated && srcIx>ut->c) { + ut->c = srcIx; + if (c==0) { + // We scanned to the end. + // Remember the actual length. + ut->b = srcIx; + ut->providerProperties &= ~I32_FLAG(UTEXT_PROVIDER_LENGTH_IS_EXPENSIVE); + } + } + return TRUE; + } + + +fillReverse: + { + // Move the incoming index to a code point boundary. + // Can only do this if the incoming index is somewhere in the interior of the string. + // If index is at the end, there is no character there to look at. + if (ix != ut->b) { + U8_SET_CP_START(s8, 0, ix); + } + + // Swap the UText buffers. + // We want to fill what was previously the alternate buffer, + // and make what was the current buffer be the new alternate. + UTF8Buf *u8b = (UTF8Buf *)ut->q; + ut->q = ut->p; + ut->p = u8b; + + UChar *buf = u8b->buf; + uint8_t *mapToNative = u8b->mapToNative; + uint8_t *mapToUChars = u8b->mapToUChars; + int32_t toUCharsMapStart = ix - (UTF8_TEXT_CHUNK_SIZE*3 + 1); + int32_t destIx = UTF8_TEXT_CHUNK_SIZE+2; // Start in the overflow region + // at end of buffer to leave room + // for a surrogate pair at the + // buffer start. + int32_t srcIx = ix; + int32_t bufNILimit = destIx; + UChar32 c; + + // Map to/from Native Indexes, fill in for the position at the end of + // the buffer. + // + mapToNative[destIx] = srcIx - toUCharsMapStart; + mapToUChars[srcIx - toUCharsMapStart] = destIx; + + // Fill the chunk buffer + // Work backwards, filling from the end of the buffer towards the front. + // + while (destIx>2 && (srcIx - toUCharsMapStart > 5) && (srcIx > 0)) { + srcIx--; + destIx--; + + // Get last byte of the UTF-8 character + c = s8[srcIx]; + if (c<0x80) { + // Special case ASCII range for speed. + buf[destIx] = c; + mapToUChars[srcIx - toUCharsMapStart] = destIx; + mapToNative[destIx] = srcIx - toUCharsMapStart; + } else { + // General case, handle everything non-ASCII. + + int32_t sIx = srcIx; // ix of last byte of multi-byte u8 char + + // Get the full character from the UTF8 string. + // use code derived from tbe macros in utf.8 + // Leaves srcIx pointing at the first byte of the UTF-8 char. + // + if (c<=0xbf) { + c=utf8_prevCharSafeBody(s8, 0, &srcIx, c, -1); + // leaves srcIx at first byte of the multi-byte char. + } else { + c=0x0fffd; + } + + // Store the character in UTF-16 buffer. + if (c<0x10000) { + buf[destIx] = c; + mapToNative[destIx] = srcIx - toUCharsMapStart; + } else { + buf[destIx] = U16_TRAIL(c); + mapToNative[destIx] = srcIx - toUCharsMapStart; + buf[--destIx] = U16_LEAD(c); + mapToNative[destIx] = srcIx - toUCharsMapStart; + } + + // Fill in the map from native indexes to UChars buf index. + do { + mapToUChars[sIx-- - toUCharsMapStart] = destIx; + } while (sIx >= srcIx); + + // Set native indexing limit to be the current position. + // We are processing a non-ascii, non-native-indexing char now; + // the limit will be here if the rest of the chars to be + // added to this buffer are ascii. + bufNILimit = destIx; + } + } + u8b->bufNativeStart = srcIx; + u8b->bufNativeLimit = ix; + u8b->bufStartIdx = destIx; + u8b->bufLimitIdx = UTF8_TEXT_CHUNK_SIZE+2; + u8b->bufNILimit = bufNILimit - u8b->bufStartIdx; + u8b->toUCharsMapStart = toUCharsMapStart; + + ut->chunkContents = &buf[u8b->bufStartIdx]; + ut->chunkLength = u8b->bufLimitIdx - u8b->bufStartIdx; + ut->chunkOffset = ut->chunkLength; + ut->chunkNativeStart = u8b->bufNativeStart; + ut->chunkNativeLimit = u8b->bufNativeLimit; + ut->nativeIndexingLimit = u8b->bufNILimit; + return TRUE; + } + +} + + + +// +// This is a slightly modified copy of u_strFromUTF8, +// Inserts a Replacement Char rather than failing on invalid UTF-8 +// Removes unnecessary features. +// +static UChar* +utext_strFromUTF8(UChar *dest, + int32_t destCapacity, + int32_t *pDestLength, + const char* src, + int32_t srcLength, // required. NUL terminated not supported. + UErrorCode *pErrorCode + ) +{ + + UChar *pDest = dest; + UChar *pDestLimit = dest+destCapacity; + UChar32 ch=0; + int32_t index = 0; + int32_t reqLength = 0; + uint8_t* pSrc = (uint8_t*) src; + + + while((index < srcLength)&&(pDest0)) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + int32_t length = ut->b; + int32_t start32 = pinIndex(start, length); + int32_t limit32 = pinIndex(limit, length); + + if(start32>limit32) { + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + + + // adjust the incoming indexes to land on code point boundaries if needed. + // adjust by no more than three, because that is the largest number of trail bytes + // in a well formed UTF8 character. + const uint8_t *buf = (const uint8_t *)ut->context; + int i; + if (start32 < ut->chunkNativeLimit) { + for (i=0; i<3; i++) { + if (U8_IS_LEAD(buf[start32]) || start32==0) { + break; + } + start32--; + } + } + + if (limit32 < ut->chunkNativeLimit) { + for (i=0; i<3; i++) { + if (U8_IS_LEAD(buf[limit32]) || limit32==0) { + break; + } + limit32--; + } + } + + // Do the actual extract. + int32_t destLength=0; + utext_strFromUTF8(dest, destCapacity, &destLength, + (const char *)ut->context+start32, limit32-start32, + pErrorCode); + return destLength; +} + +// +// utf8TextMapOffsetToNative +// +// Map a chunk (UTF-16) offset to a native index. +static int64_t U_CALLCONV +utf8TextMapOffsetToNative(const UText *ut) { + // + UTF8Buf *u8b = (UTF8Buf *)ut->p; + U_ASSERT(ut->chunkOffset>ut->nativeIndexingLimit && ut->chunkOffset<=ut->chunkLength); + int32_t nativeOffset = u8b->mapToNative[ut->chunkOffset + u8b->bufStartIdx] + u8b->toUCharsMapStart; + U_ASSERT(nativeOffset >= ut->chunkNativeStart && nativeOffset <= ut->chunkNativeLimit); + return nativeOffset; +} + +// +// Map a native index to the corrsponding chunk offset +// +static int32_t U_CALLCONV +utf8TextMapIndexToUTF16(const UText *ut, int64_t index64) { + U_ASSERT(index64 <= 0x7fffffff); + int32_t index = (int32_t)index64; + UTF8Buf *u8b = (UTF8Buf *)ut->p; + U_ASSERT(index>=ut->chunkNativeStart+ut->nativeIndexingLimit); + U_ASSERT(index<=ut->chunkNativeLimit); + int32_t mapIndex = index - u8b->toUCharsMapStart; + int32_t offset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx; + U_ASSERT(offset>=0 && offset<=ut->chunkLength); + return offset; +} + +static UText * U_CALLCONV +utf8TextClone(UText *dest, const UText *src, UBool deep, UErrorCode *status) +{ + // First do a generic shallow clone. Does everything needed for the UText struct itself. + dest = shallowTextClone(dest, src, status); + + // For deep clones, make a copy of the string. + // The copied storage is owned by the newly created clone. + // + // TODO: There is an isssue with using utext_nativeLength(). + // That function is non-const in cases where the input was NUL terminated + // and the length has not yet been determined. + // This function (clone()) is const. + // There potentially a thread safety issue lurking here. + // + if (deep && U_SUCCESS(*status)) { + int32_t len = (int32_t)utext_nativeLength((UText *)src); + char *copyStr = (char *)uprv_malloc(len+1); + if (copyStr == NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + } else { + uprv_memcpy(copyStr, src->context, len+1); + dest->context = copyStr; + dest->providerProperties |= I32_FLAG(UTEXT_PROVIDER_OWNS_TEXT); + } + } + return dest; +} + + +static void U_CALLCONV +utf8TextClose(UText *ut) { + // Most of the work of close is done by the generic UText framework close. + // All that needs to be done here is to delete the UTF8 string if the UText + // owns it. This occurs if the UText was created by cloning. + if (ut->providerProperties & I32_FLAG(UTEXT_PROVIDER_OWNS_TEXT)) { + char *s = (char *)ut->context; + uprv_free(s); + ut->context = NULL; + } +} + +U_CDECL_END + + +static struct UTextFuncs utf8Funcs = +{ + sizeof(UTextFuncs), + 0, 0, 0, // Reserved alignment padding + utf8TextClone, + utf8TextLength, + utf8TextAccess, + utf8TextExtract, + NULL, /* replace*/ + NULL, /* copy */ + utf8TextMapOffsetToNative, + utf8TextMapIndexToUTF16, + utf8TextClose, + NULL, // spare 1 + NULL, // spare 2 + NULL // spare 3 +}; + + +U_CAPI UText * U_EXPORT2 +utext_openUTF8(UText *ut, const char *s, int64_t length, UErrorCode *status) { + if(U_FAILURE(*status)) { + return NULL; + } + if(s==NULL || length<-1 || length>INT32_MAX) { + *status=U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + + ut = utext_setup(ut, sizeof(UTF8Buf) * 2, status); + if (U_FAILURE(*status)) { + return ut; + } + + ut->pFuncs = &utf8Funcs; + ut->context = s; + ut->b = (int32_t)length; + ut->c = (int32_t)length; + if (ut->c < 0) { + ut->c = 0; + ut->providerProperties |= I32_FLAG(UTEXT_PROVIDER_LENGTH_IS_EXPENSIVE); + } + ut->p = ut->pExtra; + ut->q = (char *)ut->pExtra + sizeof(UTF8Buf); + return ut; + +} + + + + + + + + +//------------------------------------------------------------------------------ +// +// UText implementation wrapper for Replaceable (read/write) +// +// Use of UText data members: +// context pointer to Replaceable. +// p pointer to Replaceable if it is owned by the UText. +// +//------------------------------------------------------------------------------ + + + +// minimum chunk size for this implementation: 3 +// to allow for possible trimming for code point boundaries +enum { REP_TEXT_CHUNK_SIZE=10 }; + +struct ReplExtra { + /* + * Chunk UChars. + * +1 to simplify filling with surrogate pair at the end. + */ + UChar s[REP_TEXT_CHUNK_SIZE+1]; +}; + + +U_CDECL_BEGIN + +static UText * U_CALLCONV +repTextClone(UText *dest, const UText *src, UBool deep, UErrorCode *status) { + // First do a generic shallow clone. Does everything needed for the UText struct itself. + dest = shallowTextClone(dest, src, status); + + // For deep clones, make a copy of the Replaceable. + // The copied Replaceable storage is owned by the newly created UText clone. + // A non-NULL pointer in UText.p is the signal to the close() function to delete + // it. + // + if (deep && U_SUCCESS(*status)) { + const Replaceable *replSrc = (const Replaceable *)src->context; + dest->context = replSrc->clone(); + dest->providerProperties |= I32_FLAG(UTEXT_PROVIDER_OWNS_TEXT); + + // with deep clone, the copy is writable, even when the source is not. + dest->providerProperties |= I32_FLAG(UTEXT_PROVIDER_WRITABLE); + } + return dest; +} + + +static void U_CALLCONV +repTextClose(UText *ut) { + // Most of the work of close is done by the generic UText framework close. + // All that needs to be done here is delete the Replaceable if the UText + // owns it. This occurs if the UText was created by cloning. + if (ut->providerProperties & I32_FLAG(UTEXT_PROVIDER_OWNS_TEXT)) { + Replaceable *rep = (Replaceable *)ut->context; + delete rep; + ut->context = NULL; + } +} + + +static int64_t U_CALLCONV +repTextLength(UText *ut) { + const Replaceable *replSrc = (const Replaceable *)ut->context; + int32_t len = replSrc->length(); + return len; +} + + +static UBool U_CALLCONV +repTextAccess(UText *ut, int64_t index, UBool forward) { + const Replaceable *rep=(const Replaceable *)ut->context; + int32_t length=rep->length(); // Full length of the input text (bigger than a chunk) + + // clip the requested index to the limits of the text. + int32_t index32 = pinIndex(index, length); + U_ASSERT(index<=INT32_MAX); + + + /* + * Compute start/limit boundaries around index, for a segment of text + * to be extracted. + * To allow for the possibility that our user gave an index to the trailing + * half of a surrogate pair, we must request one extra preceding UChar when + * going in the forward direction. This will ensure that the buffer has the + * entire code point at the specified index. + */ + if(forward) { + + if (index32>=ut->chunkNativeStart && index32chunkNativeLimit) { + // Buffer already contains the requested position. + ut->chunkOffset = (int32_t)(index - ut->chunkNativeStart); + return TRUE; + } + if (index32>=length && ut->chunkNativeLimit==length) { + // Request for end of string, and buffer already extends up to it. + // Can't get the data, but don't change the buffer. + ut->chunkOffset = length - (int32_t)ut->chunkNativeStart; + return FALSE; + } + + ut->chunkNativeLimit = index + REP_TEXT_CHUNK_SIZE - 1; + // Going forward, so we want to have the buffer with stuff at and beyond + // the requested index. The -1 gets us one code point before the + // requested index also, to handle the case of the index being on + // a trail surrogate of a surrogate pair. + if(ut->chunkNativeLimit > length) { + ut->chunkNativeLimit = length; + } + // unless buffer ran off end, start is index-1. + ut->chunkNativeStart = ut->chunkNativeLimit - REP_TEXT_CHUNK_SIZE; + if(ut->chunkNativeStart < 0) { + ut->chunkNativeStart = 0; + } + } else { + // Reverse iteration. Fill buffer with data preceding the requested index. + if (index32>ut->chunkNativeStart && index32<=ut->chunkNativeLimit) { + // Requested position already in buffer. + ut->chunkOffset = index32 - (int32_t)ut->chunkNativeStart; + return TRUE; + } + if (index32==0 && ut->chunkNativeStart==0) { + // Request for start, buffer already begins at start. + // No data, but keep the buffer as is. + ut->chunkOffset = 0; + return FALSE; + } + + // Figure out the bounds of the chunk to extract for reverse iteration. + // Need to worry about chunk not splitting surrogate pairs, and while still + // containing the data we need. + // Fix by requesting a chunk that includes an extra UChar at the end. + // If this turns out to be a lead surrogate, we can lop it off and still have + // the data we wanted. + ut->chunkNativeStart = index32 + 1 - REP_TEXT_CHUNK_SIZE; + if (ut->chunkNativeStart < 0) { + ut->chunkNativeStart = 0; + } + + ut->chunkNativeLimit = index32 + 1; + if (ut->chunkNativeLimit > length) { + ut->chunkNativeLimit = length; + } + } + + // Extract the new chunk of text from the Replaceable source. + ReplExtra *ex = (ReplExtra *)ut->pExtra; + // UnicodeString with its buffer a writable alias to the chunk buffer + UnicodeString buffer(ex->s, 0 /*buffer length*/, REP_TEXT_CHUNK_SIZE /*buffer capacity*/); + rep->extractBetween((int32_t)ut->chunkNativeStart, (int32_t)ut->chunkNativeLimit, buffer); + + ut->chunkContents = ex->s; + ut->chunkLength = (int32_t)(ut->chunkNativeLimit - ut->chunkNativeStart); + ut->chunkOffset = (int32_t)(index32 - ut->chunkNativeStart); + + // Surrogate pairs from the input text must not span chunk boundaries. + // If end of chunk could be the start of a surrogate, trim it off. + if (ut->chunkNativeLimit < length && + U16_IS_LEAD(ex->s[ut->chunkLength-1])) { + ut->chunkLength--; + ut->chunkNativeLimit--; + if (ut->chunkOffset > ut->chunkLength) { + ut->chunkOffset = ut->chunkLength; + } + } + + // if the first UChar in the chunk could be the trailing half of a surrogate pair, + // trim it off. + if(ut->chunkNativeStart>0 && U16_IS_TRAIL(ex->s[0])) { + ++(ut->chunkContents); + ++(ut->chunkNativeStart); + --(ut->chunkLength); + --(ut->chunkOffset); + } + + // adjust the index/chunkOffset to a code point boundary + U16_SET_CP_START(ut->chunkContents, 0, ut->chunkOffset); + + // Use fast indexing for get/setNativeIndex() + ut->nativeIndexingLimit = ut->chunkLength; + + return TRUE; +} + + + +static int32_t U_CALLCONV +repTextExtract(UText *ut, + int64_t start, int64_t limit, + UChar *dest, int32_t destCapacity, + UErrorCode *status) { + const Replaceable *rep=(const Replaceable *)ut->context; + int32_t length=rep->length(); + + if(U_FAILURE(*status)) { + return 0; + } + if(destCapacity<0 || (dest==NULL && destCapacity>0)) { + *status=U_ILLEGAL_ARGUMENT_ERROR; + } + if(start>limit) { + *status=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + + int32_t start32 = pinIndex(start, length); + int32_t limit32 = pinIndex(limit, length); + + // adjust start, limit if they point to trail half of surrogates + if (start32charAt(start32)) && + U_IS_SUPPLEMENTARY(rep->char32At(start32))){ + start32--; + } + if (limit32charAt(limit32)) && + U_IS_SUPPLEMENTARY(rep->char32At(limit32))){ + limit32--; + } + + length=limit32-start32; + if(length>destCapacity) { + limit32 = start32 + destCapacity; + } + UnicodeString buffer(dest, 0, destCapacity); // writable alias + rep->extractBetween(start32, limit32, buffer); + return u_terminateUChars(dest, destCapacity, length, status); +} + +static int32_t U_CALLCONV +repTextReplace(UText *ut, + int64_t start, int64_t limit, + const UChar *src, int32_t length, + UErrorCode *status) { + Replaceable *rep=(Replaceable *)ut->context; + int32_t oldLength; + + if(U_FAILURE(*status)) { + return 0; + } + if(src==NULL && length!=0) { + *status=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + oldLength=rep->length(); // will subtract from new length + if(start>limit ) { + *status=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + + int32_t start32 = pinIndex(start, oldLength); + int32_t limit32 = pinIndex(limit, oldLength); + + // Snap start & limit to code point boundaries. + if (start32charAt(start32)) && + start32>0 && U16_IS_LEAD(rep->charAt(start32-1))) + { + start32--; + } + if (limit32charAt(limit32-1)) && + U16_IS_TRAIL(rep->charAt(limit32))) + { + limit32++; + } + + // Do the actual replace operation using methods of the Replaceable class + UnicodeString replStr((UBool)(length<0), src, length); // read-only alias + rep->handleReplaceBetween(start32, limit32, replStr); + int32_t newLength = rep->length(); + int32_t lengthDelta = newLength - oldLength; + + // Is the UText chunk buffer OK? + if (ut->chunkNativeLimit > start32) { + // this replace operation may have impacted the current chunk. + // invalidate it, which will force a reload on the next access. + invalidateChunk(ut); + } + + // set the iteration position to the end of the newly inserted replacement text. + int32_t newIndexPos = limit32 + lengthDelta; + repTextAccess(ut, newIndexPos, TRUE); + + return lengthDelta; +} + + +static void U_CALLCONV +repTextCopy(UText *ut, + int64_t start, int64_t limit, + int64_t destIndex, + UBool move, + UErrorCode *status) +{ + Replaceable *rep=(Replaceable *)ut->context; + int32_t length=rep->length(); + + if(U_FAILURE(*status)) { + return; + } + if (start>limit || (startcopy(start32, limit32, destIndex32); + if(destIndex32handleReplaceBetween(start32, limit32, UnicodeString()); + } else { + // copy + rep->copy(start32, limit32, destIndex32); + } + + // If the change to the text touched the region in the chunk buffer, + // invalidate the buffer. + int32_t firstAffectedIndex = destIndex32; + if (move && start32chunkNativeLimit) { + // changes may have affected range covered by the chunk + invalidateChunk(ut); + } + + // Put iteration position at the newly inserted (moved) block, + int32_t nativeIterIndex = destIndex32 + limit32 - start32; + if (move && destIndex32>start32) { + // moved a block of text towards the end of the string. + nativeIterIndex = destIndex32; + } + + // Set position, reload chunk if needed. + repTextAccess(ut, nativeIterIndex, TRUE); +} + +static struct UTextFuncs repFuncs = +{ + sizeof(UTextFuncs), + 0, 0, 0, // Reserved alignment padding + repTextClone, + repTextLength, + repTextAccess, + repTextExtract, + repTextReplace, + repTextCopy, + NULL, // MapOffsetToNative, + NULL, // MapIndexToUTF16, + repTextClose, + NULL, // spare 1 + NULL, // spare 2 + NULL // spare 3 +}; + + +U_CAPI UText * U_EXPORT2 +utext_openReplaceable(UText *ut, Replaceable *rep, UErrorCode *status) +{ + if(U_FAILURE(*status)) { + return NULL; + } + if(rep==NULL) { + *status=U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + ut = utext_setup(ut, sizeof(ReplExtra), status); + + ut->providerProperties = I32_FLAG(UTEXT_PROVIDER_WRITABLE); + if(rep->hasMetaData()) { + ut->providerProperties |=I32_FLAG(UTEXT_PROVIDER_HAS_META_DATA); + } + + ut->pFuncs = &repFuncs; + ut->context = rep; + return ut; +} + +U_CDECL_END + + + + + + + + +//------------------------------------------------------------------------------ +// +// UText implementation for UnicodeString (read/write) and +// for const UnicodeString (read only) +// (same implementation, only the flags are different) +// +// Use of UText data members: +// context pointer to UnicodeString +// p pointer to UnicodeString IF this UText owns the string +// and it must be deleted on close(). NULL otherwise. +// +//------------------------------------------------------------------------------ + +U_CDECL_BEGIN + + +static UText * U_CALLCONV +unistrTextClone(UText *dest, const UText *src, UBool deep, UErrorCode *status) { + // First do a generic shallow clone. Does everything needed for the UText struct itself. + dest = shallowTextClone(dest, src, status); + + // For deep clones, make a copy of the UnicodeSring. + // The copied UnicodeString storage is owned by the newly created UText clone. + // A non-NULL pointer in UText.p is the signal to the close() function to delete + // the UText. + // + if (deep && U_SUCCESS(*status)) { + const UnicodeString *srcString = (const UnicodeString *)src->context; + dest->context = new UnicodeString(*srcString); + dest->providerProperties |= I32_FLAG(UTEXT_PROVIDER_OWNS_TEXT); + + // with deep clone, the copy is writable, even when the source is not. + dest->providerProperties |= I32_FLAG(UTEXT_PROVIDER_WRITABLE); + } + return dest; +} + +static void U_CALLCONV +unistrTextClose(UText *ut) { + // Most of the work of close is done by the generic UText framework close. + // All that needs to be done here is delete the UnicodeString if the UText + // owns it. This occurs if the UText was created by cloning. + if (ut->providerProperties & I32_FLAG(UTEXT_PROVIDER_OWNS_TEXT)) { + UnicodeString *str = (UnicodeString *)ut->context; + delete str; + ut->context = NULL; + } +} + + +static int64_t U_CALLCONV +unistrTextLength(UText *t) { + return ((const UnicodeString *)t->context)->length(); +} + + +static UBool U_CALLCONV +unistrTextAccess(UText *ut, int64_t index, UBool forward) { + int32_t length = ut->chunkLength; + ut->chunkOffset = pinIndex(index, length); + + // Check whether request is at the start or end + UBool retVal = (forward && index0); + return retVal; +} + + + +static int32_t U_CALLCONV +unistrTextExtract(UText *t, + int64_t start, int64_t limit, + UChar *dest, int32_t destCapacity, + UErrorCode *pErrorCode) { + const UnicodeString *us=(const UnicodeString *)t->context; + int32_t length=us->length(); + + if(U_FAILURE(*pErrorCode)) { + return 0; + } + if(destCapacity<0 || (dest==NULL && destCapacity>0)) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + } + if(start<0 || start>limit) { + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + + int32_t start32 = startgetChar32Start((int32_t)start) : length; + int32_t limit32 = limitgetChar32Start((int32_t)limit) : length; + + length=limit32-start32; + if (destCapacity>0 && dest!=NULL) { + int32_t trimmedLength = length; + if(trimmedLength>destCapacity) { + trimmedLength=destCapacity; + } + us->extract(start32, trimmedLength, dest); + } + u_terminateUChars(dest, destCapacity, length, pErrorCode); + return length; +} + +static int32_t U_CALLCONV +unistrTextReplace(UText *ut, + int64_t start, int64_t limit, + const UChar *src, int32_t length, + UErrorCode *pErrorCode) { + UnicodeString *us=(UnicodeString *)ut->context; + int32_t oldLength; + + if(U_FAILURE(*pErrorCode)) { + return 0; + } + if(src==NULL && length!=0) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + } + if(start>limit) { + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + oldLength=us->length(); + int32_t start32 = pinIndex(start, oldLength); + int32_t limit32 = pinIndex(limit, oldLength); + if (start32 < oldLength) { + start32 = us->getChar32Start(start32); + } + if (limit32 < oldLength) { + limit32 = us->getChar32Start(limit32); + } + + // replace + us->replace(start32, limit32-start32, src, length); + int32_t newLength = us->length(); + + // Update the chunk description. + ut->chunkContents = us->getBuffer(); + ut->chunkLength = newLength; + ut->chunkNativeLimit = newLength; + ut->nativeIndexingLimit = newLength; + + // Set iteration position to the point just following the newly inserted text. + int32_t lengthDelta = newLength - oldLength; + ut->chunkOffset = limit32 + lengthDelta; + + return lengthDelta; +} + +static void U_CALLCONV +unistrTextCopy(UText *ut, + int64_t start, int64_t limit, + int64_t destIndex, + UBool move, + UErrorCode *pErrorCode) { + UnicodeString *us=(UnicodeString *)ut->context; + int32_t length=us->length(); + + if(U_FAILURE(*pErrorCode)) { + return; + } + int32_t start32 = pinIndex(start, length); + int32_t limit32 = pinIndex(limit, length); + int32_t destIndex32 = pinIndex(destIndex, length); + + if( start32>limit32 || (start32copy(start32, limit32, destIndex32); + if(destIndex32replace(start32, segLength, NULL, 0); + } else { + // copy + us->copy(start32, limit32, destIndex32); + } + + // update chunk description, set iteration position. + ut->chunkContents = us->getBuffer(); + if (move==FALSE) { + // copy operation, string length grows + ut->chunkLength += limit32-start32; + ut->chunkNativeLimit = ut->chunkLength; + ut->nativeIndexingLimit = ut->chunkLength; + } + + // Iteration position to end of the newly inserted text. + ut->chunkOffset = destIndex32+limit32-start32; + if (move && destIndex32>start32) { + ut->chunkOffset = destIndex32; + } + +} + +static struct UTextFuncs unistrFuncs = +{ + sizeof(UTextFuncs), + 0, 0, 0, // Reserved alignment padding + unistrTextClone, + unistrTextLength, + unistrTextAccess, + unistrTextExtract, + unistrTextReplace, + unistrTextCopy, + NULL, // MapOffsetToNative, + NULL, // MapIndexToUTF16, + unistrTextClose, + NULL, // spare 1 + NULL, // spare 2 + NULL // spare 3 +}; + + + +U_CDECL_END + + +U_CAPI UText * U_EXPORT2 +utext_openUnicodeString(UText *ut, UnicodeString *s, UErrorCode *status) { + // TODO: use openConstUnicodeString, then add in the differences. + // + ut = utext_setup(ut, 0, status); + if (U_SUCCESS(*status)) { + ut->pFuncs = &unistrFuncs; + ut->context = s; + ut->providerProperties = I32_FLAG(UTEXT_PROVIDER_STABLE_CHUNKS)| + I32_FLAG(UTEXT_PROVIDER_WRITABLE); + + ut->chunkContents = s->getBuffer(); + ut->chunkLength = s->length(); + ut->chunkNativeStart = 0; + ut->chunkNativeLimit = ut->chunkLength; + ut->nativeIndexingLimit = ut->chunkLength; + } + return ut; +} + + + +U_CAPI UText * U_EXPORT2 +utext_openConstUnicodeString(UText *ut, const UnicodeString *s, UErrorCode *status) { + ut = utext_setup(ut, 0, status); + // note: use the standard (writable) function table for UnicodeString. + // The flag settings disable writing, so having the functions in + // the table is harmless. + if (U_SUCCESS(*status)) { + ut->pFuncs = &unistrFuncs; + ut->context = s; + ut->providerProperties = I32_FLAG(UTEXT_PROVIDER_STABLE_CHUNKS); + ut->chunkContents = s->getBuffer(); + ut->chunkLength = s->length(); + ut->chunkNativeStart = 0; + ut->chunkNativeLimit = ut->chunkLength; + ut->nativeIndexingLimit = ut->chunkLength; + } + return ut; +} + +//------------------------------------------------------------------------------ +// +// UText implementation for const UChar * strings +// +// Use of UText data members: +// context pointer to UnicodeString +// a length. -1 if not yet known. +// +// TODO: support 64 bit lengths. +// +//------------------------------------------------------------------------------ + +U_CDECL_BEGIN + + +static UText * U_CALLCONV +ucstrTextClone(UText *dest, const UText * src, UBool deep, UErrorCode * status) { + // First do a generic shallow clone. + dest = shallowTextClone(dest, src, status); + + // For deep clones, make a copy of the string. + // The copied storage is owned by the newly created clone. + // A non-NULL pointer in UText.p is the signal to the close() function to delete + // it. + // + if (deep && U_SUCCESS(*status)) { + U_ASSERT(utext_nativeLength(dest) < INT32_MAX); + int32_t len = (int32_t)utext_nativeLength(dest); + + // The cloned string IS going to be NUL terminated, whether or not the original was. + const UChar *srcStr = (const UChar *)src->context; + UChar *copyStr = (UChar *)uprv_malloc((len+1) * sizeof(UChar)); + if (copyStr == NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + } else { + int64_t i; + for (i=0; icontext = copyStr; + dest->providerProperties |= I32_FLAG(UTEXT_PROVIDER_OWNS_TEXT); + } + } + return dest; +} + + +static void U_CALLCONV +ucstrTextClose(UText *ut) { + // Most of the work of close is done by the generic UText framework close. + // All that needs to be done here is delete the string if the UText + // owns it. This occurs if the UText was created by cloning. + if (ut->providerProperties & I32_FLAG(UTEXT_PROVIDER_OWNS_TEXT)) { + UChar *s = (UChar *)ut->context; + uprv_free(s); + ut->context = NULL; + } +} + + + +static int64_t U_CALLCONV +ucstrTextLength(UText *ut) { + if (ut->a < 0) { + // null terminated, we don't yet know the length. Scan for it. + // Access is not convenient for doing this + // because the current interation postion can't be changed. + const UChar *str = (const UChar *)ut->context; + for (;;) { + if (str[ut->chunkNativeLimit] == 0) { + break; + } + ut->chunkNativeLimit++; + } + ut->a = ut->chunkNativeLimit; + ut->chunkLength = (int32_t)ut->chunkNativeLimit; + ut->nativeIndexingLimit = ut->chunkLength; + ut->providerProperties &= ~I32_FLAG(UTEXT_PROVIDER_LENGTH_IS_EXPENSIVE); + } + return ut->a; +} + + +static UBool U_CALLCONV +ucstrTextAccess(UText *ut, int64_t index, UBool forward) { + const UChar *str = (const UChar *)ut->context; + + // pin the requested index to the bounds of the string, + // and set current iteration position. + if (index<0) { + index = 0; + } else if (index < ut->chunkNativeLimit) { + // The request data is within the chunk as it is known so far. + // Put index on a code point boundary. + U16_SET_CP_START(str, 0, index); + } else if (ut->a >= 0) { + // We know the length of this string, and the user is requesting something + // at or beyond the length. Pin the requested index to the length. + index = ut->a; + } else { + // Null terminated string, length not yet known, and the requested index + // is beyond where we have scanned so far. + // Scan to 32 UChars beyond the requested index. The strategy here is + // to avoid fully scanning a long string when the caller only wants to + // see a few characters at its beginning. + int32_t scanLimit = (int32_t)index + 32; + if ((index + 32)>INT32_MAX || (index + 32)<0 ) { // note: int64 expression + scanLimit = INT32_MAX; + } + + int32_t chunkLimit = (int32_t)ut->chunkNativeLimit; + for (; chunkLimita = chunkLimit; + ut->chunkLength = chunkLimit; + ut->nativeIndexingLimit = chunkLimit; + if (index >= chunkLimit) { + index = chunkLimit; + } else { + U16_SET_CP_START(str, 0, index); + } + + ut->chunkNativeLimit = chunkLimit; + ut->providerProperties &= ~I32_FLAG(UTEXT_PROVIDER_LENGTH_IS_EXPENSIVE); + goto breakout; + } + } + // We scanned through the next batch of UChars without finding the end. + U16_SET_CP_START(str, 0, index); + if (chunkLimit == INT32_MAX) { + // Scanned to the limit of a 32 bit length. + // Forceably trim the overlength string back so length fits in int32 + // TODO: add support for 64 bit strings. + ut->a = chunkLimit; + ut->chunkLength = chunkLimit; + ut->nativeIndexingLimit = chunkLimit; + if (index > chunkLimit) { + index = chunkLimit; + } + ut->chunkNativeLimit = chunkLimit; + ut->providerProperties &= ~I32_FLAG(UTEXT_PROVIDER_LENGTH_IS_EXPENSIVE); + } else { + // The endpoint of a chunk must not be left in the middle of a surrogate pair. + // If the current end is on a lead surrogate, back the end up by one. + // It doesn't matter if the end char happens to be an unpaired surrogate, + // and it's simpler not to worry about it. + if (U16_IS_LEAD(str[chunkLimit-1])) { + --chunkLimit; + } + ut->chunkNativeLimit = chunkLimit; + } + + } +breakout: + U_ASSERT(index<=INT32_MAX); + ut->chunkOffset = (int32_t)index; + + // Check whether request is at the start or end + UBool retVal = (forward && indexchunkNativeLimit) || (!forward && index>0); + return retVal; +} + + + +static int32_t U_CALLCONV +ucstrTextExtract(UText *ut, + int64_t start, int64_t limit, + UChar *dest, int32_t destCapacity, + UErrorCode *pErrorCode) +{ + if(U_FAILURE(*pErrorCode)) { + return 0; + } + if(destCapacity<0 || (dest==NULL && destCapacity>0) || start>limit) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + const UChar *s=(const UChar *)ut->context; + int32_t si, di; + + int32_t start32; + int32_t limit32; + + // Access the start. Does two things we need: + // Pins 'start' to the length of the string, if it came in out-of-bounds. + // Snaps 'start' to the beginning of a code point. + ucstrTextAccess(ut, start, TRUE); + U_ASSERT(start <= INT32_MAX); + start32 = (int32_t)start; + + int32_t strLength=(int32_t)ut->a; + if (strLength >= 0) { + limit32 = pinIndex(limit, strLength); + } else { + limit32 = pinIndex(limit, INT32_MAX); + } + + di = 0; + for (si=start32; sia = si; // set string length for this UText + ut->chunkNativeLimit = si; + ut->chunkLength = si; + ut->nativeIndexingLimit = si; + strLength = si; + break; + } + if (di=0) { + // We have filled the destination buffer, and the string length is known. + // Cut the loop short. There is no need to scan string termination. + di = strLength; + si = limit32; + break; + } + } + di++; + } + + // If the limit index points to a lead surrogate of a pair, + // add the corresponding trail surrogate to the destination. + if (si>0 && U16_IS_LEAD(s[si-1]) && + ((sichunkOffset = si; + + // Add a terminating NUL if space in the buffer permits, + // and set the error status as required. + u_terminateUChars(dest, destCapacity, di, pErrorCode); + return di; +} + +static struct UTextFuncs ucstrFuncs = +{ + sizeof(UTextFuncs), + 0, 0, 0, // Reserved alignment padding + ucstrTextClone, + ucstrTextLength, + ucstrTextAccess, + ucstrTextExtract, + NULL, // Replace + NULL, // Copy + NULL, // MapOffsetToNative, + NULL, // MapIndexToUTF16, + ucstrTextClose, + NULL, // spare 1 + NULL, // spare 2 + NULL, // spare 3 +}; + +U_CDECL_END + + +U_CAPI UText * U_EXPORT2 +utext_openUChars(UText *ut, const UChar *s, int64_t length, UErrorCode *status) { + if (U_FAILURE(*status)) { + return NULL; + } + if (length < -1 || length>INT32_MAX) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + ut = utext_setup(ut, 0, status); + if (U_SUCCESS(*status)) { + ut->pFuncs = &ucstrFuncs; + ut->context = s; + ut->providerProperties = I32_FLAG(UTEXT_PROVIDER_STABLE_CHUNKS); + if (length==-1) { + ut->providerProperties |= I32_FLAG(UTEXT_PROVIDER_LENGTH_IS_EXPENSIVE); + } + ut->a = length; + ut->chunkContents = s; + ut->chunkNativeStart = 0; + ut->chunkNativeLimit = length>=0? length : 0; + ut->chunkLength = (int32_t)ut->chunkNativeLimit; + ut->chunkOffset = 0; + ut->nativeIndexingLimit = ut->chunkLength; + } + return ut; +} + + +//------------------------------------------------------------------------------ +// +// UText implementation for text from ICU CharacterIterators +// +// Use of UText data members: +// context pointer to the CharacterIterator +// a length of the full text. +// p pointer to buffer 1 +// b start index of local buffer 1 contents +// q pointer to buffer 2 +// c start index of local buffer 2 contents +// r pointer to the character iterator if the UText owns it. +// Null otherwise. +// +//------------------------------------------------------------------------------ +#define CIBufSize 16 + +U_CDECL_BEGIN +static void U_CALLCONV +charIterTextClose(UText *ut) { + // Most of the work of close is done by the generic UText framework close. + // All that needs to be done here is delete the CharacterIterator if the UText + // owns it. This occurs if the UText was created by cloning. + CharacterIterator *ci = (CharacterIterator *)ut->r; + delete ci; + ut->r = NULL; +} + +static int64_t U_CALLCONV +charIterTextLength(UText *ut) { + return (int32_t)ut->a; +} + +static UBool U_CALLCONV +charIterTextAccess(UText *ut, int64_t index, UBool forward) { + CharacterIterator *ci = (CharacterIterator *)ut->context; + + int32_t clippedIndex = (int32_t)index; + if (clippedIndex<0) { + clippedIndex=0; + } else if (clippedIndex>=ut->a) { + clippedIndex=(int32_t)ut->a; + } + int32_t neededIndex = clippedIndex; + if (!forward && neededIndex>0) { + // reverse iteration, want the position just before what was asked for. + neededIndex--; + } else if (forward && neededIndex==ut->a && neededIndex>0) { + // Forward iteration, don't ask for something past the end of the text. + neededIndex--; + } + + // Find the native index of the start of the buffer containing what we want. + neededIndex -= neededIndex % CIBufSize; + + UChar *buf = NULL; + UBool needChunkSetup = TRUE; + int i; + if (ut->chunkNativeStart == neededIndex) { + // The buffer we want is already the current chunk. + needChunkSetup = FALSE; + } else if (ut->b == neededIndex) { + // The first buffer (buffer p) has what we need. + buf = (UChar *)ut->p; + } else if (ut->c == neededIndex) { + // The second buffer (buffer q) has what we need. + buf = (UChar *)ut->q; + } else { + // Neither buffer already has what we need. + // Load new data from the character iterator. + // Use the buf that is not the current buffer. + buf = (UChar *)ut->p; + if (ut->p == ut->chunkContents) { + buf = (UChar *)ut->q; + } + ci->setIndex(neededIndex); + for (i=0; inextPostInc(); + if (i+neededIndex > ut->a) { + break; + } + } + } + + // We have a buffer with the data we need. + // Set it up as the current chunk, if it wasn't already. + if (needChunkSetup) { + ut->chunkContents = buf; + ut->chunkLength = CIBufSize; + ut->chunkNativeStart = neededIndex; + ut->chunkNativeLimit = neededIndex + CIBufSize; + if (ut->chunkNativeLimit > ut->a) { + ut->chunkNativeLimit = ut->a; + ut->chunkLength = (int32_t)(ut->chunkNativeLimit)-(int32_t)(ut->chunkNativeStart); + } + ut->nativeIndexingLimit = ut->chunkLength; + U_ASSERT(ut->chunkOffset>=0 && ut->chunkOffset<=CIBufSize); + } + ut->chunkOffset = clippedIndex - (int32_t)ut->chunkNativeStart; + UBool success = (forward? ut->chunkOffsetchunkLength : ut->chunkOffset>0); + return success; +} + +static UText * U_CALLCONV +charIterTextClone(UText *dest, const UText *src, UBool deep, UErrorCode * status) { + if (U_FAILURE(*status)) { + return NULL; + } + + if (deep) { + // There is no CharacterIterator API for cloning the underlying text storage. + *status = U_UNSUPPORTED_ERROR; + return NULL; + } else { + CharacterIterator *srcCI =(CharacterIterator *)src->context; + srcCI = srcCI->clone(); + dest = utext_openCharacterIterator(dest, srcCI, status); + // cast off const on getNativeIndex. + // For CharacterIterator based UTexts, this is safe, the operation is const. + int64_t ix = utext_getNativeIndex((UText *)src); + utext_setNativeIndex(dest, ix); + dest->r = srcCI; // flags that this UText owns the CharacterIterator + } + return dest; +} + +static int32_t U_CALLCONV +charIterTextExtract(UText *ut, + int64_t start, int64_t limit, + UChar *dest, int32_t destCapacity, + UErrorCode *status) +{ + if(U_FAILURE(*status)) { + return 0; + } + if(destCapacity<0 || (dest==NULL && destCapacity>0) || start>limit) { + *status=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + int32_t length = (int32_t)ut->a; + int32_t start32 = pinIndex(start, length); + int32_t limit32 = pinIndex(limit, length); + int32_t desti = 0; + int32_t srci; + + CharacterIterator *ci = (CharacterIterator *)ut->context; + ci->setIndex32(start32); // Moves ix to lead of surrogate pair, if needed. + srci = ci->getIndex(); + while (srcinext32PostInc(); + int32_t len = U16_LENGTH(c); + if (desti+len <= destCapacity) { + U16_APPEND_UNSAFE(dest, desti, c); + } else { + desti += len; + *status = U_BUFFER_OVERFLOW_ERROR; + } + srci += len; + } + + u_terminateUChars(dest, destCapacity, desti, status); + return desti; +} + +static struct UTextFuncs charIterFuncs = +{ + sizeof(UTextFuncs), + 0, 0, 0, // Reserved alignment padding + charIterTextClone, + charIterTextLength, + charIterTextAccess, + charIterTextExtract, + NULL, // Replace + NULL, // Copy + NULL, // MapOffsetToNative, + NULL, // MapIndexToUTF16, + charIterTextClose, + NULL, // spare 1 + NULL, // spare 2 + NULL // spare 3 +}; +U_CDECL_END + + +U_CAPI UText * U_EXPORT2 +utext_openCharacterIterator(UText *ut, CharacterIterator *ci, UErrorCode *status) { + if (U_FAILURE(*status)) { + return NULL; + } + + if (ci->startIndex() > 0) { + // No support for CharacterIterators that do not start indexing from zero. + *status = U_UNSUPPORTED_ERROR; + return NULL; + } + + // Extra space in UText for 2 buffers of CIBufSize UChars each. + int32_t extraSpace = 2 * CIBufSize * sizeof(UChar); + ut = utext_setup(ut, extraSpace, status); + if (U_SUCCESS(*status)) { + ut->pFuncs = &charIterFuncs; + ut->context = ci; + ut->providerProperties = 0; + ut->a = ci->endIndex(); // Length of text + ut->p = ut->pExtra; // First buffer + ut->b = -1; // Native index of first buffer contents + ut->q = (UChar*)ut->pExtra+CIBufSize; // Second buffer + ut->c = -1; // Native index of second buffer contents + + // Initialize current chunk contents to be empty. + // First access will fault something in. + // Note: The initial nativeStart and chunkOffset must sum to zero + // so that getNativeIndex() will correctly compute to zero + // if no call to Access() has ever been made. They can't be both + // zero without Access() thinking that the chunk is valid. + ut->chunkContents = (UChar *)ut->p; + ut->chunkNativeStart = -1; + ut->chunkOffset = 1; + ut->chunkNativeLimit = 0; + ut->chunkLength = 0; + ut->nativeIndexingLimit = ut->chunkOffset; // enables native indexing + } + return ut; +} + + + diff --git a/Build/source/libs/icu/icu-xetex/common/utf_impl.c b/Build/source/libs/icu/icu-xetex/common/utf_impl.c new file mode 100644 index 00000000000..8f45546bb76 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/utf_impl.c @@ -0,0 +1,364 @@ +/* +****************************************************************************** +* +* Copyright (C) 1999-2006, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: utf_impl.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 1999sep13 +* created by: Markus W. Scherer +* +* This file provides implementation functions for macros in the utfXX.h +* that would otherwise be too long as macros. +*/ + +/* set import/export definitions */ +#ifndef U_UTF8_IMPL +# define U_UTF8_IMPL +#endif + +#include "unicode/utypes.h" + +/* + * This table could be replaced on many machines by + * a few lines of assembler code using an + * "index of first 0-bit from msb" instruction and + * one or two more integer instructions. + * + * For example, on an i386, do something like + * - MOV AL, leadByte + * - NOT AL (8-bit, leave b15..b8==0..0, reverse only b7..b0) + * - MOV AH, 0 + * - BSR BX, AX (16-bit) + * - MOV AX, 6 (result) + * - JZ finish (ZF==1 if leadByte==0xff) + * - SUB AX, BX (result) + * -finish: + * (BSR: Bit Scan Reverse, scans for a 1-bit, starting from the MSB) + * + * In Unicode, all UTF-8 byte sequences with more than 4 bytes are illegal; + * lead bytes above 0xf4 are illegal. + * We keep them in this table for skipping long ISO 10646-UTF-8 sequences. + */ +U_EXPORT const uint8_t +utf8_countTrailBytes[256]={ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 3, 3, 3, 3, 3, + 3, 3, 3, /* illegal in Unicode */ + 4, 4, 4, 4, /* illegal in Unicode */ + 5, 5, /* illegal in Unicode */ + 0, 0 /* illegal bytes 0xfe and 0xff */ +}; + +static const UChar32 +utf8_minLegal[4]={ 0, 0x80, 0x800, 0x10000 }; + +static const UChar32 +utf8_errorValue[6]={ + UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_2, UTF_ERROR_VALUE, 0x10ffff, + 0x3ffffff, 0x7fffffff +}; + +/* + * Handle the non-inline part of the U8_NEXT() macro and its obsolete sibling + * UTF8_NEXT_CHAR_SAFE(). + * + * The "strict" parameter controls the error behavior: + * <0 "Safe" behavior of U8_NEXT(): All illegal byte sequences yield a negative + * code point result. + * 0 Obsolete "safe" behavior of UTF8_NEXT_CHAR_SAFE(..., FALSE): + * All illegal byte sequences yield a positive code point such that this + * result code point would be encoded with the same number of bytes as + * the illegal sequence. + * >0 Obsolete "strict" behavior of UTF8_NEXT_CHAR_SAFE(..., TRUE): + * Same as the obsolete "safe" behavior, but non-characters are also treated + * like illegal sequences. + * + * The special negative (<0) value -2 is used for lenient treatment of surrogate + * code points as legal. Some implementations use this for roundtripping of + * Unicode 16-bit strings that are not well-formed UTF-16, that is, they + * contain unpaired surrogates. + * + * Note that a UBool is the same as an int8_t. + */ +U_CAPI UChar32 U_EXPORT2 +utf8_nextCharSafeBody(const uint8_t *s, int32_t *pi, int32_t length, UChar32 c, UBool strict) { + int32_t i=*pi; + uint8_t count=UTF8_COUNT_TRAIL_BYTES(c); + if((i)+count<=(length)) { + uint8_t trail, illegal=0; + + UTF8_MASK_LEAD_BYTE((c), count); + /* count==0 for illegally leading trail bytes and the illegal bytes 0xfe and 0xff */ + switch(count) { + /* each branch falls through to the next one */ + case 5: + case 4: + /* count>=4 is always illegal: no more than 3 trail bytes in Unicode's UTF-8 */ + illegal=1; + break; + case 3: + trail=s[(i)++]; + (c)=((c)<<6)|(trail&0x3f); + if(c<0x110) { + illegal|=(trail&0xc0)^0x80; + } else { + /* code point>0x10ffff, outside Unicode */ + illegal=1; + break; + } + case 2: + trail=s[(i)++]; + (c)=((c)<<6)|(trail&0x3f); + illegal|=(trail&0xc0)^0x80; + case 1: + trail=s[(i)++]; + (c)=((c)<<6)|(trail&0x3f); + illegal|=(trail&0xc0)^0x80; + break; + case 0: + if(strict>=0) { + return UTF8_ERROR_VALUE_1; + } else { + return U_SENTINEL; + } + /* no default branch to optimize switch() - all values are covered */ + } + + /* + * All the error handling should return a value + * that needs count bytes so that UTF8_GET_CHAR_SAFE() works right. + * + * Starting with Unicode 3.0.1, non-shortest forms are illegal. + * Starting with Unicode 3.2, surrogate code points must not be + * encoded in UTF-8, and there are no irregular sequences any more. + * + * U8_ macros (new in ICU 2.4) return negative values for error conditions. + */ + + /* correct sequence - all trail bytes have (b7..b6)==(10)? */ + /* illegal is also set if count>=4 */ + if(illegal || (c)0 && UTF8_IS_TRAIL(s[i])) { + ++(i); + --count; + } + if(strict>=0) { + c=utf8_errorValue[errorCount-count]; + } else { + c=U_SENTINEL; + } + } else if((strict)>0 && UTF_IS_UNICODE_NONCHAR(c)) { + /* strict: forbid non-characters like U+fffe */ + c=utf8_errorValue[count]; + } + } else /* too few bytes left */ { + /* error handling */ + int32_t i0=i; + /* don't just set (i)=(length) in case there is an illegal sequence */ + while((i)<(length) && UTF8_IS_TRAIL(s[i])) { + ++(i); + } + if(strict>=0) { + c=utf8_errorValue[i-i0]; + } else { + c=U_SENTINEL; + } + } + *pi=i; + return c; +} + +U_CAPI int32_t U_EXPORT2 +utf8_appendCharSafeBody(uint8_t *s, int32_t i, int32_t length, UChar32 c, UBool *pIsError) { + if((uint32_t)(c)<=0x7ff) { + if((i)+1<(length)) { + (s)[(i)++]=(uint8_t)(((c)>>6)|0xc0); + (s)[(i)++]=(uint8_t)(((c)&0x3f)|0x80); + return i; + } + } else if((uint32_t)(c)<=0xffff) { + /* Starting with Unicode 3.2, surrogate code points must not be encoded in UTF-8. */ + if((i)+2<(length) && !U_IS_SURROGATE(c)) { + (s)[(i)++]=(uint8_t)(((c)>>12)|0xe0); + (s)[(i)++]=(uint8_t)((((c)>>6)&0x3f)|0x80); + (s)[(i)++]=(uint8_t)(((c)&0x3f)|0x80); + return i; + } + } else if((uint32_t)(c)<=0x10ffff) { + if((i)+3<(length)) { + (s)[(i)++]=(uint8_t)(((c)>>18)|0xf0); + (s)[(i)++]=(uint8_t)((((c)>>12)&0x3f)|0x80); + (s)[(i)++]=(uint8_t)((((c)>>6)&0x3f)|0x80); + (s)[(i)++]=(uint8_t)(((c)&0x3f)|0x80); + return i; + } + } + /* c>0x10ffff or not enough space, write an error value */ + if(pIsError!=NULL) { + *pIsError=TRUE; + } else { + length-=i; + if(length>0) { + int32_t offset; + if(length>3) { + length=3; + } + s+=i; + offset=0; + c=utf8_errorValue[length-1]; + UTF8_APPEND_CHAR_UNSAFE(s, offset, c); + i=i+offset; + } + } + return i; +} + +U_CAPI UChar32 U_EXPORT2 +utf8_prevCharSafeBody(const uint8_t *s, int32_t start, int32_t *pi, UChar32 c, UBool strict) { + int32_t i=*pi; + uint8_t b, count=1, shift=6; + + /* extract value bits from the last trail byte */ + c&=0x3f; + + for(;;) { + if(i<=start) { + /* no lead byte at all */ + if(strict>=0) { + return UTF8_ERROR_VALUE_1; + } else { + return U_SENTINEL; + } + /*break;*/ + } + + /* read another previous byte */ + b=s[--i]; + if((uint8_t)(b-0x80)<0x7e) { /* 0x80<=b<0xfe */ + if(b&0x40) { + /* lead byte, this will always end the loop */ + uint8_t shouldCount=UTF8_COUNT_TRAIL_BYTES(b); + + if(count==shouldCount) { + /* set the new position */ + *pi=i; + UTF8_MASK_LEAD_BYTE(b, count); + c|=(UChar32)b<=4 || c>0x10ffff || c0 && UTF_IS_UNICODE_NONCHAR(c))) { + /* illegal sequence or (strict and non-character) */ + if(count>=4) { + count=3; + } + if(strict>=0) { + c=utf8_errorValue[count]; + } else { + c=U_SENTINEL; + } + } else { + /* exit with correct c */ + } + } else { + /* the lead byte does not match the number of trail bytes */ + /* only set the position to the lead byte if it would + include the trail byte that we started with */ + if(count=0) { + c=utf8_errorValue[count]; + } else { + c=U_SENTINEL; + } + } else { + if(strict>=0) { + c=UTF8_ERROR_VALUE_1; + } else { + c=U_SENTINEL; + } + } + } + break; + } else if(count<5) { + /* trail byte */ + c|=(UChar32)(b&0x3f)<=0) { + c=UTF8_ERROR_VALUE_1; + } else { + c=U_SENTINEL; + } + break; + } + } else { + /* single-byte character precedes trailing bytes */ + if(strict>=0) { + c=UTF8_ERROR_VALUE_1; + } else { + c=U_SENTINEL; + } + break; + } + } + return c; +} + +U_CAPI int32_t U_EXPORT2 +utf8_back1SafeBody(const uint8_t *s, int32_t start, int32_t i) { + /* i had been decremented once before the function call */ + int32_t I=i, Z; + uint8_t b; + + /* read at most the 6 bytes s[Z] to s[i], inclusively */ + if(I-5>start) { + Z=I-5; + } else { + Z=start; + } + + /* return I if the sequence starting there is long enough to include i */ + do { + b=s[I]; + if((uint8_t)(b-0x80)>=0x7e) { /* not 0x80<=b<0xfe */ + break; + } else if(b>=0xc0) { + if(UTF8_COUNT_TRAIL_BYTES(b)>=(i-I)) { + return I; + } else { + break; + } + } + } while(Z<=--I); + + /* return i itself to be consistent with the FWD_1 macro */ + return i; +} diff --git a/Build/source/libs/icu/icu-xetex/common/util.cpp b/Build/source/libs/icu/icu-xetex/common/util.cpp new file mode 100644 index 00000000000..6b74ffe5b16 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/util.cpp @@ -0,0 +1,443 @@ +/* +********************************************************************** +* Copyright (c) 2001-2007, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 11/19/2001 aliu Creation. +********************************************************************** +*/ + +#include "util.h" +#include "unicode/unimatch.h" +#include "unicode/uniset.h" + +// Define UChar constants using hex for EBCDIC compatibility + +static const UChar BACKSLASH = 0x005C; /*\*/ +static const UChar UPPER_U = 0x0055; /*U*/ +static const UChar LOWER_U = 0x0075; /*u*/ +static const UChar APOSTROPHE = 0x0027; // '\'' +static const UChar SPACE = 0x0020; // ' ' + +// "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" +static const UChar DIGITS[] = { + 48,49,50,51,52,53,54,55,56,57, + 65,66,67,68,69,70,71,72,73,74, + 75,76,77,78,79,80,81,82,83,84, + 85,86,87,88,89,90 +}; + +U_NAMESPACE_BEGIN + +UnicodeString& ICU_Utility::appendNumber(UnicodeString& result, int32_t n, + int32_t radix, int32_t minDigits) { + if (radix < 2 || radix > 36) { + // Bogus radix + return result.append((UChar)63/*?*/); + } + // Handle negatives + if (n < 0) { + n = -n; + result.append((UChar)45/*-*/); + } + // First determine the number of digits + int32_t nn = n; + int32_t r = 1; + while (nn >= radix) { + nn /= radix; + r *= radix; + --minDigits; + } + // Now generate the digits + while (--minDigits > 0) { + result.append(DIGITS[0]); + } + while (r > 0) { + int32_t digit = n / r; + result.append(DIGITS[digit]); + n -= digit * r; + r /= radix; + } + return result; +} + +static const UChar HEX[16] = {48,49,50,51,52,53,54,55, // 0-7 + 56,57,65,66,67,68,69,70}; // 8-9 A-F + +/** + * Return true if the character is NOT printable ASCII. + */ +UBool ICU_Utility::isUnprintable(UChar32 c) { + return !(c >= 0x20 && c <= 0x7E); +} + +/** + * Escape unprintable characters using \uxxxx notation for U+0000 to + * U+FFFF and \Uxxxxxxxx for U+10000 and above. If the character is + * printable ASCII, then do nothing and return FALSE. Otherwise, + * append the escaped notation and return TRUE. + */ +UBool ICU_Utility::escapeUnprintable(UnicodeString& result, UChar32 c) { + if (isUnprintable(c)) { + result.append(BACKSLASH); + if (c & ~0xFFFF) { + result.append(UPPER_U); + result.append(HEX[0xF&(c>>28)]); + result.append(HEX[0xF&(c>>24)]); + result.append(HEX[0xF&(c>>20)]); + result.append(HEX[0xF&(c>>16)]); + } else { + result.append(LOWER_U); + } + result.append(HEX[0xF&(c>>12)]); + result.append(HEX[0xF&(c>>8)]); + result.append(HEX[0xF&(c>>4)]); + result.append(HEX[0xF&c]); + return TRUE; + } + return FALSE; +} + +/** + * Returns the index of a character, ignoring quoted text. + * For example, in the string "abc'hide'h", the 'h' in "hide" will not be + * found by a search for 'h'. + */ +// FOR FUTURE USE. DISABLE FOR NOW for coverage reasons. +/* +int32_t ICU_Utility::quotedIndexOf(const UnicodeString& text, + int32_t start, int32_t limit, + UChar charToFind) { + for (int32_t i=start; i= pos); +//? +//? if (!isForward) { +//? --pos; // pos is a limit, so back up by one +//? } +//? +//? while (pos != stop && +//? uprv_isRuleWhiteSpace(c = text.char32At(pos))) { +//? if (isForward) { +//? pos += UTF_CHAR_LENGTH(c); +//? } else { +//? pos -= UTF_CHAR_LENGTH(c); +//? } +//? } +//? +//? if (!isForward) { +//? ++pos; // make pos back into a limit +//? } +//? +//? return pos; +//?} + +/** + * Parse a single non-whitespace character 'ch', optionally + * preceded by whitespace. + * @param id the string to be parsed + * @param pos INPUT-OUTPUT parameter. On input, pos[0] is the + * offset of the first character to be parsed. On output, pos[0] + * is the index after the last parsed character. If the parse + * fails, pos[0] will be unchanged. + * @param ch the non-whitespace character to be parsed. + * @return true if 'ch' is seen preceded by zero or more + * whitespace characters. + */ +UBool ICU_Utility::parseChar(const UnicodeString& id, int32_t& pos, UChar ch) { + int32_t start = pos; + skipWhitespace(id, pos, TRUE); + if (pos == id.length() || + id.charAt(pos) != ch) { + pos = start; + return FALSE; + } + ++pos; + return TRUE; +} + +/** + * Parse a pattern string within the given Replaceable and a parsing + * pattern. Characters are matched literally and case-sensitively + * except for the following special characters: + * + * ~ zero or more uprv_isRuleWhiteSpace chars + * + * If end of pattern is reached with all matches along the way, + * pos is advanced to the first unparsed index and returned. + * Otherwise -1 is returned. + * @param pat pattern that controls parsing + * @param text text to be parsed, starting at index + * @param index offset to first character to parse + * @param limit offset after last character to parse + * @return index after last parsed character, or -1 on parse failure. + */ +int32_t ICU_Utility::parsePattern(const UnicodeString& pat, + const Replaceable& text, + int32_t index, + int32_t limit) { + int32_t ipat = 0; + + // empty pattern matches immediately + if (ipat == pat.length()) { + return index; + } + + UChar32 cpat = pat.char32At(ipat); + + while (index < limit) { + UChar32 c = text.char32At(index); + + // parse \s* + if (cpat == 126 /*~*/) { + if (uprv_isRuleWhiteSpace(c)) { + index += UTF_CHAR_LENGTH(c); + continue; + } else { + if (++ipat == pat.length()) { + return index; // success; c unparsed + } + // fall thru; process c again with next cpat + } + } + + // parse literal + else if (c == cpat) { + index += UTF_CHAR_LENGTH(c); + ipat += UTF_CHAR_LENGTH(cpat); + if (ipat == pat.length()) { + return index; // success; c parsed + } + // fall thru; get next cpat + } + + // match failure of literal + else { + return -1; + } + + cpat = pat.char32At(ipat); + } + + return -1; // text ended before end of pat +} + +/** + * Append a character to a rule that is being built up. To flush + * the quoteBuf to rule, make one final call with isLiteral == TRUE. + * If there is no final character, pass in (UChar32)-1 as c. + * @param rule the string to append the character to + * @param c the character to append, or (UChar32)-1 if none. + * @param isLiteral if true, then the given character should not be + * quoted or escaped. Usually this means it is a syntactic element + * such as > or $ + * @param escapeUnprintable if true, then unprintable characters + * should be escaped using \uxxxx or \Uxxxxxxxx. These escapes will + * appear outside of quotes. + * @param quoteBuf a buffer which is used to build up quoted + * substrings. The caller should initially supply an empty buffer, + * and thereafter should not modify the buffer. The buffer should be + * cleared out by, at the end, calling this method with a literal + * character. + */ +void ICU_Utility::appendToRule(UnicodeString& rule, + UChar32 c, + UBool isLiteral, + UBool escapeUnprintable, + UnicodeString& quoteBuf) { + // If we are escaping unprintables, then escape them outside + // quotes. \u and \U are not recognized within quotes. The same + // logic applies to literals, but literals are never escaped. + if (isLiteral || + (escapeUnprintable && ICU_Utility::isUnprintable(c))) { + if (quoteBuf.length() > 0) { + // We prefer backslash APOSTROPHE to double APOSTROPHE + // (more readable, less similar to ") so if there are + // double APOSTROPHEs at the ends, we pull them outside + // of the quote. + + // If the first thing in the quoteBuf is APOSTROPHE + // (doubled) then pull it out. + while (quoteBuf.length() >= 2 && + quoteBuf.charAt(0) == APOSTROPHE && + quoteBuf.charAt(1) == APOSTROPHE) { + rule.append(BACKSLASH).append(APOSTROPHE); + quoteBuf.remove(0, 2); + } + // If the last thing in the quoteBuf is APOSTROPHE + // (doubled) then remove and count it and add it after. + int32_t trailingCount = 0; + while (quoteBuf.length() >= 2 && + quoteBuf.charAt(quoteBuf.length()-2) == APOSTROPHE && + quoteBuf.charAt(quoteBuf.length()-1) == APOSTROPHE) { + quoteBuf.truncate(quoteBuf.length()-2); + ++trailingCount; + } + if (quoteBuf.length() > 0) { + rule.append(APOSTROPHE); + rule.append(quoteBuf); + rule.append(APOSTROPHE); + quoteBuf.truncate(0); + } + while (trailingCount-- > 0) { + rule.append(BACKSLASH).append(APOSTROPHE); + } + } + if (c != (UChar32)-1) { + /* Since spaces are ignored during parsing, they are + * emitted only for readability. We emit one here + * only if there isn't already one at the end of the + * rule. + */ + if (c == SPACE) { + int32_t len = rule.length(); + if (len > 0 && rule.charAt(len-1) != c) { + rule.append(c); + } + } else if (!escapeUnprintable || !ICU_Utility::escapeUnprintable(rule, c)) { + rule.append(c); + } + } + } + + // Escape ' and '\' and don't begin a quote just for them + else if (quoteBuf.length() == 0 && + (c == APOSTROPHE || c == BACKSLASH)) { + rule.append(BACKSLASH); + rule.append(c); + } + + // Specials (printable ascii that isn't [0-9a-zA-Z]) and + // whitespace need quoting. Also append stuff to quotes if we are + // building up a quoted substring already. + else if (quoteBuf.length() > 0 || + (c >= 0x0021 && c <= 0x007E && + !((c >= 0x0030/*'0'*/ && c <= 0x0039/*'9'*/) || + (c >= 0x0041/*'A'*/ && c <= 0x005A/*'Z'*/) || + (c >= 0x0061/*'a'*/ && c <= 0x007A/*'z'*/))) || + uprv_isRuleWhiteSpace(c)) { + quoteBuf.append(c); + // Double ' within a quote + if (c == APOSTROPHE) { + quoteBuf.append(c); + } + } + + // Otherwise just append + else { + rule.append(c); + } +} + +void ICU_Utility::appendToRule(UnicodeString& rule, + const UnicodeString& text, + UBool isLiteral, + UBool escapeUnprintable, + UnicodeString& quoteBuf) { + for (int32_t i=0; itoPattern(pat, escapeUnprintable), + TRUE, escapeUnprintable, quoteBuf); + } +} + +U_NAMESPACE_END + +U_CAPI UBool U_EXPORT2 +uprv_isRuleWhiteSpace(UChar32 c) { + /* "white space" in the sense of ICU rule parsers + This is a FIXED LIST that is NOT DEPENDENT ON UNICODE PROPERTIES. + See UAX #31 Identifier and Pattern Syntax: http://www.unicode.org/reports/tr31/ + U+0009..U+000D, U+0020, U+0085, U+200E..U+200F, and U+2028..U+2029 + Equivalent to test for Pattern_White_Space Unicode property. + */ + return (c >= 0x0009 && c <= 0x2029 && + (c <= 0x000D || c == 0x0020 || c == 0x0085 || + c == 0x200E || c == 0x200F || c >= 0x2028)); +} + +U_CAPI U_NAMESPACE_QUALIFIER UnicodeSet* U_EXPORT2 +uprv_openRuleWhiteSpaceSet(UErrorCode* ec) { + if(U_FAILURE(*ec)) { + return NULL; + } + // create a set with the Pattern_White_Space characters, + // without a pattern for fewer code dependencies + U_NAMESPACE_QUALIFIER UnicodeSet *set=new U_NAMESPACE_QUALIFIER UnicodeSet(9, 0xd); + set->UnicodeSet::add(0x20).add(0x85).add(0x200e, 0x200f).add(0x2028, 0x2029); + return set; +} + +//eof diff --git a/Build/source/libs/icu/icu-xetex/common/util.h b/Build/source/libs/icu/icu-xetex/common/util.h new file mode 100644 index 00000000000..37fb56868e4 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/util.h @@ -0,0 +1,261 @@ +/* + ********************************************************************** + * Copyright (c) 2001-2007, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + * Date Name Description + * 11/19/2001 aliu Creation. + ********************************************************************** + */ + +#ifndef ICU_UTIL_H +#define ICU_UTIL_H + +#include "unicode/utypes.h" +#include "unicode/uobject.h" +#include "unicode/unistr.h" + +//-------------------------------------------------------------------- +// class ICU_Utility +// i18n utility functions, scoped into the class ICU_Utility. +//-------------------------------------------------------------------- + +U_NAMESPACE_BEGIN + +class UnicodeMatcher; +class UnicodeSet; + +class U_COMMON_API ICU_Utility /* not : public UObject because all methods are static */ { + public: + + /** + * Append a number to the given UnicodeString in the given radix. + * Standard digits '0'-'9' are used and letters 'A'-'Z' for + * radices 11 through 36. + * @param result the digits of the number are appended here + * @param n the number to be converted to digits; may be negative. + * If negative, a '-' is prepended to the digits. + * @param radix a radix from 2 to 36 inclusive. + * @param minDigits the minimum number of digits, not including + * any '-', to produce. Values less than 2 have no effect. One + * digit is always emitted regardless of this parameter. + * @return a reference to result + */ + static UnicodeString& appendNumber(UnicodeString& result, int32_t n, + int32_t radix = 10, + int32_t minDigits = 1); + + /** + * Return true if the character is NOT printable ASCII. + * + * This method should really be in UnicodeString (or similar). For + * now, we implement it here and share it with friend classes. + */ + static UBool isUnprintable(UChar32 c); + + /** + * Escape unprintable characters using \uxxxx notation for U+0000 to + * U+FFFF and \Uxxxxxxxx for U+10000 and above. If the character is + * printable ASCII, then do nothing and return FALSE. Otherwise, + * append the escaped notation and return TRUE. + */ + static UBool escapeUnprintable(UnicodeString& result, UChar32 c); + + /** + * Returns the index of a character, ignoring quoted text. + * For example, in the string "abc'hide'h", the 'h' in "hide" will not be + * found by a search for 'h'. + * @param text text to be searched + * @param start the beginning index, inclusive; 0 <= start + * <= limit. + * @param limit the ending index, exclusive; start <= limit + * <= text.length(). + * @param c character to search for + * @return Offset of the first instance of c, or -1 if not found. + */ +//?FOR FUTURE USE. DISABLE FOR NOW for coverage reasons. +// static int32_t quotedIndexOf(const UnicodeString& text, +// int32_t start, int32_t limit, +// UChar c); + + /** + * Skip over a sequence of zero or more white space characters at pos. + * @param advance if true, advance pos to the first non-white-space + * character at or after pos, or str.length(), if there is none. + * Otherwise leave pos unchanged. + * @return the index of the first non-white-space character at or + * after pos, or str.length(), if there is none. + */ + static int32_t skipWhitespace(const UnicodeString& str, int32_t& pos, + UBool advance = FALSE); + + /** + * Skip over whitespace in a Replaceable. Whitespace is defined by + * uprv_isRuleWhiteSpace(). Skipping may be done in the forward or + * reverse direction. In either case, the leftmost index will be + * inclusive, and the rightmost index will be exclusive. That is, + * given a range defined as [start, limit), the call + * skipWhitespace(text, start, limit) will advance start past leading + * whitespace, whereas the call skipWhitespace(text, limit, start), + * will back up limit past trailing whitespace. + * @param text the text to be analyzed + * @param pos either the start or limit of a range of 'text', to skip + * leading or trailing whitespace, respectively + * @param stop either the limit or start of a range of 'text', to skip + * leading or trailing whitespace, respectively + * @return the new start or limit, depending on what was passed in to + * 'pos' + */ +//?FOR FUTURE USE. DISABLE FOR NOW for coverage reasons. +//? static int32_t skipWhitespace(const Replaceable& text, +//? int32_t pos, int32_t stop); + + /** + * Parse a single non-whitespace character 'ch', optionally + * preceded by whitespace. + * @param id the string to be parsed + * @param pos INPUT-OUTPUT parameter. On input, pos[0] is the + * offset of the first character to be parsed. On output, pos[0] + * is the index after the last parsed character. If the parse + * fails, pos[0] will be unchanged. + * @param ch the non-whitespace character to be parsed. + * @return true if 'ch' is seen preceded by zero or more + * whitespace characters. + */ + static UBool parseChar(const UnicodeString& id, int32_t& pos, UChar ch); + + /** + * Parse a pattern string starting at offset pos. Keywords are + * matched case-insensitively. Spaces may be skipped and may be + * optional or required. Integer values may be parsed, and if + * they are, they will be returned in the given array. If + * successful, the offset of the next non-space character is + * returned. On failure, -1 is returned. + * @param pattern must only contain lowercase characters, which + * will match their uppercase equivalents as well. A space + * character matches one or more required spaces. A '~' character + * matches zero or more optional spaces. A '#' character matches + * an integer and stores it in parsedInts, which the caller must + * ensure has enough capacity. + * @param parsedInts array to receive parsed integers. Caller + * must ensure that parsedInts.length is >= the number of '#' + * signs in 'pattern'. + * @return the position after the last character parsed, or -1 if + * the parse failed + */ + static int32_t parsePattern(const UnicodeString& rule, int32_t pos, int32_t limit, + const UnicodeString& pattern, int32_t* parsedInts); + + /** + * Parse a pattern string within the given Replaceable and a parsing + * pattern. Characters are matched literally and case-sensitively + * except for the following special characters: + * + * ~ zero or more uprv_isRuleWhiteSpace chars + * + * If end of pattern is reached with all matches along the way, + * pos is advanced to the first unparsed index and returned. + * Otherwise -1 is returned. + * @param pat pattern that controls parsing + * @param text text to be parsed, starting at index + * @param index offset to first character to parse + * @param limit offset after last character to parse + * @return index after last parsed character, or -1 on parse failure. + */ + static int32_t parsePattern(const UnicodeString& pat, + const Replaceable& text, + int32_t index, + int32_t limit); + + /** + * Parse an integer at pos, either of the form \d+ or of the form + * 0x[0-9A-Fa-f]+ or 0[0-7]+, that is, in standard decimal, hex, + * or octal format. + * @param pos INPUT-OUTPUT parameter. On input, the first + * character to parse. On output, the character after the last + * parsed character. + */ + static int32_t parseInteger(const UnicodeString& rule, int32_t& pos, int32_t limit); + + /** + * Parse a Unicode identifier from the given string at the given + * position. Return the identifier, or an empty string if there + * is no identifier. + * @param str the string to parse + * @param pos INPUT-OUPUT parameter. On INPUT, pos is the + * first character to examine. It must be less than str.length(), + * and it must not point to a whitespace character. That is, must + * have pos < str.length() and + * !UCharacter::isWhitespace(str.char32At(pos)). On + * OUTPUT, the position after the last parsed character. + * @return the Unicode identifier, or an empty string if there is + * no valid identifier at pos. + */ + static UnicodeString parseUnicodeIdentifier(const UnicodeString& str, int32_t& pos); + + /** + * Parse an unsigned 31-bit integer at the given offset. Use + * UCharacter.digit() to parse individual characters into digits. + * @param text the text to be parsed + * @param pos INPUT-OUTPUT parameter. On entry, pos is the + * offset within text at which to start parsing; it should point + * to a valid digit. On exit, pos is the offset after the last + * parsed character. If the parse failed, it will be unchanged on + * exit. Must be >= 0 on entry. + * @param radix the radix in which to parse; must be >= 2 and <= + * 36. + * @return a non-negative parsed number, or -1 upon parse failure. + * Parse fails if there are no digits, that is, if pos does not + * point to a valid digit on entry, or if the number to be parsed + * does not fit into a 31-bit unsigned integer. + */ + static int32_t parseNumber(const UnicodeString& text, + int32_t& pos, int8_t radix); + + static void appendToRule(UnicodeString& rule, + UChar32 c, + UBool isLiteral, + UBool escapeUnprintable, + UnicodeString& quoteBuf); + + static void appendToRule(UnicodeString& rule, + const UnicodeString& text, + UBool isLiteral, + UBool escapeUnprintable, + UnicodeString& quoteBuf); + + static void appendToRule(UnicodeString& rule, + const UnicodeMatcher* matcher, + UBool escapeUnprintable, + UnicodeString& quoteBuf); + +private: + // do not instantiate + ICU_Utility(); +}; + +U_NAMESPACE_END + +/** + * Get the set of "white space" characters in the sense of ICU rule + * parsers. Caller must close/delete result. + * Equivalent to the set of characters with the Pattern_White_Space Unicode property. + * Stable set of characters, won't change. + * See UAX #31 Identifier and Pattern Syntax: http://www.unicode.org/reports/tr31/ + * @internal + */ +U_CAPI U_NAMESPACE_QUALIFIER UnicodeSet* U_EXPORT2 +uprv_openRuleWhiteSpaceSet(UErrorCode* ec); + +/** + * Is this character a "white space" in the sense of ICU rule parsers? + * Equivalent to test for Pattern_White_Space Unicode property. + * Stable set of characters, won't change. + * See UAX #31 Identifier and Pattern Syntax: http://www.unicode.org/reports/tr31/ + * @internal + */ +U_CAPI UBool U_EXPORT2 +uprv_isRuleWhiteSpace(UChar32 c); + +#endif +//eof diff --git a/Build/source/libs/icu/icu-xetex/common/util_props.cpp b/Build/source/libs/icu/icu-xetex/common/util_props.cpp new file mode 100644 index 00000000000..f91666b4f23 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/util_props.cpp @@ -0,0 +1,214 @@ +/* +********************************************************************** +* Copyright (c) 2001-2006, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 11/19/2001 aliu Creation. +********************************************************************** +*/ + +#include "util.h" +#include "unicode/uchar.h" + +U_NAMESPACE_BEGIN + +/** + * Parse an integer at pos, either of the form \d+ or of the form + * 0x[0-9A-Fa-f]+ or 0[0-7]+, that is, in standard decimal, hex, + * or octal format. + * @param pos INPUT-OUTPUT parameter. On input, the first + * character to parse. On output, the character after the last + * parsed character. + */ +int32_t ICU_Utility::parseInteger(const UnicodeString& rule, int32_t& pos, int32_t limit) { + int32_t count = 0; + int32_t value = 0; + int32_t p = pos; + int8_t radix = 10; + + if (p < limit && rule.charAt(p) == 48 /*0*/) { + if (p+1 < limit && (rule.charAt(p+1) == 0x78 /*x*/ || rule.charAt(p+1) == 0x58 /*X*/)) { + p += 2; + radix = 16; + } + else { + p++; + count = 1; + radix = 8; + } + } + + while (p < limit) { + int32_t d = u_digit(rule.charAt(p++), radix); + if (d < 0) { + --p; + break; + } + ++count; + int32_t v = (value * radix) + d; + if (v <= value) { + // If there are too many input digits, at some point + // the value will go negative, e.g., if we have seen + // "0x8000000" already and there is another '0', when + // we parse the next 0 the value will go negative. + return 0; + } + value = v; + } + if (count > 0) { + pos = p; + } + return value; +} + +/** + * Parse a pattern string starting at offset pos. Keywords are + * matched case-insensitively. Spaces may be skipped and may be + * optional or required. Integer values may be parsed, and if + * they are, they will be returned in the given array. If + * successful, the offset of the next non-space character is + * returned. On failure, -1 is returned. + * @param pattern must only contain lowercase characters, which + * will match their uppercase equivalents as well. A space + * character matches one or more required spaces. A '~' character + * matches zero or more optional spaces. A '#' character matches + * an integer and stores it in parsedInts, which the caller must + * ensure has enough capacity. + * @param parsedInts array to receive parsed integers. Caller + * must ensure that parsedInts.length is >= the number of '#' + * signs in 'pattern'. + * @return the position after the last character parsed, or -1 if + * the parse failed + */ +int32_t ICU_Utility::parsePattern(const UnicodeString& rule, int32_t pos, int32_t limit, + const UnicodeString& pattern, int32_t* parsedInts) { + // TODO Update this to handle surrogates + int32_t p; + int32_t intCount = 0; // number of integers parsed + for (int32_t i=0; i= limit) { + return -1; + } + c = rule.charAt(pos++); + if (!uprv_isRuleWhiteSpace(c)) { + return -1; + } + // FALL THROUGH to skipWhitespace + case 126 /*'~'*/: + pos = skipWhitespace(rule, pos); + break; + case 35 /*'#'*/: + p = pos; + parsedInts[intCount++] = parseInteger(rule, p, limit); + if (p == pos) { + // Syntax error; failed to parse integer + return -1; + } + pos = p; + break; + default: + if (pos >= limit) { + return -1; + } + c = (UChar) u_tolower(rule.charAt(pos++)); + if (c != cpat) { + return -1; + } + break; + } + } + return pos; +} + +/** + * Parse a Unicode identifier from the given string at the given + * position. Return the identifier, or an empty string if there + * is no identifier. + * @param str the string to parse + * @param pos INPUT-OUPUT parameter. On INPUT, pos is the + * first character to examine. It must be less than str.length(), + * and it must not point to a whitespace character. That is, must + * have pos < str.length() and + * !uprv_isRuleWhiteSpace(str.char32At(pos)). On + * OUTPUT, the position after the last parsed character. + * @return the Unicode identifier, or an empty string if there is + * no valid identifier at pos. + */ +UnicodeString ICU_Utility::parseUnicodeIdentifier(const UnicodeString& str, int32_t& pos) { + // assert(pos < str.length()); + // assert(!uprv_isRuleWhiteSpace(str.char32At(pos))); + UnicodeString buf; + int p = pos; + while (p < str.length()) { + UChar32 ch = str.char32At(p); + if (buf.length() == 0) { + if (u_isIDStart(ch)) { + buf.append(ch); + } else { + buf.truncate(0); + return buf; + } + } else { + if (u_isIDPart(ch)) { + buf.append(ch); + } else { + break; + } + } + p += UTF_CHAR_LENGTH(ch); + } + pos = p; + return buf; +} + +/** + * Parse an unsigned 31-bit integer at the given offset. Use + * UCharacter.digit() to parse individual characters into digits. + * @param text the text to be parsed + * @param pos INPUT-OUTPUT parameter. On entry, pos[0] is the + * offset within text at which to start parsing; it should point + * to a valid digit. On exit, pos[0] is the offset after the last + * parsed character. If the parse failed, it will be unchanged on + * exit. Must be >= 0 on entry. + * @param radix the radix in which to parse; must be >= 2 and <= + * 36. + * @return a non-negative parsed number, or -1 upon parse failure. + * Parse fails if there are no digits, that is, if pos[0] does not + * point to a valid digit on entry, or if the number to be parsed + * does not fit into a 31-bit unsigned integer. + */ +int32_t ICU_Utility::parseNumber(const UnicodeString& text, + int32_t& pos, int8_t radix) { + // assert(pos[0] >= 0); + // assert(radix >= 2); + // assert(radix <= 36); + int32_t n = 0; + int32_t p = pos; + while (p < text.length()) { + UChar32 ch = text.char32At(p); + int32_t d = u_digit(ch, radix); + if (d < 0) { + break; + } + n = radix*n + d; + // ASSUME that when a 32-bit integer overflows it becomes + // negative. E.g., 214748364 * 10 + 8 => negative value. + if (n < 0) { + return -1; + } + ++p; + } + if (p == pos) { + return -1; + } + pos = p; + return n; +} + +U_NAMESPACE_END + diff --git a/Build/source/libs/icu/icu-xetex/common/utrace.c b/Build/source/libs/icu/icu-xetex/common/utrace.c new file mode 100644 index 00000000000..b2526bb2ca7 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/utrace.c @@ -0,0 +1,486 @@ +/* +******************************************************************************* +* Copyright (C) 2003-2005, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* file name: utrace.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +*/ + +#define UTRACE_IMPL +#include "unicode/utrace.h" +#include "utracimp.h" +#include "cstring.h" +#include "uassert.h" +#include "ucln_cmn.h" + + +static UTraceEntry *pTraceEntryFunc = NULL; +static UTraceExit *pTraceExitFunc = NULL; +static UTraceData *pTraceDataFunc = NULL; +static const void *gTraceContext = NULL; + +U_EXPORT int32_t +utrace_level; + +U_CAPI void U_EXPORT2 +utrace_entry(int32_t fnNumber) { + if (pTraceEntryFunc != NULL) { + (*pTraceEntryFunc)(gTraceContext, fnNumber); + } +} + + +static const char gExitFmt[] = "Returns."; +static const char gExitFmtValue[] = "Returns %d."; +static const char gExitFmtStatus[] = "Returns. Status = %d."; +static const char gExitFmtValueStatus[] = "Returns %d. Status = %d."; +static const char gExitFmtPtrStatus[] = "Returns %d. Status = %p."; + +U_CAPI void U_EXPORT2 +utrace_exit(int32_t fnNumber, int32_t returnType, ...) { + if (pTraceExitFunc != NULL) { + va_list args; + const char *fmt; + + switch (returnType) { + case 0: + fmt = gExitFmt; + break; + case UTRACE_EXITV_I32: + fmt = gExitFmtValue; + break; + case UTRACE_EXITV_STATUS: + fmt = gExitFmtStatus; + break; + case UTRACE_EXITV_I32 | UTRACE_EXITV_STATUS: + fmt = gExitFmtValueStatus; + break; + case UTRACE_EXITV_PTR | UTRACE_EXITV_STATUS: + fmt = gExitFmtPtrStatus; + break; + default: + U_ASSERT(FALSE); + fmt = gExitFmt; + } + + va_start(args, returnType); + (*pTraceExitFunc)(gTraceContext, fnNumber, fmt, args); + va_end(args); + } +} + + + +U_CAPI void U_EXPORT2 +utrace_data(int32_t fnNumber, int32_t level, const char *fmt, ...) { + if (pTraceDataFunc != NULL) { + va_list args; + va_start(args, fmt ); + (*pTraceDataFunc)(gTraceContext, fnNumber, level, fmt, args); + va_end(args); + } +} + + +static void outputChar(char c, char *outBuf, int32_t *outIx, int32_t capacity, int32_t indent) { + int32_t i; + /* Check whether a start of line indenting is needed. Three cases: + * 1. At the start of the first line (output index == 0). + * 2. At the start of subsequent lines (preceeding char in buffer == '\n') + * 3. When preflighting buffer len (buffer capacity is exceeded), when + * a \n is output. Ideally we wouldn't do the indent until the following char + * is received, but that won't work because there's no place to remember that + * the preceding char was \n. Meaning that we may overstimate the + * buffer size needed. No harm done. + */ + if (*outIx==0 || /* case 1. */ + (c!='\n' && c!=0 && *outIx < capacity && outBuf[(*outIx)-1]=='\n') || /* case 2. */ + (c=='\n' && *outIx>=capacity)) /* case 3 */ + { + /* At the start of a line. Indent. */ + for(i=0; i= 0; shiftCount-=4) { + char c = gHexChars[(val >> shiftCount) & 0xf]; + outputChar(c, outBuf, outIx, capacity, 0); + } +} + +/* Output a pointer value in hex. Work with any size of pointer */ +static void outputPtrBytes(void *val, char *outBuf, int32_t *outIx, int32_t capacity) { + int32_t i; + int32_t incVal = 1; /* +1 for big endian, -1 for little endian */ + char *p = (char *)&val; /* point to current byte to output in the ptr val */ + +#if !U_IS_BIG_ENDIAN + /* Little Endian. Move p to most significant end of the value */ + incVal = -1; + p += sizeof(void *) - 1; +#endif + + /* Loop through the bytes of the ptr as it sits in memory, from + * most significant to least significant end */ + for (i=0; i 0) { + outputHexBytes(longArg, charsToOutput, outBuf, &outIx, capacity); + outputChar(' ', outBuf, &outIx, capacity, indent); + } + if (vectorLen == -1 && longArg == 0) { + break; + } + } + } + outputChar('[', outBuf, &outIx, capacity, indent); + outputHexBytes(vectorLen, 8, outBuf, &outIx, capacity); + outputChar(']', outBuf, &outIx, capacity, indent); + } + break; + + + default: + /* %. in format string, where . is some character not in the set + * of recognized format chars. Just output it as if % wasn't there. + * (Covers "%%" outputing a single '%') + */ + outputChar(fmtC, outBuf, &outIx, capacity, indent); + } + } + outputChar(0, outBuf, &outIx, capacity, indent); /* Make sure that output is null terminated */ + return outIx + 1; /* outIx + 1 because outIx does not increment when outputing final null. */ +} + + + + +U_CAPI int32_t U_EXPORT2 +utrace_format(char *outBuf, int32_t capacity, + int32_t indent, const char *fmt, ...) { + int32_t retVal; + va_list args; + va_start(args, fmt ); + retVal = utrace_vformat(outBuf, capacity, indent, fmt, args); + va_end(args); + return retVal; +} + + +U_CAPI void U_EXPORT2 +utrace_setFunctions(const void *context, + UTraceEntry *e, UTraceExit *x, UTraceData *d) { + pTraceEntryFunc = e; + pTraceExitFunc = x; + pTraceDataFunc = d; + gTraceContext = context; +} + + +U_CAPI void U_EXPORT2 +utrace_getFunctions(const void **context, + UTraceEntry **e, UTraceExit **x, UTraceData **d) { + *e = pTraceEntryFunc; + *x = pTraceExitFunc; + *d = pTraceDataFunc; + *context = gTraceContext; +} + +U_CAPI void U_EXPORT2 +utrace_setLevel(int32_t level) { + if (level < UTRACE_OFF) { + level = UTRACE_OFF; + } + if (level > UTRACE_VERBOSE) { + level = UTRACE_VERBOSE; + } + utrace_level = level; +} + +U_CAPI int32_t U_EXPORT2 +utrace_getLevel() { + return utrace_level; +} + + +U_CFUNC UBool +utrace_cleanup() { + pTraceEntryFunc = NULL; + pTraceExitFunc = NULL; + pTraceDataFunc = NULL; + utrace_level = UTRACE_OFF; + gTraceContext = NULL; + return TRUE; +} + + +static const char * const +trFnName[] = { + "u_init", + "u_cleanup", + NULL +}; + + +static const char * const +trConvNames[] = { + "ucnv_open", + "ucnv_openPackage", + "ucnv_openAlgorithmic", + "ucnv_clone", + "ucnv_close", + "ucnv_flushCache", + "ucnv_load", + "ucnv_unload", + NULL +}; + + +static const char * const +trCollNames[] = { + "ucol_open", + "ucol_close", + "ucol_strcoll", + "ucol_getSortKey", + "ucol_getLocale", + "ucol_nextSortKeyPart", + "ucol_strcollIter", + NULL +}; + + +U_CAPI const char * U_EXPORT2 +utrace_functionName(int32_t fnNumber) { + if(UTRACE_FUNCTION_START <= fnNumber && fnNumber < UTRACE_FUNCTION_LIMIT) { + return trFnName[fnNumber]; + } else if(UTRACE_CONVERSION_START <= fnNumber && fnNumber < UTRACE_CONVERSION_LIMIT) { + return trConvNames[fnNumber - UTRACE_CONVERSION_START]; + } else if(UTRACE_COLLATION_START <= fnNumber && fnNumber < UTRACE_COLLATION_LIMIT){ + return trCollNames[fnNumber - UTRACE_COLLATION_START]; + } else { + return "[BOGUS Trace Function Number]"; + } +} + diff --git a/Build/source/libs/icu/icu-xetex/common/utracimp.h b/Build/source/libs/icu/icu-xetex/common/utracimp.h new file mode 100644 index 00000000000..130bcf34eb8 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/utracimp.h @@ -0,0 +1,384 @@ +/* +******************************************************************************* +* +* Copyright (C) 2003-2004, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: utracimp.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2003aug06 +* created by: Markus W. Scherer +* +* Internal header for ICU tracing/logging. +* +* +* Various notes: +* - using a trace level variable to only call trace functions +* when the level is sufficient +* - using the same variable for tracing on/off to never make a function +* call when off +* - the function number is put into a local variable by the entry macro +* and used implicitly to avoid copy&paste/typing mistakes by the developer +* - the application must call utrace_setFunctions() and pass in +* implementations for the trace functions +* - ICU trace macros call ICU functions that route through the function +* pointers if they have been set; +* this avoids an indirection at the call site +* (which would cost more code for another check and for the indirection) +* +* ### TODO Issues: +* - Verify that va_list is portable among compilers for the same platform. +* va_list should be portable because printf() would fail otherwise! +* - Should enum values like UTraceLevel be passed into int32_t-type arguments, +* or should enum types be used? +*/ + +#ifndef __UTRACIMP_H__ +#define __UTRACIMP_H__ + +#include "unicode/utrace.h" +#include + +U_CDECL_BEGIN + +/** + * \var utrace_level + * Trace level variable. Negative for "off". + * Use only via UTRACE_ macros. + * @internal + */ +#ifdef UTRACE_IMPL +U_EXPORT int32_t +#else +U_CFUNC U_COMMON_API int32_t +#endif +utrace_level; + + +/** + * Traced Function Exit return types. + * Flags indicating the number and types of varargs included in a call + * to a UTraceExit function. + * Bits 0-3: The function return type. First variable param. + * Bit 4: Flag for presence of U_ErrorCode status param. + * @internal + */ +typedef enum UTraceExitVal { + /** The traced function returns no value @internal */ + UTRACE_EXITV_NONE = 0, + /** The traced function returns an int32_t, or compatible, type. @internal */ + UTRACE_EXITV_I32 = 1, + /** The traced function returns a pointer @internal */ + UTRACE_EXITV_PTR = 2, + /** The traced function returns a UBool @internal */ + UTRACE_EXITV_BOOL = 3, + /** Mask to extract the return type values from a UTraceExitVal @internal */ + UTRACE_EXITV_MASK = 0xf, + /** Bit indicating that the traced function includes a UErrorCode parameter @internal */ + UTRACE_EXITV_STATUS = 0x10 +} UTraceExitVal; + +/** + * Trace function for the entry point of a function. + * Do not use directly, use UTRACE_ENTRY instead. + * @param fnNumber The UTraceFunctionNumber for the current function. + * @internal + */ +U_CAPI void U_EXPORT2 +utrace_entry(int32_t fnNumber); + +/** + * Trace function for each exit point of a function. + * Do not use directly, use UTRACE_EXIT* instead. + * @param fnNumber The UTraceFunctionNumber for the current function. + * @param returnType The type of the value returned by the function. + * @param errorCode The UErrorCode value at function exit. See UTRACE_EXIT. + * @internal + */ +U_CAPI void U_EXPORT2 +utrace_exit(int32_t fnNumber, int32_t returnType, ...); + + +/** + * Trace function used inside functions that have a UTRACE_ENTRY() statement. + * Do not use directly, use UTRACE_DATAX() macros instead. + * + * @param utraceFnNumber The number of the current function, from the local + * variable of the same name. + * @param level The trace level for this message. + * @param fmt The trace format string. + * + * @internal + */ +U_CAPI void U_EXPORT2 +utrace_data(int32_t utraceFnNumber, int32_t level, const char *fmt, ...); + +U_CDECL_END + +#if U_ENABLE_TRACING + +/** + * Boolean expression to see if ICU tracing is turned on + * to at least the specified level. + * @internal + */ +#define UTRACE_LEVEL(level) (utrace_level>=(level)) + +/** + * Flag bit in utraceFnNumber, the local variable added to each function + * with tracing code to contains the function number. + * + * Set the flag if the function's entry is traced, which will cause the + * function's exit to also be traced. utraceFnNumber is uncoditionally + * set at entry, whether or not the entry is traced, so that it will + * always be available for error trace output. + * @internal + */ +#define UTRACE_TRACED_ENTRY 0x80000000 + +/** + * Trace statement for the entry point of a function. + * Stores the function number in a local variable. + * In C code, must be placed immediately after the last variable declaration. + * Must be matched with UTRACE_EXIT() at all function exit points. + * + * Tracing should start with UTRACE_ENTRY after checking for + * U_FAILURE at function entry, so that if a function returns immediately + * because of a pre-existing error condition, it does not show up in the trace, + * consistent with ICU's error handling model. + * + * @param fnNumber The UTraceFunctionNumber for the current function. + * @internal + */ +#define UTRACE_ENTRY(fnNumber) \ + int32_t utraceFnNumber=(fnNumber); \ + if(utrace_level>=UTRACE_INFO) { \ + utrace_entry(fnNumber); \ + utraceFnNumber |= UTRACE_TRACED_ENTRY; \ + } + + +/** + * Trace statement for the entry point of open and close functions. + * Produces trace output at a less verbose setting than plain UTRACE_ENTRY + * Stores the function number in a local variable. + * In C code, must be placed immediately after the last variable declaration. + * Must be matched with UTRACE_EXIT() at all function exit points. + * + * @param fnNumber The UTraceFunctionNumber for the current function. + * @internal + */ +#define UTRACE_ENTRY_OC(fnNumber) \ + int32_t utraceFnNumber=(fnNumber); \ + if(utrace_level>=UTRACE_OPEN_CLOSE) { \ + utrace_entry(fnNumber); \ + utraceFnNumber |= UTRACE_TRACED_ENTRY; \ + } + +/** + * Trace statement for each exit point of a function that has a UTRACE_ENTRY() + * statement. + * + * @param errorCode The function's ICU UErrorCode value at function exit, + * or U_ZERO_ERROR if the function does not use a UErrorCode. + * 0==U_ZERO_ERROR indicates success, + * positive values an error (see u_errorName()), + * negative values an informational status. + * + * @internal + */ +#define UTRACE_EXIT() \ + {if(utraceFnNumber & UTRACE_TRACED_ENTRY) { \ + utrace_exit(utraceFnNumber & ~UTRACE_TRACED_ENTRY, UTRACE_EXITV_NONE); \ + }} + +/** + * Trace statement for each exit point of a function that has a UTRACE_ENTRY() + * statement, and that returns a value. + * + * @param val The function's return value, int32_t or comatible type. + * + * @internal + */ +#define UTRACE_EXIT_VALUE(val) \ + {if(utraceFnNumber & UTRACE_TRACED_ENTRY) { \ + utrace_exit(utraceFnNumber & ~UTRACE_TRACED_ENTRY, UTRACE_EXITV_I32, val); \ + }} + +#define UTRACE_EXIT_STATUS(status) \ + {if(utraceFnNumber & UTRACE_TRACED_ENTRY) { \ + utrace_exit(utraceFnNumber & ~UTRACE_TRACED_ENTRY, UTRACE_EXITV_STATUS, status); \ + }} + +#define UTRACE_EXIT_VALUE_STATUS(val, status) \ + {if(utraceFnNumber & UTRACE_TRACED_ENTRY) { \ + utrace_exit(utraceFnNumber & ~UTRACE_TRACED_ENTRY, (UTRACE_EXITV_I32 | UTRACE_EXITV_STATUS), val, status); \ + }} + +#define UTRACE_EXIT_PTR_STATUS(ptr, status) \ + {if(utraceFnNumber & UTRACE_TRACED_ENTRY) { \ + utrace_exit(utraceFnNumber & ~UTRACE_TRACED_ENTRY, (UTRACE_EXITV_PTR | UTRACE_EXITV_STATUS), ptr, status); \ + }} + +/** + * Trace statement used inside functions that have a UTRACE_ENTRY() statement. + * Takes no data arguments. + * The number of arguments for this macro must match the number of inserts + * in the format string. Vector inserts count as two arguments. + * Calls utrace_data() if the level is high enough. + * @internal + */ +#define UTRACE_DATA0(level, fmt) \ + if(UTRACE_LEVEL(level)) { \ + utrace_data(utraceFnNumber & ~UTRACE_TRACED_ENTRY, (level), (fmt)); \ + } + +/** + * Trace statement used inside functions that have a UTRACE_ENTRY() statement. + * Takes one data argument. + * The number of arguments for this macro must match the number of inserts + * in the format string. Vector inserts count as two arguments. + * Calls utrace_data() if the level is high enough. + * @internal + */ +#define UTRACE_DATA1(level, fmt, a) \ + if(UTRACE_LEVEL(level)) { \ + utrace_data(utraceFnNumber & ~UTRACE_TRACED_ENTRY , (level), (fmt), (a)); \ + } + +/** + * Trace statement used inside functions that have a UTRACE_ENTRY() statement. + * Takes two data arguments. + * The number of arguments for this macro must match the number of inserts + * in the format string. Vector inserts count as two arguments. + * Calls utrace_data() if the level is high enough. + * @internal + */ +#define UTRACE_DATA2(level, fmt, a, b) \ + if(UTRACE_LEVEL(level)) { \ + utrace_data(utraceFnNumber & ~UTRACE_TRACED_ENTRY , (level), (fmt), (a), (b)); \ + } + +/** + * Trace statement used inside functions that have a UTRACE_ENTRY() statement. + * Takes three data arguments. + * The number of arguments for this macro must match the number of inserts + * in the format string. Vector inserts count as two arguments. + * Calls utrace_data() if the level is high enough. + * @internal + */ +#define UTRACE_DATA3(level, fmt, a, b, c) \ + if(UTRACE_LEVEL(level)) { \ + utrace_data(utraceFnNumber & ~UTRACE_TRACED_ENTRY, (level), (fmt), (a), (b), (c)); \ + } + +/** + * Trace statement used inside functions that have a UTRACE_ENTRY() statement. + * Takes four data arguments. + * The number of arguments for this macro must match the number of inserts + * in the format string. Vector inserts count as two arguments. + * Calls utrace_data() if the level is high enough. + * @internal + */ +#define UTRACE_DATA4(level, fmt, a, b, c, d) \ + if(UTRACE_LEVEL(level)) { \ + utrace_data(utraceFnNumber & ~UTRACE_TRACED_ENTRY, (level), (fmt), (a), (b), (c), (d)); \ + } + +/** + * Trace statement used inside functions that have a UTRACE_ENTRY() statement. + * Takes five data arguments. + * The number of arguments for this macro must match the number of inserts + * in the format string. Vector inserts count as two arguments. + * Calls utrace_data() if the level is high enough. + * @internal + */ +#define UTRACE_DATA5(level, fmt, a, b, c, d, e) \ + if(UTRACE_LEVEL(level)) { \ + utrace_data(utraceFnNumber & ~UTRACE_TRACED_ENTRY, (level), (fmt), (a), (b), (c), (d), (e)); \ + } + +/** + * Trace statement used inside functions that have a UTRACE_ENTRY() statement. + * Takes six data arguments. + * The number of arguments for this macro must match the number of inserts + * in the format string. Vector inserts count as two arguments. + * Calls utrace_data() if the level is high enough. + * @internal + */ +#define UTRACE_DATA6(level, fmt, a, b, c, d, e, f) \ + if(UTRACE_LEVEL(level)) { \ + utrace_data(utraceFnNumber & ~UTRACE_TRACED_ENTRY, (level), (fmt), (a), (b), (c), (d), (e), (f)); \ + } + +/** + * Trace statement used inside functions that have a UTRACE_ENTRY() statement. + * Takes seven data arguments. + * The number of arguments for this macro must match the number of inserts + * in the format string. Vector inserts count as two arguments. + * Calls utrace_data() if the level is high enough. + * @internal + */ +#define UTRACE_DATA7(level, fmt, a, b, c, d, e, f, g) \ + if(UTRACE_LEVEL(level)) { \ + utrace_data(utraceFnNumber & ~UTRACE_TRACED_ENTRY, (level), (fmt), (a), (b), (c), (d), (e), (f), (g)); \ + } + +/** + * Trace statement used inside functions that have a UTRACE_ENTRY() statement. + * Takes eight data arguments. + * The number of arguments for this macro must match the number of inserts + * in the format string. Vector inserts count as two arguments. + * Calls utrace_data() if the level is high enough. + * @internal + */ +#define UTRACE_DATA8(level, fmt, a, b, c, d, e, f, g, h) \ + if(UTRACE_LEVEL(level)) { \ + utrace_data(utraceFnNumber & ~UTRACE_TRACED_ENTRY, (level), (fmt), (a), (b), (c), (d), (e), (f), (g), (h)); \ + } + +/** + * Trace statement used inside functions that have a UTRACE_ENTRY() statement. + * Takes nine data arguments. + * The number of arguments for this macro must match the number of inserts + * in the format string. Vector inserts count as two arguments. + * Calls utrace_data() if the level is high enough. + * @internal + */ +#define UTRACE_DATA9(level, fmt, a, b, c, d, e, f, g, h, i) \ + if(UTRACE_LEVEL(level)) { \ + utrace_data(utraceFnNumber & ~UTRACE_TRACED_ENTRY, (level), (fmt), (a), (b), (c), (d), (e), (f), (g), (h), (i)); \ + } + +#else + +/* + * When tracing is disabled, the following macros become empty + */ + +#define UTRACE_LEVEL(level) 0 +#define UTRACE_ENTRY(fnNumber) +#define UTRACE_ENTRY_OC(fnNumber) +#define UTRACE_EXIT() +#define UTRACE_EXIT_VALUE(val) +#define UTRACE_EXIT_STATUS(status) +#define UTRACE_EXIT_VALUE_STATUS(val, status) +#define UTRACE_EXIT_PTR_STATUS(ptr, status) +#define UTRACE_DATA0(level, fmt) +#define UTRACE_DATA1(level, fmt, a) +#define UTRACE_DATA2(level, fmt, a, b) +#define UTRACE_DATA3(level, fmt, a, b, c) +#define UTRACE_DATA4(level, fmt, a, b, c, d) +#define UTRACE_DATA5(level, fmt, a, b, c, d, e) +#define UTRACE_DATA6(level, fmt, a, b, c, d, e, f) +#define UTRACE_DATA7(level, fmt, a, b, c, d, e, f, g) +#define UTRACE_DATA8(level, fmt, a, b, c, d, e, f, g, h) +#define UTRACE_DATA9(level, fmt, a, b, c, d, e, f, g, h, i) + +#endif + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/utrie.c b/Build/source/libs/icu/icu-xetex/common/utrie.c new file mode 100644 index 00000000000..812c89db567 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/utrie.c @@ -0,0 +1,1210 @@ +/* +****************************************************************************** +* +* Copyright (C) 2001-2006, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: utrie.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2001oct20 +* created by: Markus W. Scherer +* +* This is a common implementation of a "folded" trie. +* It is a kind of compressed, serializable table of 16- or 32-bit values associated with +* Unicode code points (0..0x10ffff). +*/ + +#ifdef UTRIE_DEBUG +# include +#endif + +#include "unicode/utypes.h" +#include "cmemory.h" +#include "utrie.h" + +/* miscellaneous ------------------------------------------------------------ */ + +#undef ABS +#define ABS(x) ((x)>=0 ? (x) : -(x)) + +static U_INLINE UBool +equal_uint32(const uint32_t *s, const uint32_t *t, int32_t length) { + while(length>0 && *s==*t) { + ++s; + ++t; + --length; + } + return (UBool)(length==0); +} + +/* Building a trie ----------------------------------------------------------*/ + +U_CAPI UNewTrie * U_EXPORT2 +utrie_open(UNewTrie *fillIn, + uint32_t *aliasData, int32_t maxDataLength, + uint32_t initialValue, uint32_t leadUnitValue, + UBool latin1Linear) { + UNewTrie *trie; + int32_t i, j; + + if( maxDataLengthisAllocated= (UBool)(fillIn==NULL); + + if(aliasData!=NULL) { + trie->data=aliasData; + trie->isDataAllocated=FALSE; + } else { + trie->data=(uint32_t *)uprv_malloc(maxDataLength*4); + if(trie->data==NULL) { + uprv_free(trie); + return NULL; + } + trie->isDataAllocated=TRUE; + } + + /* preallocate and reset the first data block (block index 0) */ + j=UTRIE_DATA_BLOCK_LENGTH; + + if(latin1Linear) { + /* preallocate and reset the first block (number 0) and Latin-1 (U+0000..U+00ff) after that */ + /* made sure above that maxDataLength>=1024 */ + + /* set indexes to point to consecutive data blocks */ + i=0; + do { + /* do this at least for trie->index[0] even if that block is only partly used for Latin-1 */ + trie->index[i++]=j; + j+=UTRIE_DATA_BLOCK_LENGTH; + } while(i<(256>>UTRIE_SHIFT)); + } + + /* reset the initially allocated blocks to the initial value */ + trie->dataLength=j; + while(j>0) { + trie->data[--j]=initialValue; + } + + trie->leadUnitValue=leadUnitValue; + trie->indexLength=UTRIE_MAX_INDEX_LENGTH; + trie->dataCapacity=maxDataLength; + trie->isLatin1Linear=latin1Linear; + trie->isCompacted=FALSE; + return trie; +} + +U_CAPI UNewTrie * U_EXPORT2 +utrie_clone(UNewTrie *fillIn, const UNewTrie *other, uint32_t *aliasData, int32_t aliasDataCapacity) { + UNewTrie *trie; + UBool isDataAllocated; + + /* do not clone if other is not valid or already compacted */ + if(other==NULL || other->data==NULL || other->isCompacted) { + return NULL; + } + + /* clone data */ + if(aliasData!=NULL && aliasDataCapacity>=other->dataCapacity) { + isDataAllocated=FALSE; + } else { + aliasDataCapacity=other->dataCapacity; + aliasData=(uint32_t *)uprv_malloc(other->dataCapacity*4); + if(aliasData==NULL) { + return NULL; + } + isDataAllocated=TRUE; + } + + trie=utrie_open(fillIn, aliasData, aliasDataCapacity, + other->data[0], other->leadUnitValue, + other->isLatin1Linear); + if(trie==NULL) { + uprv_free(aliasData); + } else { + uprv_memcpy(trie->index, other->index, sizeof(trie->index)); + uprv_memcpy(trie->data, other->data, other->dataLength*4); + trie->dataLength=other->dataLength; + trie->isDataAllocated=isDataAllocated; + } + + return trie; +} + +U_CAPI void U_EXPORT2 +utrie_close(UNewTrie *trie) { + if(trie!=NULL) { + if(trie->isDataAllocated) { + uprv_free(trie->data); + trie->data=NULL; + } + if(trie->isAllocated) { + uprv_free(trie); + } + } +} + +U_CAPI uint32_t * U_EXPORT2 +utrie_getData(UNewTrie *trie, int32_t *pLength) { + if(trie==NULL || pLength==NULL) { + return NULL; + } + + *pLength=trie->dataLength; + return trie->data; +} + +static int32_t +utrie_allocDataBlock(UNewTrie *trie) { + int32_t newBlock, newTop; + + newBlock=trie->dataLength; + newTop=newBlock+UTRIE_DATA_BLOCK_LENGTH; + if(newTop>trie->dataCapacity) { + /* out of memory in the data array */ + return -1; + } + trie->dataLength=newTop; + return newBlock; +} + +/** + * No error checking for illegal arguments. + * + * @return -1 if no new data block available (out of memory in data array) + * @internal + */ +static int32_t +utrie_getDataBlock(UNewTrie *trie, UChar32 c) { + int32_t indexValue, newBlock; + + c>>=UTRIE_SHIFT; + indexValue=trie->index[c]; + if(indexValue>0) { + return indexValue; + } + + /* allocate a new data block */ + newBlock=utrie_allocDataBlock(trie); + if(newBlock<0) { + /* out of memory in the data array */ + return -1; + } + trie->index[c]=newBlock; + + /* copy-on-write for a block from a setRange() */ + uprv_memcpy(trie->data+newBlock, trie->data-indexValue, 4*UTRIE_DATA_BLOCK_LENGTH); + return newBlock; +} + +/** + * @return TRUE if the value was successfully set + */ +U_CAPI UBool U_EXPORT2 +utrie_set32(UNewTrie *trie, UChar32 c, uint32_t value) { + int32_t block; + + /* valid, uncompacted trie and valid c? */ + if(trie==NULL || trie->isCompacted || (uint32_t)c>0x10ffff) { + return FALSE; + } + + block=utrie_getDataBlock(trie, c); + if(block<0) { + return FALSE; + } + + trie->data[block+(c&UTRIE_MASK)]=value; + return TRUE; +} + +U_CAPI uint32_t U_EXPORT2 +utrie_get32(UNewTrie *trie, UChar32 c, UBool *pInBlockZero) { + int32_t block; + + /* valid, uncompacted trie and valid c? */ + if(trie==NULL || trie->isCompacted || (uint32_t)c>0x10ffff) { + if(pInBlockZero!=NULL) { + *pInBlockZero=TRUE; + } + return 0; + } + + block=trie->index[c>>UTRIE_SHIFT]; + if(pInBlockZero!=NULL) { + *pInBlockZero= (UBool)(block==0); + } + + return trie->data[ABS(block)+(c&UTRIE_MASK)]; +} + +/** + * @internal + */ +static void +utrie_fillBlock(uint32_t *block, UChar32 start, UChar32 limit, + uint32_t value, uint32_t initialValue, UBool overwrite) { + uint32_t *pLimit; + + pLimit=block+limit; + block+=start; + if(overwrite) { + while(blockisCompacted || + (uint32_t)start>0x10ffff || (uint32_t)limit>0x110000 || start>limit + ) { + return FALSE; + } + if(start==limit) { + return TRUE; /* nothing to do */ + } + + initialValue=trie->data[0]; + if(start&UTRIE_MASK) { + UChar32 nextStart; + + /* set partial block at [start..following block boundary[ */ + block=utrie_getDataBlock(trie, start); + if(block<0) { + return FALSE; + } + + nextStart=(start+UTRIE_DATA_BLOCK_LENGTH)&~UTRIE_MASK; + if(nextStart<=limit) { + utrie_fillBlock(trie->data+block, start&UTRIE_MASK, UTRIE_DATA_BLOCK_LENGTH, + value, initialValue, overwrite); + start=nextStart; + } else { + utrie_fillBlock(trie->data+block, start&UTRIE_MASK, limit&UTRIE_MASK, + value, initialValue, overwrite); + return TRUE; + } + } + + /* number of positions in the last, partial block */ + rest=limit&UTRIE_MASK; + + /* round down limit to a block boundary */ + limit&=~UTRIE_MASK; + + /* iterate over all-value blocks */ + if(value==initialValue) { + repeatBlock=0; + } else { + repeatBlock=-1; + } + while(startindex[start>>UTRIE_SHIFT]; + if(block>0) { + /* already allocated, fill in value */ + utrie_fillBlock(trie->data+block, 0, UTRIE_DATA_BLOCK_LENGTH, value, initialValue, overwrite); + } else if(trie->data[-block]!=value && (block==0 || overwrite)) { + /* set the repeatBlock instead of the current block 0 or range block */ + if(repeatBlock>=0) { + trie->index[start>>UTRIE_SHIFT]=-repeatBlock; + } else { + /* create and set and fill the repeatBlock */ + repeatBlock=utrie_getDataBlock(trie, start); + if(repeatBlock<0) { + return FALSE; + } + + /* set the negative block number to indicate that it is a repeat block */ + trie->index[start>>UTRIE_SHIFT]=-repeatBlock; + utrie_fillBlock(trie->data+repeatBlock, 0, UTRIE_DATA_BLOCK_LENGTH, value, initialValue, TRUE); + } + } + + start+=UTRIE_DATA_BLOCK_LENGTH; + } + + if(rest>0) { + /* set partial block at [last block boundary..limit[ */ + block=utrie_getDataBlock(trie, start); + if(block<0) { + return FALSE; + } + + utrie_fillBlock(trie->data+block, 0, rest, value, initialValue, overwrite); + } + + return TRUE; +} + +static int32_t +_findSameIndexBlock(const int32_t *index, int32_t indexLength, + int32_t otherBlock) { + int32_t block, i; + + for(block=UTRIE_BMP_INDEX_LENGTH; blockindex; + + /* copy the lead surrogate indexes into a temporary array */ + uprv_memcpy(leadIndexes, index+(0xd800>>UTRIE_SHIFT), 4*UTRIE_SURROGATE_BLOCK_COUNT); + + /* + * set all values for lead surrogate code *units* to leadUnitValue + * so that, by default, runtime lookups will find no data for associated + * supplementary code points, unless there is data for such code points + * which will result in a non-zero folding value below that is set for + * the respective lead units + * + * the above saved the indexes for surrogate code *points* + * fill the indexes with simplified code from utrie_setRange32() + */ + if(trie->leadUnitValue==trie->data[0]) { + block=0; /* leadUnitValue==initialValue, use all-initial-value block */ + } else { + /* create and fill the repeatBlock */ + block=utrie_allocDataBlock(trie); + if(block<0) { + /* data table overflow */ + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return; + } + utrie_fillBlock(trie->data+block, 0, UTRIE_DATA_BLOCK_LENGTH, trie->leadUnitValue, trie->data[0], TRUE); + block=-block; /* negative block number to indicate that it is a repeat block */ + } + for(c=(0xd800>>UTRIE_SHIFT); c<(0xdc00>>UTRIE_SHIFT); ++c) { + trie->index[c]=block; + } + + /* + * Fold significant index values into the area just after the BMP indexes. + * In case the first lead surrogate has significant data, + * its index block must be used first (in which case the folding is a no-op). + * Later all folded index blocks are moved up one to insert the copied + * lead surrogate indexes. + */ + indexLength=UTRIE_BMP_INDEX_LENGTH; + + /* search for any index (stage 1) entries for supplementary code points */ + for(c=0x10000; c<0x110000;) { + if(index[c>>UTRIE_SHIFT]!=0) { + /* there is data, treat the full block for a lead surrogate */ + c&=~0x3ff; + +#ifdef UTRIE_DEBUG + printf("supplementary data for lead surrogate U+%04lx\n", (long)(0xd7c0+(c>>10))); +#endif + + /* is there an identical index block? */ + block=_findSameIndexBlock(index, indexLength, c>>UTRIE_SHIFT); + + /* + * get a folded value for [c..c+0x400[ and, + * if different from the value for the lead surrogate code point, + * set it for the lead surrogate code unit + */ + value=getFoldedValue(trie, c, block+UTRIE_SURROGATE_BLOCK_COUNT); + if(value!=utrie_get32(trie, U16_LEAD(c), NULL)) { + if(!utrie_set32(trie, U16_LEAD(c), value)) { + /* data table overflow */ + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return; + } + + /* if we did not find an identical index block... */ + if(block==indexLength) { + /* move the actual index (stage 1) entries from the supplementary position to the new one */ + uprv_memmove(index+indexLength, + index+(c>>UTRIE_SHIFT), + 4*UTRIE_SURROGATE_BLOCK_COUNT); + indexLength+=UTRIE_SURROGATE_BLOCK_COUNT; + } + } + c+=0x400; + } else { + c+=UTRIE_DATA_BLOCK_LENGTH; + } + } + + /* + * index array overflow? + * This is to guarantee that a folding offset is of the form + * UTRIE_BMP_INDEX_LENGTH+n*UTRIE_SURROGATE_BLOCK_COUNT with n=0..1023. + * If the index is too large, then n>=1024 and more than 10 bits are necessary. + * + * In fact, it can only ever become n==1024 with completely unfoldable data and + * the additional block of duplicated values for lead surrogates. + */ + if(indexLength>=UTRIE_MAX_INDEX_LENGTH) { + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return; + } + + /* + * make space for the lead surrogate index block and + * insert it between the BMP indexes and the folded ones + */ + uprv_memmove(index+UTRIE_BMP_INDEX_LENGTH+UTRIE_SURROGATE_BLOCK_COUNT, + index+UTRIE_BMP_INDEX_LENGTH, + 4*(indexLength-UTRIE_BMP_INDEX_LENGTH)); + uprv_memcpy(index+UTRIE_BMP_INDEX_LENGTH, + leadIndexes, + 4*UTRIE_SURROGATE_BLOCK_COUNT); + indexLength+=UTRIE_SURROGATE_BLOCK_COUNT; + +#ifdef UTRIE_DEBUG + printf("trie index count: BMP %ld all Unicode %ld folded %ld\n", + UTRIE_BMP_INDEX_LENGTH, (long)UTRIE_MAX_INDEX_LENGTH, indexLength); +#endif + + trie->indexLength=indexLength; +} + +/* + * Set a value in the trie index map to indicate which data block + * is referenced and which one is not. + * utrie_compact() will remove data blocks that are not used at all. + * Set + * - 0 if it is used + * - -1 if it is not used + */ +static void +_findUnusedBlocks(UNewTrie *trie) { + int32_t i; + + /* fill the entire map with "not used" */ + uprv_memset(trie->map, 0xff, (UTRIE_MAX_BUILD_TIME_DATA_LENGTH>>UTRIE_SHIFT)*4); + + /* mark each block that _is_ used with 0 */ + for(i=0; iindexLength; ++i) { + trie->map[ABS(trie->index[i])>>UTRIE_SHIFT]=0; + } + + /* never move the all-initial-value block 0 */ + trie->map[0]=0; +} + +static int32_t +_findSameDataBlock(const uint32_t *data, int32_t dataLength, + int32_t otherBlock, int32_t step) { + int32_t block; + + /* ensure that we do not even partially get past dataLength */ + dataLength-=UTRIE_DATA_BLOCK_LENGTH; + + for(block=0; block<=dataLength; block+=step) { + if(equal_uint32(data+block, data+otherBlock, UTRIE_DATA_BLOCK_LENGTH)) { + return block; + } + } + return -1; +} + +/* + * Compact a folded build-time trie. + * + * The compaction + * - removes blocks that are identical with earlier ones + * - overlaps adjacent blocks as much as possible (if overlap==TRUE) + * - moves blocks in steps of the data granularity + * - moves and overlaps blocks that overlap with multiple values in the overlap region + * + * It does not + * - try to move and overlap blocks that are not already adjacent + */ +static void +utrie_compact(UNewTrie *trie, UBool overlap, UErrorCode *pErrorCode) { + int32_t i, start, newStart, overlapStart; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return; + } + + /* valid, uncompacted trie? */ + if(trie==NULL) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return; + } + if(trie->isCompacted) { + return; /* nothing left to do */ + } + + /* compaction */ + + /* initialize the index map with "block is used/unused" flags */ + _findUnusedBlocks(trie); + + /* if Latin-1 is preallocated and linear, then do not compact Latin-1 data */ + if(trie->isLatin1Linear && UTRIE_SHIFT<=8) { + overlapStart=UTRIE_DATA_BLOCK_LENGTH+256; + } else { + overlapStart=UTRIE_DATA_BLOCK_LENGTH; + } + + newStart=UTRIE_DATA_BLOCK_LENGTH; + for(start=newStart; startdataLength;) { + /* + * start: index of first entry of current block + * newStart: index where the current block is to be moved + * (right after current end of already-compacted data) + */ + + /* skip blocks that are not used */ + if(trie->map[start>>UTRIE_SHIFT]<0) { + /* advance start to the next block */ + start+=UTRIE_DATA_BLOCK_LENGTH; + + /* leave newStart with the previous block! */ + continue; + } + + /* search for an identical block */ + if( start>=overlapStart && + (i=_findSameDataBlock(trie->data, newStart, start, + overlap ? UTRIE_DATA_GRANULARITY : UTRIE_DATA_BLOCK_LENGTH)) + >=0 + ) { + /* found an identical block, set the other block's index value for the current block */ + trie->map[start>>UTRIE_SHIFT]=i; + + /* advance start to the next block */ + start+=UTRIE_DATA_BLOCK_LENGTH; + + /* leave newStart with the previous block! */ + continue; + } + + /* see if the beginning of this block can be overlapped with the end of the previous block */ + if(overlap && start>=overlapStart) { + /* look for maximum overlap (modulo granularity) with the previous, adjacent block */ + for(i=UTRIE_DATA_BLOCK_LENGTH-UTRIE_DATA_GRANULARITY; + i>0 && !equal_uint32(trie->data+(newStart-i), trie->data+start, i); + i-=UTRIE_DATA_GRANULARITY) {} + } else { + i=0; + } + + if(i>0) { + /* some overlap */ + trie->map[start>>UTRIE_SHIFT]=newStart-i; + + /* move the non-overlapping indexes to their new positions */ + start+=i; + for(i=UTRIE_DATA_BLOCK_LENGTH-i; i>0; --i) { + trie->data[newStart++]=trie->data[start++]; + } + } else if(newStartmap[start>>UTRIE_SHIFT]=newStart; + for(i=UTRIE_DATA_BLOCK_LENGTH; i>0; --i) { + trie->data[newStart++]=trie->data[start++]; + } + } else /* no overlap && newStart==start */ { + trie->map[start>>UTRIE_SHIFT]=start; + newStart+=UTRIE_DATA_BLOCK_LENGTH; + start=newStart; + } + } + + /* now adjust the index (stage 1) table */ + for(i=0; iindexLength; ++i) { + trie->index[i]=trie->map[ABS(trie->index[i])>>UTRIE_SHIFT]; + } + +#ifdef UTRIE_DEBUG + /* we saved some space */ + printf("compacting trie: count of 32-bit words %lu->%lu\n", + (long)trie->dataLength, (long)newStart); +#endif + + trie->dataLength=newStart; +} + +/* serialization ------------------------------------------------------------ */ + +/* + * Default function for the folding value: + * Just store the offset (16 bits) if there is any non-initial-value entry. + * + * The offset parameter is never 0. + * Returning the offset itself is safe for UTRIE_SHIFT>=5 because + * for UTRIE_SHIFT==5 the maximum index length is UTRIE_MAX_INDEX_LENGTH==0x8800 + * which fits into 16-bit trie values; + * for higher UTRIE_SHIFT, UTRIE_MAX_INDEX_LENGTH decreases. + * + * Theoretically, it would be safer for all possible UTRIE_SHIFT including + * those of 4 and lower to return offset>>UTRIE_SURROGATE_BLOCK_BITS + * which would always result in a value of 0x40..0x43f + * (start/end 1k blocks of supplementary Unicode code points). + * However, this would be uglier, and would not work for some existing + * binary data file formats. + * + * Also, we do not plan to change UTRIE_SHIFT because it would change binary + * data file formats, and we would probably not make it smaller because of + * the then even larger BMP index length even for empty tries. + */ +static uint32_t U_CALLCONV +defaultGetFoldedValue(UNewTrie *trie, UChar32 start, int32_t offset) { + uint32_t value, initialValue; + UChar32 limit; + UBool inBlockZero; + + initialValue=trie->data[0]; + limit=start+0x400; + while(start0 && dt==NULL)) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + if(getFoldedValue==NULL) { + getFoldedValue=defaultGetFoldedValue; + } + + data = (uint8_t*)dt; + /* fold and compact if necessary, also checks that indexLength is within limits */ + if(!trie->isCompacted) { + /* compact once without overlap to improve folding */ + utrie_compact(trie, FALSE, pErrorCode); + + /* fold the supplementary part of the index array */ + utrie_fold(trie, getFoldedValue, pErrorCode); + + /* compact again with overlap for minimum data array length */ + utrie_compact(trie, TRUE, pErrorCode); + + trie->isCompacted=TRUE; + if(U_FAILURE(*pErrorCode)) { + return 0; + } + } + + /* is dataLength within limits? */ + if( (reduceTo16Bits ? (trie->dataLength+trie->indexLength) : trie->dataLength) >= UTRIE_MAX_DATA_LENGTH) { + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + } + + length=sizeof(UTrieHeader)+2*trie->indexLength; + if(reduceTo16Bits) { + length+=2*trie->dataLength; + } else { + length+=4*trie->dataLength; + } + + if(length>capacity) { + return length; /* preflighting */ + } + + /* set the header fields */ + header=(UTrieHeader *)data; + data+=sizeof(UTrieHeader); + + header->signature=0x54726965; /* "Trie" */ + header->options=UTRIE_SHIFT | (UTRIE_INDEX_SHIFT<options|=UTRIE_OPTIONS_DATA_IS_32_BIT; + } + if(trie->isLatin1Linear) { + header->options|=UTRIE_OPTIONS_LATIN1_IS_LINEAR; + } + + header->indexLength=trie->indexLength; + header->dataLength=trie->dataLength; + + /* write the index (stage 1) array and the 16/32-bit data (stage 2) array */ + if(reduceTo16Bits) { + /* write 16-bit index values shifted right by UTRIE_INDEX_SHIFT, after adding indexLength */ + p=(uint32_t *)trie->index; + dest16=(uint16_t *)data; + for(i=trie->indexLength; i>0; --i) { + *dest16++=(uint16_t)((*p++ + trie->indexLength)>>UTRIE_INDEX_SHIFT); + } + + /* write 16-bit data values */ + p=trie->data; + for(i=trie->dataLength; i>0; --i) { + *dest16++=(uint16_t)*p++; + } + } else { + /* write 16-bit index values shifted right by UTRIE_INDEX_SHIFT */ + p=(uint32_t *)trie->index; + dest16=(uint16_t *)data; + for(i=trie->indexLength; i>0; --i) { + *dest16++=(uint16_t)(*p++ >> UTRIE_INDEX_SHIFT); + } + + /* write 32-bit data values */ + uprv_memcpy(dest16, trie->data, 4*trie->dataLength); + } + + return length; +} + +/* inverse to defaultGetFoldedValue() */ +U_CAPI int32_t U_EXPORT2 +utrie_defaultGetFoldingOffset(uint32_t data) { + return (int32_t)data; +} + +U_CAPI int32_t U_EXPORT2 +utrie_unserialize(UTrie *trie, const void *data, int32_t length, UErrorCode *pErrorCode) { + const UTrieHeader *header; + const uint16_t *p16; + uint32_t options; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return -1; + } + + /* enough data for a trie header? */ + if(lengthsignature!=0x54726965) { + *pErrorCode=U_INVALID_FORMAT_ERROR; + return -1; + } + + /* get the options and check the shift values */ + options=header->options; + if( (options&UTRIE_OPTIONS_SHIFT_MASK)!=UTRIE_SHIFT || + ((options>>UTRIE_OPTIONS_INDEX_SHIFT)&UTRIE_OPTIONS_SHIFT_MASK)!=UTRIE_INDEX_SHIFT + ) { + *pErrorCode=U_INVALID_FORMAT_ERROR; + return -1; + } + trie->isLatin1Linear= (UBool)((options&UTRIE_OPTIONS_LATIN1_IS_LINEAR)!=0); + + /* get the length values */ + trie->indexLength=header->indexLength; + trie->dataLength=header->dataLength; + + length-=(int32_t)sizeof(UTrieHeader); + + /* enough data for the index? */ + if(length<2*trie->indexLength) { + *pErrorCode=U_INVALID_FORMAT_ERROR; + return -1; + } + p16=(const uint16_t *)(header+1); + trie->index=p16; + p16+=trie->indexLength; + length-=2*trie->indexLength; + + /* get the data */ + if(options&UTRIE_OPTIONS_DATA_IS_32_BIT) { + if(length<4*trie->dataLength) { + *pErrorCode=U_INVALID_FORMAT_ERROR; + return -1; + } + trie->data32=(const uint32_t *)p16; + trie->initialValue=trie->data32[0]; + length=(int32_t)sizeof(UTrieHeader)+2*trie->indexLength+4*trie->dataLength; + } else { + if(length<2*trie->dataLength) { + *pErrorCode=U_INVALID_FORMAT_ERROR; + return -1; + } + + /* the "data16" data is used via the index pointer */ + trie->data32=NULL; + trie->initialValue=trie->index[trie->indexLength]; + length=(int32_t)sizeof(UTrieHeader)+2*trie->indexLength+2*trie->dataLength; + } + + trie->getFoldingOffset=utrie_defaultGetFoldingOffset; + + return length; +} + +U_CAPI int32_t U_EXPORT2 +utrie_unserializeDummy(UTrie *trie, + void *data, int32_t length, + uint32_t initialValue, uint32_t leadUnitValue, + UBool make16BitTrie, + UErrorCode *pErrorCode) { + uint16_t *p16; + int32_t actualLength, latin1Length, i, limit; + uint16_t block; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return -1; + } + + /* calculate the actual size of the dummy trie data */ + + /* max(Latin-1, block 0) */ + latin1Length= UTRIE_SHIFT<=8 ? 256 : UTRIE_DATA_BLOCK_LENGTH; + + trie->indexLength=UTRIE_BMP_INDEX_LENGTH+UTRIE_SURROGATE_BLOCK_COUNT; + trie->dataLength=latin1Length; + if(leadUnitValue!=initialValue) { + trie->dataLength+=UTRIE_DATA_BLOCK_LENGTH; + } + + actualLength=trie->indexLength*2; + if(make16BitTrie) { + actualLength+=trie->dataLength*2; + } else { + actualLength+=trie->dataLength*4; + } + + /* enough space for the dummy trie? */ + if(lengthisLatin1Linear=TRUE; + trie->initialValue=initialValue; + + /* fill the index and data arrays */ + p16=(uint16_t *)data; + trie->index=p16; + + if(make16BitTrie) { + /* indexes to block 0 */ + block=(uint16_t)(trie->indexLength>>UTRIE_INDEX_SHIFT); + limit=trie->indexLength; + for(i=0; i>UTRIE_INDEX_SHIFT); + i=0xd800>>UTRIE_SHIFT; + limit=0xdc00>>UTRIE_SHIFT; + for(; idata32=NULL; + + /* Latin-1 data */ + p16+=trie->indexLength; + for(i=0; iindexLength*2); + + if(leadUnitValue!=initialValue) { + /* indexes for lead surrogate code units to the block after Latin-1 */ + block=(uint16_t)(latin1Length>>UTRIE_INDEX_SHIFT); + i=0xd800>>UTRIE_SHIFT; + limit=0xdc00>>UTRIE_SHIFT; + for(; idata32=p32=(uint32_t *)(p16+trie->indexLength); + + /* Latin-1 data */ + for(i=0; igetFoldingOffset=utrie_defaultGetFoldingOffset; + + return actualLength; +} + +/* enumeration -------------------------------------------------------------- */ + +/* default UTrieEnumValue() returns the input value itself */ +static uint32_t U_CALLCONV +enumSameValue(const void *context, uint32_t value) { + return value; +} + +/** + * Enumerate all ranges of code points with the same relevant values. + * The values are transformed from the raw trie entries by the enumValue function. + */ +U_CAPI void U_EXPORT2 +utrie_enum(const UTrie *trie, + UTrieEnumValue *enumValue, UTrieEnumRange *enumRange, const void *context) { + const uint32_t *data32; + const uint16_t *index; + + uint32_t value, prevValue, initialValue; + UChar32 c, prev; + int32_t l, i, j, block, prevBlock, offset; + + /* check arguments */ + if(trie==NULL || trie->index==NULL || enumRange==NULL) { + return; + } + if(enumValue==NULL) { + enumValue=enumSameValue; + } + + index=trie->index; + data32=trie->data32; + + /* get the enumeration value that corresponds to an initial-value trie data entry */ + initialValue=enumValue(context, trie->initialValue); + + /* set variables for previous range */ + prevBlock=0; + prev=0; + prevValue=initialValue; + + /* enumerate BMP - the main loop enumerates data blocks */ + for(i=0, c=0; c<=0xffff; ++i) { + if(c==0xd800) { + /* skip lead surrogate code _units_, go to lead surr. code _points_ */ + i=UTRIE_BMP_INDEX_LENGTH; + } else if(c==0xdc00) { + /* go back to regular BMP code points */ + i=c>>UTRIE_SHIFT; + } + + block=index[i]<0) { + prevBlock=-1; + } + prev=c; + prevValue=value; + } + ++c; + } + } + } + + /* enumerate supplementary code points */ + for(l=0xd800; l<0xdc00;) { + /* lead surrogate access */ + offset=index[l>>UTRIE_SHIFT]<indexLength)) { + /* no entries for a whole block of lead surrogates */ + if(prevValue!=initialValue) { + if(prevgetFoldingOffset(value); + if(offset<=0) { + /* no data for this lead surrogate */ + if(prevValue!=initialValue) { + if(prev0) { + prevBlock=-1; + } + prev=c; + prevValue=value; + } + ++c; + } + } + } while(++i>UTRIE_SHIFT, + + /** + * Shift size for shifting left the index array values. + * Increases possible data size with 16-bit index values at the cost + * of compactability. + * This requires blocks of stage 2 data to be aligned by UTRIE_DATA_GRANULARITY. + * 0..UTRIE_SHIFT + */ + UTRIE_INDEX_SHIFT=2, + + /** The alignment size of a stage 2 data block. Also the granularity for compaction. */ + UTRIE_DATA_GRANULARITY=1<>UTRIE_SHIFT + */ + UTRIE_SURROGATE_BLOCK_COUNT=(1<>UTRIE_SHIFT +}; + +/** + * Length of the index (stage 1) array before folding. + * Maximum number of Unicode code points (0x110000) shifted right by UTRIE_SHIFT. + */ +#define UTRIE_MAX_INDEX_LENGTH (0x110000>>UTRIE_SHIFT) + +/** + * Maximum length of the runtime data (stage 2) array. + * Limited by 16-bit index values that are left-shifted by UTRIE_INDEX_SHIFT. + */ +#define UTRIE_MAX_DATA_LENGTH (0x10000<=UTRIE_BMP_INDEX_LENGTH, or 0 if there is no data for the lead surrogate + */ +typedef int32_t U_CALLCONV +UTrieGetFoldingOffset(uint32_t data); + +/** + * Run-time Trie structure. + * + * Either the data table is 16 bits wide and accessed via the index + * pointer, with each index item increased by indexLength; + * in this case, data32==NULL. + * + * Or the data table is 32 bits wide and accessed via the data32 pointer. + */ +struct UTrie { + const uint16_t *index; + const uint32_t *data32; /* NULL if 16b data is used via index */ + + /** + * This function is not used in _FROM_LEAD, _FROM_BMP, and _FROM_OFFSET_TRAIL macros. + * If convenience macros like _GET16 or _NEXT32 are used, this function must be set. + * + * utrie_unserialize() sets a default function which simply returns + * the lead surrogate's value itself - which is the inverse of the default + * folding function used by utrie_serialize(). + * + * @see UTrieGetFoldingOffset + */ + UTrieGetFoldingOffset *getFoldingOffset; + + int32_t indexLength, dataLength; + uint32_t initialValue; + UBool isLatin1Linear; +}; + +typedef struct UTrie UTrie; + +/** Internal trie getter from an offset (0 if c16 is a BMP/lead units) and a 16-bit unit */ +#define _UTRIE_GET_RAW(trie, data, offset, c16) \ + (trie)->data[ \ + ((int32_t)((trie)->index[(offset)+((c16)>>UTRIE_SHIFT)])<getFoldingOffset(result); \ +\ + /* get the real data from the folded lead/trail units */ \ + if(__offset>0) { \ + (result)=_UTRIE_GET_RAW((trie), data, __offset, (c2)&0x3ff); \ + } else { \ + (result)=(resultType)((trie)->initialValue); \ + } \ +} + +/** Internal trie getter from a BMP code point, treating a lead surrogate as a normal code point */ +#define _UTRIE_GET_FROM_BMP(trie, data, c16) \ + _UTRIE_GET_RAW(trie, data, 0xd800<=(c16) && (c16)<=0xdbff ? UTRIE_LEAD_INDEX_DISP : 0, c16); + +/** + * Internal trie getter from a code point. + * Could be faster(?) but longer with + * if((c32)<=0xd7ff) { (result)=_UTRIE_GET_RAW(trie, data, 0, c32); } + */ +#define _UTRIE_GET(trie, data, c32, result, resultType) \ + if((uint32_t)(c32)<=0xffff) { \ + /* BMP code points */ \ + (result)=_UTRIE_GET_FROM_BMP(trie, data, c32); \ + } else if((uint32_t)(c32)<=0x10ffff) { \ + /* supplementary code point */ \ + UChar __lead16=UTF16_LEAD(c32); \ + _UTRIE_GET_FROM_PAIR(trie, data, __lead16, c32, result, resultType); \ + } else { \ + /* out of range */ \ + (result)=(resultType)((trie)->initialValue); \ + } + +/** Internal next-post-increment: get the next code point (c, c2) and its data */ +#define _UTRIE_NEXT(trie, data, src, limit, c, c2, result, resultType) { \ + (c)=*(src)++; \ + if(!UTF_IS_LEAD(c)) { \ + (c2)=0; \ + (result)=_UTRIE_GET_RAW((trie), data, 0, (c)); \ + } else if((src)!=(limit) && UTF_IS_TRAIL((c2)=*(src))) { \ + ++(src); \ + _UTRIE_GET_FROM_PAIR((trie), data, (c), (c2), (result), resultType); \ + } else { \ + /* unpaired lead surrogate code point */ \ + (c2)=0; \ + (result)=_UTRIE_GET_RAW((trie), data, UTRIE_LEAD_INDEX_DISP, (c)); \ + } \ +} + +/** Internal previous: get the previous code point (c, c2) and its data */ +#define _UTRIE_PREVIOUS(trie, data, start, src, c, c2, result, resultType) { \ + (c)=*--(src); \ + if(!UTF_IS_SURROGATE(c)) { \ + (c2)=0; \ + (result)=_UTRIE_GET_RAW((trie), data, 0, (c)); \ + } else if(!UTF_IS_SURROGATE_FIRST(c)) { \ + /* trail surrogate */ \ + if((start)!=(src) && UTF_IS_LEAD((c2)=*((src)-1))) { \ + --(src); \ + (result)=(c); (c)=(c2); (c2)=(UChar)(result); /* swap c, c2 */ \ + _UTRIE_GET_FROM_PAIR((trie), data, (c), (c2), (result), resultType); \ + } else { \ + /* unpaired trail surrogate code point */ \ + (c2)=0; \ + (result)=_UTRIE_GET_RAW((trie), data, 0, (c)); \ + } \ + } else { \ + /* unpaired lead surrogate code point */ \ + (c2)=0; \ + (result)=_UTRIE_GET_RAW((trie), data, UTRIE_LEAD_INDEX_DISP, (c)); \ + } \ +} + +/* Public UTrie API ---------------------------------------------------------*/ + +/** + * Get a pointer to the contiguous part of the data array + * for the Latin-1 range (U+0000..U+00ff). + * Must be used only if the Latin-1 range is in fact linear + * (trie->isLatin1Linear). + * + * @param trie (const UTrie *, in) a pointer to the runtime trie structure + * @return (const uint16_t *) pointer to values for Latin-1 code points + */ +#define UTRIE_GET16_LATIN1(trie) ((trie)->index+(trie)->indexLength+UTRIE_DATA_BLOCK_LENGTH) + +/** + * Get a pointer to the contiguous part of the data array + * for the Latin-1 range (U+0000..U+00ff). + * Must be used only if the Latin-1 range is in fact linear + * (trie->isLatin1Linear). + * + * @param trie (const UTrie *, in) a pointer to the runtime trie structure + * @return (const uint32_t *) pointer to values for Latin-1 code points + */ +#define UTRIE_GET32_LATIN1(trie) ((trie)->data32+UTRIE_DATA_BLOCK_LENGTH) + +/** + * Get a 16-bit trie value from a BMP code point (UChar, <=U+ffff). + * c16 may be a lead surrogate, which may have a value including a folding offset. + * + * @param trie (const UTrie *, in) a pointer to the runtime trie structure + * @param c16 (UChar, in) the input BMP code point + * @return (uint16_t) trie lookup result + */ +#define UTRIE_GET16_FROM_LEAD(trie, c16) _UTRIE_GET_RAW(trie, index, 0, c16) + +/** + * Get a 32-bit trie value from a BMP code point (UChar, <=U+ffff). + * c16 may be a lead surrogate, which may have a value including a folding offset. + * + * @param trie (const UTrie *, in) a pointer to the runtime trie structure + * @param c16 (UChar, in) the input BMP code point + * @return (uint32_t) trie lookup result + */ +#define UTRIE_GET32_FROM_LEAD(trie, c16) _UTRIE_GET_RAW(trie, data32, 0, c16) + +/** + * Get a 16-bit trie value from a BMP code point (UChar, <=U+ffff). + * Even lead surrogate code points are treated as normal code points, + * with unfolded values that may differ from _FROM_LEAD() macro results for them. + * + * @param trie (const UTrie *, in) a pointer to the runtime trie structure + * @param c16 (UChar, in) the input BMP code point + * @return (uint16_t) trie lookup result + */ +#define UTRIE_GET16_FROM_BMP(trie, c16) _UTRIE_GET_FROM_BMP(trie, index, c16) + +/** + * Get a 32-bit trie value from a BMP code point (UChar, <=U+ffff). + * Even lead surrogate code points are treated as normal code points, + * with unfolded values that may differ from _FROM_LEAD() macro results for them. + * + * @param trie (const UTrie *, in) a pointer to the runtime trie structure + * @param c16 (UChar, in) the input BMP code point + * @return (uint32_t) trie lookup result + */ +#define UTRIE_GET32_FROM_BMP(trie, c16) _UTRIE_GET_FROM_BMP(trie, data32, c16) + +/** + * Get a 16-bit trie value from a code point. + * Even lead surrogate code points are treated as normal code points, + * with unfolded values that may differ from _FROM_LEAD() macro results for them. + * + * @param trie (const UTrie *, in) a pointer to the runtime trie structure + * @param c32 (UChar32, in) the input code point + * @param result (uint16_t, out) uint16_t variable for the trie lookup result + */ +#define UTRIE_GET16(trie, c32, result) _UTRIE_GET(trie, index, c32, result, uint16_t) + +/** + * Get a 32-bit trie value from a code point. + * Even lead surrogate code points are treated as normal code points, + * with unfolded values that may differ from _FROM_LEAD() macro results for them. + * + * @param trie (const UTrie *, in) a pointer to the runtime trie structure + * @param c32 (UChar32, in) the input code point + * @param result (uint32_t, out) uint32_t variable for the trie lookup result + */ +#define UTRIE_GET32(trie, c32, result) _UTRIE_GET(trie, data32, c32, result, uint32_t) + +/** + * Get the next code point (c, c2), post-increment src, + * and get a 16-bit value from the trie. + * + * @param trie (const UTrie *, in) a pointer to the runtime trie structure + * @param src (const UChar *, in/out) the source text pointer + * @param limit (const UChar *, in) the limit pointer for the text, or NULL + * @param c (UChar, out) variable for the BMP or lead code unit + * @param c2 (UChar, out) variable for 0 or the trail code unit + * @param result (uint16_t, out) uint16_t variable for the trie lookup result + */ +#define UTRIE_NEXT16(trie, src, limit, c, c2, result) _UTRIE_NEXT(trie, index, src, limit, c, c2, result, uint16_t) + +/** + * Get the next code point (c, c2), post-increment src, + * and get a 32-bit value from the trie. + * + * @param trie (const UTrie *, in) a pointer to the runtime trie structure + * @param src (const UChar *, in/out) the source text pointer + * @param limit (const UChar *, in) the limit pointer for the text, or NULL + * @param c (UChar, out) variable for the BMP or lead code unit + * @param c2 (UChar, out) variable for 0 or the trail code unit + * @param result (uint32_t, out) uint32_t variable for the trie lookup result + */ +#define UTRIE_NEXT32(trie, src, limit, c, c2, result) _UTRIE_NEXT(trie, data32, src, limit, c, c2, result, uint32_t) + +/** + * Get the previous code point (c, c2), pre-decrement src, + * and get a 16-bit value from the trie. + * + * @param trie (const UTrie *, in) a pointer to the runtime trie structure + * @param start (const UChar *, in) the start pointer for the text, or NULL + * @param src (const UChar *, in/out) the source text pointer + * @param c (UChar, out) variable for the BMP or lead code unit + * @param c2 (UChar, out) variable for 0 or the trail code unit + * @param result (uint16_t, out) uint16_t variable for the trie lookup result + */ +#define UTRIE_PREVIOUS16(trie, start, src, c, c2, result) _UTRIE_PREVIOUS(trie, index, start, src, c, c2, result, uint16_t) + +/** + * Get the previous code point (c, c2), pre-decrement src, + * and get a 32-bit value from the trie. + * + * @param trie (const UTrie *, in) a pointer to the runtime trie structure + * @param start (const UChar *, in) the start pointer for the text, or NULL + * @param src (const UChar *, in/out) the source text pointer + * @param c (UChar, out) variable for the BMP or lead code unit + * @param c2 (UChar, out) variable for 0 or the trail code unit + * @param result (uint32_t, out) uint32_t variable for the trie lookup result + */ +#define UTRIE_PREVIOUS32(trie, start, src, c, c2, result) _UTRIE_PREVIOUS(trie, data32, start, src, c, c2, result, uint32_t) + +/** + * Get a 16-bit trie value from a pair of surrogates. + * + * @param trie (const UTrie *, in) a pointer to the runtime trie structure + * @param c (UChar, in) a lead surrogate + * @param c2 (UChar, in) a trail surrogate + * @param result (uint16_t, out) uint16_t variable for the trie lookup result + */ +#define UTRIE_GET16_FROM_PAIR(trie, c, c2, result) _UTRIE_GET_FROM_PAIR(trie, index, c, c2, result, uint16_t) + +/** + * Get a 32-bit trie value from a pair of surrogates. + * + * @param trie (const UTrie *, in) a pointer to the runtime trie structure + * @param c (UChar, in) a lead surrogate + * @param c2 (UChar, in) a trail surrogate + * @param result (uint32_t, out) uint32_t variable for the trie lookup result + */ +#define UTRIE_GET32_FROM_PAIR(trie, c, c2, result) _UTRIE_GET_FROM_PAIR(trie, data32, c, c2, result, uint32_t) + +/** + * Get a 16-bit trie value from a folding offset (from the value of a lead surrogate) + * and a trail surrogate. + * + * @param trie (const UTrie *, in) a pointer to the runtime trie structure + * @param offset (int32_t, in) the folding offset from the value of a lead surrogate + * @param c2 (UChar, in) a trail surrogate (only the 10 low bits are significant) + * @return (uint16_t) trie lookup result + */ +#define UTRIE_GET16_FROM_OFFSET_TRAIL(trie, offset, c2) _UTRIE_GET_RAW(trie, index, offset, (c2)&0x3ff) + +/** + * Get a 32-bit trie value from a folding offset (from the value of a lead surrogate) + * and a trail surrogate. + * + * @param trie (const UTrie *, in) a pointer to the runtime trie structure + * @param offset (int32_t, in) the folding offset from the value of a lead surrogate + * @param c2 (UChar, in) a trail surrogate (only the 10 low bits are significant) + * @return (uint32_t) trie lookup result + */ +#define UTRIE_GET32_FROM_OFFSET_TRAIL(trie, offset, c2) _UTRIE_GET_RAW(trie, data32, offset, (c2)&0x3ff) + +/* enumeration callback types */ + +/** + * Callback from utrie_enum(), extracts a uint32_t value from a + * trie value. This value will be passed on to the UTrieEnumRange function. + * + * @param context an opaque pointer, as passed into utrie_enum() + * @param value a value from the trie + * @return the value that is to be passed on to the UTrieEnumRange function + */ +typedef uint32_t U_CALLCONV +UTrieEnumValue(const void *context, uint32_t value); + +/** + * Callback from utrie_enum(), is called for each contiguous range + * of code points with the same value as retrieved from the trie and + * transformed by the UTrieEnumValue function. + * + * The callback function can stop the enumeration by returning FALSE. + * + * @param context an opaque pointer, as passed into utrie_enum() + * @param start the first code point in a contiguous range with value + * @param limit one past the last code point in a contiguous range with value + * @param value the value that is set for all code points in [start..limit[ + * @return FALSE to stop the enumeration + */ +typedef UBool U_CALLCONV +UTrieEnumRange(const void *context, UChar32 start, UChar32 limit, uint32_t value); + +/** + * Enumerate efficiently all values in a trie. + * For each entry in the trie, the value to be delivered is passed through + * the UTrieEnumValue function. + * The value is unchanged if that function pointer is NULL. + * + * For each contiguous range of code points with a given value, + * the UTrieEnumRange function is called. + * + * @param trie a pointer to the runtime trie structure + * @param enumValue a pointer to a function that may transform the trie entry value, + * or NULL if the values from the trie are to be used directly + * @param enumRange a pointer to a function that is called for each contiguous range + * of code points with the same value + * @param context an opaque pointer that is passed on to the callback functions + */ +U_CAPI void U_EXPORT2 +utrie_enum(const UTrie *trie, + UTrieEnumValue *enumValue, UTrieEnumRange *enumRange, const void *context); + +/** + * Unserialize a trie from 32-bit-aligned memory. + * Inverse of utrie_serialize(). + * Fills the UTrie runtime trie structure with the settings for the trie data. + * + * @param trie a pointer to the runtime trie structure + * @param data a pointer to 32-bit-aligned memory containing trie data + * @param length the number of bytes available at data + * @param pErrorCode an in/out ICU UErrorCode + * @return the number of bytes at data taken up by the trie data + */ +U_CAPI int32_t U_EXPORT2 +utrie_unserialize(UTrie *trie, const void *data, int32_t length, UErrorCode *pErrorCode); + +/** + * "Unserialize" a dummy trie. + * A dummy trie is an empty runtime trie, used when a real data trie cannot + * be loaded. + * + * The input memory is filled so that the trie always returns the initialValue, + * or the leadUnitValue for lead surrogate code points. + * The Latin-1 part is always set up to be linear. + * + * @param trie a pointer to the runtime trie structure + * @param data a pointer to 32-bit-aligned memory to be filled with the dummy trie data + * @param length the number of bytes available at data (recommended to use UTRIE_DUMMY_SIZE) + * @param initialValue the initial value that is set for all code points + * @param leadUnitValue the value for lead surrogate code _units_ that do not + * have associated supplementary data + * @param pErrorCode an in/out ICU UErrorCode + * + * @see UTRIE_DUMMY_SIZE + * @see utrie_open + */ +U_CAPI int32_t U_EXPORT2 +utrie_unserializeDummy(UTrie *trie, + void *data, int32_t length, + uint32_t initialValue, uint32_t leadUnitValue, + UBool make16BitTrie, + UErrorCode *pErrorCode); + +/** + * Default implementation for UTrie.getFoldingOffset, set automatically by + * utrie_unserialize(). + * Simply returns the lead surrogate's value itself - which is the inverse + * of the default folding function used by utrie_serialize(). + * Exported for static const UTrie structures. + * + * @see UTrieGetFoldingOffset + */ +U_CAPI int32_t U_EXPORT2 +utrie_defaultGetFoldingOffset(uint32_t data); + +/* Building a trie ----------------------------------------------------------*/ + +/** + * Build-time trie structure. + * Opaque definition, here only to make fillIn parameters possible + * for utrie_open() and utrie_clone(). + */ +struct UNewTrie { + /** + * Index values at build-time are 32 bits wide for easier processing. + * Bit 31 is set if the data block is used by multiple index values (from utrie_setRange()). + */ + int32_t index[UTRIE_MAX_INDEX_LENGTH]; + uint32_t *data; + + uint32_t leadUnitValue; + int32_t indexLength, dataCapacity, dataLength; + UBool isAllocated, isDataAllocated; + UBool isLatin1Linear, isCompacted; + + /** + * Map of adjusted indexes, used in utrie_compact(). + * Maps from original indexes to new ones. + */ + int32_t map[UTRIE_MAX_BUILD_TIME_DATA_LENGTH>>UTRIE_SHIFT]; +}; + +typedef struct UNewTrie UNewTrie; + +/** + * Build-time trie callback function, used with utrie_serialize(). + * This function calculates a lead surrogate's value including a folding offset + * from the 1024 supplementary code points [start..start+1024[ . + * It is U+10000 <= start <= U+10fc00 and (start&0x3ff)==0. + * + * The folding offset is provided by the caller. + * It is offset=UTRIE_BMP_INDEX_LENGTH+n*UTRIE_SURROGATE_BLOCK_COUNT with n=0..1023. + * Instead of the offset itself, n can be stored in 10 bits - + * or fewer if it can be assumed that few lead surrogates have associated data. + * + * The returned value must be + * - not zero if and only if there is relevant data + * for the corresponding 1024 supplementary code points + * - such that UTrie.getFoldingOffset(UNewTrieGetFoldedValue(..., offset))==offset + * + * @return a folded value, or 0 if there is no relevant data for the lead surrogate. + */ +typedef uint32_t U_CALLCONV +UNewTrieGetFoldedValue(UNewTrie *trie, UChar32 start, int32_t offset); + +/** + * Open a build-time trie structure. + * The size of the build-time data array is specified to avoid allocating a large + * array in all cases. The array itself can also be passed in. + * + * Although the trie is never fully expanded to a linear array, especially when + * utrie_setRange32() is used, the data array could be large during build time. + * The maximum length is + * UTRIE_MAX_BUILD_TIME_DATA_LENGTH=0x110000+UTRIE_DATA_BLOCK_LENGTH+0x400. + * (Number of Unicode code points + one all-initial-value block + + * possible duplicate entries for 1024 lead surrogates.) + * (UTRIE_DATA_BLOCK_LENGTH<=0x200 in all cases.) + * + * @param fillIn a pointer to a UNewTrie structure to be initialized (will not be released), or + * NULL if one is to be allocated + * @param aliasData a pointer to a data array to be used (will not be released), or + * NULL if one is to be allocated + * @param maxDataLength the capacity of aliasData (if not NULL) or + * the length of the data array to be allocated + * @param initialValue the initial value that is set for all code points + * @param leadUnitValue the value for lead surrogate code _units_ that do not + * have associated supplementary data + * @param latin1Linear a flag indicating whether the Latin-1 range is to be allocated and + * kept in a linear, contiguous part of the data array + * @return a pointer to the initialized fillIn or the allocated and initialized new UNewTrie + */ +U_CAPI UNewTrie * U_EXPORT2 +utrie_open(UNewTrie *fillIn, + uint32_t *aliasData, int32_t maxDataLength, + uint32_t initialValue, uint32_t leadUnitValue, + UBool latin1Linear); + +/** + * Clone a build-time trie structure with all entries. + * + * @param fillIn like in utrie_open() + * @param other the build-time trie structure to clone + * @param aliasData like in utrie_open(), + * used if aliasDataLength>=(capacity of other's data array) + * @param aliasDataLength the length of aliasData + * @return a pointer to the initialized fillIn or the allocated and initialized new UNewTrie + */ +U_CAPI UNewTrie * U_EXPORT2 +utrie_clone(UNewTrie *fillIn, const UNewTrie *other, uint32_t *aliasData, int32_t aliasDataLength); + +/** + * Close a build-time trie structure, and release memory + * that was allocated by utrie_open() or utrie_clone(). + * + * @param trie the build-time trie + */ +U_CAPI void U_EXPORT2 +utrie_close(UNewTrie *trie); + +/** + * Get the data array of a build-time trie. + * The data may be modified, but entries that are equal before + * must still be equal after modification. + * + * @param trie the build-time trie + * @param pLength (out) a pointer to a variable that receives the number + * of entries in the data array + * @return the data array + */ +U_CAPI uint32_t * U_EXPORT2 +utrie_getData(UNewTrie *trie, int32_t *pLength); + +/** + * Set a value for a code point. + * + * @param trie the build-time trie + * @param c the code point + * @param value the value + * @return FALSE if a failure occurred (illegal argument or data array overrun) + */ +U_CAPI UBool U_EXPORT2 +utrie_set32(UNewTrie *trie, UChar32 c, uint32_t value); + +/** + * Get a value from a code point as stored in the build-time trie. + * + * @param trie the build-time trie + * @param c the code point + * @param pInBlockZero if not NULL, then *pInBlockZero is set to TRUE + * iff the value is retrieved from block 0; + * block 0 is the all-initial-value initial block + * @return the value + */ +U_CAPI uint32_t U_EXPORT2 +utrie_get32(UNewTrie *trie, UChar32 c, UBool *pInBlockZero); + +/** + * Set a value in a range of code points [start..limit[. + * All code points c with start<=c=UTRIE_DATA_BLOCK_LENGTH */ + int32_t dataLength; +} UTrieHeader; + +/** + * Constants for use with UTrieHeader.options. + * @internal + */ +enum { + /** Mask to get the UTRIE_SHIFT value from options. */ + UTRIE_OPTIONS_SHIFT_MASK=0xf, + + /** Shift options right this much to get the UTRIE_INDEX_SHIFT value. */ + UTRIE_OPTIONS_INDEX_SHIFT=4, + + /** If set, then the data (stage 2) array is 32 bits wide. */ + UTRIE_OPTIONS_DATA_IS_32_BIT=0x100, + + /** + * If set, then Latin-1 data (for U+0000..U+00ff) is stored in the data (stage 2) array + * as a simple, linear array at data+UTRIE_DATA_BLOCK_LENGTH. + */ + UTRIE_OPTIONS_LATIN1_IS_LINEAR=0x200 +}; + +U_CDECL_END + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/utypes.c b/Build/source/libs/icu/icu-xetex/common/utypes.c new file mode 100644 index 00000000000..8d180cc0384 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/utypes.c @@ -0,0 +1,199 @@ +/* +****************************************************************************** +* +* Copyright (C) 1997-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* +* FILE NAME : utypes.c (previously putil.c) +* +* Date Name Description +* 10/07/2004 grhoten split from putil.c +****************************************************************************** +*/ + +#include "unicode/utypes.h" + +/* u_errorName() ------------------------------------------------------------ */ + +static const char * const +_uErrorInfoName[U_ERROR_WARNING_LIMIT-U_ERROR_WARNING_START]={ + "U_USING_FALLBACK_WARNING", + "U_USING_DEFAULT_WARNING", + "U_SAFECLONE_ALLOCATED_WARNING", + "U_STATE_OLD_WARNING", + "U_STRING_NOT_TERMINATED_WARNING", + "U_SORT_KEY_TOO_SHORT_WARNING", + "U_AMBIGUOUS_ALIAS_WARNING", + "U_DIFFERENT_UCA_VERSION" +}; + +static const char * const +_uTransErrorName[U_PARSE_ERROR_LIMIT - U_PARSE_ERROR_START]={ + "U_BAD_VARIABLE_DEFINITION", + "U_MALFORMED_RULE", + "U_MALFORMED_SET", + "U_MALFORMED_SYMBOL_REFERENCE", + "U_MALFORMED_UNICODE_ESCAPE", + "U_MALFORMED_VARIABLE_DEFINITION", + "U_MALFORMED_VARIABLE_REFERENCE", + "U_MISMATCHED_SEGMENT_DELIMITERS", + "U_MISPLACED_ANCHOR_START", + "U_MISPLACED_CURSOR_OFFSET", + "U_MISPLACED_QUANTIFIER", + "U_MISSING_OPERATOR", + "U_MISSING_SEGMENT_CLOSE", + "U_MULTIPLE_ANTE_CONTEXTS", + "U_MULTIPLE_CURSORS", + "U_MULTIPLE_POST_CONTEXTS", + "U_TRAILING_BACKSLASH", + "U_UNDEFINED_SEGMENT_REFERENCE", + "U_UNDEFINED_VARIABLE", + "U_UNQUOTED_SPECIAL", + "U_UNTERMINATED_QUOTE", + "U_RULE_MASK_ERROR", + "U_MISPLACED_COMPOUND_FILTER", + "U_MULTIPLE_COMPOUND_FILTERS", + "U_INVALID_RBT_SYNTAX", + "U_INVALID_PROPERTY_PATTERN", + "U_MALFORMED_PRAGMA", + "U_UNCLOSED_SEGMENT", + "U_ILLEGAL_CHAR_IN_SEGMENT", + "U_VARIABLE_RANGE_EXHAUSTED", + "U_VARIABLE_RANGE_OVERLAP", + "U_ILLEGAL_CHARACTER", + "U_INTERNAL_TRANSLITERATOR_ERROR", + "U_INVALID_ID", + "U_INVALID_FUNCTION" +}; + +static const char * const +_uErrorName[U_STANDARD_ERROR_LIMIT]={ + "U_ZERO_ERROR", + + "U_ILLEGAL_ARGUMENT_ERROR", + "U_MISSING_RESOURCE_ERROR", + "U_INVALID_FORMAT_ERROR", + "U_FILE_ACCESS_ERROR", + "U_INTERNAL_PROGRAM_ERROR", + "U_MESSAGE_PARSE_ERROR", + "U_MEMORY_ALLOCATION_ERROR", + "U_INDEX_OUTOFBOUNDS_ERROR", + "U_PARSE_ERROR", + "U_INVALID_CHAR_FOUND", + "U_TRUNCATED_CHAR_FOUND", + "U_ILLEGAL_CHAR_FOUND", + "U_INVALID_TABLE_FORMAT", + "U_INVALID_TABLE_FILE", + "U_BUFFER_OVERFLOW_ERROR", + "U_UNSUPPORTED_ERROR", + "U_RESOURCE_TYPE_MISMATCH", + "U_ILLEGAL_ESCAPE_SEQUENCE", + "U_UNSUPPORTED_ESCAPE_SEQUENCE", + "U_NO_SPACE_AVAILABLE", + "U_CE_NOT_FOUND_ERROR", + "U_PRIMARY_TOO_LONG_ERROR", + "U_STATE_TOO_OLD_ERROR", + "U_TOO_MANY_ALIASES_ERROR", + "U_ENUM_OUT_OF_SYNC_ERROR", + "U_INVARIANT_CONVERSION_ERROR", + "U_INVALID_STATE_ERROR", + "U_COLLATOR_VERSION_MISMATCH", + "U_USELESS_COLLATOR_ERROR", + "U_NO_WRITE_PERMISSION" +}; +static const char * const +_uFmtErrorName[U_FMT_PARSE_ERROR_LIMIT - U_FMT_PARSE_ERROR_START] = { + "U_UNEXPECTED_TOKEN", + "U_MULTIPLE_DECIMAL_SEPARATORS", + "U_MULTIPLE_EXPONENTIAL_SYMBOLS", + "U_MALFORMED_EXPONENTIAL_PATTERN", + "U_MULTIPLE_PERCENT_SYMBOLS", + "U_MULTIPLE_PERMILL_SYMBOLS", + "U_MULTIPLE_PAD_SPECIFIERS", + "U_PATTERN_SYNTAX_ERROR", + "U_ILLEGAL_PAD_POSITION", + "U_UNMATCHED_BRACES", + "U_UNSUPPORTED_PROPERTY", + "U_UNSUPPORTED_ATTRIBUTE" +}; + +static const char * const +_uBrkErrorName[U_BRK_ERROR_LIMIT - U_BRK_ERROR_START] = { + "U_BRK_INTERNAL_ERROR", + "U_BRK_HEX_DIGITS_EXPECTED", + "U_BRK_SEMICOLON_EXPECTED", + "U_BRK_RULE_SYNTAX", + "U_BRK_UNCLOSED_SET", + "U_BRK_ASSIGN_ERROR", + "U_BRK_VARIABLE_REDFINITION", + "U_BRK_MISMATCHED_PAREN", + "U_BRK_NEW_LINE_IN_QUOTED_STRING", + "U_BRK_UNDEFINED_VARIABLE", + "U_BRK_INIT_ERROR", + "U_BRK_RULE_EMPTY_SET", + "U_BRK_UNRECOGNIZED_OPTION", + "U_BRK_MALFORMED_RULE_TAG" +}; + +static const char * const +_uRegexErrorName[U_REGEX_ERROR_LIMIT - U_REGEX_ERROR_START] = { + "U_REGEX_INTERNAL_ERROR", + "U_REGEX_RULE_SYNTAX", + "U_REGEX_INVALID_STATE", + "U_REGEX_BAD_ESCAPE_SEQUENCE", + "U_REGEX_PROPERTY_SYNTAX", + "U_REGEX_UNIMPLEMENTED", + "U_REGEX_MISMATCHED_PAREN", + "U_REGEX_NUMBER_TOO_BIG", + "U_REGEX_BAD_INTERVAL", + "U_REGEX_MAX_LT_MIN", + "U_REGEX_INVALID_BACK_REF", + "U_REGEX_INVALID_FLAG", + "U_REGEX_LOOK_BEHIND_LIMIT", + "U_REGEX_SET_CONTAINS_STRING" +}; + +static const char * const +_uIDNAErrorName[U_IDNA_ERROR_LIMIT - U_IDNA_ERROR_START] = { + "U_STRINGPREP_PROHIBITED_ERROR", + "U_STRINGPREP_UNASSIGNED_ERROR", + "U_STRINGPREP_CHECK_BIDI_ERROR", + "U_IDNA_STD3_ASCII_RULES_ERROR", + "U_IDNA_ACE_PREFIX_ERROR", + "U_IDNA_VERIFICATION_ERROR", + "U_IDNA_LABEL_TOO_LONG_ERROR", + "U_IDNA_ZERO_LENGTH_LABEL_ERROR", + "U_IDNA_DOMAIN_NAME_TOO_LONG_ERROR" +}; + +U_CAPI const char * U_EXPORT2 +u_errorName(UErrorCode code) { + if(U_ZERO_ERROR <= code && code < U_STANDARD_ERROR_LIMIT) { + return _uErrorName[code]; + } else if(U_ERROR_WARNING_START <= code && code < U_ERROR_WARNING_LIMIT) { + return _uErrorInfoName[code - U_ERROR_WARNING_START]; + } else if(U_PARSE_ERROR_START <= code && code < U_PARSE_ERROR_LIMIT){ + return _uTransErrorName[code - U_PARSE_ERROR_START]; + } else if(U_FMT_PARSE_ERROR_START <= code && code < U_FMT_PARSE_ERROR_LIMIT){ + return _uFmtErrorName[code - U_FMT_PARSE_ERROR_START]; + } else if (U_BRK_ERROR_START <= code && code < U_BRK_ERROR_LIMIT){ + return _uBrkErrorName[code - U_BRK_ERROR_START]; + } else if (U_REGEX_ERROR_START <= code && code < U_REGEX_ERROR_LIMIT) { + return _uRegexErrorName[code - U_REGEX_ERROR_START]; + } else if(U_IDNA_ERROR_START <= code && code < U_IDNA_ERROR_LIMIT) { + return _uIDNAErrorName[code - U_IDNA_ERROR_START]; + } else { + return "[BOGUS UErrorCode]"; + } +} + +/* + * Hey, Emacs, please set the following: + * + * Local Variables: + * indent-tabs-mode: nil + * End: + * + */ diff --git a/Build/source/libs/icu/icu-xetex/common/uvector.cpp b/Build/source/libs/icu/icu-xetex/common/uvector.cpp new file mode 100644 index 00000000000..028ef39f3b4 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uvector.cpp @@ -0,0 +1,472 @@ +/* +****************************************************************************** +* Copyright (C) 1999-2004, International Business Machines Corporation and * +* others. All Rights Reserved. * +****************************************************************************** +* Date Name Description +* 10/22/99 alan Creation. +********************************************************************** +*/ + +#include "uvector.h" +#include "cmemory.h" + +U_NAMESPACE_BEGIN + +#define DEFUALT_CAPACITY 8 + +/* + * Constants for hinting whether a key is an integer + * or a pointer. If a hint bit is zero, then the associated + * token is assumed to be an integer. This is needed for iSeries + */ +#define HINT_KEY_POINTER (1) +#define HINT_KEY_INTEGER (0) + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(UVector) + +UVector::UVector(UErrorCode &status) : + count(0), + capacity(0), + elements(0), + deleter(0), + comparer(0) +{ + _init(DEFUALT_CAPACITY, status); +} + +UVector::UVector(int32_t initialCapacity, UErrorCode &status) : + count(0), + capacity(0), + elements(0), + deleter(0), + comparer(0) +{ + _init(initialCapacity, status); +} + +UVector::UVector(UObjectDeleter *d, UKeyComparator *c, UErrorCode &status) : + count(0), + capacity(0), + elements(0), + deleter(d), + comparer(c) +{ + _init(DEFUALT_CAPACITY, status); +} + +UVector::UVector(UObjectDeleter *d, UKeyComparator *c, int32_t initialCapacity, UErrorCode &status) : + count(0), + capacity(0), + elements(0), + deleter(d), + comparer(c) +{ + _init(initialCapacity, status); +} + +void UVector::_init(int32_t initialCapacity, UErrorCode &status) { + if (U_FAILURE(status)) { + return; + } + // Fix bogus initialCapacity values; avoid malloc(0) + if (initialCapacity < 1) { + initialCapacity = DEFUALT_CAPACITY; + } + elements = (UHashTok *)uprv_malloc(sizeof(UHashTok)*initialCapacity); + if (elements == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + } else { + capacity = initialCapacity; + } +} + +UVector::~UVector() { + removeAllElements(); + uprv_free(elements); + elements = 0; +} + +/** + * Assign this object to another (make this a copy of 'other'). + * Use the 'assign' function to assign each element. + */ +void UVector::assign(const UVector& other, UTokenAssigner *assign, UErrorCode &ec) { + if (ensureCapacity(other.count, ec)) { + setSize(other.count); + for (int32_t i=0; iindex; --i) { + elements[i] = elements[i-1]; + } + elements[index].pointer = obj; + ++count; + } + /* else index out of range */ +} + +void UVector::insertElementAt(int32_t elem, int32_t index, UErrorCode &status) { + // must have 0 <= index <= count + if (0 <= index && index <= count && ensureCapacity(count + 1, status)) { + for (int32_t i=count; i>index; --i) { + elements[i] = elements[i-1]; + } + elements[index].pointer = NULL; + elements[index].integer = elem; + ++count; + } + /* else index out of range */ +} + +void* UVector::elementAt(int32_t index) const { + return (0 <= index && index < count) ? elements[index].pointer : 0; +} + +int32_t UVector::elementAti(int32_t index) const { + return (0 <= index && index < count) ? elements[index].integer : 0; +} + +UBool UVector::containsAll(const UVector& other) const { + for (int32_t i=0; i= 0) { + return FALSE; + } + } + return TRUE; +} + +UBool UVector::removeAll(const UVector& other) { + UBool changed = FALSE; + for (int32_t i=0; i= 0) { + removeElementAt(j); + changed = TRUE; + } + } + return changed; +} + +UBool UVector::retainAll(const UVector& other) { + UBool changed = FALSE; + for (int32_t j=size()-1; j>=0; --j) { + int32_t i = other.indexOf(elements[j]); + if (i < 0) { + removeElementAt(j); + changed = TRUE; + } + } + return changed; +} + +void UVector::removeElementAt(int32_t index) { + void* e = orphanElementAt(index); + if (e != 0 && deleter != 0) { + (*deleter)(e); + } +} + +UBool UVector::removeElement(void* obj) { + int32_t i = indexOf(obj); + if (i >= 0) { + removeElementAt(i); + return TRUE; + } + return FALSE; +} + +void UVector::removeAllElements(void) { + if (deleter != 0) { + for (int32_t i=0; icount != other.count) { + return FALSE; + } + if (comparer == 0) { + for (i=0; i= minimumCapacity) { + return TRUE; + } else { + int32_t newCap = capacity * 2; + if (newCap < minimumCapacity) { + newCap = minimumCapacity; + } + UHashTok* newElems = (UHashTok *)uprv_malloc(sizeof(UHashTok)*newCap); + if (newElems == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + return FALSE; + } + uprv_memcpy(newElems, elements, sizeof(elements[0]) * count); + uprv_free(elements); + elements = newElems; + capacity = newCap; + return TRUE; + } +} + +/** + * Change the size of this vector as follows: If newSize is smaller, + * then truncate the array, possibly deleting held elements for i >= + * newSize. If newSize is larger, grow the array, filling in new + * slots with NULL. + */ +void UVector::setSize(int32_t newSize) { + int32_t i; + if (newSize < 0) { + return; + } + if (newSize > count) { + UErrorCode ec = U_ZERO_ERROR; + if (!ensureCapacity(newSize, ec)) { + return; + } + UHashTok empty; + empty.pointer = NULL; + empty.integer = 0; + for (i=count; i=newSize; --i) { + removeElementAt(i); + } + } + count = newSize; +} + +/** + * Fill in the given array with all elements of this vector. + */ +void** UVector::toArray(void** result) const { + void** a = result; + for (int i=0; i 0) { + max = probe; + } else { + // assert(c <= 0); + min = probe + 1; + } + } + if (ensureCapacity(count + 1, ec)) { + for (int32_t i=count; i>min; --i) { + elements[i] = elements[i-1]; + } + elements[min] = tok; + ++count; + } +} + +U_NAMESPACE_END + diff --git a/Build/source/libs/icu/icu-xetex/common/uvector.h b/Build/source/libs/icu/icu-xetex/common/uvector.h new file mode 100644 index 00000000000..80a33a19156 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uvector.h @@ -0,0 +1,408 @@ +/* +********************************************************************** +* Copyright (C) 1999-2006, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 10/22/99 alan Creation. This is an internal header. +* It should not be exported. +********************************************************************** +*/ + +#ifndef UVECTOR_H +#define UVECTOR_H + +#include "unicode/utypes.h" +#include "unicode/uobject.h" +#include "uhash.h" + +U_NAMESPACE_BEGIN + +/** + * A token comparison function. + * @param tok1 A token (object or integer) + * @param tok2 A token (object or integer) + * @return 0 if the two tokens are equal, -1 if tok1 is < tok2, or + * +1 if tok1 is > tok2. + */ +typedef int8_t U_CALLCONV USortComparator(UHashTok tok1, + UHashTok tok2); + +/** + * A token assignment function. It may copy an integer, copy + * a pointer, or clone a pointer, as appropriate. + * @param dst The token to be assigned to + * @param src The token to assign from + */ +typedef void U_CALLCONV UTokenAssigner(UHashTok *dst, + UHashTok *src); + +/** + *

Ultralightweight C++ implementation of a void* vector + * that is (mostly) compatible with java.util.Vector. + * + *

This is a very simple implementation, written to satisfy an + * immediate porting need. As such, it is not completely fleshed out, + * and it aims for simplicity and conformity. Nonetheless, it serves + * its purpose (porting code from java that uses java.util.Vector) + * well, and it could be easily made into a more robust vector class. + * + *

Design notes + * + *

There is index bounds checking, but little is done about it. If + * indices are out of bounds, either nothing happens, or zero is + * returned. We do avoid indexing off into the weeds. + * + *

There is detection of out of memory, but the handling is very + * coarse-grained -- similar to UnicodeString's protocol, but even + * coarser. The class contains one static flag that is set + * when any call to new returns zero. This allows the caller + * to use several vectors and make just one check at the end to see if + * a memory failure occurred. This is more efficient than making a + * check after each call on each vector when doing many operations on + * multiple vectors. The single static flag works best when memory + * failures are infrequent, and when recovery options are limited or + * nonexistent. + * + *

Since we don't have garbage collection, UVector was given the + * option to ownits contents. To employ this, set a deleter + * function. The deleter is called on a void* pointer when that + * pointer is released by the vector, either when the vector itself is + * destructed, or when a call to setElementAt() overwrites an element, + * or when a call to remove() or one of its variants explicitly + * removes an element. If no deleter is set, or the deleter is set to + * zero, then it is assumed that the caller will delete elements as + * needed. + * + *

In order to implement methods such as contains() and indexOf(), + * UVector needs a way to compare objects for equality. To do so, it + * uses a comparison frunction, or "comparer." If the comparer is not + * set, or is set to zero, then all such methods will act as if the + * vector contains no element. That is, indexOf() will always return + * -1, contains() will always return FALSE, etc. + * + *

To do + * + *

Improve the handling of index out of bounds errors. + * + * @author Alan Liu + */ +class U_COMMON_API UVector : public UObject { + // NOTE: UVector uses the UHashKey (union of void* and int32_t) as + // its basic storage type. It uses UKeyComparator as its + // comparison function. It uses UObjectDeleter as its deleter + // function. These are named for hashtables, but used here as-is + // rather than duplicating the type. This allows sharing of + // support functions. + +private: + int32_t count; + + int32_t capacity; + + UHashTok* elements; + + UObjectDeleter *deleter; + + UKeyComparator *comparer; + +public: + UVector(UErrorCode &status); + + UVector(int32_t initialCapacity, UErrorCode &status); + + UVector(UObjectDeleter *d, UKeyComparator *c, UErrorCode &status); + + UVector(UObjectDeleter *d, UKeyComparator *c, int32_t initialCapacity, UErrorCode &status); + + virtual ~UVector(); + + /** + * Assign this object to another (make this a copy of 'other'). + * Use the 'assign' function to assign each element. + */ + void assign(const UVector& other, UTokenAssigner *assign, UErrorCode &ec); + + /** + * Compare this vector with another. They will be considered + * equal if they are of the same size and all elements are equal, + * as compared using this object's comparer. + */ + UBool operator==(const UVector& other); + + /** + * Equivalent to !operator==() + */ + inline UBool operator!=(const UVector& other); + + //------------------------------------------------------------ + // java.util.Vector API + //------------------------------------------------------------ + + void addElement(void* obj, UErrorCode &status); + + void addElement(int32_t elem, UErrorCode &status); + + void setElementAt(void* obj, int32_t index); + + void setElementAt(int32_t elem, int32_t index); + + void insertElementAt(void* obj, int32_t index, UErrorCode &status); + + void insertElementAt(int32_t elem, int32_t index, UErrorCode &status); + + void* elementAt(int32_t index) const; + + int32_t elementAti(int32_t index) const; + + UBool equals(const UVector &other) const; + + void* firstElement(void) const; + + void* lastElement(void) const; + + int32_t lastElementi(void) const; + + int32_t indexOf(void* obj, int32_t startIndex = 0) const; + + int32_t indexOf(int32_t obj, int32_t startIndex = 0) const; + + UBool contains(void* obj) const; + + UBool contains(int32_t obj) const; + + UBool containsAll(const UVector& other) const; + + UBool removeAll(const UVector& other); + + UBool retainAll(const UVector& other); + + void removeElementAt(int32_t index); + + UBool removeElement(void* obj); + + void removeAllElements(); + + int32_t size(void) const; + + UBool isEmpty(void) const; + + UBool ensureCapacity(int32_t minimumCapacity, UErrorCode &status); + + /** + * Change the size of this vector as follows: If newSize is + * smaller, then truncate the array, possibly deleting held + * elements for i >= newSize. If newSize is larger, grow the + * array, filling in new slots with NULL. + */ + void setSize(int32_t newSize); + + /** + * Fill in the given array with all elements of this vector. + */ + void** toArray(void** result) const; + + //------------------------------------------------------------ + // New API + //------------------------------------------------------------ + + UObjectDeleter *setDeleter(UObjectDeleter *d); + + UKeyComparator *setComparer(UKeyComparator *c); + + void* operator[](int32_t index) const; + + /** + * Removes the element at the given index from this vector and + * transfer ownership of it to the caller. After this call, the + * caller owns the result and must delete it and the vector entry + * at 'index' is removed, shifting all subsequent entries back by + * one index and shortening the size of the vector by one. If the + * index is out of range or if there is no item at the given index + * then 0 is returned and the vector is unchanged. + */ + void* orphanElementAt(int32_t index); + + /** + * Returns true if this vector contains none of the elements + * of the given vector. + * @param other vector to be checked for containment + * @return true if the test condition is met + */ + UBool containsNone(const UVector& other) const; + + /** + * Insert the given object into this vector at its sorted position + * as defined by 'compare'. The current elements are assumed to + * be sorted already. + */ + void sortedInsert(void* obj, USortComparator *compare, UErrorCode& ec); + + /** + * Insert the given integer into this vector at its sorted position + * as defined by 'compare'. The current elements are assumed to + * be sorted already. + */ + void sortedInsert(int32_t obj, USortComparator *compare, UErrorCode& ec); + + /** + * ICU "poor man's RTTI", returns a UClassID for this class. + */ + static UClassID U_EXPORT2 getStaticClassID(); + + /** + * ICU "poor man's RTTI", returns a UClassID for the actual class. + */ + virtual UClassID getDynamicClassID() const; + +private: + void _init(int32_t initialCapacity, UErrorCode &status); + + int32_t indexOf(UHashTok key, int32_t startIndex = 0, int8_t hint = 0) const; + + void sortedInsert(UHashTok tok, USortComparator *compare, UErrorCode& ec); + + // Disallow + UVector(const UVector&); + + // Disallow + UVector& operator=(const UVector&); + +}; + + +/** + *

Ultralightweight C++ implementation of a void* stack + * that is (mostly) compatible with java.util.Stack. As in java, this + * is merely a paper thin layer around UVector. See the UVector + * documentation for further information. + * + *

Design notes + * + *

The element at index n-1 is (of course) the top of the + * stack. + * + *

The poorly named empty() method doesn't empty the + * stack; it determines if the stack is empty. + * + * @author Alan Liu + */ +class U_COMMON_API UStack : public UVector { +public: + UStack(UErrorCode &status); + + UStack(int32_t initialCapacity, UErrorCode &status); + + UStack(UObjectDeleter *d, UKeyComparator *c, UErrorCode &status); + + UStack(UObjectDeleter *d, UKeyComparator *c, int32_t initialCapacity, UErrorCode &status); + + virtual ~UStack(); + + // It's okay not to have a virtual destructor (in UVector) + // because UStack has no special cleanup to do. + + UBool empty(void) const; + + void* peek(void) const; + + int32_t peeki(void) const; + + void* pop(void); + + int32_t popi(void); + + void* push(void* obj, UErrorCode &status); + + int32_t push(int32_t i, UErrorCode &status); + + /* + If the object o occurs as an item in this stack, + this method returns the 1-based distance from the top of the stack. + */ + int32_t search(void* obj) const; + + /** + * ICU "poor man's RTTI", returns a UClassID for this class. + */ + static UClassID U_EXPORT2 getStaticClassID(); + + /** + * ICU "poor man's RTTI", returns a UClassID for the actual class. + */ + virtual UClassID getDynamicClassID() const; + +private: + // Disallow + UStack(const UStack&); + + // Disallow + UStack& operator=(const UStack&); +}; + + +// UVector inlines + +inline int32_t UVector::size(void) const { + return count; +} + +inline UBool UVector::isEmpty(void) const { + return count == 0; +} + +inline UBool UVector::contains(void* obj) const { + return indexOf(obj) >= 0; +} + +inline UBool UVector::contains(int32_t obj) const { + return indexOf(obj) >= 0; +} + +inline void* UVector::firstElement(void) const { + return elementAt(0); +} + +inline void* UVector::lastElement(void) const { + return elementAt(count-1); +} + +inline int32_t UVector::lastElementi(void) const { + return elementAti(count-1); +} + +inline void* UVector::operator[](int32_t index) const { + return elementAt(index); +} + +inline UBool UVector::operator!=(const UVector& other) { + return !operator==(other); +} + +// UStack inlines + +inline UBool UStack::empty(void) const { + return isEmpty(); +} + +inline void* UStack::peek(void) const { + return lastElement(); +} + +inline int32_t UStack::peeki(void) const { + return lastElementi(); +} + +inline void* UStack::push(void* obj, UErrorCode &status) { + addElement(obj, status); + return obj; +} + +inline int32_t UStack::push(int32_t i, UErrorCode &status) { + addElement(i, status); + return i; +} + +U_NAMESPACE_END + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/uvectr32.cpp b/Build/source/libs/icu/icu-xetex/common/uvectr32.cpp new file mode 100644 index 00000000000..e0dd4cbb325 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uvectr32.cpp @@ -0,0 +1,273 @@ +/* +****************************************************************************** +* Copyright (C) 1999-2003, International Business Machines Corporation and * +* others. All Rights Reserved. * +****************************************************************************** +* Date Name Description +* 10/22/99 alan Creation. +********************************************************************** +*/ + +#include "uvectr32.h" +#include "cmemory.h" + +U_NAMESPACE_BEGIN + +#define DEFUALT_CAPACITY 8 + +/* + * Constants for hinting whether a key is an integer + * or a pointer. If a hint bit is zero, then the associated + * token is assumed to be an integer. This is needed for iSeries + */ + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(UVector32) + +UVector32::UVector32(UErrorCode &status) : + count(0), + capacity(0), + elements(NULL) +{ + _init(DEFUALT_CAPACITY, status); +} + +UVector32::UVector32(int32_t initialCapacity, UErrorCode &status) : + count(0), + capacity(0), + elements(0) +{ + _init(initialCapacity, status); +} + + + +void UVector32::_init(int32_t initialCapacity, UErrorCode &status) { + // Fix bogus initialCapacity values; avoid malloc(0) + if (initialCapacity < 1) { + initialCapacity = DEFUALT_CAPACITY; + } + elements = (int32_t *)uprv_malloc(sizeof(int32_t)*initialCapacity); + if (elements == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + } else { + capacity = initialCapacity; + } +} + +UVector32::~UVector32() { + uprv_free(elements); + elements = 0; +} + +/** + * Assign this object to another (make this a copy of 'other'). + */ +void UVector32::assign(const UVector32& other, UErrorCode &ec) { + if (ensureCapacity(other.count, ec)) { + setSize(other.count); + for (int32_t i=0; iindex; --i) { + elements[i] = elements[i-1]; + } + elements[index] = elem; + ++count; + } + /* else index out of range */ +} + +UBool UVector32::containsAll(const UVector32& other) const { + for (int32_t i=0; i= 0) { + return FALSE; + } + } + return TRUE; +} + +UBool UVector32::removeAll(const UVector32& other) { + UBool changed = FALSE; + for (int32_t i=0; i= 0) { + removeElementAt(j); + changed = TRUE; + } + } + return changed; +} + +UBool UVector32::retainAll(const UVector32& other) { + UBool changed = FALSE; + for (int32_t j=size()-1; j>=0; --j) { + int32_t i = other.indexOf(elements[j]); + if (i < 0) { + removeElementAt(j); + changed = TRUE; + } + } + return changed; +} + +void UVector32::removeElementAt(int32_t index) { + if (index >= 0) { + for (int32_t i=index; icount != other.count) { + return FALSE; + } + for (i=0; i= minimumCapacity) { + return TRUE; + } else { + int32_t newCap = capacity * 2; + if (newCap < minimumCapacity) { + newCap = minimumCapacity; + } + int32_t* newElems = (int32_t *)uprv_malloc(sizeof(int32_t)*newCap); + if (newElems == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + return FALSE; + } + uprv_memcpy(newElems, elements, sizeof(elements[0]) * count); + uprv_free(elements); + elements = newElems; + capacity = newCap; + return TRUE; + } +} + +/** + * Change the size of this vector as follows: If newSize is smaller, + * then truncate the array, possibly deleting held elements for i >= + * newSize. If newSize is larger, grow the array, filling in new + * slots with NULL. + */ +void UVector32::setSize(int32_t newSize) { + int32_t i; + if (newSize < 0) { + return; + } + if (newSize > count) { + UErrorCode ec = U_ZERO_ERROR; + if (!ensureCapacity(newSize, ec)) { + return; + } + for (i=count; i 0) { + if (elements[probe] > tok) { + max = probe; + } else { + // assert(c <= 0); + min = probe + 1; + } + } + if (ensureCapacity(count + 1, ec)) { + for (int32_t i=count; i>min; --i) { + elements[i] = elements[i-1]; + } + elements[min] = tok; + ++count; + } +} + + + + + +U_NAMESPACE_END + diff --git a/Build/source/libs/icu/icu-xetex/common/uvectr32.h b/Build/source/libs/icu/icu-xetex/common/uvectr32.h new file mode 100644 index 00000000000..68a3ee2e2fe --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/uvectr32.h @@ -0,0 +1,292 @@ +/* +********************************************************************** +* Copyright (C) 1999-2006, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +*/ + +// +// UVector32 is a class implementing a vector of 32 bit integers. +// It is similar to UVector, but holds int32_t values rather than pointers. +// Most of the code is unchanged from UVector. +// + +#ifndef UVECTOR32_H +#define UVECTOR32_H + +#include "unicode/utypes.h" +#include "unicode/uobject.h" +#include "uhash.h" +#include "uassert.h" + +U_NAMESPACE_BEGIN + + + +/** + *

Ultralightweight C++ implementation of a void* vector + * that is (mostly) compatible with java.util.Vector. + * + *

This is a very simple implementation, written to satisfy an + * immediate porting need. As such, it is not completely fleshed out, + * and it aims for simplicity and conformity. Nonetheless, it serves + * its purpose (porting code from java that uses java.util.Vector) + * well, and it could be easily made into a more robust vector class. + * + *

Design notes + * + *

There is index bounds checking, but little is done about it. If + * indices are out of bounds, either nothing happens, or zero is + * returned. We do avoid indexing off into the weeds. + * + *

There is detection of out of memory, but the handling is very + * coarse-grained -- similar to UnicodeString's protocol, but even + * coarser. The class contains one static flag that is set + * when any call to new returns zero. This allows the caller + * to use several vectors and make just one check at the end to see if + * a memory failure occurred. This is more efficient than making a + * check after each call on each vector when doing many operations on + * multiple vectors. The single static flag works best when memory + * failures are infrequent, and when recovery options are limited or + * nonexistent. + * + *

To do + * + *

Improve the handling of index out of bounds errors. + * + * @author Alan Liu + */ +class U_COMMON_API UVector32 : public UObject { +private: + int32_t count; + + int32_t capacity; + + int32_t* elements; + +public: + UVector32(UErrorCode &status); + + UVector32(int32_t initialCapacity, UErrorCode &status); + + virtual ~UVector32(); + + /** + * Assign this object to another (make this a copy of 'other'). + * Use the 'assign' function to assign each element. + */ + void assign(const UVector32& other, UErrorCode &ec); + + /** + * Compare this vector with another. They will be considered + * equal if they are of the same size and all elements are equal, + * as compared using this object's comparer. + */ + UBool operator==(const UVector32& other); + + /** + * Equivalent to !operator==() + */ + inline UBool operator!=(const UVector32& other); + + //------------------------------------------------------------ + // java.util.Vector API + //------------------------------------------------------------ + + void addElement(int32_t elem, UErrorCode &status); + + void setElementAt(int32_t elem, int32_t index); + + void insertElementAt(int32_t elem, int32_t index, UErrorCode &status); + + int32_t elementAti(int32_t index) const; + + UBool equals(const UVector32 &other) const; + + int32_t lastElementi(void) const; + + int32_t indexOf(int32_t elem, int32_t startIndex = 0) const; + + UBool contains(int32_t elem) const; + + UBool containsAll(const UVector32& other) const; + + UBool removeAll(const UVector32& other); + + UBool retainAll(const UVector32& other); + + void removeElementAt(int32_t index); + + void removeAllElements(); + + int32_t size(void) const; + + UBool isEmpty(void) const; + + // Inline. Use this one for speedy size check. + inline UBool ensureCapacity(int32_t minimumCapacity, UErrorCode &status); + + // Out-of-line, handles actual growth. Called by ensureCapacity() when necessary. + UBool expandCapacity(int32_t minimumCapacity, UErrorCode &status); + + /** + * Change the size of this vector as follows: If newSize is + * smaller, then truncate the array, possibly deleting held + * elements for i >= newSize. If newSize is larger, grow the + * array, filling in new slows with zero. + */ + void setSize(int32_t newSize); + + //------------------------------------------------------------ + // New API + //------------------------------------------------------------ + + /** + * Returns true if this vector contains none of the elements + * of the given vector. + * @param other vector to be checked for containment + * @return true if the test condition is met + */ + UBool containsNone(const UVector32& other) const; + + + /** + * Insert the given integer into this vector at its sorted position. + * The current elements are assumed to be sorted already. + */ + void sortedInsert(int32_t elem, UErrorCode& ec); + + /** + * Returns a pointer to the internal array holding the vector. + */ + int32_t *getBuffer() const; + + /** + * ICU "poor man's RTTI", returns a UClassID for this class. + */ + static UClassID U_EXPORT2 getStaticClassID(); + + /** + * ICU "poor man's RTTI", returns a UClassID for the actual class. + */ + virtual UClassID getDynamicClassID() const; + +private: + void _init(int32_t initialCapacity, UErrorCode &status); + + // Disallow + UVector32(const UVector32&); + + // Disallow + UVector32& operator=(const UVector32&); + + + // API Functions for Stack operations. + // In the original UVector, these were in a separate derived class, UStack. + // Here in UVector32, they are all together. +public: + UBool empty(void) const; // TODO: redundant, same as empty(). Remove it? + + int32_t peeki(void) const; + + int32_t popi(void); + + int32_t push(int32_t i, UErrorCode &status); + + int32_t *reserveBlock(int32_t size, UErrorCode &status); + int32_t *popFrame(int32_t size); +}; + + +// UVector32 inlines + +inline UBool UVector32::ensureCapacity(int32_t minimumCapacity, UErrorCode &status) { + if (capacity >= minimumCapacity) { + return TRUE; + } else { + return expandCapacity(minimumCapacity, status); + } +} + +inline int32_t UVector32::elementAti(int32_t index) const { + return (0 <= index && index < count) ? elements[index] : 0; +} + + +inline void UVector32::addElement(int32_t elem, UErrorCode &status) { + if (ensureCapacity(count + 1, status)) { + elements[count] = elem; + count++; + } +} + +inline int32_t *UVector32::reserveBlock(int32_t size, UErrorCode &status) { + ensureCapacity(count+size, status); + int32_t *rp = elements+count; + count += size; + return rp; +} + +inline int32_t *UVector32::popFrame(int32_t size) { + U_ASSERT(count >= size); + count -= size; + if (count < 0) { + count = 0; + } + return elements+count-size; +} + + + +inline int32_t UVector32::size(void) const { + return count; +} + +inline UBool UVector32::isEmpty(void) const { + return count == 0; +} + +inline UBool UVector32::contains(int32_t obj) const { + return indexOf(obj) >= 0; +} + +inline int32_t UVector32::lastElementi(void) const { + return elementAti(count-1); +} + +inline UBool UVector32::operator!=(const UVector32& other) { + return !operator==(other); +} + +inline int32_t *UVector32::getBuffer() const { + return elements; +} + + +// UStack inlines + +inline UBool UVector32::empty(void) const { + return isEmpty(); +} + +inline int32_t UVector32::peeki(void) const { + return lastElementi(); +} + +inline int32_t UVector32::push(int32_t i, UErrorCode &status) { + addElement(i, status); + return i; +} + +inline int32_t UVector32::popi(void) { + int32_t result = 0; + if (count > 0) { + count--; + result = elements[count]; + } + return result; +} + +U_NAMESPACE_END + +#endif diff --git a/Build/source/libs/icu/icu-xetex/common/wintz.c b/Build/source/libs/icu/icu-xetex/common/wintz.c new file mode 100644 index 00000000000..f64a4ad09ce --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/wintz.c @@ -0,0 +1,688 @@ +/* +******************************************************************************** +* Copyright (C) 2005-2007, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************** +* +* File WINTZ.CPP +* +******************************************************************************** +*/ + +#include "unicode/utypes.h" + +#ifdef U_WINDOWS + +#include "wintz.h" + +#include "cmemory.h" +#include "cstring.h" + +#include "unicode/ustring.h" + +# define WIN32_LEAN_AND_MEAN +# define VC_EXTRALEAN +# define NOUSER +# define NOSERVICE +# define NOIME +# define NOMCX +#include + +#define ARRAY_SIZE(array) (sizeof array / sizeof array[0]) +#define NEW_ARRAY(type,count) (type *) uprv_malloc((count) * sizeof(type)) +#define DELETE_ARRAY(array) uprv_free((void *) (array)) + +#define ICUID_STACK_BUFFER_SIZE 32 + +/* The layout of the Tzi value in the registry */ +typedef struct +{ + int32_t bias; + int32_t standardBias; + int32_t daylightBias; + SYSTEMTIME standardDate; + SYSTEMTIME daylightDate; +} TZI; + +typedef struct +{ + const char *icuid; + const char *winid; +} WindowsICUMap; + +typedef struct { + const char* winid; + const char* altwinid; +} WindowsZoneRemap; + +/** + * Various registry keys and key fragments. + */ +static const char CURRENT_ZONE_REGKEY[] = "SYSTEM\\CurrentControlSet\\Control\\TimeZoneInformation\\"; +static const char STANDARD_NAME_REGKEY[] = "StandardName"; +static const char STANDARD_TIME_REGKEY[] = " Standard Time"; +static const char TZI_REGKEY[] = "TZI"; +static const char STD_REGKEY[] = "Std"; + +/** + * HKLM subkeys used to probe for the flavor of Windows. Note that we + * specifically check for the "GMT" zone subkey; this is present on + * NT, but on XP has become "GMT Standard Time". We need to + * discriminate between these cases. + */ +static const char* const WIN_TYPE_PROBE_REGKEY[] = { + /* WIN_9X_ME_TYPE */ + "SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Time Zones", + + /* WIN_NT_TYPE */ + "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Time Zones\\GMT" + + /* otherwise: WIN_2K_XP_TYPE */ +}; + +/** + * The time zone root subkeys (under HKLM) for different flavors of + * Windows. + */ +static const char* const TZ_REGKEY[] = { + /* WIN_9X_ME_TYPE */ + "SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Time Zones\\", + + /* WIN_NT_TYPE | WIN_2K_XP_TYPE */ + "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Time Zones\\" +}; + +/** + * Flavor of Windows, from our perspective. Not a real OS version, + * but rather the flavor of the layout of the time zone information in + * the registry. + */ +enum { + WIN_9X_ME_TYPE = 0, + WIN_NT_TYPE = 1, + WIN_2K_XP_TYPE = 2 +}; + +# if 0 +/* + * ZONE_MAP from supplementalData.txt + */ +static const WindowsICUMap NEW_ZONE_MAP[] = { + {"Africa/Cairo", "Egypt"}, + {"Africa/Casablanca", "Greenwich"}, + {"Africa/Johannesburg", "South Africa"}, + {"Africa/Lagos", "W. Central Africa"}, + {"Africa/Nairobi", "E. Africa"}, + {"Africa/Windhoek", "Namibia"}, + {"America/Anchorage", "Alaskan"}, + {"America/Bogota", "SA Pacific"}, + {"America/Buenos_Aires", "SA Eastern"}, + {"America/Caracas", "SA Western"}, + {"America/Chicago", "Central"}, + {"America/Chihuahua", "Mountain Standard Time (Mexico)"}, + {"America/Denver", "Mountain"}, + {"America/Godthab", "Greenland"}, + {"America/Guatemala", "Central America"}, + {"America/Halifax", "Atlantic"}, + {"America/Indianapolis", "US Eastern"}, + {"America/Los_Angeles", "Pacific"}, + {"America/Manaus", "Central Brazilian"}, + {"America/Mexico_City", "Central Standard Time (Mexico)"}, + {"America/Montevideo", "Montevideo"}, + {"America/New_York", "Eastern"}, + {"America/Noronha", "Mid-Atlantic"}, + {"America/Phoenix", "US Mountain"}, + {"America/Regina", "Canada Central"}, + {"America/Santiago", "Pacific SA"}, + {"America/Sao_Paulo", "E. South America"}, + {"America/St_Johns", "Newfoundland"}, + {"America/Tijuana", "Pacific Standard Time (Mexico)"}, + {"Asia/Amman", "Jordan"}, + {"Asia/Baghdad", "Arabic"}, + {"Asia/Baku", "Azerbaijan"}, + {"Asia/Bangkok", "SE Asia"}, + {"Asia/Beirut", "Middle East"}, + {"Asia/Calcutta", "India"}, + {"Asia/Colombo", "Sri Lanka"}, + {"Asia/Dhaka", "Central Asia"}, + {"Asia/Jerusalem", "Israel"}, + {"Asia/Kabul", "Afghanistan"}, + {"Asia/Karachi", "West Asia"}, + {"Asia/Katmandu", "Nepal"}, + {"Asia/Krasnoyarsk", "North Asia"}, + {"Asia/Muscat", "Arabian"}, + {"Asia/Novosibirsk", "N. Central Asia"}, + {"Asia/Rangoon", "Myanmar"}, + {"Asia/Riyadh", "Arab"}, + {"Asia/Seoul", "Korea"}, + {"Asia/Shanghai", "China"}, + {"Asia/Singapore", "Singapore"}, + {"Asia/Taipei", "Taipei"}, + {"Asia/Tbilisi", "Georgian"}, + {"Asia/Tehran", "Iran"}, + {"Asia/Tokyo", "Tokyo"}, + {"Asia/Ulaanbaatar", "North Asia East"}, + {"Asia/Vladivostok", "Vladivostok"}, + {"Asia/Yakutsk", "Yakutsk"}, + {"Asia/Yekaterinburg", "Ekaterinburg"}, + {"Asia/Yerevan", "Caucasus"}, + {"Atlantic/Azores", "Azores"}, + {"Atlantic/Cape_Verde", "Cape Verde"}, + {"Australia/Adelaide", "Cen. Australia"}, + {"Australia/Brisbane", "E. Australia"}, + {"Australia/Darwin", "AUS Central"}, + {"Australia/Hobart", "Tasmania"}, + {"Australia/Perth", "W. Australia"}, + {"Australia/Sydney", "AUS Eastern"}, + {"Europe/Berlin", "W. Europe"}, + {"Europe/Helsinki", "FLE"}, + {"Europe/Istanbul", "GTB"}, + {"Europe/London", "GMT"}, + {"Europe/Minsk", "E. Europe"}, + {"Europe/Moscow", "Russian"}, + {"Europe/Paris", "Romance"}, + {"Europe/Prague", "Central Europe"}, + {"Europe/Warsaw", "Central European"}, + {"Pacific/Apia", "Samoa"}, + {"Pacific/Auckland", "New Zealand"}, + {"Pacific/Fiji", "Fiji"}, + {"Pacific/Guadalcanal", "Central Pacific"}, + {"Pacific/Guam", "West Pacific"}, + {"Pacific/Honolulu", "Hawaiian"}, + {"Pacific/Kwajalein", "Dateline"}, + {"Pacific/Tongatapu", "Tonga"} +}; +#endif + +/* NOTE: Some Windows zone ids appear more than once. In such cases the + * ICU zone id from the first one is the preferred match. + */ +static const WindowsICUMap ZONE_MAP[] = { + {"Pacific/Kwajalein", "Dateline"}, /* S (GMT-12:00) International Date Line West */ + {"Etc/GMT+12", "Dateline"}, /* S (GMT-12:00) International Date Line West */ + + {"Pacific/Apia", "Samoa"}, /* S (GMT-11:00) Midway Island, Samoa */ + + {"Pacific/Honolulu", "Hawaiian"}, /* S (GMT-10:00) Hawaii */ + + {"America/Anchorage", "Alaskan"}, /* D (GMT-09:00) Alaska */ + + {"America/Los_Angeles", "Pacific"}, /* D (GMT-08:00) Pacific Time (US & Canada) */ + {"America/Tijuana", "Pacific Standard Time (Mexico)"}, /* S (GMT-08:00) Tijuana, Baja California */ + + {"America/Phoenix", "US Mountain"}, /* S (GMT-07:00) Arizona */ + {"America/Denver", "Mountain"}, /* D (GMT-07:00) Mountain Time (US & Canada) */ + {"America/Chihuahua", "Mountain Standard Time (Mexico)"}, /* D (GMT-07:00) Chihuahua, La Paz, Mazatlan */ + + {"America/Managua", "Central America"}, /* S (GMT-06:00) Central America */ /* America/Guatemala? */ + {"America/Regina", "Canada Central"}, /* S (GMT-06:00) Saskatchewan */ + {"America/Mexico_City", "Central Standard Time (Mexico)"}, /* D (GMT-06:00) Guadalajara, Mexico City, Monterrey */ + {"America/Chicago", "Central"}, /* D (GMT-06:00) Central Time (US & Canada) */ + + {"America/Indianapolis", "US Eastern"}, /* S (GMT-05:00) Indiana (East) */ + {"America/Bogota", "SA Pacific"}, /* S (GMT-05:00) Bogota, Lima, Quito */ + {"America/New_York", "Eastern"}, /* D (GMT-05:00) Eastern Time (US & Canada) */ + + {"America/Caracas", "SA Western"}, /* S (GMT-04:00) Caracas, La Paz */ + {"America/Santiago", "Pacific SA"}, /* D (GMT-04:00) Santiago */ + {"America/Halifax", "Atlantic"}, /* D (GMT-04:00) Atlantic Time (Canada) */ + {"America/Manaus", "Central Brazilian"}, /* D (GMT-04:00 Manaus */ + + {"America/St_Johns", "Newfoundland"}, /* D (GMT-03:30) Newfoundland */ + + {"America/Buenos_Aires", "SA Eastern"}, /* S (GMT-03:00) Buenos Aires, Georgetown */ + {"America/Godthab", "Greenland"}, /* D (GMT-03:00) Greenland */ + {"America/Sao_Paulo", "E. South America"}, /* D (GMT-03:00) Brasilia */ + {"America/Montevideo", "Montevideo"}, /* S (GMT-03:00) Montevideo */ + + {"America/Noronha", "Mid-Atlantic"}, /* D (GMT-02:00) Mid-Atlantic */ + + {"Atlantic/Cape_Verde", "Cape Verde"}, /* S (GMT-01:00) Cape Verde Is. */ + {"Atlantic/Azores", "Azores"}, /* D (GMT-01:00) Azores */ + + {"Africa/Casablanca", "Greenwich"}, /* S (GMT) Casablanca, Monrovia */ + {"Europe/London", "GMT"}, /* D (GMT) Greenwich Mean Time : Dublin, Edinburgh, Lisbon, London */ + + {"Africa/Lagos", "W. Central Africa"}, /* S (GMT+01:00) West Central Africa */ + {"Europe/Berlin", "W. Europe"}, /* D (GMT+01:00) Amsterdam, Berlin, Bern, Rome, Stockholm, Vienna */ + {"Europe/Paris", "Romance"}, /* D (GMT+01:00) Brussels, Copenhagen, Madrid, Paris */ + {"Eurpoe/Warsaw", "Central European"}, /* D (GMT+01:00) Sarajevo, Skopje, Warsaw, Zagreb */ + {"Europe/Sarajevo", "Central European"}, /* D (GMT+01:00) Sarajevo, Skopje, Warsaw, Zagreb */ + {"Europe/Prague", "Central Europe"}, /* D (GMT+01:00) Belgrade, Bratislava, Budapest, Ljubljana, Prague */ + {"Europe/Belgrade", "Central Europe"}, /* D (GMT+01:00) Belgrade, Bratislava, Budapest, Ljubljana, Prague */ + + {"Africa/Johannesburg", "South Africa"}, /* S (GMT+02:00) Harare, Pretoria */ + {"Asia/Jerusalem", "Israel"}, /* S (GMT+02:00) Jerusalem */ + {"Europe/Istanbul", "GTB"}, /* D (GMT+02:00) Athens, Istanbul, Minsk */ + {"Europe/Helsinki", "FLE"}, /* D (GMT+02:00) Helsinki, Kyiv, Riga, Sofia, Tallinn, Vilnius */ + {"Africa/Cairo", "Egypt"}, /* D (GMT+02:00) Cairo */ + {"Europe/Minsk", "E. Europe"}, /* D (GMT+02:00) Bucharest */ + {"Europe/Bucharest", "E. Europe"}, /* D (GMT+02:00) Bucharest */ + {"Africa/Windhoek", "Namibia"}, /* S (GMT+02:00) Windhoek */ + {"Asia/Amman", "Jordan"}, /* S (GMT+02:00) Aman */ + {"Asia/Beirut", "Middle East"}, /* S (GMT+02:00) Beirut */ + + {"Africa/Nairobi", "E. Africa"}, /* S (GMT+03:00) Nairobi */ + {"Asia/Riyadh", "Arab"}, /* S (GMT+03:00) Kuwait, Riyadh */ + {"Europe/Moscow", "Russian"}, /* D (GMT+03:00) Moscow, St. Petersburg, Volgograd */ + {"Asia/Baghdad", "Arabic"}, /* D (GMT+03:00) Baghdad */ + + {"Asia/Tehran", "Iran"}, /* D (GMT+03:30) Tehran */ + + {"Asia/Muscat", "Arabian"}, /* S (GMT+04:00) Abu Dhabi, Muscat */ + {"Asia/Tbilisi", "Georgian"}, /* D (GMT+04:00) Tbilisi */ + {"Asia/Baku", "Azerbaijan"}, /* S (GMT+04:00) Baku */ + {"Asia/Yerevan", "Caucasus"}, /* S (GMT+04:00) Yerevan */ + {"Asia/Kabul", "Afghanistan"}, /* S (GMT+04:30) Kabul */ + + {"Asia/Karachi", "West Asia"}, /* S (GMT+05:00) Islamabad, Karachi, Tashkent */ + {"Asia/Yekaterinburg", "Ekaterinburg"}, /* D (GMT+05:00) Ekaterinburg */ + + {"Asia/Calcutta", "India"}, /* S (GMT+05:30) Chennai, Kolkata, Mumbai, New Delhi */ + + {"Asia/Katmandu", "Nepal"}, /* S (GMT+05:45) Kathmandu */ + + {"Asia/Colombo", "Sri Lanka"}, /* S (GMT+06:00) Sri Jayawardenepura */ + {"Asia/Dhaka", "Central Asia"}, /* S (GMT+06:00) Astana, Dhaka */ + {"Asia/Novosibirsk", "N. Central Asia"}, /* D (GMT+06:00) Almaty, Novosibirsk */ + + {"Asia/Rangoon", "Myanmar"}, /* S (GMT+06:30) Rangoon */ + + {"Asia/Bangkok", "SE Asia"}, /* S (GMT+07:00) Bangkok, Hanoi, Jakarta */ + {"Asia/Krasnoyarsk", "North Asia"}, /* D (GMT+07:00) Krasnoyarsk */ + + {"Australia/Perth", "W. Australia"}, /* S (GMT+08:00) Perth */ + {"Asia/Taipei", "Taipei"}, /* S (GMT+08:00) Taipei */ + {"Asia/Singapore", "Singapore"}, /* S (GMT+08:00) Kuala Lumpur, Singapore */ + {"Asia/Shanghai", "China"}, /* S (GMT+08:00) Beijing, Chongqing, Hong Kong, Urumqi */ + {"Asia/Hong_Kong", "China"}, /* S (GMT+08:00) Beijing, Chongqing, Hong Kong, Urumqi */ + {"Asia/Ulaanbaatar", "North Asia East"}, /* D (GMT+08:00) Irkutsk, Ulaan Bataar */ + {"Asia/Irkutsk", "North Asia East"}, /* D (GMT+08:00) Irkutsk, Ulaan Bataar */ + + {"Asia/Tokyo", "Tokyo"}, /* S (GMT+09:00) Osaka, Sapporo, Tokyo */ + {"Asia/Seoul", "Korea"}, /* S (GMT+09:00) Seoul */ + {"Asia/Yakutsk", "Yakutsk"}, /* D (GMT+09:00) Yakutsk */ + + {"Australia/Darwin", "AUS Central"}, /* S (GMT+09:30) Darwin */ + {"Australia/Adelaide", "Cen. Australia"}, /* D (GMT+09:30) Adelaide */ + + {"Pacific/Guam", "West Pacific"}, /* S (GMT+10:00) Guam, Port Moresby */ + {"Australia/Brisbane", "E. Australia"}, /* S (GMT+10:00) Brisbane */ + {"Asia/Vladivostok", "Vladivostok"}, /* D (GMT+10:00) Vladivostok */ + {"Australia/Hobart", "Tasmania"}, /* D (GMT+10:00) Hobart */ + {"Australia/Sydney", "AUS Eastern"}, /* D (GMT+10:00) Canberra, Melbourne, Sydney */ + + {"Asia/Guadalcanal", "Central Pacific"}, /* S (GMT+11:00) Magadan, Solomon Is., New Caledonia */ + {"Asia/Magadan", "Central Pacific"}, /* S (GMT+11:00) Magadan, Solomon Is., New Caledonia */ + + {"Pacific/Fiji", "Fiji"}, /* S (GMT+12:00) Fiji, Kamchatka, Marshall Is. */ + {"Pacific/Auckland", "New Zealand"}, /* D (GMT+12:00) Auckland, Wellington */ + + {"Pacific/Tongatapu", "Tonga"}, /* S (GMT+13:00) Nuku'alofa */ + NULL, NULL +}; + +/** + * If a lookup fails, we attempt to remap certain Windows ids to + * alternate Windows ids. If the alternate listed here begins with + * '-', we use it as is (without the '-'). If it begins with '+', we + * append a " Standard Time" if appropriate. + */ +static const WindowsZoneRemap ZONE_REMAP[] = { + "Central European", "-Warsaw", + "Central Europe", "-Prague Bratislava", + "China", "-Beijing", + + "Greenwich", "+GMT", + "GTB", "+GFT", + "Arab", "+Saudi Arabia", + "SE Asia", "+Bangkok", + "AUS Eastern", "+Sydney", + "Mountain Standard Time (Mexico)", "-Mexico Standard Time 2", + "Central Standard Time (Mexico)", "+Mexico", + NULL, NULL, +}; + +static int32_t fWinType = -1; + +static int32_t detectWindowsType() +{ + int32_t winType; + LONG result; + HKEY hkey; + + /* Detect the version of windows by trying to open a sequence of + probe keys. We don't use the OS version API because what we + really want to know is how the registry is laid out. + Specifically, is it 9x/Me or not, and is it "GMT" or "GMT + Standard Time". */ + for (winType = 0; winType < 2; winType += 1) { + result = RegOpenKeyExA(HKEY_LOCAL_MACHINE, + WIN_TYPE_PROBE_REGKEY[winType], + 0, + KEY_QUERY_VALUE, + &hkey); + RegCloseKey(hkey); + + if (result == ERROR_SUCCESS) { + break; + } + } + + return winType; +} + +/* + * TODO: Binary search sorted ZONE_MAP... + * (u_detectWindowsTimeZone() needs them sorted by offset...) + */ +static const char *findWindowsZoneID(const UChar *icuid, int32_t length) +{ + char stackBuffer[ICUID_STACK_BUFFER_SIZE]; + char *buffer = stackBuffer; + const char *result = NULL; + int i; + + /* + * NOTE: >= because length doesn't include + * trailing null. + */ + if (length >= ICUID_STACK_BUFFER_SIZE) { + buffer = NEW_ARRAY(char, length + 1); + } + + u_UCharsToChars(icuid, buffer, length); + buffer[length] = '\0'; + + for (i = 0; ZONE_MAP[i].icuid != NULL; i += 1) { + if (uprv_strcmp(buffer, ZONE_MAP[i].icuid) == 0) { + result = ZONE_MAP[i].winid; + break; + } + } + + if (buffer != stackBuffer) { + DELETE_ARRAY(buffer); + } + + return result; +} + +static LONG openTZRegKey(HKEY *hkey, const char *winid) +{ + char subKeyName[96]; /* TODO: why 96?? */ + char *name; + LONG result; + + /* TODO: This isn't thread safe, but it's probably good enough. */ + if (fWinType < 0) { + fWinType = detectWindowsType(); + } + + uprv_strcpy(subKeyName, TZ_REGKEY[(fWinType == WIN_9X_ME_TYPE) ? 0 : 1]); + name = &subKeyName[strlen(subKeyName)]; + uprv_strcat(subKeyName, winid); + + if (fWinType != WIN_9X_ME_TYPE && + (winid[strlen(winid) - 1] != '2') && + (winid[strlen(winid) - 1] != ')') && + !(fWinType == WIN_NT_TYPE && strcmp(winid, "GMT") == 0)) { + uprv_strcat(subKeyName, STANDARD_TIME_REGKEY); + } + + result = RegOpenKeyExA(HKEY_LOCAL_MACHINE, + subKeyName, + 0, + KEY_QUERY_VALUE, + hkey); + + if (result != ERROR_SUCCESS) { + int i; + + /* If the primary lookup fails, try to remap the Windows zone + ID, according to the remapping table. */ + for (i=0; ZONE_REMAP[i].winid; i++) { + if (uprv_strcmp(winid, ZONE_REMAP[i].winid) == 0) { + uprv_strcpy(name, ZONE_REMAP[i].altwinid + 1); + if (*(ZONE_REMAP[i].altwinid) == '+' && fWinType != WIN_9X_ME_TYPE) { + uprv_strcat(subKeyName, STANDARD_TIME_REGKEY); + } + return RegOpenKeyExA(HKEY_LOCAL_MACHINE, + subKeyName, + 0, + KEY_QUERY_VALUE, + hkey); + } + } + } + + return result; +} + +static LONG getTZI(const char *winid, TZI *tzi) +{ + DWORD cbData = sizeof(TZI); + LONG result; + HKEY hkey; + + result = openTZRegKey(&hkey, winid); + + if (result == ERROR_SUCCESS) { + result = RegQueryValueExA(hkey, + TZI_REGKEY, + NULL, + NULL, + (LPBYTE)tzi, + &cbData); + + } + + RegCloseKey(hkey); + + return result; +} + +U_CAPI UBool U_EXPORT2 +uprv_getWindowsTimeZoneInfo(TIME_ZONE_INFORMATION *zoneInfo, const UChar *icuid, int32_t length) +{ + const char *winid; + TZI tzi; + LONG result; + + winid = findWindowsZoneID(icuid, length); + + if (winid != NULL) { + result = getTZI(winid, &tzi); + + if (result == ERROR_SUCCESS) { + zoneInfo->Bias = tzi.bias; + zoneInfo->DaylightBias = tzi.daylightBias; + zoneInfo->StandardBias = tzi.standardBias; + zoneInfo->DaylightDate = tzi.daylightDate; + zoneInfo->StandardDate = tzi.standardDate; + + return TRUE; + } + } + + return FALSE; +} + +/* + This code attempts to detect the Windows time zone, as set in the + Windows Date and Time control panel. It attempts to work on + multiple flavors of Windows (9x, Me, NT, 2000, XP) and on localized + installs. It works by directly interrogating the registry and + comparing the data there with the data returned by the + GetTimeZoneInformation API, along with some other strategies. The + registry contains time zone data under one of two keys (depending on + the flavor of Windows): + + HKLM\SOFTWARE\Microsoft\Windows\CurrentVersion\Time Zones\ + HKLM\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Time Zones\ + + Under this key are several subkeys, one for each time zone. These + subkeys are named "Pacific" on Win9x/Me and "Pacific Standard Time" + on WinNT/2k/XP. There are some other wrinkles; see the code for + details. The subkey name is NOT LOCALIZED, allowing us to support + localized installs. + + Under the subkey are data values. We care about: + + Std Standard time display name, localized + TZI Binary block of data + + The TZI data is of particular interest. It contains the offset, two + more offsets for standard and daylight time, and the start and end + rules. This is the same data returned by the GetTimeZoneInformation + API. The API may modify the data on the way out, so we have to be + careful, but essentially we do a binary comparison against the TZI + blocks of various registry keys. When we find a match, we know what + time zone Windows is set to. Since the registry key is not + localized, we can then translate the key through a simple table + lookup into the corresponding ICU time zone. + + This strategy doesn't always work because there are zones which + share an offset and rules, so more than one TZI block will match. + For example, both Tokyo and Seoul are at GMT+9 with no DST rules; + their TZI blocks are identical. For these cases, we fall back to a + name lookup. We attempt to match the display name as stored in the + registry for the current zone to the display name stored in the + registry for various Windows zones. By comparing the registry data + directly we avoid conversion complications. + + Author: Alan Liu + Since: ICU 2.6 + Based on original code by Carl Brown +*/ + +/** + * Main Windows time zone detection function. Returns the Windows + * time zone, translated to an ICU time zone, or NULL upon failure. + */ +U_CFUNC const char* U_EXPORT2 +uprv_detectWindowsTimeZone() { + LONG result; + HKEY hkey; + TZI tziKey; + TZI tziReg; + TIME_ZONE_INFORMATION apiTZI; + int firstMatch, lastMatch; + int j; + + /* Obtain TIME_ZONE_INFORMATION from the API, and then convert it + to TZI. We could also interrogate the registry directly; we do + this below if needed. */ + uprv_memset(&apiTZI, 0, sizeof(apiTZI)); + uprv_memset(&tziKey, 0, sizeof(tziKey)); + uprv_memset(&tziReg, 0, sizeof(tziReg)); + GetTimeZoneInformation(&apiTZI); + tziKey.bias = apiTZI.Bias; + uprv_memcpy((char *)&tziKey.standardDate, (char*)&apiTZI.StandardDate, + sizeof(apiTZI.StandardDate)); + uprv_memcpy((char *)&tziKey.daylightDate, (char*)&apiTZI.DaylightDate, + sizeof(apiTZI.DaylightDate)); + + /* For each zone that can be identified by Offset+Rules, see if we + have a match. Continue scanning after finding a match, + recording the index of the first and the last match. We have + to do this because some zones are not unique under + Offset+Rules. */ + firstMatch = -1; + lastMatch = -1; + for (j=0; ZONE_MAP[j].icuid; j++) { + result = getTZI(ZONE_MAP[j].winid, &tziReg); + + if (result == ERROR_SUCCESS) { + /* Assume that offsets are grouped together, and bail out + when we've scanned everything with a matching + offset. */ + if (firstMatch >= 0 && tziKey.bias != tziReg.bias) { + break; + } + + /* Windows alters the DaylightBias in some situations. + Using the bias and the rules suffices, so overwrite + these unreliable fields. */ + tziKey.standardBias = tziReg.standardBias; + tziKey.daylightBias = tziReg.daylightBias; + + if (uprv_memcmp((char *)&tziKey, (char*)&tziReg, sizeof(tziKey)) == 0) { + if (firstMatch < 0) { + firstMatch = j; + } + + lastMatch = j; + } + } + } + + /* This should never happen; if it does it means our table doesn't + match Windows AT ALL, perhaps because this is post-XP? */ + if (firstMatch < 0) { + return NULL; + } + + if (firstMatch != lastMatch) { + char stdName[32]; + DWORD stdNameSize; + char stdRegName[64]; + DWORD stdRegNameSize; + + /* Offset+Rules lookup yielded >= 2 matches. Try to match the + localized display name. Get the name from the registry + (not the API). This avoids conversion issues. Use the + standard name, since Windows modifies the daylight name to + match the standard name if there is no DST. */ + if (RegOpenKeyExA(HKEY_LOCAL_MACHINE, + CURRENT_ZONE_REGKEY, + 0, + KEY_QUERY_VALUE, + &hkey) == ERROR_SUCCESS) + { + stdNameSize = sizeof(stdName); + result = RegQueryValueExA(hkey, + STANDARD_NAME_REGKEY, + NULL, + NULL, + (LPBYTE)stdName, + &stdNameSize); + RegCloseKey(hkey); + + /* + * Scan through the Windows time zone data in the registry + * again (just the range of zones with matching TZIs) and + * look for a standard display name match. + */ + for (j = firstMatch; j <= lastMatch; j += 1) { + stdRegNameSize = sizeof(stdRegName); + result = openTZRegKey(&hkey, ZONE_MAP[j].winid); + + if (result == ERROR_SUCCESS) { + result = RegQueryValueExA(hkey, + STD_REGKEY, + NULL, + NULL, + (LPBYTE)stdRegName, + &stdRegNameSize); + } + + RegCloseKey(hkey); + + if (result == ERROR_SUCCESS && + stdRegNameSize == stdNameSize && + uprv_memcmp(stdName, stdRegName, stdNameSize) == 0) + { + firstMatch = j; /* record the match */ + break; + } + } + } else { + RegCloseKey(hkey); /* should never get here */ + } + } + + return ZONE_MAP[firstMatch].icuid; +} + +#endif /* #ifdef U_WINDOWS */ diff --git a/Build/source/libs/icu/icu-xetex/common/wintz.h b/Build/source/libs/icu/icu-xetex/common/wintz.h new file mode 100644 index 00000000000..0704953f46e --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/wintz.h @@ -0,0 +1,37 @@ +/* +******************************************************************************** +* Copyright (C) 2005-2007, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************** +* +* File WINTZ.H +* +******************************************************************************** +*/ + +#ifndef __WINTZ +#define __WINTZ + +#include "unicode/utypes.h" + +#ifdef U_WINDOWS + +/** + * \file + * \brief C API: Utilities for dealing w/ Windows time zones. + */ + +U_CDECL_BEGIN +/* Forward declarations for Windows types... */ +typedef struct _TIME_ZONE_INFORMATION TIME_ZONE_INFORMATION; +U_CDECL_END + +U_CAPI UBool U_EXPORT2 +uprv_getWindowsTimeZoneInfo(TIME_ZONE_INFORMATION *zoneInfo, const UChar *icuid, int32_t length); + +U_CFUNC const char* U_EXPORT2 +uprv_detectWindowsTimeZone(); + +#endif /* #ifdef U_WINDOWS */ + +#endif /* __WINTZ */ -- cgit v1.2.3