diff options
Diffstat (limited to 'Master/tlpkg/tlperl/lib/CORE/utfebcdic.h')
-rw-r--r-- | Master/tlpkg/tlperl/lib/CORE/utfebcdic.h | 73 |
1 files changed, 16 insertions, 57 deletions
diff --git a/Master/tlpkg/tlperl/lib/CORE/utfebcdic.h b/Master/tlpkg/tlperl/lib/CORE/utfebcdic.h index c2f0788cc4e..97b8f7001a5 100644 --- a/Master/tlpkg/tlperl/lib/CORE/utfebcdic.h +++ b/Master/tlpkg/tlperl/lib/CORE/utfebcdic.h @@ -121,22 +121,8 @@ START_EXTERN_C -#ifdef DOINIT - #include "ebcdic_tables.h" -#else -EXTCONST U8 PL_utf8skip[]; -EXTCONST U8 PL_e2utf[]; -EXTCONST U8 PL_utf2e[]; -EXTCONST U8 PL_e2a[]; -EXTCONST U8 PL_a2e[]; -EXTCONST U8 PL_fold[]; -EXTCONST U8 PL_fold_latin1[]; -EXTCONST U8 PL_latin1_lc[]; -EXTCONST U8 PL_mod_latin1_uc[]; -#endif - END_EXTERN_C /* EBCDIC-happy ways of converting native code to UTF-8 */ @@ -216,58 +202,33 @@ possible to UTF-8-encode a single code point in different ways, but that is explicitly forbidden, and the shortest possible encoding should always be used (and that is what Perl does). */ -/* This is a fundamental property of UTF-EBCDIC */ -#define OFFUNI_IS_INVARIANT(c) (((UV)(c)) < 0xA0) - -/* It turns out that on EBCDIC platforms, the invariants are the characters - * that have ASCII equivalents, plus the C1 controls. Since the C0 controls - * and DELETE are ASCII, this is the same as: (isASCII(uv) || isCNTRL_L1(uv)) - * */ -#define UVCHR_IS_INVARIANT(uv) cBOOL(FITS_IN_8_BITS(uv) \ - && (PL_charclass[(U8) (uv)] & (_CC_mask(_CC_ASCII) | _CC_mask(_CC_CNTRL)))) - -/* UTF-EBCDIC semantic macros - We used to transform back into I8 and then - * compare, but now only have to do a single lookup by using a bit in - * l1_char_class_tab.h. - * Comments as to the meaning of each are given at their corresponding utf8.h - * definitions. */ +/* It turns out that just this one number is sufficient to derive all the basic + * macros for UTF-8 and UTF-EBCDIC. Everything follows from the fact that + * there are 6 bits of real information in a UTF-8 continuation byte vs. 5 bits + * in a UTF-EBCDIC one. */ -#define UTF8_IS_START(c) _generic_isCC(c, _CC_UTF8_IS_START) - -#define UTF_IS_CONTINUATION_MASK 0xE0 - -#define UTF8_IS_CONTINUATION(c) _generic_isCC(c, _CC_UTF8_IS_CONTINUATION) - -/* The above instead could be written as this: -#define UTF8_IS_CONTINUATION(c) \ - (((NATIVE_UTF8_TO_I8(c) & UTF_IS_CONTINUATION_MASK) \ - == UTF_CONTINUATION_MARK) - */ - -/* Equivalent to ! UVCHR_IS_INVARIANT(c) */ -#define UTF8_IS_CONTINUED(c) cBOOL(FITS_IN_8_BITS(c) \ - && ! (PL_charclass[(U8) (c)] & (_CC_mask(_CC_ASCII) | _CC_mask(_CC_CNTRL)))) +#define UTF_ACCUMULATION_SHIFT 5 -#define UTF8_IS_DOWNGRADEABLE_START(c) _generic_isCC(c, \ - _CC_UTF8_IS_DOWNGRADEABLE_START) +/* Also needed is how perl handles a start byte of 8 one bits. The decision + * was made to just append the minimal number of bytes after that so that code + * points up to 64 bits wide could be represented. In UTF-8, that was an extra + * 5 bytes, and in UTF-EBCDIC it's 6. The result is in UTF8_MAXBYTES defined + * above. This implementation has the advantage that you have everything you + * need in the first byte. Other ways of extending UTF-8 have been devised, + * some to arbitrarily high code points. But they require looking at the next + * byte(s) when the first one is 8 one bits. */ -/* Equivalent to (UTF8_IS_START(c) && ! UTF8_IS_DOWNGRADEABLE_START(c)) - * Makes sure that the START bit is set and the DOWNGRADEABLE bit isn't */ -#define UTF8_IS_ABOVE_LATIN1(c) cBOOL(FITS_IN_8_BITS(c) \ - && ((PL_charclass[(U8) (c)] & ( _CC_mask(_CC_UTF8_IS_START) \ - |_CC_mask(_CC_UTF8_IS_DOWNGRADEABLE_START))) \ - == _CC_mask(_CC_UTF8_IS_START))) +/* These others are for efficiency or for other decisions we've made */ #define isUTF8_POSSIBLY_PROBLEMATIC(c) \ _generic_isCC(c, _CC_UTF8_START_BYTE_IS_FOR_AT_LEAST_SURROGATE) -#define UTF_CONTINUATION_MARK 0xA0 -#define UTF_ACCUMULATION_SHIFT 5 - /* ^? is defined to be APC on EBCDIC systems. See the definition of toCTRL() * for more */ #define QUESTION_MARK_CTRL LATIN1_TO_NATIVE(0x9F) +#define UNICODE_IS_PERL_EXTENDED(uv) UNLIKELY((UV) (uv) > 0x3FFFFFFF) + /* Helper macros for isUTF8_CHAR_foo, so use those instead of this. These were * generated by regen/regcharclass.pl, and then moved here. Then they were * hand-edited to add some LIKELY() calls, presuming that malformations are @@ -511,8 +472,6 @@ explicitly forbidden, and the shortest possible encoding should always be used * has this start byte (expressed in I8) as the maximum */ #define _IS_UTF8_CHAR_HIGHEST_START_BYTE 0xF9 -#define UNICODE_IS_PERL_EXTENDED(uv) UNLIKELY((UV) (uv) > 0x3FFFFFFF) - /* * ex: set ts=8 sts=4 sw=4 et: */ |