diff options
Diffstat (limited to 'Master/tlpkg/tlperl/lib/CORE/utfebcdic.h')
-rw-r--r-- | Master/tlpkg/tlperl/lib/CORE/utfebcdic.h | 161 |
1 files changed, 94 insertions, 67 deletions
diff --git a/Master/tlpkg/tlperl/lib/CORE/utfebcdic.h b/Master/tlpkg/tlperl/lib/CORE/utfebcdic.h index 24101edf6f0..10b666afe27 100644 --- a/Master/tlpkg/tlperl/lib/CORE/utfebcdic.h +++ b/Master/tlpkg/tlperl/lib/CORE/utfebcdic.h @@ -11,10 +11,12 @@ * http://www.unicode.org/unicode/reports/tr16 * * To summarize, the way it works is: - * To convert an EBCDIC character to UTF-EBCDIC: - * 1) convert to Unicode. The table in the generated file 'ebcdic_tables.h' - * that does this for EBCDIC bytes is PL_e2a (with inverse PL_a2e). The - * 'a' stands for ASCII platform, meaning latin1. + * To convert an EBCDIC code point to UTF-EBCDIC: + * 1) convert to Unicode. No conversion is necesary for code points above + * 255, as Unicode and EBCDIC are identical in this range. For smaller + * code points, the conversion is done by lookup in the PL_e2a table (with + * inverse PL_a2e) in the generated file 'ebcdic_tables.h'. The 'a' + * stands for ASCII platform, meaning 0-255 Unicode. * 2) convert that to a utf8-like string called I8 ('I' stands for * intermediate) with variant characters occupying multiple bytes. This * step is similar to the utf8-creating step from Unicode, but the details @@ -25,17 +27,25 @@ * invariant byte starts with 0 starts with 0 or 100 * continuation byte starts with 10 starts with 101 * start byte same in both: if the code point requires N bytes, - * then the leading N bits are 1, followed by a 0. (No - * trailing 0 for the very largest possible allocation - * in I8, far beyond the current Unicode standard's - * max, as shown in the comment later in this file.) + * then the leading N bits are 1, followed by a 0. If + * all 8 bits in the first byte are 1, the code point + * will occupy 14 bytes (compared to 13 in Perl's + * extended UTF-8). This is incompatible with what + * tr16 implies should be the representation of code + * points 2**30 and above, but allows Perl to be able + * to represent all code points that fit in a 64-bit + * word in either our extended UTF-EBCDIC or UTF-8. * 3) Use the algorithm in tr16 to convert each byte from step 2 into * final UTF-EBCDIC. This is done by table lookup from a table * constructed from the algorithm, reproduced in ebcdic_tables.h as * PL_utf2e, with its inverse being PL_e2utf. They are constructed so that * all EBCDIC invariants remain invariant, but no others do, and the first * byte of a variant will always have its upper bit set. But note that - * the upper bit of some invariants is also 1. + * the upper bit of some invariants is also 1. The table also is designed + * so that lexically comparing two UTF-EBCDIC-variant characters yields + * the Unicode code point order. (To get native code point order, one has + * to convert the latin1-range characters to their native code point + * value.) * * For example, the ordinal value of 'A' is 193 in EBCDIC, and also is 193 in * UTF-EBCDIC. Step 1) converts it to 65, Step 2 leaves it at 65, and Step 3 @@ -87,7 +97,8 @@ * aren't equivalent to ASCII characters nor C1 controls form the set of * continuation bytes; the remaining 64 non-ASCII, non-control code points form * the potential start bytes, in order. (However, the first 5 of these lead to - * malformed overlongs, so there really are only 59 start bytes.) Hence the + * malformed overlongs, so there really are only 59 start bytes, and the first + * three of the 59 are the start bytes for the Latin1 range.) Hence the * UTF-EBCDIC for the smallest variant code point, 0x160, will have likely 0x41 * as its continuation byte, provided 0x41 isn't an ASCII or C1 equivalent. * And its start byte will be the code point that is 37 (32+5) non-ASCII, @@ -130,20 +141,33 @@ END_EXTERN_C /* EBCDIC-happy ways of converting native code to UTF-8 */ -#define NATIVE_TO_LATIN1(ch) PL_e2a[(U8)(ch)] -#define LATIN1_TO_NATIVE(ch) PL_a2e[(U8)(ch)] +/* Use these when ch is known to be < 256 */ +#define NATIVE_TO_LATIN1(ch) (__ASSERT_(FITS_IN_8_BITS(ch)) PL_e2a[(U8)(ch)]) +#define LATIN1_TO_NATIVE(ch) (__ASSERT_(FITS_IN_8_BITS(ch)) PL_a2e[(U8)(ch)]) -#define NATIVE_UTF8_TO_I8(ch) PL_e2utf[(U8)(ch)] -#define I8_TO_NATIVE_UTF8(ch) PL_utf2e[(U8)(ch)] +/* Use these on bytes */ +#define NATIVE_UTF8_TO_I8(b) (__ASSERT_(FITS_IN_8_BITS(b)) PL_e2utf[(U8)(b)]) +#define I8_TO_NATIVE_UTF8(b) (__ASSERT_(FITS_IN_8_BITS(b)) PL_utf2e[(U8)(b)]) /* Transforms in wide UV chars */ -#define NATIVE_TO_UNI(ch) (((ch) > 255) ? (ch) : NATIVE_TO_LATIN1(ch)) -#define UNI_TO_NATIVE(ch) (((ch) > 255) ? (ch) : LATIN1_TO_NATIVE(ch)) +#define NATIVE_TO_UNI(ch) (FITS_IN_8_BITS(ch) ? NATIVE_TO_LATIN1(ch) : (UV) (ch)) +#define UNI_TO_NATIVE(ch) (FITS_IN_8_BITS(ch) ? LATIN1_TO_NATIVE(ch) : (UV) (ch)) + +/* How wide can a single UTF-8 encoded character become in bytes. */ +/* NOTE: Strictly speaking Perl's UTF-8 should not be called UTF-8 since UTF-8 + * is an encoding of Unicode, and Unicode's upper limit, 0x10FFFF, can be + * expressed with 5 bytes. However, Perl thinks of UTF-8 as a way to encode + * non-negative integers in a binary format, even those above Unicode. 14 is + * the smallest number that covers 2**64 + * + * WARNING: This number must be in sync with the value in + * regen/charset_translations.pl. */ +#define UTF8_MAXBYTES 14 /* - The following table is adapted from tr16, it shows I8 encoding of Unicode code points. + The following table is adapted from tr16, it shows the I8 encoding of Unicode code points. - Unicode Bit pattern 1st Byte 2nd Byte 3rd Byte 4th Byte 5th Byte 6th Byte 7th byte + Unicode U32 Bit pattern 1st Byte 2nd Byte 3rd Byte 4th Byte 5th Byte 6th Byte 7th Byte U+0000..U+007F 000000000xxxxxxx 0xxxxxxx U+0080..U+009F 00000000100xxxxx 100xxxxx U+00A0..U+03FF 000000yyyyyxxxxx 110yyyyy 101xxxxx @@ -151,68 +175,71 @@ END_EXTERN_C U+4000..U+3FFFF 0wwwzzzzzyyyyyxxxxx 11110www 101zzzzz 101yyyyy 101xxxxx U+40000..U+3FFFFF 0vvwwwwwzzzzzyyyyyxxxxx 111110vv 101wwwww 101zzzzz 101yyyyy 101xxxxx U+400000..U+3FFFFFF 0uvvvvvwwwwwzzzzzyyyyyxxxxx 1111110u 101vvvvv 101wwwww 101zzzzz 101yyyyy 101xxxxx - U+4000000..U+7FFFFFFF 0tuuuuuvvvvvwwwwwzzzzzyyyyyxxxxx 1111111t 101uuuuu 101vvvvv 101wwwww 101zzzzz 101yyyyy 101xxxxx + U+4000000..U+3FFFFFFF 00uuuuuvvvvvwwwwwzzzzzyyyyyxxxxx 11111110 101uuuuu 101vvvvv 101wwwww 101zzzzz 101yyyyy 101xxxxx - Note: The I8 transformation is valid for UCS-4 values X'0' to - X'7FFFFFFF' (the full extent of ISO/IEC 10646 coding space). +Beyond this, Perl uses an incompatible extension, similar to the one used in +regular UTF-8. There are now 14 bytes. A full 32 bits of information thus looks like this: + 1st Byte 2nd-7th 8th Byte 9th Byte 10th B 11th B 12th B 13th B 14th B +U+40000000..U+FFFFFFFF ttuuuuuvvvvvwwwwwzzzzzyyyyyxxxxx 11111111 10100000 101000tt 101uuuuu 101vvvvv 101wwwww 101zzzzz 101yyyyy 101xxxxx - */ +For 32-bit words, the 2nd through 7th bytes effectively function as leading +zeros. Above 32 bits, these fill up, with each byte yielding 5 bits of +information, so that with 13 continuation bytes, we can handle 65 bits, just +above what a 64 bit word can hold */ -/* Input is a true Unicode (not-native) code point */ -#define OFFUNISKIP(uv) ( (uv) < 0xA0 ? 1 : \ - (uv) < 0x400 ? 2 : \ - (uv) < 0x4000 ? 3 : \ - (uv) < 0x40000 ? 4 : \ - (uv) < 0x400000 ? 5 : \ - (uv) < 0x4000000 ? 6 : 7 ) -#define UNI_IS_INVARIANT(c) (((UV)(c)) < 0xA0) +/* This is a fundamental property of UTF-EBCDIC */ +#define OFFUNI_IS_INVARIANT(c) (((UV)(c)) < 0xA0) -/* UTF-EBCDIC semantic macros - transform back into I8 and then compare +/* It turns out that on EBCDIC platforms, the invariants are the characters + * that have ASCII equivalents, plus the C1 controls. Since the C0 controls + * and DELETE are ASCII, this is the same as: (isASCII(uv) || isCNTRL_L1(uv)) + * */ +#define UVCHR_IS_INVARIANT(uv) cBOOL(FITS_IN_8_BITS(uv) \ + && (PL_charclass[(U8) (uv)] & (_CC_mask(_CC_ASCII) | _CC_mask(_CC_CNTRL)))) + +/* UTF-EBCDIC semantic macros - We used to transform back into I8 and then + * compare, but now only have to do a single lookup by using a bit in + * l1_char_class_tab.h. * Comments as to the meaning of each are given at their corresponding utf8.h - * definitions */ - -#define UTF8_IS_START(c) (NATIVE_UTF8_TO_I8(c) >= 0xC5 \ - && NATIVE_UTF8_TO_I8(c) != 0xE0) -#define UTF8_IS_CONTINUATION(c) ((NATIVE_UTF8_TO_I8(c) & 0xE0) == 0xA0) -#define UTF8_IS_CONTINUED(c) (NATIVE_UTF8_TO_I8(c) >= 0xA0) - -#define UTF8_IS_DOWNGRADEABLE_START(c) (NATIVE_UTF8_TO_I8(c) >= 0xC5 \ - && NATIVE_UTF8_TO_I8(c) <= 0xC7) -/* Saying it this way adds a runtime test, but removes 2 run-time lookups */ -/*#define UTF8_IS_DOWNGRADEABLE_START(c) ((c) == I8_TO_NATIVE_UTF8(0xC5) \ - || (c) == I8_TO_NATIVE_UTF8(0xC6) \ - || (c) == I8_TO_NATIVE_UTF8(0xC7)) -*/ -#define UTF8_IS_ABOVE_LATIN1(c) (NATIVE_UTF8_TO_I8(c) >= 0xC8) - -/* Can't exceed 7 on EBCDIC platforms */ -#define UTF_START_MARK(len) (0xFF & (0xFE << (7-(len)))) - -#define UTF_START_MASK(len) (((len) >= 6) ? 0x01 : (0x1F >> ((len)-2))) -#define UTF_CONTINUATION_MARK 0xA0 -#define UTF_CONTINUATION_MASK ((U8)0x1f) -#define UTF_ACCUMULATION_SHIFT 5 + * definitions. */ -/* How wide can a single UTF-8 encoded character become in bytes. */ -/* NOTE: Strictly speaking Perl's UTF-8 should not be called UTF-8 since UTF-8 - * is an encoding of Unicode, and Unicode's upper limit, 0x10FFFF, can be - * expressed with 5 bytes. However, Perl thinks of UTF-8 as a way to encode - * non-negative integers in a binary format, even those above Unicode */ -#define UTF8_MAXBYTES 7 +#define UTF8_IS_START(c) _generic_isCC(c, _CC_UTF8_IS_START) + +#define UTF_IS_CONTINUATION_MASK 0xE0 + +#define UTF8_IS_CONTINUATION(c) _generic_isCC(c, _CC_UTF8_IS_CONTINUATION) + +/* The above instead could be written as this: +#define UTF8_IS_CONTINUATION(c) \ + (((NATIVE_UTF8_TO_I8(c) & UTF_IS_CONTINUATION_MASK) \ + == UTF_CONTINUATION_MARK) + */ + +/* Equivalent to ! UVCHR_IS_INVARIANT(c) */ +#define UTF8_IS_CONTINUED(c) cBOOL(FITS_IN_8_BITS(c) \ + && ! (PL_charclass[(U8) (c)] & (_CC_mask(_CC_ASCII) | _CC_mask(_CC_CNTRL)))) -/* The maximum number of UTF-8 bytes a single Unicode character can - * uppercase/lowercase/fold into. Unicode guarantees that the maximum - * expansion is 3 characters. On EBCDIC platforms, the highest Unicode - * character occupies 5 bytes, therefore this number is 15 */ -#define UTF8_MAXBYTES_CASE 15 +#define UTF8_IS_DOWNGRADEABLE_START(c) _generic_isCC(c, \ + _CC_UTF8_IS_DOWNGRADEABLE_START) + +/* Equivalent to (UTF8_IS_START(c) && ! UTF8_IS_DOWNGRADEABLE_START(c)) + * Makes sure that the START bit is set and the DOWNGRADEABLE bit isn't */ +#define UTF8_IS_ABOVE_LATIN1(c) cBOOL(FITS_IN_8_BITS(c) \ + && ((PL_charclass[(U8) (c)] & ( _CC_mask(_CC_UTF8_IS_START) \ + |_CC_mask(_CC_UTF8_IS_DOWNGRADEABLE_START))) \ + == _CC_mask(_CC_UTF8_IS_START))) + +#define isUTF8_POSSIBLY_PROBLEMATIC(c) \ + _generic_isCC(c, _CC_UTF8_START_BYTE_IS_FOR_AT_LEAST_SURROGATE) + +#define UTF_CONTINUATION_MARK 0xA0 +#define UTF_ACCUMULATION_SHIFT 5 /* ^? is defined to be APC on EBCDIC systems. See the definition of toCTRL() * for more */ #define QUESTION_MARK_CTRL LATIN1_TO_NATIVE(0x9F) -#define MAX_UTF8_TWO_BYTE 0x3FF - /* * ex: set ts=8 sts=4 sw=4 et: */ |