summaryrefslogtreecommitdiff
path: root/Master/tlpkg/tlperl/lib/CORE/utfebcdic.h
diff options
context:
space:
mode:
Diffstat (limited to 'Master/tlpkg/tlperl/lib/CORE/utfebcdic.h')
-rw-r--r--Master/tlpkg/tlperl/lib/CORE/utfebcdic.h73
1 files changed, 16 insertions, 57 deletions
diff --git a/Master/tlpkg/tlperl/lib/CORE/utfebcdic.h b/Master/tlpkg/tlperl/lib/CORE/utfebcdic.h
index c2f0788cc4e..97b8f7001a5 100644
--- a/Master/tlpkg/tlperl/lib/CORE/utfebcdic.h
+++ b/Master/tlpkg/tlperl/lib/CORE/utfebcdic.h
@@ -121,22 +121,8 @@
START_EXTERN_C
-#ifdef DOINIT
-
#include "ebcdic_tables.h"
-#else
-EXTCONST U8 PL_utf8skip[];
-EXTCONST U8 PL_e2utf[];
-EXTCONST U8 PL_utf2e[];
-EXTCONST U8 PL_e2a[];
-EXTCONST U8 PL_a2e[];
-EXTCONST U8 PL_fold[];
-EXTCONST U8 PL_fold_latin1[];
-EXTCONST U8 PL_latin1_lc[];
-EXTCONST U8 PL_mod_latin1_uc[];
-#endif
-
END_EXTERN_C
/* EBCDIC-happy ways of converting native code to UTF-8 */
@@ -216,58 +202,33 @@ possible to UTF-8-encode a single code point in different ways, but that is
explicitly forbidden, and the shortest possible encoding should always be used
(and that is what Perl does). */
-/* This is a fundamental property of UTF-EBCDIC */
-#define OFFUNI_IS_INVARIANT(c) (((UV)(c)) < 0xA0)
-
-/* It turns out that on EBCDIC platforms, the invariants are the characters
- * that have ASCII equivalents, plus the C1 controls. Since the C0 controls
- * and DELETE are ASCII, this is the same as: (isASCII(uv) || isCNTRL_L1(uv))
- * */
-#define UVCHR_IS_INVARIANT(uv) cBOOL(FITS_IN_8_BITS(uv) \
- && (PL_charclass[(U8) (uv)] & (_CC_mask(_CC_ASCII) | _CC_mask(_CC_CNTRL))))
-
-/* UTF-EBCDIC semantic macros - We used to transform back into I8 and then
- * compare, but now only have to do a single lookup by using a bit in
- * l1_char_class_tab.h.
- * Comments as to the meaning of each are given at their corresponding utf8.h
- * definitions. */
+/* It turns out that just this one number is sufficient to derive all the basic
+ * macros for UTF-8 and UTF-EBCDIC. Everything follows from the fact that
+ * there are 6 bits of real information in a UTF-8 continuation byte vs. 5 bits
+ * in a UTF-EBCDIC one. */
-#define UTF8_IS_START(c) _generic_isCC(c, _CC_UTF8_IS_START)
-
-#define UTF_IS_CONTINUATION_MASK 0xE0
-
-#define UTF8_IS_CONTINUATION(c) _generic_isCC(c, _CC_UTF8_IS_CONTINUATION)
-
-/* The above instead could be written as this:
-#define UTF8_IS_CONTINUATION(c) \
- (((NATIVE_UTF8_TO_I8(c) & UTF_IS_CONTINUATION_MASK) \
- == UTF_CONTINUATION_MARK)
- */
-
-/* Equivalent to ! UVCHR_IS_INVARIANT(c) */
-#define UTF8_IS_CONTINUED(c) cBOOL(FITS_IN_8_BITS(c) \
- && ! (PL_charclass[(U8) (c)] & (_CC_mask(_CC_ASCII) | _CC_mask(_CC_CNTRL))))
+#define UTF_ACCUMULATION_SHIFT 5
-#define UTF8_IS_DOWNGRADEABLE_START(c) _generic_isCC(c, \
- _CC_UTF8_IS_DOWNGRADEABLE_START)
+/* Also needed is how perl handles a start byte of 8 one bits. The decision
+ * was made to just append the minimal number of bytes after that so that code
+ * points up to 64 bits wide could be represented. In UTF-8, that was an extra
+ * 5 bytes, and in UTF-EBCDIC it's 6. The result is in UTF8_MAXBYTES defined
+ * above. This implementation has the advantage that you have everything you
+ * need in the first byte. Other ways of extending UTF-8 have been devised,
+ * some to arbitrarily high code points. But they require looking at the next
+ * byte(s) when the first one is 8 one bits. */
-/* Equivalent to (UTF8_IS_START(c) && ! UTF8_IS_DOWNGRADEABLE_START(c))
- * Makes sure that the START bit is set and the DOWNGRADEABLE bit isn't */
-#define UTF8_IS_ABOVE_LATIN1(c) cBOOL(FITS_IN_8_BITS(c) \
- && ((PL_charclass[(U8) (c)] & ( _CC_mask(_CC_UTF8_IS_START) \
- |_CC_mask(_CC_UTF8_IS_DOWNGRADEABLE_START))) \
- == _CC_mask(_CC_UTF8_IS_START)))
+/* These others are for efficiency or for other decisions we've made */
#define isUTF8_POSSIBLY_PROBLEMATIC(c) \
_generic_isCC(c, _CC_UTF8_START_BYTE_IS_FOR_AT_LEAST_SURROGATE)
-#define UTF_CONTINUATION_MARK 0xA0
-#define UTF_ACCUMULATION_SHIFT 5
-
/* ^? is defined to be APC on EBCDIC systems. See the definition of toCTRL()
* for more */
#define QUESTION_MARK_CTRL LATIN1_TO_NATIVE(0x9F)
+#define UNICODE_IS_PERL_EXTENDED(uv) UNLIKELY((UV) (uv) > 0x3FFFFFFF)
+
/* Helper macros for isUTF8_CHAR_foo, so use those instead of this. These were
* generated by regen/regcharclass.pl, and then moved here. Then they were
* hand-edited to add some LIKELY() calls, presuming that malformations are
@@ -511,8 +472,6 @@ explicitly forbidden, and the shortest possible encoding should always be used
* has this start byte (expressed in I8) as the maximum */
#define _IS_UTF8_CHAR_HIGHEST_START_BYTE 0xF9
-#define UNICODE_IS_PERL_EXTENDED(uv) UNLIKELY((UV) (uv) > 0x3FFFFFFF)
-
/*
* ex: set ts=8 sts=4 sw=4 et:
*/