summaryrefslogtreecommitdiff
path: root/systems/texlive/tlnet/tlpkg/tlperl/lib/CORE/utf8.h
diff options
context:
space:
mode:
Diffstat (limited to 'systems/texlive/tlnet/tlpkg/tlperl/lib/CORE/utf8.h')
-rw-r--r--systems/texlive/tlnet/tlpkg/tlperl/lib/CORE/utf8.h852
1 files changed, 550 insertions, 302 deletions
diff --git a/systems/texlive/tlnet/tlpkg/tlperl/lib/CORE/utf8.h b/systems/texlive/tlnet/tlpkg/tlperl/lib/CORE/utf8.h
index 3bec01989f..3256cf649a 100644
--- a/systems/texlive/tlnet/tlpkg/tlperl/lib/CORE/utf8.h
+++ b/systems/texlive/tlnet/tlpkg/tlperl/lib/CORE/utf8.h
@@ -12,11 +12,103 @@
* You may distribute under the terms of either the GNU General Public
* License or the Artistic License, as specified in the README file.
*
+ * A note on nomenclature: The term UTF-8 is used loosely and inconsistently
+ * in Perl documentation. For one, perl uses an extension of UTF-8 to
+ * represent code points that Unicode considers illegal. For another, ASCII
+ * platform UTF-8 is usually conflated with EBCDIC platform UTF-EBCDIC, because
+ * outside some of the macros in this this file, the differences are hopefully
+ * invisible at the semantic level.
+ *
+ * UTF-EBCDIC has an isomorphic translation named I8 (for "Intermediate eight")
+ * which differs from UTF-8 only in a few details. It is often useful to
+ * translate UTF-EBCDIC into this form for processing. In general, macros and
+ * functions that are expecting their inputs to be either in I8 or UTF-8 are
+ * named UTF_foo (without an '8'), to indicate this.
+ *
+ * Unfortunately there are inconsistencies.
+ *
*/
#ifndef PERL_UTF8_H_ /* Guard against recursive inclusion */
#define PERL_UTF8_H_ 1
+/*
+=for apidoc Ay||utf8ness_t
+
+This typedef is used by several core functions that return PV strings, to
+indicate the UTF-8ness of those strings.
+
+(If you write a new function, you probably should instead return the PV in an
+SV with the UTF-8 flag of the SV properly set, rather than use this mechanism.)
+
+The possible values this can be are:
+
+=over
+
+=item C<UTF8NESS_YES>
+
+This means the string definitely should be treated as a sequence of
+UTF-8-encoded characters.
+
+Most code that needs to handle this typedef should be of the form:
+
+ if (utf8ness_flag == UTF8NESS_YES) {
+ treat as utf8; // like turning on an SV UTF-8 flag
+ }
+
+=item C<UTF8NESS_NO>
+
+This means the string definitely should be treated as a sequence of bytes, not
+encoded as UTF-8.
+
+=item C<UTF8NESS_IMMATERIAL>
+
+This means it is equally valid to treat the string as bytes, or as UTF-8
+characters; use whichever way you want. This happens when the string consists
+entirely of characters which have the same representation whether encoded in
+UTF-8 or not.
+
+=item C<UTF8NESS_UNKNOWN>
+
+This means it is unknown how the string should be treated. No core function
+will ever return this value to a non-core caller. Instead, it is used by the
+caller to initialize a variable to a non-legal value. A typical call will look like:
+
+ utf8ness_t string_is_utf8 = UTF8NESS_UNKNOWN
+ const char * string = foo(arg1, arg2, ..., &string_is_utf8);
+ if (string_is_utf8 == UTF8NESS_YES) {
+ do something for UTF-8;
+ }
+
+=back
+
+The following relationships hold between the enum values:
+
+=over
+
+=item S<C<0 E<lt>= I<enum value> E<lt>= UTF8NESS_IMMATERIAL>>
+
+the string may be treated in code as non-UTF8
+
+=item S<C<UTF8NESS_IMMATERIAL E<lt>= <I<enum value>>>
+
+the string may be treated in code as encoded in UTF-8
+
+=back
+
+=cut
+*/
+
+typedef enum {
+ UTF8NESS_NO = 0, /* Definitely not UTF-8 */
+ UTF8NESS_IMMATERIAL = 1, /* Representation is the same in UTF-8 as
+ not, so the UTF8ness doesn't actually
+ matter */
+ UTF8NESS_YES = 2, /* Defintely is UTF-8, wideness
+ unspecified */
+ UTF8NESS_UNKNOWN = -1, /* Undetermined so far */
+} utf8ness_t;
+
/* Use UTF-8 as the default script encoding?
* Turning this on will break scripts having non-UTF-8 binary
* data (such as Latin-1) in string literals. */
@@ -67,6 +159,15 @@ the string is invariant.
#define utf8n_to_uvchr_error(s, len, lenp, flags, errors) \
utf8n_to_uvchr_msgs(s, len, lenp, flags, errors, 0)
+#define utf16_to_utf8(p, d, bytelen, newlen) \
+ utf16_to_utf8_base(p, d, bytelen, newlen, 0, 1)
+#define utf16_to_utf8_reversed(p, d, bytelen, newlen) \
+ utf16_to_utf8_base(p, d, bytelen, newlen, 1, 0)
+#define utf8_to_utf16(p, d, bytelen, newlen) \
+ utf8_to_utf16_base(p, d, bytelen, newlen, 0, 1)
+#define utf8_to_utf16_reversed(p, d, bytelen, newlen) \
+ utf8_to_utf16_base(p, d, bytelen, newlen, 1, 0)
+
#define to_uni_fold(c, p, lenp) _to_uni_fold_flags(c, p, lenp, FOLD_FLAGS_FULL)
#define foldEQ_utf8(s1, pe1, l1, u1, s2, pe2, l2, u2) \
@@ -78,32 +179,31 @@ the string is invariant.
#define FOLDEQ_S1_FOLDS_SANE (1 << 4)
#define FOLDEQ_S2_FOLDS_SANE (1 << 5)
-#ifdef EBCDIC
-/* The equivalent of these macros but implementing UTF-EBCDIC
- are in the following header file:
- */
-
-#include "utfebcdic.h"
+/* This will be described more fully below, but it turns out that the
+ * fundamental difference between UTF-8 and UTF-EBCDIC is that the former has
+ * the upper 2 bits of a continuation byte be '10', and the latter has the
+ * upper 3 bits be '101', leaving 6 and 5 significant bits respectively.
+ *
+ * It is helpful to know the EBCDIC value on ASCII platforms, mainly to avoid
+ * some #ifdef's */
+#define UTF_EBCDIC_CONTINUATION_BYTE_INFO_BITS 5
-#else /* ! EBCDIC */
-START_EXTERN_C
+/* See explanation below at 'UTF8_MAXBYTES' */
+#define ASCII_PLATFORM_UTF8_MAXBYTES 13
-/*
+#ifdef EBCDIC
-=for apidoc AmnU|STRLEN|UTF8_MAXBYTES
+/* The equivalent of the next few macros but implementing UTF-EBCDIC are in the
+ * following header file: */
+# include "utfebcdic.h"
-The maximum width of a single UTF-8 encoded character, in bytes.
+# else /* ! EBCDIC */
-NOTE: Strictly speaking Perl's UTF-8 should not be called UTF-8 since UTF-8
-is an encoding of Unicode, and Unicode's upper limit, 0x10FFFF, can be
-expressed with 4 bytes. However, Perl thinks of UTF-8 as a way to encode
-non-negative integers in a binary format, even those above Unicode.
-
-=cut
- */
-#define UTF8_MAXBYTES 13
+START_EXTERN_C
-#ifdef DOINIT
+# ifndef DOINIT
+EXTCONST unsigned char PL_utf8skip[];
+# else
EXTCONST unsigned char PL_utf8skip[] = {
/* 0x00 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
/* 0x10 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
@@ -125,11 +225,9 @@ EXTCONST unsigned char PL_utf8skip[] = {
/* Perl extended (never was official UTF-8). Up to 36 bit */
/* 0xFE */ 7,
/* More extended, Up to 72 bits (64-bit + reserved) */
-/* 0xFF */ UTF8_MAXBYTES
+/* 0xFF */ ASCII_PLATFORM_UTF8_MAXBYTES
};
-#else
-EXTCONST unsigned char PL_utf8skip[];
-#endif
+# endif
END_EXTERN_C
@@ -178,8 +276,8 @@ adding no time nor space requirements to the implementation.
=cut
*/
-#define NATIVE_TO_LATIN1(ch) (__ASSERT_(FITS_IN_8_BITS(ch)) ((U8) ((ch) | 0)))
-#define LATIN1_TO_NATIVE(ch) (__ASSERT_(FITS_IN_8_BITS(ch)) ((U8) ((ch) | 0)))
+#define NATIVE_TO_LATIN1(ch) (__ASSERT_(FITS_IN_8_BITS(ch)) ((U8) (ch)))
+#define LATIN1_TO_NATIVE(ch) (__ASSERT_(FITS_IN_8_BITS(ch)) ((U8) (ch)))
/* I8 is an intermediate version of UTF-8 used only in UTF-EBCDIC. We thus
* consider it to be identical to UTF-8 on ASCII platforms. Strictly speaking
@@ -187,11 +285,11 @@ adding no time nor space requirements to the implementation.
* because they are 8-bit encodings that serve the same purpose in Perl, and
* rarely do we need to distinguish them. The term "NATIVE_UTF8" applies to
* whichever one is applicable on the current platform */
-#define NATIVE_UTF8_TO_I8(ch) (__ASSERT_(FITS_IN_8_BITS(ch)) ((U8) ((ch) | 0)))
-#define I8_TO_NATIVE_UTF8(ch) (__ASSERT_(FITS_IN_8_BITS(ch)) ((U8) ((ch) | 0)))
+#define NATIVE_UTF8_TO_I8(ch) (__ASSERT_(FITS_IN_8_BITS(ch)) ((U8) (ch)))
+#define I8_TO_NATIVE_UTF8(ch) (__ASSERT_(FITS_IN_8_BITS(ch)) ((U8) (ch)))
-#define UNI_TO_NATIVE(ch) ((UV) ((ch) | 0))
-#define NATIVE_TO_UNI(ch) ((UV) ((ch) | 0))
+#define UNI_TO_NATIVE(ch) ((UV) ASSERT_NOT_PTR(ch))
+#define NATIVE_TO_UNI(ch) ((UV) ASSERT_NOT_PTR(ch))
/*
@@ -226,10 +324,7 @@ possible to UTF-8-encode a single code point in different ways, but that is
explicitly forbidden, and the shortest possible encoding should always be used
(and that is what Perl does). The non-shortest ones are called 'overlongs'.
- */
-
-/*
- Another way to look at it, as bits:
+Another way to look at it, as bits:
Code Points 1st Byte 2nd Byte 3rd Byte 4th Byte
@@ -252,27 +347,30 @@ are in the character. */
* sequence contributes to the specification of the code point. In the bit
* maps above, you see that the first 2 bits are a constant '10', leaving 6 of
* real information */
-#define UTF_ACCUMULATION_SHIFT 6
+# define UTF_CONTINUATION_BYTE_INFO_BITS 6
/* ^? is defined to be DEL on ASCII systems. See the definition of toCTRL()
* for more */
-#define QUESTION_MARK_CTRL DEL_NATIVE
+# define QUESTION_MARK_CTRL DEL_NATIVE
-/* Surrogates, non-character code points and above-Unicode code points are
- * problematic in some contexts. This allows code that needs to check for
- * those to quickly exclude the vast majority of code points it will
- * encounter */
-#define isUTF8_POSSIBLY_PROBLEMATIC(c) (__ASSERT_(FITS_IN_8_BITS(c)) \
- (U8) c >= 0xED)
+#endif /* EBCDIC vs ASCII */
-#define UNICODE_IS_PERL_EXTENDED(uv) UNLIKELY((UV) (uv) > 0x7FFFFFFF)
+/* It turns out that in a number of cases, that handling ASCII vs EBCDIC is a
+ * matter of being off-by-one. So this is a convenience macro, used to avoid
+ * some #ifdefs. */
+#define ONE_IF_EBCDIC_ZERO_IF_NOT \
+ (UTF_CONTINUATION_BYTE_INFO_BITS == UTF_EBCDIC_CONTINUATION_BYTE_INFO_BITS)
-#endif /* EBCDIC vs ASCII */
+/* Since the significant bits in a continuation byte are stored in the
+ * least-significant positions, we often find ourselves shifting by that
+ * amount. This is a clearer name in such situations */
+#define UTF_ACCUMULATION_SHIFT UTF_CONTINUATION_BYTE_INFO_BITS
-/* 2**UTF_ACCUMULATION_SHIFT - 1. This masks out all but the bits that carry
- * real information in a continuation byte. This turns out to be 0x3F in
- * UTF-8, 0x1F in UTF-EBCDIC. */
-#define UTF_CONTINUATION_MASK ((U8) (nBIT_MASK(UTF_ACCUMULATION_SHIFT)))
+/* 2**info_bits - 1. This masks out all but the bits that carry real
+ * information in a continuation byte. This turns out to be 0x3F in UTF-8,
+ * 0x1F in UTF-EBCDIC. */
+#define UTF_CONTINUATION_MASK \
+ ((U8) nBIT_MASK(UTF_CONTINUATION_BYTE_INFO_BITS))
/* For use in UTF8_IS_CONTINUATION(). This turns out to be 0xC0 in UTF-8,
* E0 in UTF-EBCDIC */
@@ -281,9 +379,13 @@ are in the character. */
/* This defines the bits that are to be in the continuation bytes of a
* multi-byte UTF-8 encoded character that mark it is a continuation byte.
* This turns out to be 0x80 in UTF-8, 0xA0 in UTF-EBCDIC. (khw doesn't know
- * the underlying reason that B0 works here) */
+ * the underlying reason that B0 works here, except it just happens to work.
+ * One could solve for two linear equations and come up with it.) */
#define UTF_CONTINUATION_MARK (UTF_IS_CONTINUATION_MASK & 0xB0)
+/* This value is clearer in some contexts */
+#define UTF_MIN_CONTINUATION_BYTE UTF_CONTINUATION_MARK
+
/* Is the byte 'c' part of a multi-byte UTF8-8 encoded sequence, and not the
* first byte thereof? */
#define UTF8_IS_CONTINUATION(c) (__ASSERT_(FITS_IN_8_BITS(c)) \
@@ -293,7 +395,8 @@ are in the character. */
/* Is the representation of the Unicode code point 'cp' the same regardless of
* being encoded in UTF-8 or not? This is a fundamental property of
* UTF-8,EBCDIC */
-#define OFFUNI_IS_INVARIANT(c) (((WIDEST_UTYPE)(c)) < UTF_CONTINUATION_MARK)
+#define OFFUNI_IS_INVARIANT(c) \
+ (((WIDEST_UTYPE)(c)) < UTF_MIN_CONTINUATION_BYTE)
/*
=for apidoc Am|bool|UVCHR_IS_INVARIANT|UV cp
@@ -307,54 +410,142 @@ C<cp> is Unicode if above 255; otherwise is platform-native.
*/
#define UVCHR_IS_INVARIANT(cp) (OFFUNI_IS_INVARIANT(NATIVE_TO_UNI(cp)))
-/* Internal macro to be used only in this file to aid in constructing other
- * publicly accessible macros.
- * The number of bytes required to express this uv in UTF-8, for just those
- * uv's requiring 2 through 6 bytes, as these are common to all platforms and
- * word sizes. The number of bytes needed is given by the number of leading 1
- * bits in the start byte. There are 32 start bytes that have 2 initial 1 bits
- * (C0-DF); there are 16 that have 3 initial 1 bits (E0-EF); 8 that have 4
- * initial 1 bits (F0-F8); 4 that have 5 initial 1 bits (F9-FB), and 2 that
- * have 6 initial 1 bits (FC-FD). The largest number a string of n bytes can
- * represent is (the number of possible start bytes for 'n')
- * * (the number of possiblities for each start byte
- * The latter in turn is
- * 2 ** ( (how many continuation bytes there are)
- * * (the number of bits of information each
- * continuation byte holds))
+/* This defines the 1-bits that are to be in the first byte of a multi-byte
+ * UTF-8 encoded character that mark it as a start byte and give the number of
+ * bytes that comprise the character. 'len' is that number.
*
- * If we were on a platform where we could use a fast find first set bit
- * instruction (or count leading zeros instruction) this could be replaced by
- * using that to find the log2 of the uv, and divide that by the number of bits
- * of information in each continuation byte, adjusting for large cases and how
- * much information is in a start byte for that length */
-#define __COMMON_UNI_SKIP(uv) \
- (UV) (uv) < (32 * (1U << ( UTF_ACCUMULATION_SHIFT))) ? 2 : \
- (UV) (uv) < (16 * (1U << (2 * UTF_ACCUMULATION_SHIFT))) ? 3 : \
- (UV) (uv) < ( 8 * (1U << (3 * UTF_ACCUMULATION_SHIFT))) ? 4 : \
- (UV) (uv) < ( 4 * (1U << (4 * UTF_ACCUMULATION_SHIFT))) ? 5 : \
- (UV) (uv) < ( 2 * (1U << (5 * UTF_ACCUMULATION_SHIFT))) ? 6 :
-
-/* Internal macro to be used only in this file.
- * This adds to __COMMON_UNI_SKIP the details at this platform's upper range.
- * For any-sized EBCDIC platforms, or 64-bit ASCII ones, we need one more test
- * to see if just 7 bytes is needed, or if the maximum is needed. For 32-bit
- * ASCII platforms, everything is representable by 7 bytes */
-#if defined(UV_IS_QUAD) || defined(EBCDIC)
-# define __BASE_UNI_SKIP(uv) (__COMMON_UNI_SKIP(uv) \
- LIKELY((UV) (uv) < ((UV) 1U << (6 * UTF_ACCUMULATION_SHIFT))) \
- ? 7 \
- : UTF8_MAXBYTES)
-#else
-# define __BASE_UNI_SKIP(uv) (__COMMON_UNI_SKIP(uv) 7)
-#endif
+ * To illustrate: len = 2 => ((U8) ~ 0b0011_1111) or 1100_0000
+ * 7 => ((U8) ~ 0b0000_0001) or 1111_1110
+ * > 7 => 0xFF
+ *
+ * This is not to be used on a single-byte character. As in many places in
+ * perl, U8 must be 8 bits
+ */
+#define UTF_START_MARK(len) ((U8) ~(0xFF >> (len)))
-/* The next two macros use the base macro defined above, and add in the tests
- * at the low-end of the range, for just 1 byte, yielding complete macros,
- * publicly accessible. */
+/* Masks out the initial one bits in a start byte, leaving the following 0 bit
+ * and the real data bits. 'len' is the number of bytes in the multi-byte
+ * sequence that comprises the character.
+ *
+ * To illustrate: len = 2 => 0b0011_1111 works on start byte 110xxxxx
+ * 6 => 0b0000_0011 works on start byte 1111110x
+ * >= 7 => There are no data bits in the start byte
+ * Note that on ASCII platforms, this can be passed a len=1 byte; and all the
+ * real data bits will be returned:
+ len = 1 => 0b0111_1111
+ * This isn't true on EBCDIC platforms, where some len=1 bytes are of the form
+ * 0b101x_xxxx, so this can't be used there on single-byte characters. */
+#define UTF_START_MASK(len) (0xFF >> (len))
-/* Input is a true Unicode (not-native) code point */
-#define OFFUNISKIP(uv) (OFFUNI_IS_INVARIANT(uv) ? 1 : __BASE_UNI_SKIP(uv))
+/*
+
+=for apidoc AmnU|STRLEN|UTF8_MAXBYTES
+
+The maximum width of a single UTF-8 encoded character, in bytes.
+
+NOTE: Strictly speaking Perl's UTF-8 should not be called UTF-8 since UTF-8
+is an encoding of Unicode, and Unicode's upper limit, 0x10FFFF, can be
+expressed with 4 bytes. However, Perl thinks of UTF-8 as a way to encode
+non-negative integers in a binary format, even those above Unicode.
+
+=cut
+
+The start byte 0xFE, never used in any ASCII platform UTF-8 specification, has
+an obvious meaning, namely it has its upper 7 bits set, so it should start a
+sequence of 7 bytes. And in fact, this is exactly what standard UTF-EBCDIC
+does.
+
+The start byte FF, on the other hand could have several different plausible
+meanings:
+ 1) The meaning in standard UTF-EBCDIC, namely as an FE start byte, with the
+ bottom bit that should be a fixed '0' to form FE, instead acting as an
+ info bit, 0 or 1.
+ 2) That the sequence should have exactly 8 bytes.
+ 3) That the next byte is to be treated as a sort of extended start byte,
+ which in combination with this one gives the total length of the sequence.
+ There are published UTF-8 extensions that do this, some string together
+ multiple initial FF start bytes to achieve arbitrary precision.
+ 4) That the sequence has exactly n bytes, where n is what the implementation
+ chooses.
+
+Perl has chosen 4).
+The goal is to be able to represent 64-bit values in UTF-8 or UTF-EBCDIC. That
+rules out items 1) and 2). Item 3) has the deal-breaking disadvantage of
+requiring one to read more than one byte to determine the total length of the
+sequence. So in Perl, a start byte of FF indicates a UTF-8 string consisting
+of the start byte, plus enough continuation bytes to encode a 64 bit value.
+This turns out to be 13 total bytes in UTF-8 and 14 in UTF-EBCDIC. This is
+because we get zero info bits from the start byte, plus
+ 12 * 6 bits of info per continuation byte (could encode 72-bit numbers) on
+ UTF-8 (khw knows not why 11, which would encode 66 bits wasn't
+ chosen instead); and
+ 13 * 5 bits of info per byte (could encode 65-bit numbers) on UTF-EBCDIC
+
+The disadvantages of this method are:
+ 1) There's potentially a lot of wasted bytes for all but the largest values.
+ For example, something that could be represented by 7 continuation bytes,
+ instead requires the full 12 or 13.
+ 2) There would be problems should larger values, 128-bit say, ever need to be
+ represented.
+
+WARNING: This number must be in sync with the value in
+regen/charset_translations.pl. */
+#define UTF8_MAXBYTES \
+ (ASCII_PLATFORM_UTF8_MAXBYTES + ONE_IF_EBCDIC_ZERO_IF_NOT)
+
+/* Calculate how many bytes are necessary to represent a value whose most
+ * significant 1 bit is in bit position 'pos' of the word. For 0x1, 'pos would
+ * be 0; and for 0x400, 'pos' would be 10, and the result would be:
+ * EBCDIC floor((-1 + (10 + 5 - 1 - 1)) / (5 - 1))
+ * = floor((-1 + (13)) / 4)
+ * = floor(12 / 4)
+ * = 3
+ * ASCII floor(( 0 + (10 + 6 - 1 - 1)) / (6 - 1))
+ * = floor(14 / 5)
+ * = 2
+ * The reason this works is because the number of bits needed to represent a
+ * value is proportional to (UTF_CONTINUATION_BYTE_INFO_BITS - 1). The -1 is
+ * because each new continuation byte removes one bit of information from the
+ * start byte.
+ *
+ * This is a step function (we need to allocate a full extra byte if we
+ * overflow by just a single bit)
+ *
+ * The caller is responsible for making sure 'pos' is at least 8 (occupies 9
+ * bits), as it breaks down at the lower edge. At the high end, if it returns
+ * 8 or more, Perl instead anomalously uses MAX_BYTES, so this would be wrong.
+ * */
+#define UNISKIP_BY_MSB_(pos) \
+ ( ( -ONE_IF_EBCDIC_ZERO_IF_NOT /* platform break pos's are off-by-one */ \
+ + (pos) + ((UTF_CONTINUATION_BYTE_INFO_BITS - 1) - 1)) /* Step fcn */ \
+ / (UTF_CONTINUATION_BYTE_INFO_BITS - 1)) /* take floor of */
+
+/* Compute the number of UTF-8 bytes required for representing the input uv,
+ * which must be a Unicode, not native value.
+ *
+ * This uses msbit_pos() which doesn't work on NUL, and UNISKIP_BY_MSB_ breaks
+ * down for small code points. So first check if the input is invariant to get
+ * around that, and use a helper for high code points to accommodate the fact
+ * that above 7 btyes, the value is anomalous. The helper is empty on
+ * platforms that don't go that high */
+#define OFFUNISKIP(uv) \
+ ((OFFUNI_IS_INVARIANT(uv)) \
+ ? 1 \
+ : (OFFUNISKIP_helper_(uv) UNISKIP_BY_MSB_(msbit_pos(uv))))
+
+/* We need to go to MAX_BYTES when we can't represent 'uv' by the number of
+ * information bits in 6 continuation bytes (when we get to 6, the start byte
+ * has no information bits to add to the total). But on 32-bit ASCII
+ * platforms, that doesn't happen until 6*6 bits, so on those platforms, this
+ * will always be false */
+#if UVSIZE * CHARBITS > (6 * UTF_CONTINUATION_BYTE_INFO_BITS)
+# define HAS_EXTRA_LONG_UTF8
+# define OFFUNISKIP_helper_(uv) \
+ UNLIKELY(uv > nBIT_UMAX(6 * UTF_CONTINUATION_BYTE_INFO_BITS)) \
+ ? UTF8_MAXBYTES :
+#else
+# define OFFUNISKIP_helper_(uv)
+#endif
/*
@@ -365,10 +556,56 @@ encoded as UTF-8. C<cp> is a native (ASCII or EBCDIC) code point if less than
=cut
*/
-#define UVCHR_SKIP(uv) ( UVCHR_IS_INVARIANT(uv) ? 1 : __BASE_UNI_SKIP(uv))
+#define UVCHR_SKIP(uv) OFFUNISKIP(NATIVE_TO_UNI(uv))
+
+#define NATIVE_SKIP(uv) UVCHR_SKIP(uv) /* Old terminology */
+
+/* Most code which says UNISKIP is really thinking in terms of native code
+ * points (0-255) plus all those beyond. This is an imprecise term, but having
+ * it means existing code continues to work. For precision, use UVCHR_SKIP,
+ * NATIVE_SKIP, or OFFUNISKIP */
+#define UNISKIP(uv) UVCHR_SKIP(uv)
+
+/* Compute the start byte for a given code point. This requires the log2 of
+ * the code point, which is hard to compute at compile time, which this macro
+ * wants to be. (Perhaps deBruijn sequences could be used.) So a parameter
+ * for the number of bits the value occupies is passed in, which the programmer
+ * has had to figure out to get compile-time effect. And asserts are used to
+ * make sure the value is correct.
+ *
+ * Since we are interested only in the start byte, we ignore the lower bits
+ * accounted for by the continuation bytes. Each continuation byte eats up
+ * UTF_CONTINUATION_BYTE_INFO_BITS bits, so the number of continuation bytes
+ * needed is floor(bits / UTF_CONTINUATION_BYTE_INFO_BITS). That number is fed
+ * to UTF_START_MARK() to get the upper part of the start byte. The left over
+ * bits form the lower part which is OR'd with the mark
+ *
+ * Note that on EBCDIC platforms, this is actually the I8 */
+#define UTF_START_BYTE(uv, bits) \
+ (__ASSERT_((uv) >> ((bits) - 1)) /* At least 'bits' */ \
+ __ASSERT_(((uv) & ~nBIT_MASK(bits)) == 0) /* No extra bits */ \
+ UTF_START_MARK(UNISKIP_BY_MSB_((bits) - 1)) \
+ | ((uv) >> (((bits) / UTF_CONTINUATION_BYTE_INFO_BITS) \
+ * UTF_CONTINUATION_BYTE_INFO_BITS)))
+
+/* Compute the first continuation byte for a given code point. This is mostly
+ * for compile-time, so how many bits it occupies is also passed in).
+ *
+ * We are interested in the first continuation byte, so we ignore the lower
+ * bits accounted for by the rest of the continuation bytes by right shifting
+ * out their info bit, and mask out the higher bits that will go into the start
+ * byte.
+ *
+ * Note that on EBCDIC platforms, this is actually the I8 */
+#define UTF_FIRST_CONT_BYTE(uv, bits) \
+ (__ASSERT_((uv) >> ((bits) - 1)) /* At least 'bits' */ \
+ __ASSERT_(((uv) & ~nBIT_MASK(bits)) == 0) /* No extra bits */ \
+ UTF_CONTINUATION_MARK \
+ | ( UTF_CONTINUATION_MASK \
+ & ((uv) >> ((((bits) / UTF_CONTINUATION_BYTE_INFO_BITS) - 1) \
+ * UTF_CONTINUATION_BYTE_INFO_BITS))))
-#define UTF_MIN_START_BYTE \
- ((UTF_CONTINUATION_MARK >> UTF_ACCUMULATION_SHIFT) | UTF_START_MARK(2))
+#define UTF_MIN_START_BYTE UTF_START_BYTE(UTF_MIN_CONTINUATION_BYTE, 8)
/* Is the byte 'c' the first byte of a multi-byte UTF8-8 encoded sequence?
* This excludes invariants (they are single-byte). It also excludes the
@@ -385,8 +622,7 @@ encoded as UTF-8. C<cp> is a native (ASCII or EBCDIC) code point if less than
# define UTF8_IS_START(c) UTF8_IS_START_base(c)
#endif
-#define UTF_MIN_ABOVE_LATIN1_BYTE \
- ((0x100 >> UTF_ACCUMULATION_SHIFT) | UTF_START_MARK(2))
+#define UTF_MIN_ABOVE_LATIN1_BYTE UTF_START_BYTE(0x100, 9)
/* Is the UTF8-encoded byte 'c' the first byte of a sequence of bytes that
* represent a code point > 255? */
@@ -397,18 +633,27 @@ encoded as UTF-8. C<cp> is a native (ASCII or EBCDIC) code point if less than
* UTF8_IS_NEXT_CHAR_DOWNGRADEABLE() instead if the input isn't known to
* be well-formed. */
#define UTF8_IS_DOWNGRADEABLE_START(c) (__ASSERT_(FITS_IN_8_BITS(c)) \
- inRANGE(NATIVE_UTF8_TO_I8(c), \
+ inRANGE_helper_(U8, NATIVE_UTF8_TO_I8(c), \
UTF_MIN_START_BYTE, UTF_MIN_ABOVE_LATIN1_BYTE - 1))
/* The largest code point representable by two UTF-8 bytes on this platform.
- * As explained in the comments for __COMMON_UNI_SKIP, 32 start bytes with
- * UTF_ACCUMULATION_SHIFT bits of information each */
-#define MAX_UTF8_TWO_BYTE (32 * (1U << UTF_ACCUMULATION_SHIFT) - 1)
+ * The binary for that code point is:
+ * 1101_1111 10xx_xxxx in UTF-8, and
+ * 1101_1111 101y_yyyy in UTF-EBCDIC I8.
+ * where both x and y are 1, and shown this way to indicate there is one more x
+ * than there is y. The number of x and y bits are their platform's respective
+ * UTF_CONTINUATION_BYTE_INFO_BITS. Squeezing out the bits that don't
+ * contribute to the value, these evaluate to:
+ * 1_1111 xx_xxxx in UTF-8, and
+ * 1_1111 y_yyyy in UTF-EBCDIC I8.
+ * or, the maximum value of an unsigned with (5 + info_bit_count) bits */
+#define MAX_UTF8_TWO_BYTE nBIT_UMAX(5 + UTF_CONTINUATION_BYTE_INFO_BITS)
/* The largest code point representable by two UTF-8 bytes on any platform that
- * Perl runs on. This value is constrained by EBCDIC which has 5 bits per
- * continuation byte */
-#define MAX_PORTABLE_UTF8_TWO_BYTE (32 * nBIT_UMAX(5))
+ * Perl runs on. */
+#define MAX_PORTABLE_UTF8_TWO_BYTE \
+ nBIT_UMAX(5 + MIN( UTF_CONTINUATION_BYTE_INFO_BITS, \
+ UTF_EBCDIC_CONTINUATION_BYTE_INFO_BITS))
/*
@@ -428,8 +673,8 @@ uppercase/lowercase/titlecase/fold into.
*
=cut
*/
-#define UTF8_MAXBYTES_CASE \
- MAX(UTF8_MAXBYTES, UTF8_MAX_FOLD_CHAR_EXPAND * OFFUNISKIP(0x10FFFF))
+#define UTF8_MAXBYTES_CASE \
+ MAX(UTF8_MAXBYTES, UTF8_MAX_FOLD_CHAR_EXPAND * UNISKIP_BY_MSB_(20))
/* Rest of these are attributes of Unicode and perl's internals rather than the
* encoding, or happen to be the same in both ASCII and EBCDIC (at least at
@@ -448,19 +693,6 @@ uppercase/lowercase/titlecase/fold into.
#define I8_TO_NATIVE(ch) I8_TO_NATIVE_UTF8(ch)
#define NATIVE8_TO_UNI(ch) NATIVE_TO_LATIN1(ch)
-/* This defines the 1-bits that are to be in the first byte of a multi-byte
- * UTF-8 encoded character that mark it as a start byte and give the number of
- * bytes that comprise the character. 'len' is the number of bytes in the
- * multi-byte sequence. */
-#define UTF_START_MARK(len) (UNLIKELY((len) > 7) \
- ? 0xFF \
- : ((U8) (0xFE << (7-(len)))))
-
-/* Masks out the initial one bits in a start byte, leaving the real data ones.
- * Doesn't work on an invariant byte. 'len' is the number of bytes in the
- * multi-byte sequence that comprises the character. */
-#define UTF_START_MASK(len) (UNLIKELY((len) >= 7) ? 0x00 : (0x1F >> ((len)-2)))
-
/* Adds a UTF8 continuation byte 'new' of information to a running total code
* point 'old' of all the continuation bytes so far. This is designed to be
* used in a loop to convert from UTF-8 to the code point represented. Note
@@ -474,19 +706,10 @@ uppercase/lowercase/titlecase/fold into.
/* This works in the face of malformed UTF-8. */
#define UTF8_IS_NEXT_CHAR_DOWNGRADEABLE(s, e) \
- ( UTF8_IS_DOWNGRADEABLE_START(*(s)) \
- && ( (e) - (s) > 1) \
+ ( ( (e) - (s) > 1) \
+ && UTF8_IS_DOWNGRADEABLE_START(*(s)) \
&& UTF8_IS_CONTINUATION(*((s)+1)))
-/* Number of bytes a code point occupies in UTF-8. */
-#define NATIVE_SKIP(uv) UVCHR_SKIP(uv)
-
-/* Most code which says UNISKIP is really thinking in terms of native code
- * points (0-255) plus all those beyond. This is an imprecise term, but having
- * it means existing code continues to work. For precision, use UVCHR_SKIP,
- * NATIVE_SKIP, or OFFUNISKIP */
-#define UNISKIP(uv) UVCHR_SKIP(uv)
-
/* Longer, but more accurate name */
#define UTF8_IS_ABOVE_LATIN1_START(c) UTF8_IS_ABOVE_LATIN1(c)
@@ -513,7 +736,7 @@ uppercase/lowercase/titlecase/fold into.
#define TWO_BYTE_UTF8_TO_NATIVE(HI, LO) \
(__ASSERT_(FITS_IN_8_BITS(HI)) \
__ASSERT_(FITS_IN_8_BITS(LO)) \
- __ASSERT_(PL_utf8skip[HI] == 2) \
+ __ASSERT_(PL_utf8skip[(U8) HI] == 2) \
__ASSERT_(UTF8_IS_CONTINUATION(LO)) \
UNI_TO_NATIVE(UTF8_ACCUMULATE((NATIVE_UTF8_TO_I8(HI) & UTF_START_MASK(2)), \
(LO))))
@@ -545,7 +768,7 @@ of the input buffer.
=cut
*/
-#define UTF8SKIP(s) PL_utf8skip[*(const U8*)(s)]
+#define UTF8SKIP(s) PL_utf8skip[*(const U8*)(ASSERT_IS_PTR(s))]
/*
=for apidoc Am|STRLEN|UTF8_SKIP|char* s
@@ -615,10 +838,8 @@ The reason it works on both UTF-8 encoded strings and non-UTF-8 encoded, is
that it returns TRUE in each for the exact same set of bit patterns. It is
valid on a subset of what UVCHR_IS_INVARIANT is valid on, so can just use that;
and the compiler should optimize out anything extraneous given the
-implementation of the latter. The |0 makes sure this isn't mistakenly called
-with a ptr argument.
-*/
-#define UTF8_IS_INVARIANT(c) UVCHR_IS_INVARIANT((c) | 0)
+implementation of the latter. */
+#define UTF8_IS_INVARIANT(c) UVCHR_IS_INVARIANT(ASSERT_NOT_PTR(c))
/* Like the above, but its name implies a non-UTF8 input, which as the comments
* above show, doesn't matter as to its implementation */
@@ -695,12 +916,12 @@ with a ptr argument.
: isWORDCHAR_utf8_safe((U8 *) p, (U8 *) e))
#define isALNUM_lazy_if_safe(p, e, UTF) isWORDCHAR_lazy_if_safe(p, e, UTF)
-#define UTF8_MAXLEN UTF8_MAXBYTES
+#define UTF8_MAXLEN UTF8_MAXBYTES
/* A Unicode character can fold to up to 3 characters */
-#define UTF8_MAX_FOLD_CHAR_EXPAND 3
+#define UTF8_MAX_FOLD_CHAR_EXPAND 3
-#define IN_BYTES UNLIKELY(CopHINTS_get(PL_curcop) & HINT_BYTES)
+#define IN_BYTES UNLIKELY(CopHINTS_get(PL_curcop) & HINT_BYTES)
/*
@@ -717,8 +938,8 @@ case any call to string overloading updates the internal UTF-8 encoding flag.
/* Should all strings be treated as Unicode, and not just UTF-8 encoded ones?
* Is so within 'feature unicode_strings' or 'locale :not_characters', and not
- * within 'use bytes'. UTF-8 locales are not tested for here, but perhaps
- * could be */
+ * within 'use bytes'. UTF-8 locales are not tested for here, because it gets
+ * complicated by the probability of having categories in different locales. */
#define IN_UNI_8_BIT \
(( ( (CopHINTS_get(PL_curcop) & HINT_UNI_8_BIT)) \
|| ( CopHINTS_get(PL_curcop) & HINT_LOCALE_PARTIAL \
@@ -726,6 +947,188 @@ case any call to string overloading updates the internal UTF-8 encoding flag.
&& _is_in_locale_category(FALSE, -1))) \
&& (! IN_BYTES))
+#define UNICODE_SURROGATE_FIRST 0xD800
+#define UNICODE_SURROGATE_LAST 0xDFFF
+
+/*
+=for apidoc Am|bool|UNICODE_IS_SURROGATE|const UV uv
+
+Returns a boolean as to whether or not C<uv> is one of the Unicode surrogate
+code points
+
+=for apidoc Am|bool|UTF8_IS_SURROGATE|const U8 *s|const U8 *e
+
+Evaluates to non-zero if the first few bytes of the string starting at C<s> and
+looking no further than S<C<e - 1>> are well-formed UTF-8 that represents one
+of the Unicode surrogate code points; otherwise it evaluates to 0. If
+non-zero, the value gives how many bytes starting at C<s> comprise the code
+point's representation.
+
+=cut
+ */
+
+#define UNICODE_IS_SURROGATE(uv) UNLIKELY(inRANGE(uv, UNICODE_SURROGATE_FIRST, \
+ UNICODE_SURROGATE_LAST))
+#define UTF8_IS_SURROGATE(s, e) is_SURROGATE_utf8_safe(s, e)
+
+/*
+
+=for apidoc AmnU|UV|UNICODE_REPLACEMENT
+
+Evaluates to 0xFFFD, the code point of the Unicode REPLACEMENT CHARACTER
+
+=for apidoc Am|bool|UNICODE_IS_REPLACEMENT|const UV uv
+
+Returns a boolean as to whether or not C<uv> is the Unicode REPLACEMENT
+CHARACTER
+
+=for apidoc Am|bool|UTF8_IS_REPLACEMENT|const U8 *s|const U8 *e
+
+Evaluates to non-zero if the first few bytes of the string starting at C<s> and
+looking no further than S<C<e - 1>> are well-formed UTF-8 that represents the
+Unicode REPLACEMENT CHARACTER; otherwise it evaluates to 0. If non-zero, the
+value gives how many bytes starting at C<s> comprise the code point's
+representation.
+
+=cut
+ */
+#define UNICODE_REPLACEMENT 0xFFFD
+#define UNICODE_IS_REPLACEMENT(uv) UNLIKELY((UV) (uv) == UNICODE_REPLACEMENT)
+#define UTF8_IS_REPLACEMENT(s, send) \
+ UNLIKELY( \
+ ((send) - (s)) >= ((SSize_t)(sizeof(REPLACEMENT_CHARACTER_UTF8) - 1))\
+ && memEQ((s), REPLACEMENT_CHARACTER_UTF8, \
+ sizeof(REPLACEMENT_CHARACTER_UTF8) - 1))
+
+/* Max legal code point according to Unicode */
+#define PERL_UNICODE_MAX 0x10FFFF
+
+/*
+
+=for apidoc Am|bool|UNICODE_IS_SUPER|const UV uv
+
+Returns a boolean as to whether or not C<uv> is above the maximum legal Unicode
+code point of U+10FFFF.
+
+=cut
+*/
+
+#define UNICODE_IS_SUPER(uv) UNLIKELY((UV) (uv) > PERL_UNICODE_MAX)
+
+/*
+=for apidoc Am|bool|UTF8_IS_SUPER|const U8 *s|const U8 *e
+
+Recall that Perl recognizes an extension to UTF-8 that can encode code
+points larger than the ones defined by Unicode, which are 0..0x10FFFF.
+
+This macro evaluates to non-zero if the first few bytes of the string starting
+at C<s> and looking no further than S<C<e - 1>> are from this UTF-8 extension;
+otherwise it evaluates to 0. If non-zero, the return is how many bytes
+starting at C<s> comprise the code point's representation.
+
+0 is returned if the bytes are not well-formed extended UTF-8, or if they
+represent a code point that cannot fit in a UV on the current platform. Hence
+this macro can give different results when run on a 64-bit word machine than on
+one with a 32-bit word size.
+
+Note that it is illegal in Perl to have code points that are larger than what can
+fit in an IV on the current machine; and illegal in Unicode to have any that
+this macro matches
+
+=cut
+
+ * ASCII EBCDIC I8
+ * U+10FFFF: \xF4\x8F\xBF\xBF \xF9\xA1\xBF\xBF\xBF max legal Unicode
+ * U+110000: \xF4\x90\x80\x80 \xF9\xA2\xA0\xA0\xA0
+ * U+110001: \xF4\x90\x80\x81 \xF9\xA2\xA0\xA0\xA1
+ */
+#define UTF_START_BYTE_110000_ UTF_START_BYTE(PERL_UNICODE_MAX + 1, 21)
+#define UTF_FIRST_CONT_BYTE_110000_ \
+ UTF_FIRST_CONT_BYTE(PERL_UNICODE_MAX + 1, 21)
+#define UTF8_IS_SUPER(s, e) \
+ ( ((e) - (s)) >= UNISKIP_BY_MSB_(20) \
+ && ( NATIVE_UTF8_TO_I8(s[0]) >= UTF_START_BYTE_110000_ \
+ && ( NATIVE_UTF8_TO_I8(s[0]) > UTF_START_BYTE_110000_ \
+ || NATIVE_UTF8_TO_I8(s[1]) >= UTF_FIRST_CONT_BYTE_110000_))) \
+ ? isUTF8_CHAR(s, e) \
+ : 0
+
+/*
+=for apidoc Am|bool|UNICODE_IS_NONCHAR|const UV uv
+
+Returns a boolean as to whether or not C<uv> is one of the Unicode
+non-character code points
+
+=cut
+*/
+
+/* Is 'uv' one of the 32 contiguous-range noncharacters? */
+#define UNICODE_IS_32_CONTIGUOUS_NONCHARS(uv) \
+ UNLIKELY(inRANGE(uv, 0xFDD0, 0xFDEF))
+
+/* Is 'uv' one of the 34 plane-ending noncharacters 0xFFFE, 0xFFFF, 0x1FFFE,
+ * 0x1FFFF, ... 0x10FFFE, 0x10FFFF, given that we know that 'uv' is not above
+ * the Unicode legal max */
+#define UNICODE_IS_END_PLANE_NONCHAR_GIVEN_NOT_SUPER(uv) \
+ UNLIKELY(((UV) (uv) & 0xFFFE) == 0xFFFE)
+
+#define UNICODE_IS_NONCHAR(uv) \
+ ( UNLIKELY(UNICODE_IS_32_CONTIGUOUS_NONCHARS(uv)) \
+ || ( UNLIKELY(UNICODE_IS_END_PLANE_NONCHAR_GIVEN_NOT_SUPER(uv)) \
+ && LIKELY(! UNICODE_IS_SUPER(uv))))
+
+/*
+=for apidoc Am|bool|UTF8_IS_NONCHAR|const U8 *s|const U8 *e
+
+Evaluates to non-zero if the first few bytes of the string starting at C<s> and
+looking no further than S<C<e - 1>> are well-formed UTF-8 that represents one
+of the Unicode non-character code points; otherwise it evaluates to 0. If
+non-zero, the value gives how many bytes starting at C<s> comprise the code
+point's representation.
+
+=cut
+*/
+#define UTF8_IS_NONCHAR(s, e) is_NONCHAR_utf8_safe(s,e)
+
+/* This is now machine generated, and the 'given' clause is no longer
+ * applicable */
+#define UTF8_IS_NONCHAR_GIVEN_THAT_NON_SUPER_AND_GE_PROBLEMATIC(s, e) \
+ UTF8_IS_NONCHAR(s, e)
+
+/* Surrogates, non-character code points and above-Unicode code points are
+ * problematic in some contexts. These macros allow code that needs to check
+ * for those to quickly exclude the vast majority of code points it will
+ * encounter.
+ *
+ * The lowest such code point is the smallest surrogate, U+D800. We calculate
+ * the start byte of that. 0xD800 occupies 16 bits. */
+#define isUNICODE_POSSIBLY_PROBLEMATIC(uv) ((uv) >= UNICODE_SURROGATE_FIRST)
+#define isUTF8_POSSIBLY_PROBLEMATIC(c) \
+ (NATIVE_UTF8_TO_I8(c) >= UTF_START_BYTE(UNICODE_SURROGATE_FIRST, 16))
+
+/* Perl extends Unicode so that it is possible to encode (as extended UTF-8 or
+ * UTF-EBCDIC) any 64-bit value. No standard known to khw ever encoded higher
+ * than a 31 bit value. On ASCII platforms this just meant arbitrarily saying
+ * nothing could be higher than this. On these the start byte FD gets you to
+ * 31 bits, and FE and FF are forbidden as start bytes. On EBCDIC platforms,
+ * FD gets you only to 26 bits; adding FE to mean 7 total bytes gets you to 30
+ * bits. To get to 31 bits, they treated an initial FF byte idiosyncratically.
+ * It was considered to be the start byte FE meaning it had 7 total bytes, and
+ * the final 1 was treated as an information bit, getting you to 31 bits.
+ *
+ * Perl used to accept this idiosyncratic interpretation of FF, but now rejects
+ * it in order to get to being able to encode 64 bits. The bottom line is that
+ * it is a Perl extension to use the start bytes FE and FF on ASCII platforms,
+ * and the start byte FF on EBCDIC ones. That translates into that it is a
+ * Perl extension to represent anything occupying more than 31 bits on ASCII
+ * platforms; 30 bits on EBCDIC. */
+#define UNICODE_IS_PERL_EXTENDED(uv) \
+ UNLIKELY((UV) (uv) > nBIT_UMAX(31 - ONE_IF_EBCDIC_ZERO_IF_NOT))
+#define UTF8_IS_PERL_EXTENDED(s) \
+ (UTF8SKIP(s) > 6 + ONE_IF_EBCDIC_ZERO_IF_NOT)
+
+/* Largest code point we accept from external sources */
+#define MAX_LEGAL_CP ((UV)IV_MAX)
#define UTF8_ALLOW_EMPTY 0x0001 /* Allow a zero length string */
#define UTF8_GOT_EMPTY UTF8_ALLOW_EMPTY
@@ -822,100 +1225,6 @@ case any call to string overloading updates the internal UTF-8 encoding flag.
#define UTF8_ALLOW_ANYUV 0
#define UTF8_ALLOW_DEFAULT UTF8_ALLOW_ANYUV
-/*
-=for apidoc Am|bool|UTF8_IS_SURROGATE|const U8 *s|const U8 *e
-
-Evaluates to non-zero if the first few bytes of the string starting at C<s> and
-looking no further than S<C<e - 1>> are well-formed UTF-8 that represents one
-of the Unicode surrogate code points; otherwise it evaluates to 0. If
-non-zero, the value gives how many bytes starting at C<s> comprise the code
-point's representation.
-
-=cut
- */
-#define UTF8_IS_SURROGATE(s, e) is_SURROGATE_utf8_safe(s, e)
-
-
-#define UTF8_IS_REPLACEMENT(s, send) is_REPLACEMENT_utf8_safe(s,send)
-
-#define MAX_LEGAL_CP ((UV)IV_MAX)
-
-/*
-=for apidoc Am|bool|UTF8_IS_SUPER|const U8 *s|const U8 *e
-
-Recall that Perl recognizes an extension to UTF-8 that can encode code
-points larger than the ones defined by Unicode, which are 0..0x10FFFF.
-
-This macro evaluates to non-zero if the first few bytes of the string starting
-at C<s> and looking no further than S<C<e - 1>> are from this UTF-8 extension;
-otherwise it evaluates to 0. If non-zero, the value gives how many bytes
-starting at C<s> comprise the code point's representation.
-
-0 is returned if the bytes are not well-formed extended UTF-8, or if they
-represent a code point that cannot fit in a UV on the current platform. Hence
-this macro can give different results when run on a 64-bit word machine than on
-one with a 32-bit word size.
-
-Note that it is illegal to have code points that are larger than what can
-fit in an IV on the current machine.
-
-=cut
-
- * ASCII EBCDIC I8
- * U+10FFFF: \xF4\x8F\xBF\xBF \xF9\xA1\xBF\xBF\xBF max legal Unicode
- * U+110000: \xF4\x90\x80\x80 \xF9\xA2\xA0\xA0\xA0
- * U+110001: \xF4\x90\x80\x81 \xF9\xA2\xA0\xA0\xA1
- */
-#ifdef EBCDIC
-# define UTF8_IS_SUPER(s, e) \
- (( ((e) > (s) + 4) \
- && (NATIVE_UTF8_TO_I8(*(s)) >= 0xF9) \
- && UNLIKELY( NATIVE_UTF8_TO_I8(*(s)) > 0xF9 \
- || (NATIVE_UTF8_TO_I8(*((s) + 1)) >= 0xA2)) \
- && LIKELY((s) + UTF8SKIP(s) <= (e))) \
- ? is_utf8_char_helper(s, s + UTF8SKIP(s), 0) : 0)
-#else
-# define UTF8_IS_SUPER(s, e) \
- (( ((e) > (s) + 3) \
- && (*(U8*) (s)) >= 0xF4 \
- && (UNLIKELY( ((*(U8*) (s)) > 0xF4) \
- || (*((U8*) (s) + 1) >= 0x90))) \
- && LIKELY((s) + UTF8SKIP(s) <= (e))) \
- ? is_utf8_char_helper(s, s + UTF8SKIP(s), 0) : 0)
-#endif
-
-/* These are now machine generated, and the 'given' clause is no longer
- * applicable */
-#define UTF8_IS_NONCHAR_GIVEN_THAT_NON_SUPER_AND_GE_PROBLEMATIC(s, e) \
- cBOOL(is_NONCHAR_utf8_safe(s,e))
-
-/*
-=for apidoc Am|bool|UTF8_IS_NONCHAR|const U8 *s|const U8 *e
-
-Evaluates to non-zero if the first few bytes of the string starting at C<s> and
-looking no further than S<C<e - 1>> are well-formed UTF-8 that represents one
-of the Unicode non-character code points; otherwise it evaluates to 0. If
-non-zero, the value gives how many bytes starting at C<s> comprise the code
-point's representation.
-
-=for apidoc AmnU|UV|UNICODE_REPLACEMENT
-
-Evaluates to 0xFFFD, the code point of the Unicode REPLACEMENT CHARACTER
-
-=cut
- */
-#define UTF8_IS_NONCHAR(s, e) \
- UTF8_IS_NONCHAR_GIVEN_THAT_NON_SUPER_AND_GE_PROBLEMATIC(s, e)
-
-#define UNICODE_SURROGATE_FIRST 0xD800
-#define UNICODE_SURROGATE_LAST 0xDFFF
-#define UNICODE_REPLACEMENT 0xFFFD
-#define UNICODE_BYTE_ORDER_MARK 0xFEFF
-
-/* Though our UTF-8 encoding can go beyond this,
- * let's be conservative and do as Unicode says. */
-#define PERL_UNICODE_MAX 0x10FFFF
-
#define UNICODE_WARN_SURROGATE 0x0001 /* UTF-16 surrogates */
#define UNICODE_WARN_NONCHAR 0x0002 /* Non-char code points */
#define UNICODE_WARN_SUPER 0x0004 /* Above 0x10FFFF */
@@ -950,31 +1259,9 @@ Evaluates to 0xFFFD, the code point of the Unicode REPLACEMENT CHARACTER
#define UNICODE_ALLOW_SUPER 0
#define UNICODE_ALLOW_ANY 0
-/* This matches the 2048 code points between UNICODE_SURROGATE_FIRST (0xD800) and
- * UNICODE_SURROGATE_LAST (0xDFFF) */
-#define UNICODE_IS_SURROGATE(uv) UNLIKELY(((UV) (uv) & (~0xFFFF | 0xF800)) \
- == 0xD800)
-
-#define UNICODE_IS_REPLACEMENT(uv) UNLIKELY((UV) (uv) == UNICODE_REPLACEMENT)
-#define UNICODE_IS_BYTE_ORDER_MARK(uv) UNLIKELY((UV) (uv) \
- == UNICODE_BYTE_ORDER_MARK)
-
-/* Is 'uv' one of the 32 contiguous-range noncharacters? */
-#define UNICODE_IS_32_CONTIGUOUS_NONCHARS(uv) UNLIKELY((UV) (uv) >= 0xFDD0 \
- && (UV) (uv) <= 0xFDEF)
-
-/* Is 'uv' one of the 34 plane-ending noncharacters 0xFFFE, 0xFFFF, 0x1FFFE,
- * 0x1FFFF, ... 0x10FFFE, 0x10FFFF, given that we know that 'uv' is not above
- * the Unicode legal max */
-#define UNICODE_IS_END_PLANE_NONCHAR_GIVEN_NOT_SUPER(uv) \
- UNLIKELY(((UV) (uv) & 0xFFFE) == 0xFFFE)
-
-#define UNICODE_IS_NONCHAR(uv) \
- ( UNICODE_IS_32_CONTIGUOUS_NONCHARS(uv) \
- || ( LIKELY( ! UNICODE_IS_SUPER(uv)) \
- && UNICODE_IS_END_PLANE_NONCHAR_GIVEN_NOT_SUPER(uv)))
-
-#define UNICODE_IS_SUPER(uv) UNLIKELY((UV) (uv) > PERL_UNICODE_MAX)
+#define UNICODE_BYTE_ORDER_MARK 0xFEFF
+#define UNICODE_IS_BYTE_ORDER_MARK(uv) UNLIKELY((UV) (uv) \
+ == UNICODE_BYTE_ORDER_MARK)
#define LATIN_SMALL_LETTER_SHARP_S LATIN_SMALL_LETTER_SHARP_S_NATIVE
#define LATIN_SMALL_LETTER_Y_WITH_DIAERESIS \
@@ -1020,45 +1307,6 @@ Evaluates to 0xFFFD, the code point of the Unicode REPLACEMENT CHARACTER
#define bytes_from_utf8(s, lenp, is_utf8p) \
bytes_from_utf8_loc(s, lenp, is_utf8p, 0)
-/*
-
-=for apidoc Am|STRLEN|isUTF8_CHAR_flags|const U8 *s|const U8 *e| const U32 flags
-
-Evaluates to non-zero if the first few bytes of the string starting at C<s> and
-looking no further than S<C<e - 1>> are well-formed UTF-8, as extended by Perl,
-that represents some code point, subject to the restrictions given by C<flags>;
-otherwise it evaluates to 0. If non-zero, the value gives how many bytes
-starting at C<s> comprise the code point's representation. Any bytes remaining
-before C<e>, but beyond the ones needed to form the first code point in C<s>,
-are not examined.
-
-If C<flags> is 0, this gives the same results as C<L</isUTF8_CHAR>>;
-if C<flags> is C<UTF8_DISALLOW_ILLEGAL_INTERCHANGE>, this gives the same results
-as C<L</isSTRICT_UTF8_CHAR>>;
-and if C<flags> is C<UTF8_DISALLOW_ILLEGAL_C9_INTERCHANGE>, this gives
-the same results as C<L</isC9_STRICT_UTF8_CHAR>>.
-Otherwise C<flags> may be any combination of the C<UTF8_DISALLOW_I<foo>> flags
-understood by C<L</utf8n_to_uvchr>>, with the same meanings.
-
-The three alternative macros are for the most commonly needed validations; they
-are likely to run somewhat faster than this more general one, as they can be
-inlined into your code.
-
-Use L</is_utf8_string_flags>, L</is_utf8_string_loc_flags>, and
-L</is_utf8_string_loclen_flags> to check entire strings.
-
-=cut
-*/
-
-#define isUTF8_CHAR_flags(s, e, flags) \
- (UNLIKELY((e) <= (s)) \
- ? 0 \
- : (UTF8_IS_INVARIANT(*s)) \
- ? 1 \
- : UNLIKELY(((e) - (s)) < UTF8SKIP(s)) \
- ? 0 \
- : is_utf8_char_helper(s, e, flags))
-
/* Do not use; should be deprecated. Use isUTF8_CHAR() instead; this is
* retained solely for backwards compatibility */
#define IS_UTF8_CHAR(p, n) (isUTF8_CHAR(p, (p) + (n)) == n)