summaryrefslogtreecommitdiff
path: root/Master/tlpkg/tlperl/lib/CORE/utf8.h
diff options
context:
space:
mode:
Diffstat (limited to 'Master/tlpkg/tlperl/lib/CORE/utf8.h')
-rw-r--r--Master/tlpkg/tlperl/lib/CORE/utf8.h253
1 files changed, 22 insertions, 231 deletions
diff --git a/Master/tlpkg/tlperl/lib/CORE/utf8.h b/Master/tlpkg/tlperl/lib/CORE/utf8.h
index e0fc6b61419..d0b8742d5a8 100644
--- a/Master/tlpkg/tlperl/lib/CORE/utf8.h
+++ b/Master/tlpkg/tlperl/lib/CORE/utf8.h
@@ -34,11 +34,6 @@
#define FOLD_FLAGS_FULL 0x2
#define FOLD_FLAGS_NOMIX_ASCII 0x4
-/* For _core_swash_init(), internal core use only */
-#define _CORE_SWASH_INIT_USER_DEFINED_PROPERTY 0x1
-#define _CORE_SWASH_INIT_RETURN_IF_UNDEF 0x2
-#define _CORE_SWASH_INIT_ACCEPT_INVLIST 0x4
-
/*
=head1 Unicode Support
L<perlguts/Unicode Support> has an introduction to this API.
@@ -74,8 +69,9 @@ the string is invariant.
#define uvchr_to_utf8_flags_msgs(d,uv,flags,msgs) \
uvoffuni_to_utf8_flags_msgs(d,NATIVE_TO_UNI(uv),flags, msgs)
#define utf8_to_uvchr_buf(s, e, lenp) \
- utf8n_to_uvchr(s, (U8*)(e) - (U8*)(s), lenp, \
- ckWARN_d(WARN_UTF8) ? 0 : UTF8_ALLOW_ANY)
+ (__ASSERT_((U8*) (e) > (U8*) (s)) \
+ utf8n_to_uvchr(s, (U8*)(e) - (U8*)(s), lenp, \
+ ckWARN_d(WARN_UTF8) ? 0 : UTF8_ALLOW_ANY))
#define utf8n_to_uvchr(s, len, lenp, flags) \
utf8n_to_uvchr_error(s, len, lenp, flags, 0)
#define utf8n_to_uvchr_error(s, len, lenp, flags, errors) \
@@ -315,104 +311,6 @@ C<cp> is Unicode if above 255; otherwise is platform-native.
#define isUTF8_POSSIBLY_PROBLEMATIC(c) (__ASSERT_(FITS_IN_8_BITS(c)) \
(U8) c >= 0xED)
-/* A helper macro for isUTF8_CHAR, so use that one instead of this. This was
- * generated by regen/regcharclass.pl, and then moved here. Then it was
- * hand-edited to add some LIKELY() calls, presuming that malformations are
- * unlikely. The lines that generated it were then commented out. This was
- * done because it takes on the order of 10 minutes to generate, and is never
- * going to change, unless the generated code is improved, and figuring out
- * the LIKELYs there would be hard.
- *
- UTF8_CHAR: Matches legal UTF-8 variant code points up through 0x1FFFFFF
-
- 0x80 - 0x1FFFFF
-*/
-/*** GENERATED CODE ***/
-#define is_UTF8_CHAR_utf8_no_length_checks(s) \
-( ( 0xC2 <= ((const U8*)s)[0] && ((const U8*)s)[0] <= 0xDF ) ? \
- ( LIKELY( ( ((const U8*)s)[1] & 0xC0 ) == 0x80 ) ? 2 : 0 ) \
-: ( 0xE0 == ((const U8*)s)[0] ) ? \
- ( LIKELY( ( ( ((const U8*)s)[1] & 0xE0 ) == 0xA0 ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) ? 3 : 0 )\
-: ( 0xE1 <= ((const U8*)s)[0] && ((const U8*)s)[0] <= 0xEF ) ? \
- ( LIKELY( ( ( ((const U8*)s)[1] & 0xC0 ) == 0x80 ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) ? 3 : 0 )\
-: ( 0xF0 == ((const U8*)s)[0] ) ? \
- ( LIKELY( ( ( 0x90 <= ((const U8*)s)[1] && ((const U8*)s)[1] <= 0xBF ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) && ( ( ((const U8*)s)[3] & 0xC0 ) == 0x80 ) ) ? 4 : 0 )\
-: ( ( ( ( 0xF1 <= ((const U8*)s)[0] && ((const U8*)s)[0] <= 0xF7 ) && LIKELY( ( ((const U8*)s)[1] & 0xC0 ) == 0x80 ) ) && LIKELY( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) && LIKELY( ( ((const U8*)s)[3] & 0xC0 ) == 0x80 ) ) ? 4 : 0 )
-
-/* The above macro handles UTF-8 that has this start byte as the maximum */
-#define _IS_UTF8_CHAR_HIGHEST_START_BYTE 0xF7
-
-/* A helper macro for isSTRICT_UTF8_CHAR, so use that one instead of this.
- * Like is_UTF8_CHAR_utf8_no_length_checks(), this was moved here and LIKELYs
- * added manually.
- *
- STRICT_UTF8_CHAR: Matches legal Unicode UTF-8 variant code points, no
- surrrogates nor non-character code points
-*/
-/*** GENERATED CODE ***/
-#define is_STRICT_UTF8_CHAR_utf8_no_length_checks(s) \
-( ( 0xC2 <= ((const U8*)s)[0] && ((const U8*)s)[0] <= 0xDF ) ? \
- ( LIKELY( ( ((const U8*)s)[1] & 0xC0 ) == 0x80 ) ? 2 : 0 ) \
-: ( 0xE0 == ((const U8*)s)[0] ) ? \
- ( LIKELY( ( ( ((const U8*)s)[1] & 0xE0 ) == 0xA0 ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) ? 3 : 0 )\
-: ( ( 0xE1 <= ((const U8*)s)[0] && ((const U8*)s)[0] <= 0xEC ) || 0xEE == ((const U8*)s)[0] ) ?\
- ( ( ( ( ((const U8*)s)[1] & 0xC0 ) == 0x80 ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) ? 3 : 0 )\
-: ( 0xED == ((const U8*)s)[0] ) ? \
- ( LIKELY( ( ( ((const U8*)s)[1] & 0xE0 ) == 0x80 ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) ? 3 : 0 )\
-: ( 0xEF == ((const U8*)s)[0] ) ? \
- ( ( ( 0x80 <= ((const U8*)s)[1] && ((const U8*)s)[1] <= 0xB6 ) || ( 0xB8 <= ((const U8*)s)[1] && ((const U8*)s)[1] <= 0xBE ) ) ?\
- ( LIKELY( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ? 3 : 0 ) \
- : ( 0xB7 == ((const U8*)s)[1] ) ? \
- ( LIKELY( ( ((const U8*)s)[2] & 0xF0 ) == 0x80 || ( ((const U8*)s)[2] & 0xF0 ) == 0xB0 ) ? 3 : 0 )\
- : ( ( 0xBF == ((const U8*)s)[1] ) && ( 0x80 <= ((const U8*)s)[2] && ((const U8*)s)[2] <= 0xBD ) ) ? 3 : 0 )\
-: ( 0xF0 == ((const U8*)s)[0] ) ? \
- ( ( ( 0x90 <= ((const U8*)s)[1] && ((const U8*)s)[1] <= 0x9E ) || ( 0xA0 <= ((const U8*)s)[1] && ((const U8*)s)[1] <= 0xAE ) || ( 0xB0 <= ((const U8*)s)[1] && ((const U8*)s)[1] <= 0xBE ) ) ?\
- ( LIKELY( ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) && ( ( ((const U8*)s)[3] & 0xC0 ) == 0x80 ) ) ? 4 : 0 )\
- : ( ((const U8*)s)[1] == 0x9F || ( ( ((const U8*)s)[1] & 0xEF ) == 0xAF ) ) ? \
- ( ( 0x80 <= ((const U8*)s)[2] && ((const U8*)s)[2] <= 0xBE ) ? \
- ( LIKELY( ( ((const U8*)s)[3] & 0xC0 ) == 0x80 ) ? 4 : 0 ) \
- : LIKELY( ( 0xBF == ((const U8*)s)[2] ) && ( 0x80 <= ((const U8*)s)[3] && ((const U8*)s)[3] <= 0xBD ) ) ? 4 : 0 )\
- : 0 ) \
-: ( 0xF1 <= ((const U8*)s)[0] && ((const U8*)s)[0] <= 0xF3 ) ? \
- ( ( ( ( ((const U8*)s)[1] & 0xC8 ) == 0x80 ) || ( ( ((const U8*)s)[1] & 0xCC ) == 0x88 ) || ( ( ((const U8*)s)[1] & 0xCE ) == 0x8C ) || ( ( ((const U8*)s)[1] & 0xCF ) == 0x8E ) ) ?\
- ( LIKELY( ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) && ( ( ((const U8*)s)[3] & 0xC0 ) == 0x80 ) ) ? 4 : 0 )\
- : ( ( ((const U8*)s)[1] & 0xCF ) == 0x8F ) ? \
- ( ( 0x80 <= ((const U8*)s)[2] && ((const U8*)s)[2] <= 0xBE ) ? \
- ( LIKELY( ( ((const U8*)s)[3] & 0xC0 ) == 0x80 ) ? 4 : 0 ) \
- : LIKELY( ( 0xBF == ((const U8*)s)[2] ) && ( 0x80 <= ((const U8*)s)[3] && ((const U8*)s)[3] <= 0xBD ) ) ? 4 : 0 )\
- : 0 ) \
-: ( 0xF4 == ((const U8*)s)[0] ) ? \
- ( ( 0x80 <= ((const U8*)s)[1] && ((const U8*)s)[1] <= 0x8E ) ? \
- ( LIKELY( ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) && ( ( ((const U8*)s)[3] & 0xC0 ) == 0x80 ) ) ? 4 : 0 )\
- : ( 0x8F == ((const U8*)s)[1] ) ? \
- ( ( 0x80 <= ((const U8*)s)[2] && ((const U8*)s)[2] <= 0xBE ) ? \
- ( LIKELY( ( ((const U8*)s)[3] & 0xC0 ) == 0x80 ) ? 4 : 0 ) \
- : LIKELY( ( 0xBF == ((const U8*)s)[2] ) && ( 0x80 <= ((const U8*)s)[3] && ((const U8*)s)[3] <= 0xBD ) ) ? 4 : 0 )\
- : 0 ) \
-: 0 )
-
-/* Similarly,
- C9_STRICT_UTF8_CHAR: Matches legal Unicode UTF-8 variant code
- points, no surrogates
- 0x0080 - 0xD7FF
- 0xE000 - 0x10FFFF
-*/
-/*** GENERATED CODE ***/
-#define is_C9_STRICT_UTF8_CHAR_utf8_no_length_checks(s) \
-( ( 0xC2 <= ((const U8*)s)[0] && ((const U8*)s)[0] <= 0xDF ) ? \
- ( LIKELY( ( ((const U8*)s)[1] & 0xC0 ) == 0x80 ) ? 2 : 0 ) \
-: ( 0xE0 == ((const U8*)s)[0] ) ? \
- ( LIKELY( ( ( ((const U8*)s)[1] & 0xE0 ) == 0xA0 ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) ? 3 : 0 )\
-: ( ( 0xE1 <= ((const U8*)s)[0] && ((const U8*)s)[0] <= 0xEC ) || ( ((const U8*)s)[0] & 0xFE ) == 0xEE ) ?\
- ( LIKELY( ( ( ((const U8*)s)[1] & 0xC0 ) == 0x80 ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) ? 3 : 0 )\
-: ( 0xED == ((const U8*)s)[0] ) ? \
- ( LIKELY( ( ( ((const U8*)s)[1] & 0xE0 ) == 0x80 ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) ? 3 : 0 )\
-: ( 0xF0 == ((const U8*)s)[0] ) ? \
- ( LIKELY( ( ( 0x90 <= ((const U8*)s)[1] && ((const U8*)s)[1] <= 0xBF ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) && ( ( ((const U8*)s)[3] & 0xC0 ) == 0x80 ) ) ? 4 : 0 )\
-: ( 0xF1 <= ((const U8*)s)[0] && ((const U8*)s)[0] <= 0xF3 ) ? \
- ( LIKELY( ( ( ( ((const U8*)s)[1] & 0xC0 ) == 0x80 ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) && ( ( ((const U8*)s)[3] & 0xC0 ) == 0x80 ) ) ? 4 : 0 )\
-: LIKELY( ( ( ( 0xF4 == ((const U8*)s)[0] ) && ( ( ((const U8*)s)[1] & 0xF0 ) == 0x80 ) ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) && ( ( ((const U8*)s)[3] & 0xC0 ) == 0x80 ) ) ? 4 : 0 )
-
#define UNICODE_IS_PERL_EXTENDED(uv) UNLIKELY((UV) (uv) > 0x7FFFFFFF)
#endif /* EBCDIC vs ASCII */
@@ -600,6 +498,20 @@ only) byte is pointed to by C<s>.
#define UTF8SKIP(s) PL_utf8skip[*(const U8*)(s)]
#define UTF8_SKIP(s) UTF8SKIP(s)
+/*
+
+=for apidoc Am|STRLEN|UTF8_SAFE_SKIP|char* s|char* e
+returns 0 if S<C<s E<gt>= e>>; otherwise returns the number of bytes in the
+UTF-8 encoded character whose first byte is pointed to by C<s>. But it never
+returns beyond C<e>. On DEBUGGING builds, it asserts that S<C<s E<lt>= e>>.
+
+=cut
+ */
+#define UTF8_SAFE_SKIP(s, e) (__ASSERT_((e) >= (s)) \
+ ((e) - (s)) <= 0 \
+ ? 0 \
+ : MIN(((e) - (s)), UTF8_SKIP(s)))
+
/* Most code that says 'UNI_' really means the native value for code points up
* through 255 */
#define UNI_IS_INVARIANT(cp) UVCHR_IS_INVARIANT(cp)
@@ -751,7 +663,7 @@ case any call to string overloading updates the internal UTF-8 encoding flag.
#define UTF8_ALLOW_CONTINUATION 0x0002
#define UTF8_GOT_CONTINUATION UTF8_ALLOW_CONTINUATION
-/* Unexpected continuation byte */
+/* Unexpected non-continuation byte */
#define UTF8_ALLOW_NON_CONTINUATION 0x0004
#define UTF8_GOT_NON_CONTINUATION UTF8_ALLOW_NON_CONTINUATION
@@ -835,8 +747,7 @@ case any call to string overloading updates the internal UTF-8 encoding flag.
|UTF8_ALLOW_OVERFLOW)
/* Accept any Perl-extended UTF-8 that evaluates to any UV on the platform, but
- * not any malformed. This is the default. (Note that UVs above IV_MAX are
- * deprecated. */
+ * not any malformed. This is the default. */
#define UTF8_ALLOW_ANYUV 0
#define UTF8_ALLOW_DEFAULT UTF8_ALLOW_ANYUV
@@ -856,6 +767,8 @@ point's representation.
#define UTF8_IS_REPLACEMENT(s, send) is_REPLACEMENT_utf8_safe(s,send)
+#define MAX_LEGAL_CP ((UV)IV_MAX)
+
/*
=for apidoc Am|bool|UTF8_IS_SUPER|const U8 *s|const U8 *e
@@ -872,7 +785,7 @@ represent a code point that cannot fit in a UV on the current platform. Hence
this macro can give different results when run on a 64-bit word machine than on
one with a 32-bit word size.
-Note that it is deprecated to have code points that are larger than what can
+Note that it is illegal to have code points that are larger than what can
fit in an IV on the current machine.
=cut
@@ -1022,134 +935,12 @@ point's representation.
#define SHARP_S_SKIP 2
-/*
-
-=for apidoc Am|STRLEN|isUTF8_CHAR|const U8 *s|const U8 *e
-
-Evaluates to non-zero if the first few bytes of the string starting at C<s> and
-looking no further than S<C<e - 1>> are well-formed UTF-8, as extended by Perl,
-that represents some code point; otherwise it evaluates to 0. If non-zero, the
-value gives how many bytes starting at C<s> comprise the code point's
-representation. Any bytes remaining before C<e>, but beyond the ones needed to
-form the first code point in C<s>, are not examined.
-
-The code point can be any that will fit in a UV on this machine, using Perl's
-extension to official UTF-8 to represent those higher than the Unicode maximum
-of 0x10FFFF. That means that this macro is used to efficiently decide if the
-next few bytes in C<s> is legal UTF-8 for a single character.
-
-Use C<L</isSTRICT_UTF8_CHAR>> to restrict the acceptable code points to those
-defined by Unicode to be fully interchangeable across applications;
-C<L</isC9_STRICT_UTF8_CHAR>> to use the L<Unicode Corrigendum
-#9|http://www.unicode.org/versions/corrigendum9.html> definition of allowable
-code points; and C<L</isUTF8_CHAR_flags>> for a more customized definition.
-
-Use C<L</is_utf8_string>>, C<L</is_utf8_string_loc>>, and
-C<L</is_utf8_string_loclen>> to check entire strings.
-
-Note that it is deprecated to use code points higher than what will fit in an
-IV. This macro does not raise any warnings for such code points, treating them
-as valid.
-
-Note also that a UTF-8 INVARIANT character (i.e. ASCII on non-EBCDIC machines)
-is a valid UTF-8 character.
-
-=cut
-*/
-
-#define isUTF8_CHAR(s, e) \
- (UNLIKELY((e) <= (s)) \
- ? 0 \
- : (UTF8_IS_INVARIANT(*s)) \
- ? 1 \
- : UNLIKELY(((e) - (s)) < UTF8SKIP(s)) \
- ? 0 \
- : LIKELY(NATIVE_UTF8_TO_I8(*s) <= _IS_UTF8_CHAR_HIGHEST_START_BYTE) \
- ? is_UTF8_CHAR_utf8_no_length_checks(s) \
- : _is_utf8_char_helper(s, e, 0))
-
#define is_utf8_char_buf(buf, buf_end) isUTF8_CHAR(buf, buf_end)
#define bytes_from_utf8(s, lenp, is_utf8p) \
bytes_from_utf8_loc(s, lenp, is_utf8p, 0)
/*
-=for apidoc Am|STRLEN|isSTRICT_UTF8_CHAR|const U8 *s|const U8 *e
-
-Evaluates to non-zero if the first few bytes of the string starting at C<s> and
-looking no further than S<C<e - 1>> are well-formed UTF-8 that represents some
-Unicode code point completely acceptable for open interchange between all
-applications; otherwise it evaluates to 0. If non-zero, the value gives how
-many bytes starting at C<s> comprise the code point's representation. Any
-bytes remaining before C<e>, but beyond the ones needed to form the first code
-point in C<s>, are not examined.
-
-The largest acceptable code point is the Unicode maximum 0x10FFFF, and must not
-be a surrogate nor a non-character code point. Thus this excludes any code
-point from Perl's extended UTF-8.
-
-This is used to efficiently decide if the next few bytes in C<s> is
-legal Unicode-acceptable UTF-8 for a single character.
-
-Use C<L</isC9_STRICT_UTF8_CHAR>> to use the L<Unicode Corrigendum
-#9|http://www.unicode.org/versions/corrigendum9.html> definition of allowable
-code points; C<L</isUTF8_CHAR>> to check for Perl's extended UTF-8;
-and C<L</isUTF8_CHAR_flags>> for a more customized definition.
-
-Use C<L</is_strict_utf8_string>>, C<L</is_strict_utf8_string_loc>>, and
-C<L</is_strict_utf8_string_loclen>> to check entire strings.
-
-=cut
-*/
-
-#define isSTRICT_UTF8_CHAR(s, e) \
- (UNLIKELY((e) <= (s)) \
- ? 0 \
- : (UTF8_IS_INVARIANT(*s)) \
- ? 1 \
- : UNLIKELY(((e) - (s)) < UTF8SKIP(s)) \
- ? 0 \
- : is_STRICT_UTF8_CHAR_utf8_no_length_checks(s))
-
-/*
-
-=for apidoc Am|STRLEN|isC9_STRICT_UTF8_CHAR|const U8 *s|const U8 *e
-
-Evaluates to non-zero if the first few bytes of the string starting at C<s> and
-looking no further than S<C<e - 1>> are well-formed UTF-8 that represents some
-Unicode non-surrogate code point; otherwise it evaluates to 0. If non-zero,
-the value gives how many bytes starting at C<s> comprise the code point's
-representation. Any bytes remaining before C<e>, but beyond the ones needed to
-form the first code point in C<s>, are not examined.
-
-The largest acceptable code point is the Unicode maximum 0x10FFFF. This
-differs from C<L</isSTRICT_UTF8_CHAR>> only in that it accepts non-character
-code points. This corresponds to
-L<Unicode Corrigendum #9|http://www.unicode.org/versions/corrigendum9.html>.
-which said that non-character code points are merely discouraged rather than
-completely forbidden in open interchange. See
-L<perlunicode/Noncharacter code points>.
-
-Use C<L</isUTF8_CHAR>> to check for Perl's extended UTF-8; and
-C<L</isUTF8_CHAR_flags>> for a more customized definition.
-
-Use C<L</is_c9strict_utf8_string>>, C<L</is_c9strict_utf8_string_loc>>, and
-C<L</is_c9strict_utf8_string_loclen>> to check entire strings.
-
-=cut
-*/
-
-#define isC9_STRICT_UTF8_CHAR(s, e) \
- (UNLIKELY((e) <= (s)) \
- ? 0 \
- : (UTF8_IS_INVARIANT(*s)) \
- ? 1 \
- : UNLIKELY(((e) - (s)) < UTF8SKIP(s)) \
- ? 0 \
- : is_C9_STRICT_UTF8_CHAR_utf8_no_length_checks(s))
-
-/*
-
=for apidoc Am|STRLEN|isUTF8_CHAR_flags|const U8 *s|const U8 *e| const U32 flags
Evaluates to non-zero if the first few bytes of the string starting at C<s> and