diff options
Diffstat (limited to 'Master/tlpkg/tlperl/lib/CORE/utf8.h')
-rw-r--r-- | Master/tlpkg/tlperl/lib/CORE/utf8.h | 253 |
1 files changed, 22 insertions, 231 deletions
diff --git a/Master/tlpkg/tlperl/lib/CORE/utf8.h b/Master/tlpkg/tlperl/lib/CORE/utf8.h index e0fc6b61419..d0b8742d5a8 100644 --- a/Master/tlpkg/tlperl/lib/CORE/utf8.h +++ b/Master/tlpkg/tlperl/lib/CORE/utf8.h @@ -34,11 +34,6 @@ #define FOLD_FLAGS_FULL 0x2 #define FOLD_FLAGS_NOMIX_ASCII 0x4 -/* For _core_swash_init(), internal core use only */ -#define _CORE_SWASH_INIT_USER_DEFINED_PROPERTY 0x1 -#define _CORE_SWASH_INIT_RETURN_IF_UNDEF 0x2 -#define _CORE_SWASH_INIT_ACCEPT_INVLIST 0x4 - /* =head1 Unicode Support L<perlguts/Unicode Support> has an introduction to this API. @@ -74,8 +69,9 @@ the string is invariant. #define uvchr_to_utf8_flags_msgs(d,uv,flags,msgs) \ uvoffuni_to_utf8_flags_msgs(d,NATIVE_TO_UNI(uv),flags, msgs) #define utf8_to_uvchr_buf(s, e, lenp) \ - utf8n_to_uvchr(s, (U8*)(e) - (U8*)(s), lenp, \ - ckWARN_d(WARN_UTF8) ? 0 : UTF8_ALLOW_ANY) + (__ASSERT_((U8*) (e) > (U8*) (s)) \ + utf8n_to_uvchr(s, (U8*)(e) - (U8*)(s), lenp, \ + ckWARN_d(WARN_UTF8) ? 0 : UTF8_ALLOW_ANY)) #define utf8n_to_uvchr(s, len, lenp, flags) \ utf8n_to_uvchr_error(s, len, lenp, flags, 0) #define utf8n_to_uvchr_error(s, len, lenp, flags, errors) \ @@ -315,104 +311,6 @@ C<cp> is Unicode if above 255; otherwise is platform-native. #define isUTF8_POSSIBLY_PROBLEMATIC(c) (__ASSERT_(FITS_IN_8_BITS(c)) \ (U8) c >= 0xED) -/* A helper macro for isUTF8_CHAR, so use that one instead of this. This was - * generated by regen/regcharclass.pl, and then moved here. Then it was - * hand-edited to add some LIKELY() calls, presuming that malformations are - * unlikely. The lines that generated it were then commented out. This was - * done because it takes on the order of 10 minutes to generate, and is never - * going to change, unless the generated code is improved, and figuring out - * the LIKELYs there would be hard. - * - UTF8_CHAR: Matches legal UTF-8 variant code points up through 0x1FFFFFF - - 0x80 - 0x1FFFFF -*/ -/*** GENERATED CODE ***/ -#define is_UTF8_CHAR_utf8_no_length_checks(s) \ -( ( 0xC2 <= ((const U8*)s)[0] && ((const U8*)s)[0] <= 0xDF ) ? \ - ( LIKELY( ( ((const U8*)s)[1] & 0xC0 ) == 0x80 ) ? 2 : 0 ) \ -: ( 0xE0 == ((const U8*)s)[0] ) ? \ - ( LIKELY( ( ( ((const U8*)s)[1] & 0xE0 ) == 0xA0 ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) ? 3 : 0 )\ -: ( 0xE1 <= ((const U8*)s)[0] && ((const U8*)s)[0] <= 0xEF ) ? \ - ( LIKELY( ( ( ((const U8*)s)[1] & 0xC0 ) == 0x80 ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) ? 3 : 0 )\ -: ( 0xF0 == ((const U8*)s)[0] ) ? \ - ( LIKELY( ( ( 0x90 <= ((const U8*)s)[1] && ((const U8*)s)[1] <= 0xBF ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) && ( ( ((const U8*)s)[3] & 0xC0 ) == 0x80 ) ) ? 4 : 0 )\ -: ( ( ( ( 0xF1 <= ((const U8*)s)[0] && ((const U8*)s)[0] <= 0xF7 ) && LIKELY( ( ((const U8*)s)[1] & 0xC0 ) == 0x80 ) ) && LIKELY( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) && LIKELY( ( ((const U8*)s)[3] & 0xC0 ) == 0x80 ) ) ? 4 : 0 ) - -/* The above macro handles UTF-8 that has this start byte as the maximum */ -#define _IS_UTF8_CHAR_HIGHEST_START_BYTE 0xF7 - -/* A helper macro for isSTRICT_UTF8_CHAR, so use that one instead of this. - * Like is_UTF8_CHAR_utf8_no_length_checks(), this was moved here and LIKELYs - * added manually. - * - STRICT_UTF8_CHAR: Matches legal Unicode UTF-8 variant code points, no - surrrogates nor non-character code points -*/ -/*** GENERATED CODE ***/ -#define is_STRICT_UTF8_CHAR_utf8_no_length_checks(s) \ -( ( 0xC2 <= ((const U8*)s)[0] && ((const U8*)s)[0] <= 0xDF ) ? \ - ( LIKELY( ( ((const U8*)s)[1] & 0xC0 ) == 0x80 ) ? 2 : 0 ) \ -: ( 0xE0 == ((const U8*)s)[0] ) ? \ - ( LIKELY( ( ( ((const U8*)s)[1] & 0xE0 ) == 0xA0 ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) ? 3 : 0 )\ -: ( ( 0xE1 <= ((const U8*)s)[0] && ((const U8*)s)[0] <= 0xEC ) || 0xEE == ((const U8*)s)[0] ) ?\ - ( ( ( ( ((const U8*)s)[1] & 0xC0 ) == 0x80 ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) ? 3 : 0 )\ -: ( 0xED == ((const U8*)s)[0] ) ? \ - ( LIKELY( ( ( ((const U8*)s)[1] & 0xE0 ) == 0x80 ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) ? 3 : 0 )\ -: ( 0xEF == ((const U8*)s)[0] ) ? \ - ( ( ( 0x80 <= ((const U8*)s)[1] && ((const U8*)s)[1] <= 0xB6 ) || ( 0xB8 <= ((const U8*)s)[1] && ((const U8*)s)[1] <= 0xBE ) ) ?\ - ( LIKELY( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ? 3 : 0 ) \ - : ( 0xB7 == ((const U8*)s)[1] ) ? \ - ( LIKELY( ( ((const U8*)s)[2] & 0xF0 ) == 0x80 || ( ((const U8*)s)[2] & 0xF0 ) == 0xB0 ) ? 3 : 0 )\ - : ( ( 0xBF == ((const U8*)s)[1] ) && ( 0x80 <= ((const U8*)s)[2] && ((const U8*)s)[2] <= 0xBD ) ) ? 3 : 0 )\ -: ( 0xF0 == ((const U8*)s)[0] ) ? \ - ( ( ( 0x90 <= ((const U8*)s)[1] && ((const U8*)s)[1] <= 0x9E ) || ( 0xA0 <= ((const U8*)s)[1] && ((const U8*)s)[1] <= 0xAE ) || ( 0xB0 <= ((const U8*)s)[1] && ((const U8*)s)[1] <= 0xBE ) ) ?\ - ( LIKELY( ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) && ( ( ((const U8*)s)[3] & 0xC0 ) == 0x80 ) ) ? 4 : 0 )\ - : ( ((const U8*)s)[1] == 0x9F || ( ( ((const U8*)s)[1] & 0xEF ) == 0xAF ) ) ? \ - ( ( 0x80 <= ((const U8*)s)[2] && ((const U8*)s)[2] <= 0xBE ) ? \ - ( LIKELY( ( ((const U8*)s)[3] & 0xC0 ) == 0x80 ) ? 4 : 0 ) \ - : LIKELY( ( 0xBF == ((const U8*)s)[2] ) && ( 0x80 <= ((const U8*)s)[3] && ((const U8*)s)[3] <= 0xBD ) ) ? 4 : 0 )\ - : 0 ) \ -: ( 0xF1 <= ((const U8*)s)[0] && ((const U8*)s)[0] <= 0xF3 ) ? \ - ( ( ( ( ((const U8*)s)[1] & 0xC8 ) == 0x80 ) || ( ( ((const U8*)s)[1] & 0xCC ) == 0x88 ) || ( ( ((const U8*)s)[1] & 0xCE ) == 0x8C ) || ( ( ((const U8*)s)[1] & 0xCF ) == 0x8E ) ) ?\ - ( LIKELY( ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) && ( ( ((const U8*)s)[3] & 0xC0 ) == 0x80 ) ) ? 4 : 0 )\ - : ( ( ((const U8*)s)[1] & 0xCF ) == 0x8F ) ? \ - ( ( 0x80 <= ((const U8*)s)[2] && ((const U8*)s)[2] <= 0xBE ) ? \ - ( LIKELY( ( ((const U8*)s)[3] & 0xC0 ) == 0x80 ) ? 4 : 0 ) \ - : LIKELY( ( 0xBF == ((const U8*)s)[2] ) && ( 0x80 <= ((const U8*)s)[3] && ((const U8*)s)[3] <= 0xBD ) ) ? 4 : 0 )\ - : 0 ) \ -: ( 0xF4 == ((const U8*)s)[0] ) ? \ - ( ( 0x80 <= ((const U8*)s)[1] && ((const U8*)s)[1] <= 0x8E ) ? \ - ( LIKELY( ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) && ( ( ((const U8*)s)[3] & 0xC0 ) == 0x80 ) ) ? 4 : 0 )\ - : ( 0x8F == ((const U8*)s)[1] ) ? \ - ( ( 0x80 <= ((const U8*)s)[2] && ((const U8*)s)[2] <= 0xBE ) ? \ - ( LIKELY( ( ((const U8*)s)[3] & 0xC0 ) == 0x80 ) ? 4 : 0 ) \ - : LIKELY( ( 0xBF == ((const U8*)s)[2] ) && ( 0x80 <= ((const U8*)s)[3] && ((const U8*)s)[3] <= 0xBD ) ) ? 4 : 0 )\ - : 0 ) \ -: 0 ) - -/* Similarly, - C9_STRICT_UTF8_CHAR: Matches legal Unicode UTF-8 variant code - points, no surrogates - 0x0080 - 0xD7FF - 0xE000 - 0x10FFFF -*/ -/*** GENERATED CODE ***/ -#define is_C9_STRICT_UTF8_CHAR_utf8_no_length_checks(s) \ -( ( 0xC2 <= ((const U8*)s)[0] && ((const U8*)s)[0] <= 0xDF ) ? \ - ( LIKELY( ( ((const U8*)s)[1] & 0xC0 ) == 0x80 ) ? 2 : 0 ) \ -: ( 0xE0 == ((const U8*)s)[0] ) ? \ - ( LIKELY( ( ( ((const U8*)s)[1] & 0xE0 ) == 0xA0 ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) ? 3 : 0 )\ -: ( ( 0xE1 <= ((const U8*)s)[0] && ((const U8*)s)[0] <= 0xEC ) || ( ((const U8*)s)[0] & 0xFE ) == 0xEE ) ?\ - ( LIKELY( ( ( ((const U8*)s)[1] & 0xC0 ) == 0x80 ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) ? 3 : 0 )\ -: ( 0xED == ((const U8*)s)[0] ) ? \ - ( LIKELY( ( ( ((const U8*)s)[1] & 0xE0 ) == 0x80 ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) ? 3 : 0 )\ -: ( 0xF0 == ((const U8*)s)[0] ) ? \ - ( LIKELY( ( ( 0x90 <= ((const U8*)s)[1] && ((const U8*)s)[1] <= 0xBF ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) && ( ( ((const U8*)s)[3] & 0xC0 ) == 0x80 ) ) ? 4 : 0 )\ -: ( 0xF1 <= ((const U8*)s)[0] && ((const U8*)s)[0] <= 0xF3 ) ? \ - ( LIKELY( ( ( ( ((const U8*)s)[1] & 0xC0 ) == 0x80 ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) && ( ( ((const U8*)s)[3] & 0xC0 ) == 0x80 ) ) ? 4 : 0 )\ -: LIKELY( ( ( ( 0xF4 == ((const U8*)s)[0] ) && ( ( ((const U8*)s)[1] & 0xF0 ) == 0x80 ) ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) && ( ( ((const U8*)s)[3] & 0xC0 ) == 0x80 ) ) ? 4 : 0 ) - #define UNICODE_IS_PERL_EXTENDED(uv) UNLIKELY((UV) (uv) > 0x7FFFFFFF) #endif /* EBCDIC vs ASCII */ @@ -600,6 +498,20 @@ only) byte is pointed to by C<s>. #define UTF8SKIP(s) PL_utf8skip[*(const U8*)(s)] #define UTF8_SKIP(s) UTF8SKIP(s) +/* + +=for apidoc Am|STRLEN|UTF8_SAFE_SKIP|char* s|char* e +returns 0 if S<C<s E<gt>= e>>; otherwise returns the number of bytes in the +UTF-8 encoded character whose first byte is pointed to by C<s>. But it never +returns beyond C<e>. On DEBUGGING builds, it asserts that S<C<s E<lt>= e>>. + +=cut + */ +#define UTF8_SAFE_SKIP(s, e) (__ASSERT_((e) >= (s)) \ + ((e) - (s)) <= 0 \ + ? 0 \ + : MIN(((e) - (s)), UTF8_SKIP(s))) + /* Most code that says 'UNI_' really means the native value for code points up * through 255 */ #define UNI_IS_INVARIANT(cp) UVCHR_IS_INVARIANT(cp) @@ -751,7 +663,7 @@ case any call to string overloading updates the internal UTF-8 encoding flag. #define UTF8_ALLOW_CONTINUATION 0x0002 #define UTF8_GOT_CONTINUATION UTF8_ALLOW_CONTINUATION -/* Unexpected continuation byte */ +/* Unexpected non-continuation byte */ #define UTF8_ALLOW_NON_CONTINUATION 0x0004 #define UTF8_GOT_NON_CONTINUATION UTF8_ALLOW_NON_CONTINUATION @@ -835,8 +747,7 @@ case any call to string overloading updates the internal UTF-8 encoding flag. |UTF8_ALLOW_OVERFLOW) /* Accept any Perl-extended UTF-8 that evaluates to any UV on the platform, but - * not any malformed. This is the default. (Note that UVs above IV_MAX are - * deprecated. */ + * not any malformed. This is the default. */ #define UTF8_ALLOW_ANYUV 0 #define UTF8_ALLOW_DEFAULT UTF8_ALLOW_ANYUV @@ -856,6 +767,8 @@ point's representation. #define UTF8_IS_REPLACEMENT(s, send) is_REPLACEMENT_utf8_safe(s,send) +#define MAX_LEGAL_CP ((UV)IV_MAX) + /* =for apidoc Am|bool|UTF8_IS_SUPER|const U8 *s|const U8 *e @@ -872,7 +785,7 @@ represent a code point that cannot fit in a UV on the current platform. Hence this macro can give different results when run on a 64-bit word machine than on one with a 32-bit word size. -Note that it is deprecated to have code points that are larger than what can +Note that it is illegal to have code points that are larger than what can fit in an IV on the current machine. =cut @@ -1022,134 +935,12 @@ point's representation. #define SHARP_S_SKIP 2 -/* - -=for apidoc Am|STRLEN|isUTF8_CHAR|const U8 *s|const U8 *e - -Evaluates to non-zero if the first few bytes of the string starting at C<s> and -looking no further than S<C<e - 1>> are well-formed UTF-8, as extended by Perl, -that represents some code point; otherwise it evaluates to 0. If non-zero, the -value gives how many bytes starting at C<s> comprise the code point's -representation. Any bytes remaining before C<e>, but beyond the ones needed to -form the first code point in C<s>, are not examined. - -The code point can be any that will fit in a UV on this machine, using Perl's -extension to official UTF-8 to represent those higher than the Unicode maximum -of 0x10FFFF. That means that this macro is used to efficiently decide if the -next few bytes in C<s> is legal UTF-8 for a single character. - -Use C<L</isSTRICT_UTF8_CHAR>> to restrict the acceptable code points to those -defined by Unicode to be fully interchangeable across applications; -C<L</isC9_STRICT_UTF8_CHAR>> to use the L<Unicode Corrigendum -#9|http://www.unicode.org/versions/corrigendum9.html> definition of allowable -code points; and C<L</isUTF8_CHAR_flags>> for a more customized definition. - -Use C<L</is_utf8_string>>, C<L</is_utf8_string_loc>>, and -C<L</is_utf8_string_loclen>> to check entire strings. - -Note that it is deprecated to use code points higher than what will fit in an -IV. This macro does not raise any warnings for such code points, treating them -as valid. - -Note also that a UTF-8 INVARIANT character (i.e. ASCII on non-EBCDIC machines) -is a valid UTF-8 character. - -=cut -*/ - -#define isUTF8_CHAR(s, e) \ - (UNLIKELY((e) <= (s)) \ - ? 0 \ - : (UTF8_IS_INVARIANT(*s)) \ - ? 1 \ - : UNLIKELY(((e) - (s)) < UTF8SKIP(s)) \ - ? 0 \ - : LIKELY(NATIVE_UTF8_TO_I8(*s) <= _IS_UTF8_CHAR_HIGHEST_START_BYTE) \ - ? is_UTF8_CHAR_utf8_no_length_checks(s) \ - : _is_utf8_char_helper(s, e, 0)) - #define is_utf8_char_buf(buf, buf_end) isUTF8_CHAR(buf, buf_end) #define bytes_from_utf8(s, lenp, is_utf8p) \ bytes_from_utf8_loc(s, lenp, is_utf8p, 0) /* -=for apidoc Am|STRLEN|isSTRICT_UTF8_CHAR|const U8 *s|const U8 *e - -Evaluates to non-zero if the first few bytes of the string starting at C<s> and -looking no further than S<C<e - 1>> are well-formed UTF-8 that represents some -Unicode code point completely acceptable for open interchange between all -applications; otherwise it evaluates to 0. If non-zero, the value gives how -many bytes starting at C<s> comprise the code point's representation. Any -bytes remaining before C<e>, but beyond the ones needed to form the first code -point in C<s>, are not examined. - -The largest acceptable code point is the Unicode maximum 0x10FFFF, and must not -be a surrogate nor a non-character code point. Thus this excludes any code -point from Perl's extended UTF-8. - -This is used to efficiently decide if the next few bytes in C<s> is -legal Unicode-acceptable UTF-8 for a single character. - -Use C<L</isC9_STRICT_UTF8_CHAR>> to use the L<Unicode Corrigendum -#9|http://www.unicode.org/versions/corrigendum9.html> definition of allowable -code points; C<L</isUTF8_CHAR>> to check for Perl's extended UTF-8; -and C<L</isUTF8_CHAR_flags>> for a more customized definition. - -Use C<L</is_strict_utf8_string>>, C<L</is_strict_utf8_string_loc>>, and -C<L</is_strict_utf8_string_loclen>> to check entire strings. - -=cut -*/ - -#define isSTRICT_UTF8_CHAR(s, e) \ - (UNLIKELY((e) <= (s)) \ - ? 0 \ - : (UTF8_IS_INVARIANT(*s)) \ - ? 1 \ - : UNLIKELY(((e) - (s)) < UTF8SKIP(s)) \ - ? 0 \ - : is_STRICT_UTF8_CHAR_utf8_no_length_checks(s)) - -/* - -=for apidoc Am|STRLEN|isC9_STRICT_UTF8_CHAR|const U8 *s|const U8 *e - -Evaluates to non-zero if the first few bytes of the string starting at C<s> and -looking no further than S<C<e - 1>> are well-formed UTF-8 that represents some -Unicode non-surrogate code point; otherwise it evaluates to 0. If non-zero, -the value gives how many bytes starting at C<s> comprise the code point's -representation. Any bytes remaining before C<e>, but beyond the ones needed to -form the first code point in C<s>, are not examined. - -The largest acceptable code point is the Unicode maximum 0x10FFFF. This -differs from C<L</isSTRICT_UTF8_CHAR>> only in that it accepts non-character -code points. This corresponds to -L<Unicode Corrigendum #9|http://www.unicode.org/versions/corrigendum9.html>. -which said that non-character code points are merely discouraged rather than -completely forbidden in open interchange. See -L<perlunicode/Noncharacter code points>. - -Use C<L</isUTF8_CHAR>> to check for Perl's extended UTF-8; and -C<L</isUTF8_CHAR_flags>> for a more customized definition. - -Use C<L</is_c9strict_utf8_string>>, C<L</is_c9strict_utf8_string_loc>>, and -C<L</is_c9strict_utf8_string_loclen>> to check entire strings. - -=cut -*/ - -#define isC9_STRICT_UTF8_CHAR(s, e) \ - (UNLIKELY((e) <= (s)) \ - ? 0 \ - : (UTF8_IS_INVARIANT(*s)) \ - ? 1 \ - : UNLIKELY(((e) - (s)) < UTF8SKIP(s)) \ - ? 0 \ - : is_C9_STRICT_UTF8_CHAR_utf8_no_length_checks(s)) - -/* - =for apidoc Am|STRLEN|isUTF8_CHAR_flags|const U8 *s|const U8 *e| const U32 flags Evaluates to non-zero if the first few bytes of the string starting at C<s> and |