diff options
author | Karl Berry <karl@freefriends.org> | 2016-04-05 22:27:26 +0000 |
---|---|---|
committer | Karl Berry <karl@freefriends.org> | 2016-04-05 22:27:26 +0000 |
commit | b56b320b5e2515160073fa1b469514002688fe11 (patch) | |
tree | 965a7100c5e45fca8ec803d22b8b6ce14fca4633 /Master/tlpkg/tlperl/lib/Unicode/Collate.pm | |
parent | d26c206452d2e285c3bbf949f34011e4a55fd8f9 (diff) |
tlperl 5.22.1 from siep
git-svn-id: svn://tug.org/texlive/trunk@40252 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/tlpkg/tlperl/lib/Unicode/Collate.pm')
-rw-r--r-- | Master/tlpkg/tlperl/lib/Unicode/Collate.pm | 105 |
1 files changed, 77 insertions, 28 deletions
diff --git a/Master/tlpkg/tlperl/lib/Unicode/Collate.pm b/Master/tlpkg/tlperl/lib/Unicode/Collate.pm index 0fd29517265..f939b66d550 100644 --- a/Master/tlpkg/tlperl/lib/Unicode/Collate.pm +++ b/Master/tlpkg/tlperl/lib/Unicode/Collate.pm @@ -17,7 +17,7 @@ use File::Spec; no warnings 'utf8'; -our $VERSION = '1.04'; +our $VERSION = '1.12'; our $PACKAGE = __PACKAGE__; ### begin XS only ### @@ -89,9 +89,9 @@ my $DefaultRearrange = [ 0x0E40..0x0E44, 0x0EC0..0x0EC4 ]; my $HighestVCE = pack(VCE_TEMPLATE, 0, 0xFFFE, 0x20, 0x5, 0xFFFF); my $minimalVCE = pack(VCE_TEMPLATE, 0, 1, 0x20, 0x5, 0xFFFE); -sub UCA_Version { "28" } +sub UCA_Version { "30" } -sub Base_Unicode_Version { "6.3.0" } +sub Base_Unicode_Version { "7.0.0" } ###### @@ -99,11 +99,9 @@ sub pack_U { return pack('U*', @_); } -### begin XS only ### -*unpack_U = exists &Unicode::Collate::bootstrap && - $] < 5.008 && \&unpackUfor56 && 0x41 == unpackUfor56('A') - ? \&unpackUfor56 : sub { return unpack('U*', shift(@_).pack('U*')) }; -### end XS only ### +sub unpack_U { + return unpack('U*', shift(@_).pack('U*')); +} ###### @@ -117,6 +115,7 @@ our @ChangeOK = qw/ katakana_before_hiragana upper_before_lower ignore_level2 overrideCJK overrideHangul overrideOut preprocess UCA_Version hangul_terminator variable identical highestFFFF minimalFFFE + long_contraction /; our @ChangeNG = qw/ @@ -189,6 +188,7 @@ my %DerivCode = ( 24 => \&_derivCE_24, 26 => \&_derivCE_24, # 26 == 24 28 => \&_derivCE_24, # 28 == 24 + 30 => \&_derivCE_24, # 30 == 24 ); sub checkCollator { @@ -285,6 +285,7 @@ sub new } } + # only in new(), not in change() $self->{level} ||= MaxLevel; $self->{UCA_Version} ||= UCA_Version(); @@ -299,7 +300,10 @@ sub new if ! exists $self->{rearrange}; $self->{backwards} = $self->{backwardsTable} if ! exists $self->{backwards}; + exists $self->{long_contraction} or $self->{long_contraction} + = 22 <= $self->{UCA_Version} && $self->{UCA_Version} <= 24; + # checkCollator() will be called in change() $self->checkCollator(); return $self; @@ -441,12 +445,10 @@ sub parseEntry $self->{maxlength}{$uv[0]} = @uv; } } - if (@uv > 2) { - while (@uv) { - pop @uv; - my $fake_entry = join(CODE_SEP, @uv); # in JCPS - $self->{contraction}{$fake_entry} = 1; - } + while (@uv > 2) { + pop @uv; + my $fake_entry = join(CODE_SEP, @uv); # in JCPS + $self->{contraction}{$fake_entry} = 1; } } @@ -486,6 +488,7 @@ sub splitEnt my $reH = $self->{rearrangeHash}; my $vers = $self->{UCA_Version}; my $ver9 = $vers >= 9 && $vers <= 11; + my $long = $self->{long_contraction}; my $uXS = $self->{__useXS}; ### XS only my @buf; @@ -566,6 +569,15 @@ sub splitEnt last unless $curCC; my $tail = CODE_SEP . $src[$p]; + if ($preCC != $curCC && $map->{$jcps.$tail}) { + $jcps .= $tail; + push @out, $p; + } else { + $preCC = $curCC; + } + + next if !$long; + if ($preCC_uc != $curCC && ($map->{$jcps_uc.$tail} || $cont->{$jcps_uc.$tail})) { $jcps_uc .= $tail; @@ -573,16 +585,9 @@ sub splitEnt } else { $preCC_uc = $curCC; } - - if ($preCC != $curCC && $map->{$jcps.$tail}) { - $jcps .= $tail; - push @out, $p; - } else { - $preCC = $curCC; - } } - if ($map->{$jcps_uc}) { + if (@out_uc && $map->{$jcps_uc}) { $jcps = $jcps_uc; $src[$_] = undef for @out_uc; } else { @@ -1068,6 +1073,7 @@ with no parameters, the collator should do the default collation. ignore_level2 => $bool, katakana_before_hiragana => $bool, level => $collationLevel, + long_contraction => $bool, minimalFFFE => $bool, normalization => $normalization_form, overrideCJK => \&overrideCJK, @@ -1091,7 +1097,7 @@ If the revision (previously "tracking version") number of UCA is given, behavior of that revision is emulated on collating. If omitted, the return value of C<UCA_Version()> is used. -The following revisions are supported. The default is 28. +The following revisions are supported. The default is 30. UCA Unicode Standard DUCET (@version) ------------------------------------------------------- @@ -1106,6 +1112,9 @@ The following revisions are supported. The default is 28. 24 6.1.0 6.1.0 (6.1.0) 26 6.2.0 6.2.0 (6.2.0) 28 6.3.0 6.3.0 (6.3.0) + 30 7.0.0 7.0.0 (7.0.0) + +* See below C<long_contraction> with C<UCA_Version> 22 and 24. * Noncharacters (e.g. U+FFFF) are not ignored, and can be overridden since C<UCA_Version> 22. @@ -1144,7 +1153,7 @@ forwards at all the levels. =item entry --- see 5 Tailoring; 3.6.1 File Format, UTS #10. +-- see 5 Tailoring; 9.1 Allkeys File Format, UTS #10. If the same character (or a sequence of characters) exists in the collation element table through C<table>, @@ -1261,7 +1270,7 @@ of the string after them (in NFD by default) are used. =item ignoreName --- see 3.6.2 Variable Weighting, UTS #10. +-- see 3.6 Variable Weighting, UTS #10. Makes the entry in the table completely ignorable; i.e. as if the weights were zero at all level. @@ -1322,6 +1331,46 @@ and 'shift-trimmed'), the level 4 may be unreliable. See also C<identical>. +=item long_contraction + +-- see 3.8.2 Well-Formedness of the DUCET, 4.2 Produce Array, UTS #10. + +If the parameter is made true, for a contraction with three or more +characters (here nicknamed "long contraction"), initial substrings +will be handled. +For example, a contraction ABC, where A is a starter, and B and C +are non-starters (character with non-zero combining character class), +will be detected even if there is not AB as a contraction. + +B<Default:> Usually false. +If C<UCA_Version> is 22 or 24, and the value of C<long_contraction> +is not specified in C<new()>, a true value is set implicitly. +This is a workaround to pass Conformance Tests for Unicode 6.0.0 and 6.1.0. + +C<change()> handles C<long_contraction> explicitly only. +If C<long_contraction> is not specified in C<change()>, even though +C<UCA_Version> is changed, C<long_contraction> will not be changed. + +B<Limitation:> Scanning non-starters is one-way (no back tracking). +If AB is found but not ABC is not found, other long contraction where +the first character is A and the second is not B may not be found. + +Under C<(normalization =E<gt> undef)>, detection step of discontiguous +contractions will be skipped. + +B<Note:> The following contractions in DUCET are not considered +in steps S2.1.1 to S2.1.3, where they are discontiguous. + + 0FB2 0F71 0F80 (TIBETAN VOWEL SIGN VOCALIC RR) + 0FB3 0F71 0F80 (TIBETAN VOWEL SIGN VOCALIC LL) + +For example C<TIBETAN VOWEL SIGN VOCALIC RR> with C<COMBINING TILDE OVERLAY> +(C<U+0344>) is C<0FB2 0344 0F71 0F80> in NFD. +In this case C<0FB2 0F80> (C<TIBETAN VOWEL SIGN VOCALIC R>) is detected, +instead of C<0FB2 0F71 0F80>. +Inserted C<0344> makes C<0FB2 0F71 0F80> discontiguous and lack of +contraction C<0FB2 0F71> prohibits C<0FB2 0F71 0F80> from being detected. + =item minimalFFFE -- see 5.14 Collation Elements, UTS #35. @@ -1615,7 +1664,7 @@ B<NOTE>: Contractions via C<entry> are not be suppressed. =item table --- see 3.6 Default Unicode Collation Element Table, UTS #10. +-- see 3.8 Default Unicode Collation Element Table, UTS #10. You can use another collation element table if desired. @@ -1694,7 +1743,7 @@ this parameter doesn't work validly. =item variable --- see 3.6.2 Variable Weighting, UTS #10. +-- see 3.6 Variable Weighting, UTS #10. This key allows for variable weighting of variable collation elements, which are marked with an ASTERISK in the table @@ -2029,7 +2078,7 @@ B<Unicode::Normalize is required to try The Conformance Test.> =head1 AUTHOR, COPYRIGHT AND LICENSE The Unicode::Collate module for perl was written by SADAHIRO Tomoyuki, -<SADAHIRO@cpan.org>. This module is Copyright(C) 2001-2013, +<SADAHIRO@cpan.org>. This module is Copyright(C) 2001-2014, SADAHIRO Tomoyuki. Japan. All rights reserved. This module is free software; you can redistribute it and/or |