diff options
Diffstat (limited to 'Master/tlpkg/tlperl/lib/Encode')
21 files changed, 1754 insertions, 79 deletions
diff --git a/Master/tlpkg/tlperl/lib/Encode/Alias.pm b/Master/tlpkg/tlperl/lib/Encode/Alias.pm index 9e15fe3c8ca..28d3cad34de 100644 --- a/Master/tlpkg/tlperl/lib/Encode/Alias.pm +++ b/Master/tlpkg/tlperl/lib/Encode/Alias.pm @@ -2,10 +2,10 @@ package Encode::Alias; use strict; use warnings; no warnings 'redefine'; -our $VERSION = do { my @r = ( q$Revision: 2.16 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; +our $VERSION = do { my @r = ( q$Revision: 2.18 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; use constant DEBUG => !!$ENV{PERL_ENCODE_DEBUG}; -use base qw(Exporter); +use Exporter 'import'; # Public, encouraged API is exported by default @@ -211,7 +211,7 @@ sub init_aliases { define_alias( qr/^macintosh$/i => '"MacRoman"' ); # https://rt.cpan.org/Ticket/Display.html?id=78125 define_alias( qr/^macce$/i => '"MacCentralEurRoman"' ); - # Ououououou. gone. They are differente! + # Ououououou. gone. They are different! # define_alias( qr/\bmacRomanian$/i => '"macRumanian"'); # Standardize on the dashed versions. @@ -338,7 +338,7 @@ As of Encode 1.87, the older form no longer works. -Encode up to 1.86 internally used "local $_" to implement ths older +Encode up to 1.86 internally used "local $_" to implement this older form. But consider the code below; use Encode; diff --git a/Master/tlpkg/tlperl/lib/Encode/CN/HZ.pm b/Master/tlpkg/tlperl/lib/Encode/CN/HZ.pm index 98c16a9053a..f035d821f57 100644 --- a/Master/tlpkg/tlperl/lib/Encode/CN/HZ.pm +++ b/Master/tlpkg/tlperl/lib/Encode/CN/HZ.pm @@ -5,11 +5,11 @@ use warnings; use utf8 (); use vars qw($VERSION); -$VERSION = do { my @r = ( q$Revision: 2.5 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; +$VERSION = do { my @r = ( q$Revision: 2.7 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; use Encode qw(:fallbacks); -use base qw(Encode::Encoding); +use parent qw(Encode::Encoding); __PACKAGE__->Define('hz'); # HZ is a combination of ASCII and escaped GB, so we implement it @@ -23,7 +23,7 @@ sub decode ($$;$) { my ( $obj, $str, $chk ) = @_; my $GB = Encode::find_encoding('gb2312-raw'); - my $ret = ''; + my $ret = substr($str, 0, 0); # to propagate taintedness my $in_ascii = 1; # default mode is ASCII. while ( length $str ) { @@ -133,10 +133,10 @@ sub cat_decode { } sub encode($$;$) { - my ( $obj, $str, $chk ) = @_; + my ( $obj, $str, $chk ) = @_; my $GB = Encode::find_encoding('gb2312-raw'); - my $ret = ''; + my $ret = substr($str, 0, 0); # to propagate taintedness; my $in_ascii = 1; # default mode is ASCII. no warnings 'utf8'; # $str may be malformed UTF8 at the end of a chunk. diff --git a/Master/tlpkg/tlperl/lib/Encode/Encoder.pm b/Master/tlpkg/tlperl/lib/Encode/Encoder.pm index 2cde1e865f5..23e0349bf3c 100644 --- a/Master/tlpkg/tlperl/lib/Encode/Encoder.pm +++ b/Master/tlpkg/tlperl/lib/Encode/Encoder.pm @@ -1,10 +1,10 @@ # -# $Id: Encoder.pm,v 2.2 2011/08/09 07:49:44 dankogai Exp $ +# $Id: Encoder.pm,v 2.3 2013/09/14 07:51:59 dankogai Exp $ # package Encode::Encoder; use strict; use warnings; -our $VERSION = do { my @r = ( q$Revision: 2.2 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; +our $VERSION = do { my @r = ( q$Revision: 2.3 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; require Exporter; our @ISA = qw(Exporter); @@ -211,7 +211,7 @@ To make the Base64 transcoder example above really work, you could write a module like this: package Encode::Base64; - use base 'Encode::Encoding'; + use parent 'Encode::Encoding'; __PACKAGE__->Define('base64'); use MIME::Base64; sub encode{ diff --git a/Master/tlpkg/tlperl/lib/Encode/Encoding.pm b/Master/tlpkg/tlperl/lib/Encode/Encoding.pm index 768d6d126ac..39d2e0ab64e 100644 --- a/Master/tlpkg/tlperl/lib/Encode/Encoding.pm +++ b/Master/tlpkg/tlperl/lib/Encode/Encoding.pm @@ -3,7 +3,7 @@ package Encode::Encoding; # Base class for classes which implement encodings use strict; use warnings; -our $VERSION = do { my @r = ( q$Revision: 2.5 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; +our $VERSION = do { my @r = ( q$Revision: 2.7 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; require Encode; @@ -81,7 +81,7 @@ Encode::Encoding - Encode Implementation Base Class =head1 SYNOPSIS package Encode::MyEncoding; - use base qw(Encode::Encoding); + use parent qw(Encode::Encoding); __PACKAGE__->Define(qw(myCanonical myAlias)); @@ -121,14 +121,14 @@ fragment. If perlio_ok() is true, SHOULD becomes MUST. =item * -If I<$check> is is false then C<encode> MUST make a "best effort" to +If I<$check> is false then C<encode> MUST make a "best effort" to convert the string - for example, by using a replacement character. =back =item -E<gt>decode($octets [,$check]) -MUST return the string that I<$octets> represents. +MUST return the string that I<$octets> represents. =over 2 @@ -251,7 +251,7 @@ is assumed. package Encode::ROT13; use strict; - use base qw(Encode::Encoding); + use parent qw(Encode::Encoding); __PACKAGE__->Define('rot13'); @@ -290,7 +290,7 @@ C<Encode::Encoding> as a base class. This allows that class to define additional behaviour for all encoding objects. package Encode::MyEncoding; - use base qw(Encode::Encoding); + use parent qw(Encode::Encoding); __PACKAGE__->Define(qw(myCanonical myAlias)); diff --git a/Master/tlpkg/tlperl/lib/Encode/GSM0338.pm b/Master/tlpkg/tlperl/lib/Encode/GSM0338.pm index aeff6379789..20257a1cbd9 100644 --- a/Master/tlpkg/tlperl/lib/Encode/GSM0338.pm +++ b/Master/tlpkg/tlperl/lib/Encode/GSM0338.pm @@ -1,5 +1,5 @@ # -# $Id: GSM0338.pm,v 2.2 2012/08/15 05:36:16 dankogai Exp $ +# $Id: GSM0338.pm,v 2.5 2013/09/14 07:51:59 dankogai Exp $ # package Encode::GSM0338; @@ -8,11 +8,11 @@ use warnings; use Carp; use vars qw($VERSION); -$VERSION = do { my @r = ( q$Revision: 2.2 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; +$VERSION = do { my @r = ( q$Revision: 2.5 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; use Encode qw(:fallbacks); -use base qw(Encode::Encoding); +use parent qw(Encode::Encoding); __PACKAGE__->Define('gsm0338'); sub needs_lines { 1 } @@ -171,7 +171,7 @@ our $NBSP = "\x{00A0}"; sub decode ($$;$) { my ( $obj, $bytes, $chk ) = @_; - my $str; + my $str = substr($bytes, 0, 0); # to propagate taintedness; while ( length $bytes ) { my $c = substr( $bytes, 0, 1, '' ); my $u; @@ -216,7 +216,7 @@ sub decode ($$;$) { sub encode($$;$) { my ( $obj, $str, $chk ) = @_; - my $bytes; + my $bytes = substr($str, 0, 0); # to propagate taintedness while ( length $str ) { my $u = substr( $str, 0, 1, '' ); my $c; @@ -259,7 +259,7 @@ this module. =head1 NOTES -Unlike most other encodings, the following aways croaks on error +Unlike most other encodings, the following always croaks on error for any $chk that evaluates to true. $gsm0338 = encode("gsm0338", $utf8 $chk); diff --git a/Master/tlpkg/tlperl/lib/Encode/Guess.pm b/Master/tlpkg/tlperl/lib/Encode/Guess.pm index 31ec58f4fca..b44daf59eb5 100644 --- a/Master/tlpkg/tlperl/lib/Encode/Guess.pm +++ b/Master/tlpkg/tlperl/lib/Encode/Guess.pm @@ -2,7 +2,7 @@ package Encode::Guess; use strict; use warnings; use Encode qw(:fallbacks find_encoding); -our $VERSION = do { my @r = ( q$Revision: 2.5 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; +our $VERSION = do { my @r = ( q$Revision: 2.6 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; my $Canon = 'Guess'; use constant DEBUG => !!$ENV{PERL_ENCODE_DEBUG}; @@ -12,7 +12,7 @@ $Encode::Encoding{$Canon} = bless { Suspects => {%DEF_SUSPECTS}, } => __PACKAGE__; -use base qw(Encode::Encoding); +use parent qw(Encode::Encoding); sub needs_lines { 1 } sub perlio_ok { 0 } diff --git a/Master/tlpkg/tlperl/lib/Encode/JP/JIS7.pm b/Master/tlpkg/tlperl/lib/Encode/JP/JIS7.pm index 3b5699c7fd0..588389a034a 100644 --- a/Master/tlpkg/tlperl/lib/Encode/JP/JIS7.pm +++ b/Master/tlpkg/tlperl/lib/Encode/JP/JIS7.pm @@ -1,7 +1,7 @@ package Encode::JP::JIS7; use strict; use warnings; -our $VERSION = do { my @r = ( q$Revision: 2.4 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; +our $VERSION = do { my @r = ( q$Revision: 2.5 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; use Encode qw(:fallbacks); @@ -16,7 +16,7 @@ for my $name ( '7bit-jis', 'iso-2022-jp', 'iso-2022-jp-1' ) { } => __PACKAGE__; } -use base qw(Encode::Encoding); +use parent qw(Encode::Encoding); # we override this to 1 so PerlIO works sub needs_lines { 1 } diff --git a/Master/tlpkg/tlperl/lib/Encode/KR/2022_KR.pm b/Master/tlpkg/tlperl/lib/Encode/KR/2022_KR.pm index 0218d971ee2..44373e5d589 100644 --- a/Master/tlpkg/tlperl/lib/Encode/KR/2022_KR.pm +++ b/Master/tlpkg/tlperl/lib/Encode/KR/2022_KR.pm @@ -1,11 +1,11 @@ package Encode::KR::2022_KR; use strict; use warnings; -our $VERSION = do { my @r = ( q$Revision: 2.2 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; +our $VERSION = do { my @r = ( q$Revision: 2.3 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; use Encode qw(:fallbacks); -use base qw(Encode::Encoding); +use parent qw(Encode::Encoding); __PACKAGE__->Define('iso-2022-kr'); sub needs_lines { 1 } diff --git a/Master/tlpkg/tlperl/lib/Encode/Locale.pm b/Master/tlpkg/tlperl/lib/Encode/Locale.pm index b14e10a01a8..c901c62a07f 100644 --- a/Master/tlpkg/tlperl/lib/Encode/Locale.pm +++ b/Master/tlpkg/tlperl/lib/Encode/Locale.pm @@ -1,7 +1,7 @@ package Encode::Locale; use strict; -our $VERSION = "1.03"; +our $VERSION = "1.04"; use base 'Exporter'; our @EXPORT_OK = qw( @@ -25,31 +25,60 @@ sub _init { unless ($ENCODING_LOCALE) { # Try to obtain what the Windows ANSI code page is eval { - unless (defined &GetACP) { + unless (defined &GetConsoleCP) { + require Win32; + # no point falling back to Win32::GetConsoleCP from this + # as added same time, 0.45 + eval { Win32::GetConsoleCP() }; + # manually "import" it since Win32->import refuses + *GetConsoleCP = sub { &Win32::GetConsoleCP } unless $@; + } + unless (defined &GetConsoleCP) { require Win32::API; - Win32::API->Import('kernel32', 'int GetACP()'); - }; - if (defined &GetACP) { - my $cp = GetACP(); + Win32::API->Import('kernel32', 'int GetConsoleCP()'); + } + if (defined &GetConsoleCP) { + my $cp = GetConsoleCP(); $ENCODING_LOCALE = "cp$cp" if $cp; } }; } unless ($ENCODING_CONSOLE_IN) { - # If we have the Win32::Console module installed we can ask - # it for the code set to use - eval { - require Win32::Console; - my $cp = Win32::Console::InputCP(); - $ENCODING_CONSOLE_IN = "cp$cp" if $cp; - $cp = Win32::Console::OutputCP(); - $ENCODING_CONSOLE_OUT = "cp$cp" if $cp; - }; - # Invoking the 'chcp' program might also work - if (!$ENCODING_CONSOLE_IN && (qx(chcp) || '') =~ /^Active code page: (\d+)/) { - $ENCODING_CONSOLE_IN = "cp$1"; + # only test one since set together + unless (defined &GetInputCP) { + eval { + require Win32; + eval { Win32::GetConsoleCP() }; + # manually "import" it since Win32->import refuses + *GetInputCP = sub { &Win32::GetConsoleCP } unless $@; + *GetOutputCP = sub { &Win32::GetConsoleOutputCP } unless $@; + }; + unless (defined &GetInputCP) { + eval { + # try Win32::Console module for codepage to use + require Win32::Console; + eval { Win32::Console::InputCP() }; + *GetInputCP = sub { &Win32::Console::InputCP } + unless $@; + *GetOutputCP = sub { &Win32::Console::OutputCP } + unless $@; + }; + } + unless (defined &GetInputCP) { + # final fallback + *GetInputCP = *GetOutputCP = sub { + # another fallback that could work is: + # reg query HKLM\System\CurrentControlSet\Control\Nls\CodePage /v ACP + ((qx(chcp) || '') =~ /^Active code page: (\d+)/) + ? $1 : (); + }; + } } + my $cp = GetInputCP(); + $ENCODING_CONSOLE_IN = "cp$cp" if $cp; + $cp = GetOutputCP(); + $ENCODING_CONSOLE_OUT = "cp$cp" if $cp; } } @@ -206,8 +235,7 @@ C<Encode::Locale> will do that if available and make these encodings known under the C<Encode> aliases "console_in" and "console_out". For systems where we can't determine the terminal encoding these will be aliased as the same encoding as "locale". The advice is to use "console_in" for input known to -come from the terminal and "console_out" for output known to go from the -terminal. +come from the terminal and "console_out" for output to the terminal. In addition to arranging for various Encode aliases the following functions and variables are provided: @@ -266,7 +294,7 @@ L<Encode> know this encoding as "locale". =item $ENCODING_LOCALE_FS -The encoding name determined to be suiteable for file system interfaces +The encoding name determined to be suitable for file system interfaces involving file names. L<Encode> know this encoding as "locale_fs". @@ -336,7 +364,7 @@ Users are advised to always specify UTF-8 as the locale charset. =head1 SEE ALSO -L<I18N::Langinfo>, L<Encode> +L<I18N::Langinfo>, L<Encode>, L<Term::Encoding> =head1 AUTHOR diff --git a/Master/tlpkg/tlperl/lib/Encode/MIME/Header.pm b/Master/tlpkg/tlperl/lib/Encode/MIME/Header.pm index c41797c7035..090a177d113 100644 --- a/Master/tlpkg/tlperl/lib/Encode/MIME/Header.pm +++ b/Master/tlpkg/tlperl/lib/Encode/MIME/Header.pm @@ -3,7 +3,7 @@ use strict; use warnings; no warnings 'redefine'; -our $VERSION = do { my @r = ( q$Revision: 2.13 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; +our $VERSION = do { my @r = ( q$Revision: 2.15 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; use Encode qw(find_encoding encode_utf8 decode_utf8); use MIME::Base64; use Carp; @@ -31,7 +31,7 @@ $Encode::Encoding{'MIME-Q'} = bless { Name => 'MIME-Q', } => __PACKAGE__; -use base qw(Encode::Encoding); +use parent qw(Encode::Encoding); sub needs_lines { 1 } sub perlio_ok { 0 } @@ -135,7 +135,7 @@ sub encode($$;$) { $subline .= ' ' if ($subline =~ /\?=$/ and $word =~ /^=\?/); $subline .= $word; } - $subline and push @subline, $subline; + length($subline) and push @subline, $subline; push @line, join( "\n " => @subline ); } $_[1] = '' if $chk; diff --git a/Master/tlpkg/tlperl/lib/Encode/MIME/Header/ISO_2022_JP.pm b/Master/tlpkg/tlperl/lib/Encode/MIME/Header/ISO_2022_JP.pm index 4abfbd05ef8..86955c83f1f 100644 --- a/Master/tlpkg/tlperl/lib/Encode/MIME/Header/ISO_2022_JP.pm +++ b/Master/tlpkg/tlperl/lib/Encode/MIME/Header/ISO_2022_JP.pm @@ -3,7 +3,7 @@ package Encode::MIME::Header::ISO_2022_JP; use strict; use warnings; -use base qw(Encode::MIME::Header); +use parent qw(Encode::MIME::Header); $Encode::Encoding{'MIME-Header-ISO_2022_JP'} = bless { encode => 'B', bpl => 76, Name => 'MIME-Header-ISO_2022_JP' } => @@ -14,7 +14,7 @@ use constant TAIL => '?='; use Encode::CJKConstants qw(%RE); -our $VERSION = do { my @r = ( q$Revision: 1.3 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; +our $VERSION = do { my @r = ( q$Revision: 1.4 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; # I owe the below codes totally to # Jcode by Dan Kogai & http://www.din.or.jp/~ohzaki/perl.htm#JP_Base64 diff --git a/Master/tlpkg/tlperl/lib/Encode/Makefile_PL.e2x b/Master/tlpkg/tlperl/lib/Encode/Makefile_PL.e2x index ece58830fea..c17a5096f79 100644 --- a/Master/tlpkg/tlperl/lib/Encode/Makefile_PL.e2x +++ b/Master/tlpkg/tlperl/lib/Encode/Makefile_PL.e2x @@ -83,7 +83,7 @@ sub post_initialize if $^O eq 'MacOS' && $self->{SOURCE} !~ /\b$name\.c\b/; # $self->{'H'} = [$self->catfile($self->updir,'encode.h')]; my %xs; - foreach my $table (keys %tables) { + foreach my $table (sort keys %tables) { push (@{$self->{'C'}},"$table.c"); # Do NOT add $table.h etc. to H_FILES unless we own up as to how they # get built. @@ -98,10 +98,9 @@ sub post_initialize #include <EXTERN.h> #include <perl.h> #include <XSUB.h> -#define U8 U8 #include "encode.h" END - foreach my $table (keys %tables) { + foreach my $table (sort keys %tables) { print XS qq[#include "${table}.h"\n]; } print XS <<"END"; @@ -111,8 +110,14 @@ Encode_XSEncoding(pTHX_ encode_t *enc) { dSP; HV *stash = gv_stashpv("Encode::XS", TRUE); - SV *sv = sv_bless(newRV_noinc(newSViv(PTR2IV(enc))),stash); + SV *iv = newSViv(PTR2IV(enc)); + SV *sv = sv_bless(newRV_noinc(iv),stash); int i = 0; + /* with the SvLEN() == 0 hack, PVX won't be freed. We cast away name's + constness, in the hope that perl won't mess with it. */ + assert(SvTYPE(iv) >= SVt_PV); assert(SvLEN(iv) == 0); + SvFLAGS(iv) |= SVp_POK; + SvPVX(iv) = (char*) enc->name[0]; PUSHMARK(sp); XPUSHs(sv); while (enc->name[i]) @@ -130,7 +135,7 @@ PROTOTYPES: DISABLE BOOT: { END - foreach my $table (keys %tables) { + foreach my $table (sort keys %tables) { print XS qq[#include "${table}.exh"\n]; } print XS "}\n"; @@ -144,14 +149,14 @@ sub postamble my $dir = "."; # $self->catdir('Encode'); my $str = "# $name\$(OBJ_EXT) depends on .h and .exh files not .c files - but all written by enc2xs\n"; $str .= "$name.c : $name.xs "; - foreach my $table (keys %tables) + foreach my $table (sort keys %tables) { $str .= " $table.c"; } $str .= "\n\n"; $str .= "$name\$(OBJ_EXT) : $name.c\n\n"; - foreach my $table (keys %tables) + foreach my $table (sort keys %tables) { my $numlines = 1; my $lengthsofar = length($str); diff --git a/Master/tlpkg/tlperl/lib/Encode/PerlIO.pod b/Master/tlpkg/tlperl/lib/Encode/PerlIO.pod new file mode 100644 index 00000000000..1a9269a9b0c --- /dev/null +++ b/Master/tlpkg/tlperl/lib/Encode/PerlIO.pod @@ -0,0 +1,167 @@ +=head1 NAME + +Encode::PerlIO -- a detailed document on Encode and PerlIO + +=head1 Overview + +It is very common to want to do encoding transformations when +reading or writing files, network connections, pipes etc. +If Perl is configured to use the new 'perlio' IO system then +C<Encode> provides a "layer" (see L<PerlIO>) which can transform +data as it is read or written. + +Here is how the blind poet would modernise the encoding: + + use Encode; + open(my $iliad,'<:encoding(iso-8859-7)','iliad.greek'); + open(my $utf8,'>:utf8','iliad.utf8'); + my @epic = <$iliad>; + print $utf8 @epic; + close($utf8); + close($illiad); + +In addition, the new IO system can also be configured to read/write +UTF-8 encoded characters (as noted above, this is efficient): + + open(my $fh,'>:utf8','anything'); + print $fh "Any \x{0021} string \N{SMILEY FACE}\n"; + +Either of the above forms of "layer" specifications can be made the default +for a lexical scope with the C<use open ...> pragma. See L<open>. + +Once a handle is open, its layers can be altered using C<binmode>. + +Without any such configuration, or if Perl itself is built using the +system's own IO, then write operations assume that the file handle +accepts only I<bytes> and will C<die> if a character larger than 255 is +written to the handle. When reading, each octet from the handle becomes +a byte-in-a-character. Note that this default is the same behaviour +as bytes-only languages (including Perl before v5.6) would have, +and is sufficient to handle native 8-bit encodings e.g. iso-8859-1, +EBCDIC etc. and any legacy mechanisms for handling other encodings +and binary data. + +In other cases, it is the program's responsibility to transform +characters into bytes using the API above before doing writes, and to +transform the bytes read from a handle into characters before doing +"character operations" (e.g. C<lc>, C</\W+/>, ...). + +You can also use PerlIO to convert larger amounts of data you don't +want to bring into memory. For example, to convert between ISO-8859-1 +(Latin 1) and UTF-8 (or UTF-EBCDIC in EBCDIC machines): + + open(F, "<:encoding(iso-8859-1)", "data.txt") or die $!; + open(G, ">:utf8", "data.utf") or die $!; + while (<F>) { print G } + + # Could also do "print G <F>" but that would pull + # the whole file into memory just to write it out again. + +More examples: + + open(my $f, "<:encoding(cp1252)") + open(my $g, ">:encoding(iso-8859-2)") + open(my $h, ">:encoding(latin9)") # iso-8859-15 + +See also L<encoding> for how to change the default encoding of the +data in your script. + +=head1 How does it work? + +Here is a crude diagram of how filehandle, PerlIO, and Encode +interact. + + filehandle <-> PerlIO PerlIO <-> scalar (read/printed) + \ / + Encode + +When PerlIO receives data from either direction, it fills a buffer +(currently with 1024 bytes) and passes the buffer to Encode. +Encode tries to convert the valid part and passes it back to PerlIO, +leaving invalid parts (usually a partial character) in the buffer. +PerlIO then appends more data to the buffer, calls Encode again, +and so on until the data stream ends. + +To do so, PerlIO always calls (de|en)code methods with CHECK set to 1. +This ensures that the method stops at the right place when it +encounters partial character. The following is what happens when +PerlIO and Encode tries to encode (from utf8) more than 1024 bytes +and the buffer boundary happens to be in the middle of a character. + + A B C .... ~ \x{3000} .... + 41 42 43 .... 7E e3 80 80 .... + <- buffer ---------------> + << encoded >>>>>>>>>> + <- next buffer ------ + +Encode converts from the beginning to \x7E, leaving \xe3 in the buffer +because it is invalid (partial character). + +Unfortunately, this scheme does not work well with escape-based +encodings such as ISO-2022-JP. + +=head1 Line Buffering + +Now let's see what happens when you try to decode from ISO-2022-JP and +the buffer ends in the middle of a character. + + JIS208-ESC \x{5f3e} + A B C .... ~ \e $ B |DAN | .... + 41 42 43 .... 7E 1b 24 41 43 46 .... + <- buffer ---------------------------> + << encoded >>>>>>>>>>>>>>>>>>>>>>> + +As you see, the next buffer begins with \x43. But \x43 is 'C' in +ASCII, which is wrong in this case because we are now in JISX 0208 +area so it has to convert \x43\x46, not \x43. Unlike utf8 and EUC, +in escape-based encodings you can't tell if a given octet is a whole +character or just part of it. + +Fortunately PerlIO also supports line buffer if you tell PerlIO to use +one instead of fixed buffer. Since ISO-2022-JP is guaranteed to revert to ASCII at the end of the line, partial +character will never happen when line buffer is used. + +To tell PerlIO to use line buffer, implement -E<gt>needs_lines method +for your encoding object. See L<Encode::Encoding> for details. + +Thanks to these efforts most encodings that come with Encode support +PerlIO but that still leaves following encodings. + + iso-2022-kr + MIME-B + MIME-Header + MIME-Q + +Fortunately iso-2022-kr is hardly used (according to Jungshik) and +MIME-* are very unlikely to be fed to PerlIO because they are for mail +headers. See L<Encode::MIME::Header> for details. + +=head2 How can I tell whether my encoding fully supports PerlIO ? + +As of this writing, any encoding whose class belongs to Encode::XS and +Encode::Unicode works. The Encode module has a C<perlio_ok> method +which you can use before applying PerlIO encoding to the filehandle. +Here is an example: + + my $use_perlio = perlio_ok($enc); + my $layer = $use_perlio ? "<:raw" : "<:encoding($enc)"; + open my $fh, $layer, $file or die "$file : $!"; + while(<$fh>){ + $_ = decode($enc, $_) unless $use_perlio; + # .... + } + +=head1 SEE ALSO + +L<Encode::Encoding>, +L<Encode::Supported>, +L<Encode::PerlIO>, +L<encoding>, +L<perlebcdic>, +L<perlfunc/open>, +L<perlunicode>, +L<utf8>, +the Perl Unicode Mailing List E<lt>perl-unicode@perl.orgE<gt> + +=cut + diff --git a/Master/tlpkg/tlperl/lib/Encode/Supported.pod b/Master/tlpkg/tlperl/lib/Encode/Supported.pod new file mode 100644 index 00000000000..8efa4ff6f89 --- /dev/null +++ b/Master/tlpkg/tlperl/lib/Encode/Supported.pod @@ -0,0 +1,901 @@ +=head1 NAME + +Encode::Supported -- Encodings supported by Encode + +=head1 DESCRIPTION + +=head2 Encoding Names + +Encoding names are case insensitive. White space in names +is ignored. In addition, an encoding may have aliases. +Each encoding has one "canonical" name. The "canonical" +name is chosen from the names of the encoding by picking +the first in the following sequence (with a few exceptions). + +=over 2 + +=item * + +The name used by the Perl community. That includes 'utf8' and 'ascii'. +Unlike aliases, canonical names directly reach the method so such +frequently used words like 'utf8' don't need to do alias lookups. + +=item * + +The MIME name as defined in IETF RFCs. This includes all "iso-"s. + +=item * + +The name in the IANA registry. + +=item * + +The name used by the organization that defined it. + +=back + +In case I<de jure> canonical names differ from that of the Encode +module, they are always aliased if it ever be implemented. So you can +safely tell if a given encoding is implemented or not just by passing +the canonical name. + +Because of all the alias issues, and because in the general case +encodings have state, "Encode" uses an encoding object internally +once an operation is in progress. + +=head1 Supported Encodings + +As of Perl 5.8.0, at least the following encodings are recognized. +Note that unless otherwise specified, they are all case insensitive +(via alias) and all occurrence of spaces are replaced with '-'. +In other words, "ISO 8859 1" and "iso-8859-1" are identical. + +Encodings are categorized and implemented in several different modules +but you don't have to C<use Encode::XX> to make them available for +most cases. Encode.pm will automatically load those modules on demand. + +=head2 Built-in Encodings + +The following encodings are always available. + + Canonical Aliases Comments & References + ---------------------------------------------------------------- + ascii US-ascii ISO-646-US [ECMA] + ascii-ctrl Special Encoding + iso-8859-1 latin1 [ISO] + null Special Encoding + utf8 UTF-8 [RFC2279] + ---------------------------------------------------------------- + +I<null> and I<ascii-ctrl> are special. "null" fails for all character +so when you set fallback mode to PERLQQ, HTMLCREF or XMLCREF, ALL +CHARACTERS will fall back to character references. Ditto for +"ascii-ctrl" except for control characters. For fallback modes, see +L<Encode>. + +=head2 Encode::Unicode -- other Unicode encodings + +Unicode coding schemes other than native utf8 are supported by +Encode::Unicode, which will be autoloaded on demand. + + ---------------------------------------------------------------- + UCS-2BE UCS-2, iso-10646-1 [IANA, UC] + UCS-2LE [UC] + UTF-16 [UC] + UTF-16BE [UC] + UTF-16LE [UC] + UTF-32 [UC] + UTF-32BE UCS-4 [UC] + UTF-32LE [UC] + UTF-7 [RFC2152] + ---------------------------------------------------------------- + +To find how (UCS-2|UTF-(16|32))(LE|BE)? differ from one another, +see L<Encode::Unicode>. + +UTF-7 is a special encoding which "re-encodes" UTF-16BE into a 7-bit +encoding. It is implemented separately by Encode::Unicode::UTF7. + +=head2 Encode::Byte -- Extended ASCII + +Encode::Byte implements most single-byte encodings except for +Symbols and EBCDIC. The following encodings are based on single-byte +encodings implemented as extended ASCII. Most of them map +\x80-\xff (upper half) to non-ASCII characters. + +=over 2 + +=item ISO-8859 and corresponding vendor mappings + +Since there are so many, they are presented in table format with +languages and corresponding encoding names by vendors. Note that +the table is sorted in order of ISO-8859 and the corresponding vendor +mappings are slightly different from that of ISO. See +L<http://czyborra.com/charsets/iso8859.html> for details. + + Lang/Regions ISO/Other Std. DOS Windows Macintosh Others + ---------------------------------------------------------------- + N. America (ASCII) cp437 AdobeStandardEncoding + cp863 (DOSCanadaF) + W. Europe iso-8859-1 cp850 cp1252 MacRoman nextstep + hp-roman8 + cp860 (DOSPortuguese) + Cntrl. Europe iso-8859-2 cp852 cp1250 MacCentralEurRoman + MacCroatian + MacRomanian + MacRumanian + Latin3[1] iso-8859-3 + Latin4[2] iso-8859-4 + Cyrillics iso-8859-5 cp855 cp1251 MacCyrillic + (See also next section) cp866 MacUkrainian + Arabic iso-8859-6 cp864 cp1256 MacArabic + cp1006 MacFarsi + Greek iso-8859-7 cp737 cp1253 MacGreek + cp869 (DOSGreek2) + Hebrew iso-8859-8 cp862 cp1255 MacHebrew + Turkish iso-8859-9 cp857 cp1254 MacTurkish + Nordics iso-8859-10 cp865 + cp861 MacIcelandic + MacSami + Thai iso-8859-11[3] cp874 MacThai + (iso-8859-12 is nonexistent. Reserved for Indics?) + Baltics iso-8859-13 cp775 cp1257 + Celtics iso-8859-14 + Latin9 [4] iso-8859-15 + Latin10 iso-8859-16 + Vietnamese viscii cp1258 MacVietnamese + ---------------------------------------------------------------- + + [1] Esperanto, Maltese, and Turkish. Turkish is now on 8859-9. + [2] Baltics. Now on 8859-10, except for Latvian. + [3] TIS 620 + Non-Breaking Space (0xA0 / U+00A0) + [4] Nicknamed Latin0; the Euro sign as well as French and Finnish + letters that are missing from 8859-1 were added. + +All cp* are also available as ibm-*, ms-*, and windows-* . See also +L<http://czyborra.com/charsets/codepages.html>. + +Macintosh encodings don't seem to be registered in such entities as +IANA. "Canonical" names in Encode are based upon Apple's Tech Note +1150. See L<http://developer.apple.com/technotes/tn/tn1150.html> +for details. + +=item KOI8 - De Facto Standard for the Cyrillic world + +Though ISO-8859 does have ISO-8859-5, the KOI8 series is far more +popular in the Net. L<Encode> comes with the following KOI charsets. +For gory details, see L<http://czyborra.com/charsets/cyrillic.html> + + ---------------------------------------------------------------- + koi8-f + koi8-r cp878 [RFC1489] + koi8-u [RFC2319] + ---------------------------------------------------------------- + +=back + +=head2 gsm0338 - Hentai Latin 1 + +GSM0338 is for GSM handsets. Though it shares alphanumerals with +ASCII, control character ranges and other parts are mapped very +differently, mainly to store Greek characters. There are also escape +sequences (starting with 0x1B) to cover e.g. the Euro sign. + +This was once handled by L<Encode::Bytes> but because of all those +unusual specifications, Encode 2.20 has relocated the support to +L<Encode::GSM0338>. See L<Encode::GSM0338> for details. + +=over 2 + +=item gsm0338 support before 2.19 + +Some special cases like a trailing 0x00 byte or a lone 0x1B byte are not +well-defined and decode() will return an empty string for them. +One possible workaround is + + $gsm =~ s/\x00\z/\x00\x00/; + $uni = decode("gsm0338", $gsm); + $uni .= "\xA0" if $gsm =~ /\x1B\z/; + +Note that the Encode implementation of GSM0338 does not implement the +reuse of Latin capital letters as Greek capital letters (for example, +the 0x5A is U+005A (LATIN CAPITAL LETTER Z), not U+0396 (GREEK CAPITAL +LETTER ZETA). + +The GSM0338 is also covered in Encode::Byte even though it is not +an "extended ASCII" encoding. + +=back + +=head2 CJK: Chinese, Japanese, Korean (Multibyte) + +Note that Vietnamese is listed above. Also read "Encoding vs Charset" +below. Also note that these are implemented in distinct modules by +countries, due to the size concerns (simplified Chinese is mapped +to 'CN', continental China, while traditional Chinese is mapped to +'TW', Taiwan). Please refer to their respective documentation pages. + +=over 2 + +=item Encode::CN -- Continental China + + Standard DOS/Win Macintosh Comment/Reference + ---------------------------------------------------------------- + euc-cn [1] MacChineseSimp + (gbk) cp936 [2] + gb12345-raw { GB12345 without CES } + gb2312-raw { GB2312 without CES } + hz + iso-ir-165 + ---------------------------------------------------------------- + + [1] GB2312 is aliased to this. See L<Microsoft-related naming mess> + [2] gbk is aliased to this. See L<Microsoft-related naming mess> + +=item Encode::JP -- Japan + + Standard DOS/Win Macintosh Comment/Reference + ---------------------------------------------------------------- + euc-jp + shiftjis cp932 macJapanese + 7bit-jis + iso-2022-jp [RFC1468] + iso-2022-jp-1 [RFC2237] + jis0201-raw { JIS X 0201 (roman + halfwidth kana) without CES } + jis0208-raw { JIS X 0208 (Kanji + fullwidth kana) without CES } + jis0212-raw { JIS X 0212 (Extended Kanji) without CES } + ---------------------------------------------------------------- + +=item Encode::KR -- Korea + + Standard DOS/Win Macintosh Comment/Reference + ---------------------------------------------------------------- + euc-kr MacKorean [RFC1557] + cp949 [1] + iso-2022-kr [RFC1557] + johab [KS X 1001:1998, Annex 3] + ksc5601-raw { KSC5601 without CES } + ---------------------------------------------------------------- + + [1] ks_c_5601-1987, (x-)?windows-949, and uhc are aliased to this. + See below. + +=item Encode::TW -- Taiwan + + Standard DOS/Win Macintosh Comment/Reference + ---------------------------------------------------------------- + big5-eten cp950 MacChineseTrad {big5 aliased to big5-eten} + big5-hkscs + ---------------------------------------------------------------- + +=item Encode::HanExtra -- More Chinese via CPAN + +Due to the size concerns, additional Chinese encodings below are +distributed separately on CPAN, under the name Encode::HanExtra. + + Standard DOS/Win Macintosh Comment/Reference + ---------------------------------------------------------------- + big5ext CMEX's Big5e Extension + big5plus CMEX's Big5+ Extension + cccii Chinese Character Code for Information Interchange + euc-tw EUC (Extended Unix Character) + gb18030 GBK with Traditional Characters + ---------------------------------------------------------------- + +=item Encode::JIS2K -- JIS X 0213 encodings via CPAN + +Due to size concerns, additional Japanese encodings below are +distributed separately on CPAN, under the name Encode::JIS2K. + + Standard DOS/Win Macintosh Comment/Reference + ---------------------------------------------------------------- + euc-jisx0213 + shiftjisx0123 + iso-2022-jp-3 + jis0213-1-raw + jis0213-2-raw + ---------------------------------------------------------------- + +=back + +=head2 Miscellaneous encodings + +=over 2 + +=item Encode::EBCDIC + +See L<perlebcdic> for details. + + ---------------------------------------------------------------- + cp37 + cp500 + cp875 + cp1026 + cp1047 + posix-bc + ---------------------------------------------------------------- + +=item Encode::Symbols + +For symbols and dingbats. + + ---------------------------------------------------------------- + symbol + dingbats + MacDingbats + AdobeZdingbat + AdobeSymbol + ---------------------------------------------------------------- + +=item Encode::MIME::Header + +Strictly speaking, MIME header encoding documented in RFC 2047 is more +of encapsulation than encoding. However, their support in modern +world is imperative so they are supported. + + ---------------------------------------------------------------- + MIME-Header [RFC2047] + MIME-B [RFC2047] + MIME-Q [RFC2047] + ---------------------------------------------------------------- + +=item Encode::Guess + +This one is not a name of encoding but a utility that lets you pick up +the most appropriate encoding for a data out of given I<suspects>. See +L<Encode::Guess> for details. + +=back + +=head1 Unsupported encodings + +The following encodings are not supported as yet; some because they +are rarely used, some because of technical difficulties. They may +be supported by external modules via CPAN in the future, however. + +=over 2 + +=item ISO-2022-JP-2 [RFC1554] + +Not very popular yet. Needs Unicode Database or equivalent to +implement encode() (because it includes JIS X 0208/0212, KSC5601, and +GB2312 simultaneously, whose code points in Unicode overlap. So you +need to lookup the database to determine to what character set a given +Unicode character should belong). + +=item ISO-2022-CN [RFC1922] + +Not very popular. Needs CNS 11643-1 and -2 which are not available in +this module. CNS 11643 is supported (via euc-tw) in Encode::HanExtra. +Autrijus Tang may add support for this encoding in his module in future. + +=item Various HP-UX encodings + +The following are unsupported due to the lack of mapping data. + + '8' - arabic8, greek8, hebrew8, kana8, thai8, and turkish8 + '15' - japanese15, korean15, and roi15 + +=item Cyrillic encoding ISO-IR-111 + +Anton Tagunov doubts its usefulness. + +=item ISO-8859-8-1 [Hebrew] + +None of the Encode team knows Hebrew enough (ISO-8859-8, cp1255 and +MacHebrew are supported because and just because there were mappings +available at L<http://www.unicode.org/>). Contributions welcome. + +=item ISIRI 3342, Iran System, ISIRI 2900 [Farsi] + +Ditto. + +=item Thai encoding TCVN + +Ditto. + +=item Vietnamese encodings VPS + +Though Jungshik Shin has reported that Mozilla supports this encoding, +it was too late before 5.8.0 for us to add it. In the future, it +may be available via a separate module. See +L<http://lxr.mozilla.org/seamonkey/source/intl/uconv/ucvlatin/vps.uf> +and +L<http://lxr.mozilla.org/seamonkey/source/intl/uconv/ucvlatin/vps.ut> +if you are interested in helping us. + +=item Various Mac encodings + +The following are unsupported due to the lack of mapping data. + + MacArmenian, MacBengali, MacBurmese, MacEthiopic + MacExtArabic, MacGeorgian, MacKannada, MacKhmer + MacLaotian, MacMalayalam, MacMongolian, MacOriya + MacSinhalese, MacTamil, MacTelugu, MacTibetan + MacVietnamese + +The rest which are already available are based upon the vendor mappings +at L<http://www.unicode.org/Public/MAPPINGS/VENDORS/APPLE/> . + +=item (Mac) Indic encodings + +The maps for the following are available at L<http://www.unicode.org/> +but remain unsupported because those encodings need an algorithmical +approach, currently unsupported by F<enc2xs>: + + MacDevanagari + MacGurmukhi + MacGujarati + +For details, please see C<Unicode mapping issues and notes:> at +L<http://www.unicode.org/Public/MAPPINGS/VENDORS/APPLE/DEVANAGA.TXT> . + +I believe this issue is prevalent not only for Mac Indics but also in +other Indic encodings, but the above were the only Indic encodings +maps that I could find at L<http://www.unicode.org/> . + +=back + +=head1 Encoding vs. Charset -- terminology + +We are used to using the term (character) I<encoding> and I<character +set> interchangeably. But just as confusing the terms byte and +character is dangerous and the terms should be differentiated when +needed, we need to differentiate I<encoding> and I<character set>. + +To understand that, here is a description of how we make computers +grok our characters. + +=over 2 + +=item * + +First we start with which characters to include. We call this +collection of characters I<character repertoire>. + +=item * + +Then we have to give each character a unique ID so your computer can +tell the difference between 'a' and 'A'. This itemized character +repertoire is now a I<character set>. + +=item * + +If your computer can grow the character set without further +processing, you can go ahead and use it. This is called a I<coded +character set> (CCS) or I<raw character encoding>. ASCII is used this +way for most cases. + +=item * + +But in many cases, especially multi-byte CJK encodings, you have to +tweak a little more. Your network connection may not accept any data +with the Most Significant Bit set, and your computer may not be able to +tell if a given byte is a whole character or just half of it. So you +have to I<encode> the character set to use it. + +A I<character encoding scheme> (CES) determines how to encode a given +character set, or a set of multiple character sets. 7bit ISO-2022 is +an example of a CES. You switch between character sets via I<escape +sequences>. + +=back + +Technically, or mathematically, speaking, a character set encoded in +such a CES that maps character by character may form a CCS. EUC is such +an example. The CES of EUC is as follows: + +=over 2 + +=item * + +Map ASCII unchanged. + +=item * + +Map such a character set that consists of 94 or 96 powered by N +members by adding 0x80 to each byte. + +=item * + +You can also use 0x8e and 0x8f to indicate that the following sequence of +characters belongs to yet another character set. To each following byte +is added the value 0x80. + +=back + +By carefully looking at the encoded byte sequence, you can find that the +byte sequence conforms a unique number. In that sense, EUC is a CCS +generated by a CES above from up to four CCS (complicated?). UTF-8 +falls into this category. See L<perlUnicode/"UTF-8"> to find out how +UTF-8 maps Unicode to a byte sequence. + +You may also have found out by now why 7bit ISO-2022 cannot comprise +a CCS. If you look at a byte sequence \x21\x21, you can't tell if +it is two !'s or IDEOGRAPHIC SPACE. EUC maps the latter to \xA1\xA1 +so you have no trouble differentiating between "!!". and S<" ">. + +=head1 Encoding Classification (by Anton Tagunov and Dan Kogai) + +This section tries to classify the supported encodings by their +applicability for information exchange over the Internet and to +choose the most suitable aliases to name them in the context of +such communication. + +=over 2 + +=item * + +To (en|de)code encodings marked by C<(**)>, you need +C<Encode::HanExtra>, available from CPAN. + +=back + +Encoding names + + US-ASCII UTF-8 ISO-8859-* KOI8-R + Shift_JIS EUC-JP ISO-2022-JP ISO-2022-JP-1 + EUC-KR Big5 GB2312 + +are registered with IANA as preferred MIME names and may +be used over the Internet. + +C<Shift_JIS> has been officialized by JIS X 0208:1997. +L<Microsoft-related naming mess> gives details. + +C<GB2312> is the IANA name for C<EUC-CN>. +See L<Microsoft-related naming mess> for details. + +C<GB_2312-80> I<raw> encoding is available as C<gb2312-raw> +with Encode. See L<Encode::CN> for details. + + EUC-CN + KOI8-U [RFC2319] + +have not been registered with IANA (as of March 2002) but +seem to be supported by major web browsers. +The IANA name for C<EUC-CN> is C<GB2312>. + + KS_C_5601-1987 + +is heavily misused. +See L<Microsoft-related naming mess> for details. + +C<KS_C_5601-1987> I<raw> encoding is available as C<kcs5601-raw> +with Encode. See L<Encode::KR> for details. + + UTF-16 UTF-16BE UTF-16LE + +are IANA-registered C<charset>s. See [RFC 2781] for details. +Jungshik Shin reports that UTF-16 with a BOM is well accepted +by MS IE 5/6 and NS 4/6. Beware however that + +=over 2 + +=item * + +C<UTF-16> support in any software you're going to be +using/interoperating with has probably been less tested +then C<UTF-8> support + +=item * + +C<UTF-8> coded data seamlessly passes traditional +command piping (C<cat>, C<more>, etc.) while C<UTF-16> coded +data is likely to cause confusion (with its zero bytes, +for example) + +=item * + +it is beyond the power of words to describe the way HTML browsers +encode non-C<ASCII> form data. To get a general impression, visit +L<http://www.alanflavell.org.uk/charset/form-i18n.html>. +While encoding of form data has stabilized for C<UTF-8> encoded pages +(at least IE 5/6, NS 6, and Opera 6 behave consistently), be sure to +expect fun (and cross-browser discrepancies) with C<UTF-16> encoded +pages! + +=back + +The rule of thumb is to use C<UTF-8> unless you know what +you're doing and unless you really benefit from using C<UTF-16>. + + ISO-IR-165 [RFC1345] + VISCII + GB 12345 + GB 18030 (**) (see links bellow) + EUC-TW (**) + +are totally valid encodings but not registered at IANA. +The names under which they are listed here are probably the +most widely-known names for these encodings and are recommended +names. + + BIG5PLUS (**) + +is a proprietary name. + +=head2 Microsoft-related naming mess + +Microsoft products misuse the following names: + +=over 2 + +=item KS_C_5601-1987 + +Microsoft extension to C<EUC-KR>. + +Proper names: C<CP949>, C<UHC>, C<x-windows-949> (as used by Mozilla). + +See L<http://lists.w3.org/Archives/Public/ietf-charsets/2001AprJun/0033.html> +for details. + +Encode aliases C<KS_C_5601-1987> to C<cp949> to reflect this common +misusage. I<Raw> C<KS_C_5601-1987> encoding is available as +C<kcs5601-raw>. + +See L<Encode::KR> for details. + +=item GB2312 + +Microsoft extension to C<EUC-CN>. + +Proper names: C<CP936>, C<GBK>. + +C<GB2312> has been registered in the C<EUC-CN> meaning at +IANA. This has partially repaired the situation: Microsoft's +C<GB2312> has become a superset of the official C<GB2312>. + +Encode aliases C<GB2312> to C<euc-cn> in full agreement with +IANA registration. C<cp936> is supported separately. +I<Raw> C<GB_2312-80> encoding is available as C<gb2312-raw>. + +See L<Encode::CN> for details. + +=item Big5 + +Microsoft extension to C<Big5>. + +Proper name: C<CP950>. + +Encode separately supports C<Big5> and C<cp950>. + +=item Shift_JIS + +Microsoft's understanding of C<Shift_JIS>. + +JIS has not endorsed the full Microsoft standard however. +The official C<Shift_JIS> includes only JIS X 0201 and JIS X 0208 +character sets, while Microsoft has always used C<Shift_JIS> +to encode a wider character repertoire. See C<IANA> registration for +C<Windows-31J>. + +As a historical predecessor, Microsoft's variant +probably has more rights for the name, though it may be objected +that Microsoft shouldn't have used JIS as part of the name +in the first place. + +Unambiguous name: C<CP932>. C<IANA> name (also used by Mozilla, and +provided as an alias by Encode): C<Windows-31J>. + +Encode separately supports C<Shift_JIS> and C<cp932>. + +=back + +=head1 Glossary + +=over 2 + +=item character repertoire + +A collection of unique characters. A I<character> set in the strictest +sense. At this stage, characters are not numbered. + +=item coded character set (CCS) + +A character set that is mapped in a way computers can use directly. +Many character encodings, including EUC, fall in this category. + +=item character encoding scheme (CES) + +An algorithm to map a character set to a byte sequence. You don't +have to be able to tell which character set a given byte sequence +belongs. 7-bit ISO-2022 is a CES but it cannot be a CCS. EUC is an +example of being both a CCS and CES. + +=item charset (in MIME context) + +has long been used in the meaning of C<encoding>, CES. + +While the word combination C<character set> has lost this meaning +in MIME context since [RFC 2130], the C<charset> abbreviation has +retained it. This is how [RFC 2277] and [RFC 2278] bless C<charset>: + + This document uses the term "charset" to mean a set of rules for + mapping from a sequence of octets to a sequence of characters, such + as the combination of a coded character set and a character encoding + scheme; this is also what is used as an identifier in MIME "charset=" + parameters, and registered in the IANA charset registry ... (Note + that this is NOT a term used by other standards bodies, such as ISO). + [RFC 2277] + +=item EUC + +Extended Unix Character. See ISO-2022. + +=item ISO-2022 + +A CES that was carefully designed to coexist with ASCII. There are a 7 +bit version and an 8 bit version. + +The 7 bit version switches character set via escape sequence so it +cannot form a CCS. Since this is more difficult to handle in programs +than the 8 bit version, the 7 bit version is not very popular except for +iso-2022-jp, the I<de facto> standard CES for e-mails. + +The 8 bit version can form a CCS. EUC and ISO-8859 are two examples +thereof. Pre-5.6 perl could use them as string literals. + +=item UCS + +Short for I<Universal Character Set>. When you say just UCS, it means +I<Unicode>. + +=item UCS-2 + +ISO/IEC 10646 encoding form: Universal Character Set coded in two +octets. + +=item Unicode + +A character set that aims to include all character repertoires of the +world. Many character sets in various national as well as industrial +standards have become, in a way, just subsets of Unicode. + +=item UTF + +Short for I<Unicode Transformation Format>. Determines how to map a +Unicode character into a byte sequence. + +=item UTF-16 + +A UTF in 16-bit encoding. Can either be in big endian or little +endian. The big endian version is called UTF-16BE (equal to UCS-2 + +surrogate support) and the little endian version is called UTF-16LE. + +=back + +=head1 See Also + +L<Encode>, +L<Encode::Byte>, +L<Encode::CN>, L<Encode::JP>, L<Encode::KR>, L<Encode::TW>, +L<Encode::EBCDIC>, L<Encode::Symbol> +L<Encode::MIME::Header>, L<Encode::Guess> + +=head1 References + +=over 2 + +=item ECMA + +European Computer Manufacturers Association +L<http://www.ecma.ch> + +=over 2 + +=item ECMA-035 (eq C<ISO-2022>) + +L<http://www.ecma.ch/ecma1/STAND/ECMA-035.HTM> + +The specification of ISO-2022 is available from the link above. + +=back + +=item IANA + +Internet Assigned Numbers Authority +L<http://www.iana.org/> + +=over 2 + +=item Assigned Charset Names by IANA + +L<http://www.iana.org/assignments/character-sets> + +Most of the C<canonical names> in Encode derive from this list +so you can directly apply the string you have extracted from MIME +header of mails and web pages. + +=back + +=item ISO + +International Organization for Standardization +L<http://www.iso.ch/> + +=item RFC + +Request For Comments -- need I say more? +L<http://www.rfc-editor.org/>, L<http://www.ietf.org/rfc.html>, +L<http://www.faqs.org/rfcs/> + +=item UC + +Unicode Consortium +L<http://www.unicode.org/> + +=over 2 + +=item Unicode Glossary + +L<http://www.unicode.org/glossary/> + +The glossary of this document is based upon this site. + +=back + +=back + +=head2 Other Notable Sites + +=over 2 + +=item czyborra.com + +L<http://czyborra.com/> + +Contains a lot of useful information, especially gory details of ISO +vs. vendor mappings. + +=item CJK.inf + +L<http://examples.oreilly.com/cjkvinfo/doc/cjk.inf> + +Somewhat obsolete (last update in 1996), but still useful. Also try + +L<ftp://ftp.oreilly.com/pub/examples/nutshell/cjkv/pdf/GB18030_Summary.pdf> + +You will find brief info on C<EUC-CN>, C<GBK> and mostly on C<GB 18030>. + +=item Jungshik Shin's Hangul FAQ + +L<http://jshin.net/faq> + +And especially its subject 8. + +L<http://jshin.net/faq/qa8.html> + +A comprehensive overview of the Korean (C<KS *>) standards. + +=item debian.org: "Introduction to i18n" + +A brief description for most of the mentioned CJK encodings is +contained in +L<http://www.debian.org/doc/manuals/intro-i18n/ch-codes.en.html> + +=back + +=head2 Offline sources + +=over 2 + +=item C<CJKV Information Processing> by Ken Lunde + +CJKV Information Processing +1999 O'Reilly & Associates, ISBN : 1-56592-224-7 + +The modern successor of C<CJK.inf>. + +Features a comprehensive coverage of CJKV character sets and +encodings along with many other issues faced by anyone trying +to better support CJKV languages/scripts in all the areas of +information processing. + +To purchase this book, visit +L<http://oreilly.com/catalog/9780596514471/> +or your favourite bookstore. + +=back + +=cut diff --git a/Master/tlpkg/tlperl/lib/Encode/Unicode.pm b/Master/tlpkg/tlperl/lib/Encode/Unicode.pm index f71567c6196..6b35cb7566a 100644 --- a/Master/tlpkg/tlperl/lib/Encode/Unicode.pm +++ b/Master/tlpkg/tlperl/lib/Encode/Unicode.pm @@ -4,7 +4,7 @@ use strict; use warnings; no warnings 'redefine'; -our $VERSION = do { my @r = ( q$Revision: 2.7 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; +our $VERSION = do { my @r = ( q$Revision: 2.9 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; use XSLoader; XSLoader::load( __PACKAGE__, $VERSION ); @@ -42,7 +42,7 @@ for my $name ( } => __PACKAGE__; } -use base qw(Encode::Encoding); +use parent qw(Encode::Encoding); sub renew { my $self = shift; @@ -52,7 +52,7 @@ sub renew { return $clone; } -# There used to be a perl implemntation of (en|de)code but with +# There used to be a perl implementation of (en|de)code but with # XS version is ripe, perl version is zapped for optimal speed *decode = \&decode_xs; diff --git a/Master/tlpkg/tlperl/lib/Encode/Unicode/UTF7.pm b/Master/tlpkg/tlperl/lib/Encode/Unicode/UTF7.pm index ea9257b47b5..d5d86e2f903 100644 --- a/Master/tlpkg/tlperl/lib/Encode/Unicode/UTF7.pm +++ b/Master/tlpkg/tlperl/lib/Encode/Unicode/UTF7.pm @@ -1,13 +1,13 @@ # -# $Id: UTF7.pm,v 2.6 2012/08/05 23:08:49 dankogai Exp $ +# $Id: UTF7.pm,v 2.8 2013/09/14 07:51:59 dankogai Exp $ # package Encode::Unicode::UTF7; use strict; use warnings; no warnings 'redefine'; -use base qw(Encode::Encoding); +use parent qw(Encode::Encoding); __PACKAGE__->Define('UTF-7'); -our $VERSION = do { my @r = ( q$Revision: 2.6 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; +our $VERSION = do { my @r = ( q$Revision: 2.8 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; use MIME::Base64; use Encode; @@ -32,7 +32,7 @@ sub encode($$;$) { my ( $obj, $str, $chk ) = @_; my $len = length($str); pos($str) = 0; - my $bytes = ''; + my $bytes = substr($str, 0, 0); # to propagate taintedness while ( pos($str) < $len ) { if ( $str =~ /\G($re_asis+)/ogc ) { my $octets = $1; @@ -59,9 +59,10 @@ sub encode($$;$) { } sub decode($$;$) { + use re 'taint'; my ( $obj, $bytes, $chk ) = @_; my $len = length($bytes); - my $str = ""; + my $str = substr($bytes, 0, 0); # to propagate taintedness; pos($bytes) = 0; no warnings 'uninitialized'; while ( pos($bytes) < $len ) { diff --git a/Master/tlpkg/tlperl/lib/Encode/compat.pm b/Master/tlpkg/tlperl/lib/Encode/compat.pm new file mode 100644 index 00000000000..39432b8f9d4 --- /dev/null +++ b/Master/tlpkg/tlperl/lib/Encode/compat.pm @@ -0,0 +1,87 @@ +# $File: //member/autrijus/Encode-compat/lib/Encode/compat.pm $ $Author: autrijus $ +# $Revision: #7 $ $Change: 10735 $ $DateTime: 2004/06/03 14:08:57 $ + +package Encode::compat; +$Encode::compat::VERSION = '0.07'; + +use strict; + +if ($] >= 5.007001 or $INC{'Encode.pm'}) { + # nothing happens -- Encode.pm already available. +} +elsif ($] >= 5.006001 and $] <= 5.007) { + require Encode::compat::Alias; + $INC{'Encode/Alias.pm'} = $INC{'Encode/compat/Alias.pm'}; + + require Encode::compat::common; + require Encode::compat::5006001; + $INC{'Encode.pm'} = __FILE__; +} +else { + die "Encode.pm compatibility layer for $] not yet available."; +} + +1; + +__END__ + +=head1 NAME + +Encode::compat - Encode.pm emulation layer + +=head1 VERSION + +This document describes version 0.07 of Encode::compat, released +June 3, 2004. + +=head1 SYNOPSIS + + use Encode::compat; # a no-op for Perl v5.7.1+ + use Encode qw(...); # all constants and imports works transparently + + # use Encode functions as normal + +=head1 DESCRIPTION + +WARNING: THIS IS A PROOF-OF-CONCEPT. Most functions are incomplete. +All implementation details are subject to change! + +This module provide a compatibility layer for B<Encode.pm> users on perl +versions earlier than v5.7.1. It translates whatever call it receives +into B<Text::Iconv>, or (in the future) B<Unicode::MapUTF8> to perform +the actual work. + +The C<is_utf8()>, C<_utf8_on()> and C<_utf8_off()> calls are performed +by the method native to the perl version -- 5.6.1 would use +C<pack>/C<unpack>, 5.6.0 uses C<tr//CU>, etc. + +Theoretically, it could be backported to 5.005 and earlier, with none of +the unicode-related semantics available, and serves only as a +abstraction layer above C<Text::Iconv>, C<Unicode::MapUTF8> and possibly +other transcoding modules. + +=head1 CAVEATS + +Currently, this module only support 5.6.1+, and merely provides the three +utility function above (C<encode()>, C<decode()> and C<from_to()>), with +a very kludgy C<FB_HTMLCREF> fallback against C<latin-1> in +C<from_to()>. + +=head1 SEE ALSO + +L<Encode>, L<perlunicode> + +=head1 AUTHORS + +Autrijus Tang E<lt>autrijus@autrijus.orgE<gt> + +=head1 COPYRIGHT + +Copyright 2002, 2003, 2004 by Autrijus Tang E<lt>autrijus@autrijus.orgE<gt>. + +This program is free software; you can redistribute it and/or +modify it under the same terms as Perl itself. + +See L<http://www.perl.com/perl/misc/Artistic.html> + +=cut diff --git a/Master/tlpkg/tlperl/lib/Encode/compat/5006001.pm b/Master/tlpkg/tlperl/lib/Encode/compat/5006001.pm new file mode 100644 index 00000000000..cbe2d50180e --- /dev/null +++ b/Master/tlpkg/tlperl/lib/Encode/compat/5006001.pm @@ -0,0 +1,32 @@ +# $File: //member/autrijus/Encode-compat/lib/Encode/compat/5006001.pm $ $Author: autrijus $ +# $Revision: #3 $ $Change: 2534 $ $DateTime: 2002/12/02 00:33:16 $ + +package Encode::compat::5006001; +our $VERSION = '0.05'; + +1; + +package Encode; + +use strict; +use base 'Exporter'; +no warnings 'redefine'; + +sub _utf8_on { + $_[0] = pack('U*', unpack('U0U*', $_[0])) +} + +sub _utf8_off { + $_[0] = pack('C*', unpack('C*', $_[0])) +} + +sub is_utf8 { + # XXX: got any better ideas? + use utf8; + foreach my $char (split(//, $_[0])) { + return 1 if ord($char) > 255; + } + return 0; +} + +1; diff --git a/Master/tlpkg/tlperl/lib/Encode/compat/Alias.pm b/Master/tlpkg/tlperl/lib/Encode/compat/Alias.pm new file mode 100644 index 00000000000..6e8d74e6a6f --- /dev/null +++ b/Master/tlpkg/tlperl/lib/Encode/compat/Alias.pm @@ -0,0 +1,330 @@ +# $File: //member/autrijus/.vimrc $ $Author: autrijus $ +# $Revision: #1 $ $Change: 1649 $ $DateTime: 2002/10/24 15:21:23 $ + +package Encode::compat::Alias; +our $VERSION = '0.05'; + +1; + +package Encode::Alias; +use strict; +our $VERSION = '0.05'; +our $DEBUG = 0; + +use base qw(Exporter); + +# Public, encouraged API is exported by default + +our @EXPORT = + qw ( + define_alias + find_alias + ); + +our @Alias; # ordered matching list +our %Alias; # cached known aliases + +sub find_alias +{ + my $class = shift; + local $_ = shift; + unless (exists $Alias{$_}) + { + $Alias{$_} = undef; # Recursion guard + for (my $i=0; $i < @Alias; $i += 2) + { + my $alias = $Alias[$i]; + my $val = $Alias[$i+1]; + my $new; + if (ref($alias) eq 'Regexp' && $_ =~ $alias) + { + $DEBUG and warn "eval $val"; + $new = eval $val; + # $@ and warn "$val, $@"; + } + elsif (ref($alias) eq 'CODE') + { + $DEBUG and warn "$alias", "->", "($val)"; + $new = $alias->($val); + } + elsif (lc($_) eq lc($alias)) + { + $new = $val; + } + if (defined($new)) + { + next if $new eq $_; # avoid (direct) recursion on bugs + $DEBUG and warn "$alias, $new"; + my $enc = (ref($new)) ? $new : Encode::find_encoding($new); + if ($enc) + { + $Alias{$_} = $enc; + last; + } + } + } + } + if ($DEBUG){ + my $name; + if (my $e = $Alias{$_}){ + $name = $e->name; + }else{ + $name = ""; + } + warn "find_alias($class, $_)->name = $name"; + } + return $Alias{$_}; +} + +sub define_alias +{ + while (@_) + { + my ($alias,$name) = splice(@_,0,2); + unshift(@Alias, $alias => $name); # newer one has precedence + # clear %Alias cache to allow overrides + if (ref($alias)){ + my @a = keys %Alias; + for my $k (@a){ + if (ref($alias) eq 'Regexp' && $k =~ $alias) + { + $DEBUG and warn "delete \$Alias\{$k\}"; + delete $Alias{$k}; + } + elsif (ref($alias) eq 'CODE') + { + $DEBUG and warn "delete \$Alias\{$k\}"; + delete $Alias{$alias->($name)}; + } + } + }else{ + $DEBUG and warn "delete \$Alias\{$alias\}"; + delete $Alias{$alias}; + } + } +} + +# Allow latin-1 style names as well + # 0 1 2 3 4 5 6 7 8 9 10 +our @Latin2iso = ( 0, 1, 2, 3, 4, 9, 10, 13, 14, 15, 16 ); +# Allow winlatin1 style names as well +our %Winlatin2cp = ( + 'latin1' => 1252, + 'latin2' => 1250, + 'cyrillic' => 1251, + 'greek' => 1253, + 'turkish' => 1254, + 'hebrew' => 1255, + 'arabic' => 1256, + 'baltic' => 1257, + 'vietnamese' => 1258, + ); + +init_aliases(); + +sub undef_aliases{ + @Alias = (); + %Alias = (); +} + +sub init_aliases +{ + undef_aliases(); + + # Try all-lower-case version should all else fails + define_alias( qr/^(.*)$/ => '"\L$1"' ); + + # UTF/UCS stuff + define_alias( qr/^UCS-?2-?LE$/i => '"UCS-2LE"' ); + define_alias( qr/^UCS-?2-?(BE)?$/i => '"UCS-2BE"', + qr/^UCS-?4-?(BE|LE)?$/i => 'uc("UTF-32$1")', + qr/^iso-10646-1$/i => '"UCS-2BE"' ); + define_alias( qr/^UTF(16|32)-?BE$/i => '"UTF-$1BE"', + qr/^UTF(16|32)-?LE$/i => '"UTF-$1LE"', + qr/^UTF(16|32)$/i => '"UTF-$1"', + ); + # ASCII + define_alias(qr/^(?:US-?)ascii$/i => '"ascii"'); + define_alias('C' => 'ascii'); + define_alias(qr/\bISO[-_]?646[-_]?US$/i => '"ascii"'); + # Allow variants of iso-8859-1 etc. + define_alias( qr/\biso[-_]?(\d+)[-_](\d+)$/i => '"iso-$1-$2"' ); + + # At least HP-UX has these. + define_alias( qr/\biso8859(\d+)$/i => '"iso-8859-$1"' ); + + # More HP stuff. + define_alias( qr/\b(?:hp-)?(arabic|greek|hebrew|kana|roman|thai|turkish)8$/i => '"${1}8"' ); + + # The Official name of ASCII. + define_alias( qr/\bANSI[-_]?X3\.4[-_]?1968$/i => '"ascii"' ); + + # This is a font issue, not an encoding issue. + # (The currency symbol of the Latin 1 upper half + # has been redefined as the euro symbol.) + define_alias( qr/^(.+)\@euro$/i => '"$1"' ); + + define_alias( qr/\b(?:iso[-_]?)?latin[-_]?(\d+)$/i + => 'defined $Encode::Alias::Latin2iso[$1] ? "iso-8859-$Encode::Alias::Latin2iso[$1]" : undef' ); + + define_alias( qr/\bwin(latin[12]|cyrillic|baltic|greek|turkish| + hebrew|arabic|baltic|vietnamese)$/ix => + '"cp" . $Encode::Alias::Winlatin2cp{lc($1)}' ); + + # Common names for non-latin prefered MIME names + define_alias( 'ascii' => 'US-ascii', + 'cyrillic' => 'iso-8859-5', + 'arabic' => 'iso-8859-6', + 'greek' => 'iso-8859-7', + 'hebrew' => 'iso-8859-8', + 'thai' => 'iso-8859-11', + 'tis620' => 'iso-8859-11', + ); + + # At least AIX has IBM-NNN (surprisingly...) instead of cpNNN. + # And Microsoft has their own naming (again, surprisingly). + # And windows-* is registered in IANA! + define_alias( qr/\b(?:cp|ibm|ms|windows)[-_ ]?(\d{2,4})$/i => '"cp$1"'); + + # Sometimes seen with a leading zero. + # define_alias( qr/\bcp037\b/i => '"cp37"'); + + # Mac Mappings + # predefined in *.ucm; unneeded + # define_alias( qr/\bmacIcelandic$/i => '"macIceland"'); + define_alias( qr/^mac_(.*)$/i => '"mac$1"'); + # Ououououou. gone. They are differente! + # define_alias( qr/\bmacRomanian$/i => '"macRumanian"'); + + # Standardize on the dashed versions. + # define_alias( qr/\butf8$/i => 'utf-8' ); + define_alias( qr/\bkoi8r$/i => 'koi8-r' ); + define_alias( qr/\bkoi8u$/i => 'koi8-u' ); + + unless ($Encode::ON_EBCDIC){ + # for Encode::CN + define_alias( qr/\beuc.*cn$/i => '"euc-cn"' ); + define_alias( qr/\bcn.*euc$/i => '"euc-cn"' ); + # define_alias( qr/\bGB[- ]?(\d+)$/i => '"euc-cn"' ) + # CP936 doesn't have vendor-addon for GBK, so they're identical. + define_alias( qr/^gbk$/i => '"cp936"'); + # This fixes gb2312 vs. euc-cn confusion, practically + define_alias( qr/\bGB[-_ ]?2312(?:\D.*$|$)/i => '"euc-cn"' ); + # for Encode::JP + define_alias( qr/\bjis$/i => '"7bit-jis"' ); + define_alias( qr/\beuc.*jp$/i => '"euc-jp"' ); + define_alias( qr/\bjp.*euc$/i => '"euc-jp"' ); + define_alias( qr/\bujis$/i => '"euc-jp"' ); + define_alias( qr/\bshift.*jis$/i => '"shiftjis"' ); + define_alias( qr/\bsjis$/i => '"shiftjis"' ); + # for Encode::KR + define_alias( qr/\beuc.*kr$/i => '"euc-kr"' ); + define_alias( qr/\bkr.*euc$/i => '"euc-kr"' ); + # This fixes ksc5601 vs. euc-kr confusion, practically + define_alias( qr/(?:x-)?uhc$/i => '"cp949"' ); + define_alias( qr/(?:x-)?windows-949$/i => '"cp949"' ); + define_alias( qr/\bks_c_5601-1987$/i => '"cp949"' ); + # for Encode::TW + define_alias( qr/\bbig-?5$/i => '"big5-eten"' ); + define_alias( qr/\bbig5-?et(?:en)?$/i => '"big5-eten"' ); + define_alias( qr/\btca[-_]?big5$/i => '"big5-eten"' ); + define_alias( qr/\bbig5-?hk(?:scs)?$/i => '"big5-hkscs"' ); + define_alias( qr/\bhk(?:scs)?[-_]?big5$/i => '"big5-hkscs"' ); + } + # utf8 is blessed :) + define_alias( qr/^UTF-8$/i => '"utf8"',); + # At last, Map white space and _ to '-' + define_alias( qr/^(\S+)[\s_]+(.*)$/i => '"$1-$2"' ); +} + +1; +__END__ + +# TODO: HP-UX '8' encodings arabic8 greek8 hebrew8 kana8 thai8 turkish8 +# TODO: HP-UX '15' encodings japanese15 korean15 roi15 +# TODO: Cyrillic encoding ISO-IR-111 (useful?) +# TODO: Armenian encoding ARMSCII-8 +# TODO: Hebrew encoding ISO-8859-8-1 +# TODO: Thai encoding TCVN +# TODO: Vietnamese encodings VPS +# TODO: Mac Asian+African encodings: Arabic Armenian Bengali Burmese +# ChineseSimp ChineseTrad Devanagari Ethiopic ExtArabic +# Farsi Georgian Gujarati Gurmukhi Hebrew Japanese +# Kannada Khmer Korean Laotian Malayalam Mongolian +# Oriya Sinhalese Symbol Tamil Telugu Tibetan Vietnamese + +=head1 NAME + +Encode::Alias - alias definitions to encodings + +=head1 SYNOPSIS + + use Encode; + use Encode::Alias; + define_alias( newName => ENCODING); + +=head1 DESCRIPTION + +Allows newName to be used as an alias for ENCODING. ENCODING may be +either the name of an encoding or an encoding object (as described +in L<Encode>). + +Currently I<newName> can be specified in the following ways: + +=over 4 + +=item As a simple string. + +=item As a qr// compiled regular expression, e.g.: + + define_alias( qr/^iso8859-(\d+)$/i => '"iso-8859-$1"' ); + +In this case, if I<ENCODING> is not a reference, it is C<eval>-ed +in order to allow C<$1> etc. to be substituted. The example is one +way to alias names as used in X11 fonts to the MIME names for the +iso-8859-* family. Note the double quotes inside the single quotes. + +If you are using a regex here, you have to use the quotes as shown or +it won't work. Also note that regex handling is tricky even for the +experienced. Use it with caution. + +=item As a code reference, e.g.: + + define_alias( sub { return /^iso8859-(\d+)$/i ? "iso-8859-$1" : undef } , ''); + +In this case, C<$_> will be set to the name that is being looked up and +I<ENCODING> is passed to the sub as its first argument. The example +is another way to alias names as used in X11 fonts to the MIME names +for the iso-8859-* family. + +=back + +=head2 Alias overloading + +You can override predefined aliases by simply applying define_alias(). +The new alias is always evaluated first, and when neccessary, +define_alias() flushes the internal cache to make the new definition +available. + + # redirect SHIFT_JIS to MS/IBM Code Page 932, which is a + # superset of SHIFT_JIS + + define_alias( qr/shift.*jis$/i => '"cp932"' ); + define_alias( qr/sjis$/i => '"cp932"' ); + +If you want to zap all predefined aliases, you can use + + Encode::Alias->undef_aliases; + +to do so. And + + Encode::Alias->init_aliases; + +gets the factory settings back. + +=head1 SEE ALSO + +L<Encode>, L<Encode::Supported> + +=cut + diff --git a/Master/tlpkg/tlperl/lib/Encode/compat/common.pm b/Master/tlpkg/tlperl/lib/Encode/compat/common.pm new file mode 100644 index 00000000000..f9ba21cb67d --- /dev/null +++ b/Master/tlpkg/tlperl/lib/Encode/compat/common.pm @@ -0,0 +1,127 @@ +# $File: //member/autrijus/Encode-compat/lib/Encode/compat/common.pm $ $Author: autrijus $ +# $Revision: #7 $ $Change: 10024 $ $DateTime: 2004/02/13 21:42:35 $ + +package Encode::compat::common; +our $VERSION = '0.06'; + +1; + +package Encode; + +use strict; +our $VERSION = '0.06'; + +our @EXPORT = qw( + decode decode_utf8 encode encode_utf8 + encodings find_encoding +); + +use constant DIE_ON_ERR => 1; +use constant WARN_ON_ERR => 2; +use constant RETURN_ON_ERR => 4; +use constant LEAVE_SRC => 8; + +use constant PERLQQ => 256; +use constant HTMLCREF => 512; +use constant XMLCREF => 1024; + +use constant FB_DEFAULT => 0; +use constant FB_CROAK => 1; +use constant FB_QUIET => 4; +use constant FB_WARN => 6; +use constant FB_PERLQQ => 256; +use constant FB_HTMLCREF => 512; +use constant FB_XMLCREF => 1024; + +our @FB_FLAGS = qw(DIE_ON_ERR WARN_ON_ERR RETURN_ON_ERR LEAVE_SRC + PERLQQ HTMLCREF XMLCREF); +our @FB_CONSTS = qw(FB_DEFAULT FB_CROAK FB_QUIET FB_WARN + FB_PERLQQ FB_HTMLCREF FB_XMLCREF); + +our @EXPORT_OK = + ( + qw( + _utf8_off _utf8_on define_encoding from_to is_16bit is_8bit + is_utf8 perlio_ok resolve_alias utf8_downgrade utf8_upgrade + ), + @FB_FLAGS, @FB_CONSTS, + ); + +our %EXPORT_TAGS = + ( + all => [ @EXPORT, @EXPORT_OK ], + fallbacks => [ @FB_CONSTS ], + fallback_all => [ @FB_CONSTS, @FB_FLAGS ], + ); + +sub from_to ($$$;$) { + use utf8; + + # XXX: bad hack + if ($_[3] and $_[3] == FB_HTMLCREF() and lc($_[2]) eq 'latin1') { + $_[0] = join('', map { + ord($_) < 128 + ? $_ : '&#' . ord($_) . ';' + } split(//, decode($_[1], $_[0]))); + } + else { + $_[0] = _convert(@_[0..2]); + } +} + +sub encodings { + # XXX: revisit + require Encode::Alias; + return sort values %Encode::Alias::Alias; +} + +sub find_encoding { + return $_[0]; +} + +sub decode_utf8($;$) { + return decode("utf-8", @_); +} + +sub encode_utf8($;$) { + return encode("utf-8", @_); +} + +sub decode($$;$) { + my $result = ($_[0] =~ /utf-?8/i) + ? $_[1] : _convert($_[1], $_[0] => 'utf-8'); + _utf8_on($result); + return $result; +} + +sub encode($$;$) { + my $result = ($_[0] =~ /utf-?8/i) + ? $_[1] : _convert($_[1], 'utf-8' => $_[0]); + _utf8_off($result); + return $result; +} + +{ + my %decoder; + sub _convert { + require Text::Iconv; + Text::Iconv->raise_error(1); + + require Encode::Alias; + my ($from, $to) = map { + s/^utf8$/utf-8/i; + s/^big5-eten$/big5/i; + $_; + } map { + Encode::Alias->find_alias($_) || lc($_) + } ($_[1], $_[2]); + + my $result = ($from eq $to) ? $_[0] : ( + $decoder{$from, $to} ||= Text::Iconv->new( $from, $to ) + )->convert($_[0]); + + return $result; + } +} + +1; diff --git a/Master/tlpkg/tlperl/lib/Encode/encode.h b/Master/tlpkg/tlperl/lib/Encode/encode.h index 0dcf83d32bc..5fbcf76ad38 100644 --- a/Master/tlpkg/tlperl/lib/Encode/encode.h +++ b/Master/tlpkg/tlperl/lib/Encode/encode.h @@ -1,11 +1,8 @@ #ifndef ENCODE_H #define ENCODE_H -#ifndef U8 -/* - A tad devious this: - perl normally has a #define for U8 - if that isn't present then we - typedef it - leaving it #ifndef so we can do data parts without +#ifndef H_PERL +/* check whether we're "in perl" so that we can do data parts without getting extern references to the code parts */ typedef unsigned char U8; @@ -74,7 +71,7 @@ struct encode_s const char *const name[2]; /* name(s) of this encoding */ }; -#ifdef U8 +#ifdef H_PERL /* See comment at top of file for deviousness */ extern int do_encode(const encpage_t *enc, const U8 *src, STRLEN *slen, @@ -83,7 +80,7 @@ extern int do_encode(const encpage_t *enc, const U8 *src, STRLEN *slen, extern void Encode_DefineEncoding(encode_t *enc); -#endif /* U8 */ +#endif /* H_PERL */ #define ENCODE_NOSPACE 1 #define ENCODE_PARTIAL 2 |