diff options
author | Karl Berry <karl@freefriends.org> | 2012-05-21 00:15:27 +0000 |
---|---|---|
committer | Karl Berry <karl@freefriends.org> | 2012-05-21 00:15:27 +0000 |
commit | a4c42bfb2337d37da89d789cb8cc226367994e32 (patch) | |
tree | c3eabdef5d565a4e515d2be0d9d4d0540bde0250 /Master/tlpkg/tlperl/lib/LWP/RobotUA.pm | |
parent | 8274475057f024d35332ac47c2e2f23ea156e6ed (diff) |
perl 5.14.2 from siep
git-svn-id: svn://tug.org/texlive/trunk@26525 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/tlpkg/tlperl/lib/LWP/RobotUA.pm')
-rw-r--r-- | Master/tlpkg/tlperl/lib/LWP/RobotUA.pm | 21 |
1 files changed, 8 insertions, 13 deletions
diff --git a/Master/tlpkg/tlperl/lib/LWP/RobotUA.pm b/Master/tlpkg/tlperl/lib/LWP/RobotUA.pm index 82c99fe26c9..695fac9a86d 100644 --- a/Master/tlpkg/tlperl/lib/LWP/RobotUA.pm +++ b/Master/tlpkg/tlperl/lib/LWP/RobotUA.pm @@ -2,7 +2,7 @@ package LWP::RobotUA; require LWP::UserAgent; @ISA = qw(LWP::UserAgent); -$VERSION = "5.835"; +$VERSION = "6.03"; require WWW::RobotRules; require HTTP::Request; @@ -126,21 +126,16 @@ sub simple_request $self->{'rules'}->parse($robot_url, ""); my $robot_req = HTTP::Request->new('GET', $robot_url); + my $parse_head = $self->parse_head(0); my $robot_res = $self->request($robot_req); + $self->parse_head($parse_head); my $fresh_until = $robot_res->fresh_until; - if ($robot_res->is_success) { - my $c = $robot_res->content; - if ($robot_res->content_type =~ m,^text/, && $c =~ /^\s*Disallow\s*:/mi) { - $self->{'rules'}->parse($robot_url, $c, $fresh_until); - } - else { - $self->{'rules'}->parse($robot_url, "", $fresh_until); - } - - } - else { - $self->{'rules'}->parse($robot_url, "", $fresh_until); + my $content = ""; + if ($robot_res->is_success && $robot_res->content_is_text) { + $content = $robot_res->decoded_content; + $content = "" unless $content && $content =~ /^\s*Disallow\s*:/mi; } + $self->{'rules'}->parse($robot_url, $content, $fresh_until); # recalculate allowed... $allowed = $self->{'rules'}->allowed($request->uri); |