more work, currently incomplete

git-svn-id: svn://tug.org/texlive/trunk@47806 c570f23f-e606-0410-a88d-b1316a301751

2 files changed, 81 insertions, 25 deletions

diff --git a/Master/tlpkg/TeXLive/TLConfig.pm b/Master/tlpkg/TeXLive/TLConfig.pm
index c697418f103..4588e29dbf8 100644
--- a/Master/tlpkg/TeXLive/TLConfig.pm
+++ b/Master/tlpkg/TeXLive/TLConfig.pm
@@ -22,6 +22,10 @@ BEGIN {
     $MetaCategoriesRegexp
     $CategoriesRegexp
     $DefaultCategory
+    $DefaultFallbackDownloader
+    @AcceptedFallbackDownloaders
+    %FallbackDownloaderProgram
+    %FallbackDownloaderArgs
     $DefaultCompressorFormat
     $DefaultContainerExtension
     @AcceptedCompressors
@@ -110,6 +114,14 @@ if ($^O =~ /^MSWin/i) {
   $CriticalPackagesRegexp = '^(texlive\.infra|tlperl\.win32$)';
 }
 
+#
+our $DefaultFallbackDownloader = "wget";
+our @AcceptedFallbackDownloaders = qw/wget curl/;
+our %FallbackDownloaderProgram = ( 'wget' => 'wget', 'curl' => 'curl');
+our %FallbackDownloaderArgs = (
+  'wget' => ['--user-agent=texlive/wget', '--tries=10', "--timeout=$NetworkTimeout", '-q', '-O'],
+  'curl' => ['--user-agent', 'texlive/curl', '--retry', '10', '--connect-timeout', "$NetworkTimeout", '--silent', '--output']
+);
 # the way we package things on the web
 our $DefaultCompressorFormat = "xz";
 our $DefaultContainerExtension = "tar.$DefaultCompressorFormat";
diff --git a/Master/tlpkg/TeXLive/TLUtils.pm b/Master/tlpkg/TeXLive/TLUtils.pm
index 33d10629d28..d24d3ecf9e7 100644
--- a/Master/tlpkg/TeXLive/TLUtils.pm
+++ b/Master/tlpkg/TeXLive/TLUtils.pm
@@ -54,7 +54,7 @@ C<TeXLive::TLUtils> -- utilities used in TeX Live infrastructure
   TeXLive::TLUtils::touch(@files);
   TeXLive::TLUtils::collapse_dirs(@files);
   TeXLive::TLUtils::removed_dirs(@files);
-  TeXLive::TLUtils::download_file($path, $destination [, $progs ]);
+  TeXLive::TLUtils::download_file($path, $destination);
   TeXLive::TLUtils::setup_programs($bindir, $platform);
   TeXLive::TLUtils::setup_compressor();
   TeXLive::TLUtils::tlcmp($file, $file);
@@ -2332,13 +2332,34 @@ sub setup_programs {
   $::progs{'xz'} = "xz";
   $::progs{'tar'} = "tar";
   $::progs{'lz4'} = "lz4";
-
+  $::progs{'gzip'} = "gzip";
+  $::progs{'downloader'} = $::progs{$DefaultFallbackDownloader};
+  $::progs{'downloader-args'} = @{$FallbackDownloaderArgs{$DefaultFallbackDownloader}};
   if ($^O =~ /^MSWin/i) {
     $::progs{'wget'}  = conv_to_w32_path("$bindir/wget/wget.exe");
     $::progs{'tar'}   = conv_to_w32_path("$bindir/tar.exe");
     $::progs{'xz'}    = conv_to_w32_path("$bindir/xz/xz.exe");
     $::progs{'lz4'}   = conv_to_w32_path("$bindir/lz4/lz4.exe");
-    for my $prog ("xz", "wget") {
+    # we leave gzip as is, needs to be provided by the PATH
+    # for windows we don't expect to have curl available, use our own
+    # shipped wget in all cases
+    my $dltype = 'wget';
+    $::progs{'downloader'} = $FallbackDownloaderProgram{$dltype};
+    $::progs{'downloader-args'} = $FallbackDownloaderArgs{$dltype};
+    # allow for hit-her-to override env vars
+    if ($ENV{"TL_DOWNLOAD_PROGRAM"}) {
+      $::progs{'downloader'} = $ENV{"TL_DOWNLOAD_PROGRAM"};
+    }
+    if ($ENV{"TL_DOWNLOAD_ARGS"}) {
+      $::progs{'downloader-args'} = [ split (" ", $ENV{"TL_DOWNLOAD_ARGS"}) ];
+    }
+    my $comptype = setup_program_with_env('TEXLIVE_COMPRESSOR', $DefaultCompressorFormat, @AcceptedCompressors);
+    $::progs{'compressor'} = $CompressorProgram{$comptype}:
+    $::progs{'compressor-args'} = $CompressorArgs{$comptype}:
+    $::progs{'decompressor'} = $DecompressorProgram{$comptype}:
+    $::progs{'compressor-args'} = $DecompressorArgs{$comptype}:
+
+    for my $prog ("downloader", "compressor") {
       my $ret = system("$::progs{$prog} --version >nul 2>&1"); # on windows
       if ($ret != 0) {
         warn "TeXLive::TLUtils::setup_programs (w32) failed";  # no nl for perl
@@ -2358,7 +2379,37 @@ sub setup_programs {
       $::installerdir = "$bindir/../..";
       $platform = platform();
     }
-    my $s = 0;
+    # setup of the fallback downloaders
+    for my $dltype (@AcceptedFallbackDownloaders) {
+      my $defprog = $FallbackDownloadProgram{$dltype};
+      # do not warn on errors
+      setup_unix_one($defprog, "$bindir/$dltype/$defprog.$platform", "--version", 1);
+    }
+    for my $comptype (@AcceptedCompressors) {
+      my $defprog = $CompressorProgram{$comptype};
+      # do not warn on errors
+      setup_unix_one($defprog, "$bindir/$comptype/$defprog.$platform", "--version", 1);
+    }
+    # in the case of downloaders keys and prog names agree
+    my $dltype = setup_program_with_env('TEXLIVE_DOWNLOADER', 
+                                        $DefaultFallbackDownloader,
+                                        @AcceptedFallbackDownloaders);
+    $::progs{'downloader'} = $FallbackDownloaderProgram{$dltype};
+    $::progs{'downloader-args'} = $FallbackDownloaderArgs{$dltype};
+    if ($ENV{"TL_DOWNLOAD_PROGRAM"}) {
+      $::progs{'downloader'} = $ENV{"TL_DOWNLOAD_PROGRAM"};
+    }
+    if ($ENV{"TL_DOWNLOAD_ARGS"}) {
+      $::progs{'downloader-args'} = [ split (" ", $ENV{"TL_DOWNLOAD_ARGS"}) ];
+    }
+
+    # TODO TODO TODO
+    die("NNED TO UPDATE MORE STUFF HERE IN setup_programs!");
+    my $comptype = setup_program_with_env('TEXLIVE_COMPRESSOR', $DefaultCompressorFormat, @AcceptedCompressors);
+
+
+
+
     $s += setup_unix_one('wget', "$bindir/wget/wget.$platform", "--version");
     $s += setup_unix_one('xz',   "$bindir/xz/xz.$platform", "notest");
     $ok = ($s == 2);  # failure return unless all are present.
@@ -2477,38 +2528,33 @@ sub setup_unix_one {
 }
 
 
-=item C<setup_compressor()>
+=item C<setup_program_with_env($envvar, $default, @accepted)>
 
-Setup compression method for containers.
+Setup a program based on configuration in the environment of default.
 
 =cut
 
-# TODO TODO needs improvement, check also whether compressor prog is available!!!
-sub setup_compressor {
-  my $compressor = "xz";
-  if ($ENV{'TEXLIVE_COMPRESSOR'}) {
-    if (TeXLive::TLUtils::member($ENV{'TEXLIVE_COMPRESSOR'}, @TeXLive::TLConfig::AcceptedCompressors)) {
-      $compressor = $ENV{'TEXLIVE_COMPRESSOR'};
+sub setup_program_with_env {
+  my ($envvar, $default, @accepted) = @_;
+  my $sel = $default;
+  if ($ENV{$envvar}) {
+    if (TeXLive::TLUtils::member($ENV{$envvar}, @accepted)) {
+      $sel = $ENV{$envvar};
     } else {
-      tlwarn("$prog: unknown compressor in TEXLIVE_COMPRESSOR=$ENV{'TEXLIVE_COMPRESSOR'}, falling back to xz\n");
+      tlwarn("$prog: unknown prog in $envvar=$ENV{$envvar}, falling back to $default\n");
     }
   }
-  my $compressorextension = $TeXLive::TLConfig::CompressorExtension{$compressor};
-  return ($compressor, $compressorextension);
+  return $sel;
 }
 
 
-=item C<download_file( $relpath, $destination [, $progs ] )>
+
+=item C<download_file( $relpath, $destination )>
 
 Try to download the file given in C<$relpath> from C<$TeXLiveURL>
 into C<$destination>, which can be either
 a filename of simply C<|>. In the latter case a file handle is returned.
 
-The optional argument C<$progs> is a reference to a hash giving full
-paths to the respective programs, at least C<wget>.  If C<$progs> is not
-given the C<%::progs> hash is consulted, and if this also does not exist
-we try a literal C<wget>.
-
 Downloading honors two environment variables: C<TL_DOWNLOAD_PROGRAM> and
 C<TL_DOWNLOAD_ARGS>. The former overrides the above specification
 devolving to C<wget>, and the latter overrides the default wget
@@ -2521,11 +2567,9 @@ would end in C<-O>.  Use with care.
 =cut
 
 sub download_file {
-  my ($relpath, $dest, $progs) = @_;
+  my ($relpath, $dest) = @_;
   my $wget;
-  if (defined($progs) && defined($progs->{'wget'})) {
-    $wget = $progs->{'wget'};
-  } elsif (defined($::progs{'wget'})) {
+  if (defined($::progs{'wget'})) {
     $wget = $::progs{'wget'};
   } else {
     tlwarn ("download_file: Programs not set up, trying literal wget\n");