versioned .tar.xz containers for the tlnet repository

git-svn-id: svn://tug.org/texlive/trunk@52931 c570f23f-e606-0410-a88d-b1316a301751

4 files changed, 246 insertions, 29 deletions

diff --git a/Master/texmf-dist/scripts/texlive/tlmgr.pl b/Master/texmf-dist/scripts/texlive/tlmgr.pl
index 130154eca41..3e4717fdbbb 100755
--- a/Master/texmf-dist/scripts/texlive/tlmgr.pl
+++ b/Master/texmf-dist/scripts/texlive/tlmgr.pl
@@ -1095,9 +1095,10 @@ sub backup_and_remove_package {
   }
   if ($opts{"backup"}) {
     $tlp->make_container($::progs{'compressor'}, $localtlpdb->root,
-                         $opts{"backupdir"}, 
-                         "${pkg}.r" . $tlp->revision,
-                         $tlp->relocated);
+                         destdir => $opts{"backupdir"}, 
+                         containername => "${pkg}.r" . $tlp->revision,
+                         relative => $tlp->relocated,
+                         user => 1);
     if ($autobackup) {
       # in case we do auto backups we remove older backups
       clear_old_backups($pkg, $opts{"backupdir"}, $autobackup);
@@ -2166,7 +2167,9 @@ sub action_backup {
            . $tlp->revision . ".tar.$compressorextension\n");
       if (!$opts{"dry-run"}) {
         $tlp->make_container($::progs{'compressor'}, $localtlpdb->root,
-                             $opts{"backupdir"}, "${pkg}.r" . $tlp->revision);
+                             destdir => $opts{"backupdir"},
+                             containername => "${pkg}.r" . $tlp->revision,
+                             user => 1);
       }
     }
   }
@@ -2281,8 +2284,12 @@ sub write_w32_updater {
     push (@rst_tlpobj, "tlpkg\\tlpobj\\$pkg.tlpobj");
     push (@rst_info, "$pkg ^($oldrev^)");
     next if ($opts{"dry-run"});
-    # create backup; make_container expects file name in a format: some-name.r[0-9]+
-    my ($size, undef, $fullname) = $localtlp->make_container("tar", $root, $temp, "__BACKUP_$pkg.r$oldrev");
+    # create backup; make_container expects filename in format:
+    #   some-name.r[0-9]+
+    my ($size, undef, $fullname) = $localtlp->make_container("tar", $root,
+                                     destdir => $temp,
+                                     containername => "__BACKUP_$pkg.r$oldrev",
+                                     user => 1);
     if ($size <= 0) {
       tlwarn("$prg: creation of backup container failed for: $pkg\n");
       return 1; # backup failed? abort
@@ -3202,8 +3209,10 @@ sub action_update {
       if ($opts{"backup"} && !$opts{"dry-run"}) {
         my $compressorextension = $Compressors{$::progs{'compressor'}}{'extension'};
         $tlp->make_container($::progs{'compressor'}, $root,
-                             $opts{"backupdir"}, "${pkg}.r" . $tlp->revision,
-                             $tlp->relocated);
+                             destdir => $opts{"backupdir"},
+                             containername => "${pkg}.r" . $tlp->revision,
+                             relative => $tlp->relocated,
+                             user => 1);
         $unwind_package =
             "$opts{'backupdir'}/${pkg}.r" . $tlp->revision . ".tar.$compressorextension";
         
@@ -3241,9 +3250,11 @@ sub action_update {
         # no backup was made, so let us create a temporary .tar file
         # of the package
         my $tlp = $localtlpdb->get_package($pkg);
-        my ($s, undef, $fullname) = $tlp->make_container("tar", $root, $temp,
-                                      "__BACKUP_${pkg}.r" . $tlp->revision,
-                                      $tlp->relocated);
+        my ($s, undef, $fullname) = $tlp->make_container("tar", $root,
+                         destdir => $temp,
+                         containername => "__BACKUP_${pkg}.r" . $tlp->revision,
+                         relative => $tlp->relocated,
+                         user => 1);
         if ($s <= 0) {
           tlwarn("\n$prg: creation of backup container failed for: $pkg\n");
           tlwarn("$prg: continuing to update other packages, please retry...\n");
diff --git a/Master/tlpkg/TeXLive/TLPOBJ.pm b/Master/tlpkg/TeXLive/TLPOBJ.pm
index 55ead17b769..a4db6d4c9ce 100644
--- a/Master/tlpkg/TeXLive/TLPOBJ.pm
+++ b/Master/tlpkg/TeXLive/TLPOBJ.pm
@@ -559,7 +559,12 @@ sub common_texmf_tree {
 
 
 sub make_container {
-  my ($self,$type,$instroot,$destdir,$containername,$relative) = @_;
+  my ($self, $type, $instroot, %other) = @_;
+  my $destdir = ($other{'destdir'} || undef);
+  my $containername = ($other{'containername'} || undef);
+  my $relative = ($other{'relative'} || undef);
+  my $user = ($other{'user'} || undef);
+  my $copy_instead_of_link = ($other{'copy_instead_of_link'} || undef);
   if (!($type eq 'tar' ||
         TeXLive::TLUtils::member($type, @{$::progs{'working_compressors'}}))) {
     tlwarn "$0: TLPOBJ supports @{$::progs{'working_compressors'}} and tar containers, not $type\n";
@@ -627,7 +632,11 @@ sub make_container {
   $selfcopy->writeout(\*TMP);
   close(TMP);
   push(@files, "$tlpobjdir/$self->{'name'}.tlpobj");
-  $tarname = "$containername.tar";
+  # Switch to versioned containers
+  # $tarname = "$containername.tar";
+  $tarname = "$containername.r" . $self->revision . ".tar";
+  my $unversionedtar;
+  $unversionedtar = "$containername.tar" if (! $user);
 
   # start the fun
   my $tar = $::progs{'tar'};
@@ -649,7 +658,9 @@ sub make_container {
   # overflow standard tar format and result in special things being
   # done.  We don't want the GNU-specific special things.
   #
-  my $is_user_container = ( $containername =~ /\.r[0-9]/ );
+  # We use versioned containers throughout, user mode is determined by
+  # argument.
+  my $is_user_container = $user;
   my @attrs
     = $is_user_container
       ? ()
@@ -701,7 +712,7 @@ sub make_container {
       # A complication, as always.  collapse_dirs returns absolute paths.
       # We want to change them back to relative so that the backup tar
       # has the same structure.
-      # in relative mode we have to remove the texmf-dist prefix, too
+      # In relative mode we have to remove the texmf-dist prefix, too.
       s,^$instroot/,, foreach @files_to_backup;
       if ($relative) {
         s,^$RelocTree/,, foreach @files_to_backup;
@@ -712,6 +723,7 @@ sub make_container {
 
   # Run tar. Unlink both here in case the container is also plain tar.
   unlink("$destdir/$tarname");
+  unlink("$destdir/$unversionedtar") if (! $user);
   unlink("$destdir/$containername");
   xsystem(@cmdline);
 
@@ -743,6 +755,19 @@ sub make_container {
       # most strange cases.
       unlink("$destdir/$tarname")
         if ((-r "$destdir/$tarname") && (-r "$destdir/$containername"));
+      # in case of system containers also create the links to the 
+      # versioned containers
+      if (! $user) {
+        my $linkname = "$destdir/$unversionedtar.$compressorextension";
+        unlink($linkname) if (-r $linkname);
+        if ($copy_instead_of_link) {
+          TeXLive::TLUtils::copy("-f", "$destdir/$containername", $linkname)
+        } else {
+          if (!symlink($containername, $linkname)) {
+            tlwarn("$0: Couldn't generate link $linkname -> $containername?\n");
+          }
+        }
+      }
     } else {
       tlwarn("$0: Couldn't find $destdir/$tarname to run $compressor\n");
       return (0, 0, "");
@@ -1663,13 +1688,13 @@ if all files of the package are from the same texmf tree, this tree
 is returned, otherwise an undefined value. That is also a check
 whether a package is relocatable.
 
-=item C<make_container($type,$instroot[, $destdir[, $containername[, $relative]]])>
+=item C<make_container($type,$instroot, [ destdir => $destdir, containername => $containername, relative => 0|1, user => 0|1 ])>
 
 creates a container file of the all files in the C<TLPOBJ>
 in C<$destdir> (if not defined then C<< TLPOBJ->containerdir >> is used).
 
 The C<$type> variable specifies the type of container to be used.
-Currently only C<zip> or C<xz> are allowed, and are generating
+Currently only C<zip> or C<xz> are allowed, and generate
 zip files and tar.xz files, respectively.
 
 The file name of the created container file is C<$containername.extension>,
@@ -1683,7 +1708,7 @@ C<TLPOBJ> file in C<tlpkg/tlpobj/$name.tlpobj>.
 The argument C<$instroot> specifies the root of the installation from
 which the files should be taken.
 
-If the argument C<$relative> is present and true (perlish true) AND the
+If the argument C<$relative> is passed and true (perlish true) AND the
 packages does not span multiple texmf trees (i.e., all the first path
 components of all files are the same) then a relative packages is created,
 i.e., the first path component is stripped. In this case the tlpobj file
@@ -1692,6 +1717,9 @@ is placed into the root of the installation.
 This is used to distribute packages which can be installed in any arbitrary
 texmf tree (of other distributions, too).
 
+If user is present and true, no extra arguments for container generation are
+passed to tar (to make sure that user tar doesn't break).
+
 Return values are the size, the checksum, and the full name of the container.
 
 =item C<recompute_sizes($tltree)>
diff --git a/Master/tlpkg/bin/tl-update-containers b/Master/tlpkg/bin/tl-update-containers
index b42ff669e41..4cb41812038 100755
--- a/Master/tlpkg/bin/tl-update-containers
+++ b/Master/tlpkg/bin/tl-update-containers
@@ -241,12 +241,18 @@ sub main {
   # get list of packages.
   PACKS: for my $pkg (sort @todopacks) {
     next if $pkg =~ /00texlive/;
+    # we make links from unversioned to versioned containers; this
+    # variable is an override for debugging:
+    my $copy_instead_of_link = 0;
+    #
     foreach my $manualpkgre (@TeXLive::TLConfig::CriticalPackagesList) {
       # we match the initial string of the package name, so that all the
       # .arch packages are skipped, too
       if ($pkg =~ m/^$manualpkgre/) {
         if ($opt_all || $opt_recreate) {
           tlwarn("$0: Updating critical $pkg due to -all\n");
+          # in general, we want links for everything, but for testing maybe:
+          #$copy_instead_of_link = 1;
           last; # of the manualpkgre checks
 
         } else {
@@ -299,7 +305,11 @@ sub main {
         $objcopy->clear_srcfiles;
         if ($objsrc) {
           my ($s,$m) = $objsrc->make_container($type, $Master,
-                               $opt_containerdir, "$pkg.source", $do_relative);
+                                               destdir => $opt_containerdir,
+                                               containername => "$pkg.source", 
+                                               relative => $do_relative,
+                                               user => 0,
+                                copy_instead_of_link => $copy_instead_of_link);
           if ($s > 0) {
             # something was created 
             # important, we have to add it to the original $obj
@@ -323,7 +333,11 @@ sub main {
         $objcopy->clear_docfiles;
         if ($objdoc) {
           my ($s,$m) = $objdoc->make_container($type, $Master,
-                                 $opt_containerdir, "$pkg.doc", $do_relative);
+                                               destdir => $opt_containerdir,
+                                               containername => "$pkg.doc", 
+                                               relative => $do_relative,
+                                               user => 0,
+                                copy_instead_of_link => $copy_instead_of_link);
           if ($s > 0) {
             # something was created
             $obj->doccontainersize($s);
@@ -341,8 +355,12 @@ sub main {
       push @removecontainers, "$pkg.doc";
     }
     if (!$opt_dry) {
-      my ($s,$m) = $objcopy->make_container($type, $Master, $opt_containerdir,
-                                            $pkg, $do_relative);
+      my ($s,$m) = $objcopy->make_container($type, $Master,
+                                            destdir => $opt_containerdir,
+                                            containername => $pkg, 
+                                            relative => $do_relative,
+                                            user => 0,
+                                copy_instead_of_link => $copy_instead_of_link);
       if ($s > 0) {
         $obj->containersize($s);
       }
@@ -357,9 +375,9 @@ sub main {
     # add the updated (or new) TLPOBJ to NET TLPDB
     # that way the other container sizes are not destroyed
     $nettlpdb->add_tlpobj($obj) unless $opt_dry;
-  }
+  } # end of big loop on all todopacks
 
-  # remove old containers
+  # remove source/doc containers that have become obsolete.
   for my $op (@removecontainers) {
     if (-r "$opt_containerdir/$op.tar.xz") {
       info("$0: $op container is old, removing it\n");
@@ -392,6 +410,9 @@ sub main {
     $count{"removed"}++;
   }
 
+  # remove non-symlinked .rNNN files.
+  &rcheck_containerdir ($opt_containerdir);
+  
   if ($opt_recreate) {
     info("$0: all packages recreated.\n");
   } else {
@@ -400,7 +421,7 @@ sub main {
       info("$0: $count{new} new, $count{removed} removed, " .
             "$count{updated} updated, $count{unchanged} unchanged.\n");
     } else {
-      info("$0: nothing to be done.\n");
+      info("$0: no containers to be updated.\n");
     }
   }
 
@@ -438,6 +459,56 @@ sub main {
   return 0;
 }
 
+
+# With versioned containers, we have to remove older .rNNN tarballs when
+# a new NNN gets made and symlinked. Otherwise we'd be distributing
+# every version of every package.
+# 
+# That is, if we previously had foo.tar.xz -> foo.r500.tar.xz,
+# but now we've just created foo.r501.tar.xz (and updated the symlink).
+# so we need to delete the .r500.
+# 
+# We do this purely by looking at the contents of DIR: every file
+# whose name has a .rNNN pattern is checked for a corresponding symlink,
+# and if no symlink, it is deleted.
+# 
+# There should be no stray files with other names, so we report them if
+# we find them, but do nothing with them.
+# 
+sub rcheck_containerdir {
+  my ($dir) = @_;
+  
+  opendir (my $dh, $dir) || die "opendir($dir) failed: $!";
+  for my $fname (sort readdir $dh) {
+    if ($fname =~ /(.*)\.r[0-9]+\./) {
+      my $linkname = "$dir/$1.tar.xz";
+      if (-l $linkname) {
+        my $linktarget = readlink ($linkname);
+        if ($linktarget ne $fname) {
+          if ($opt_dry) {
+            print "would unlink($fname) [because have $linktarget]\n";
+          } else {
+            debug("unlinking $dir/$fname [because have $linktarget]\n");
+            unlink("$dir/$fname")
+            || tlwarn("$0: non-symlinked unlink($dir/$fname) failed: $!");
+          }
+        } # (else it's the current version, do nothing.)
+      } else {
+        # this can happen if we accidentally created unlinked containers.
+        tlwarn("$0: have file $fname, but no link $linkname!\n");
+      }
+    } elsif ($fname =~ /^([^.]+(\.[^.]+)*\.tar\.xz|\.\.?)$/) {
+      ; # unversioned name, ok. Either . or .. or
+        # matching PKGNAME[.SUBPKG]*.tar.xz, where .SUBPKG might
+        # be .doc, .source, .x86_64-linux, etc., or
+        # .infra.x86_64-linux (for texlive.infra), win32.doc (for wintools).
+    } else {
+      tlwarn("$0: unexpected file: $fname (in $dir)\n");
+    }
+  }
+  closedir ($dh) || warn "closedir($dir) failed: $!";
+}
+
 __END__
 
 =head1 NAME
diff --git a/Master/tlpkg/doc/packages.txt b/Master/tlpkg/doc/packages.txt
index 089415eb20d..d9587f09bee 100644
--- a/Master/tlpkg/doc/packages.txt
+++ b/Master/tlpkg/doc/packages.txt
@@ -44,10 +44,10 @@ Do not fail to revert the tlpdb, or it will create an svn conflict.
  rm -rf tlcritical
  mkdir tlcritical
  tl-update-tlcritical --recreate
- 
-
-To branch before a release, in the unlikely event of separate
-  development work before we want to switch trunk over:
+
+
+ To branch before a release,
+   in the unlikely event of separate development work before switching trunk:
 svn copy -r 12205 -m'texlive-2009-dev tag based on r12205' \
   svn://tug.org/texlive/trunk svn://tug.org/texlive/tags/texlive-2009-dev
 svn copy -r 12205 -m'branch2009-dev branch based on r12205' \
@@ -55,3 +55,110 @@ svn copy -r 12205 -m'branch2009-dev branch based on r12205' \
 
 (Use -dev instead of .something because - will sort first, keeping the
 listings in a nice chronological order.)
+
+
+ To build tlnet in a separate location,
+   e.g., if making changes to its structure.
+   (e.g., in Oct 2019 we changed to versioned containers .rNNN.tar.xz.)
+
+# First we build the test repository and test installation from scratch.
+# Do it with no (significant) changes from the current source, for sanity.
+# 
+# Use subdir of /home/ftp/texlive or another dir ignored for backups
+# (see /etc/rsnapshot.conf).
+destdir=/home/ftp/texlive/test-tlnet
+#
+# since we're building into a new hierarchy for testing, have to touch tlpdb.
+rm -rf $destdir
+mkdir -p $destdir/tlpkg
+touch !$/texlive.tlpdb
+
+# Update (populate, the first time) the new repo.
+# should take about 15 minutes; updating with the full scheme (which
+# should be done at least once) instead of infraonly takes about 35 min.
+tl-update-tlnet --critical --scheme infraonly --dest $destdir
+
+# if there are errors, the build hierarchy will not be moved to the
+# destination; can do it by hand if desired:
+cd /home/ftp
+rm -rf texlive/test-tlnet/ # just removing the empty file we created
+mv tlnet-trial-*/tlsrc.try texlive/test-tlnet/
+rmdir tlnet-trial-*
+
+# If ever desired, could recreate every package from scratch.
+# This will take hours.
+#tl-update-tlnet --critical --recreate --scheme infraonly --dest $destdir
+
+# Now we can test the new repo: do an installation into /tmp/ki (hardwired).
+# The default scheme here is again infraonly.
+tl-try-install --repo $destdir
+
+# We'll want to test network, local disk, and from-development repositories.
+# So move /tmp/ki, set the repository, and do it again.
+mv /tmp/ki /tmp/tldsk
+tl-try-install --repo $destdir
+/tmp/tldsk/bin/x86_64-linux/tlmgr option repo $destdir
+#
+# For ftp access, want to use ftpmirror to log in, since anonymous ftp
+# will often fail. See tug.org:~karl/.netrc or the ftpd setup on tug.
+mv /tmp/ki /tmp/tlftp
+/tmp/tlftp/bin/x86_64-linux/tlmgr option repo ftp://fm.tug.org/texlive/test-tlnet
+#
+# Create test installation from development tree.
+# This exercises yet another code path.
+Master=...
+$Master/install-tl --profile=$Master/tlpkg/dev/profiles/TLinfra.pro 
+mv /tmp/ki /tmp/tldev
+
+# Now, from those repos, initial check with current tlmgr;
+# update list should be empty at first.
+/tmp/tldsk/bin/x86_64-linux/tlmgr update --list
+/tmp/tlftp/bin/x86_64-linux/tlmgr update --list
+/tmp/tldev/bin/x86_64-linux/tlmgr update --list
+
+# So much for the setup and sanity checks. Now it's time to test
+# whatever the real changes are. Make the changes (without
+# committing), and update our test repo just as above (no need to recreate):
+tl-update-tlnet --critical --scheme infraonly --dest $destdir
+
+# Then re-update the installations:
+/tmp/tldsk/bin/x86_64-linux/tlmgr update --self --all
+/tmp/tlftp/bin/x86_64-linux/tlmgr update --self --all
+/tmp/tldev/bin/x86_64-linux/tlmgr update --self --all
+
+# Also try an older tlmgr against the test repo (readonly, don't want to
+# update /usr/local/texlive/2019):
+env PATH=/usr/local/texlive/2019/bin/x86_64-linux:/usr/bin \
+  tlmgr -repo $destdir update --list | head
+#
+# Also must try creating and updating a 2019 install from the test-
+# repo; best to do a full install here for complete testing.
+rm -rf /tmp/ki
+pro=--profile=/home/texlive/trunk/Master/tlpkg/dev/profiles/TLfull.pro
+env PATH=/usr/bin \
+  /usr/local/texlive/2019/install-tl $pro
+test19=/tmp/tlt19
+mv /tmp/ki $test19 # so we don't accidentally lose it with rm /tmp/ki
+#
+env PATH=$test19/bin/x86_64-linux:/usr/bin \
+  tlmgr -repo $destdir update --self --all
+
+# To create the disaster recovery scripts in the test repo:
+tl-update-tlcritical --tlcrit $destdir --no-containers
+
+# Repeat until satisfied: all changes are in test-tlnet and
+# installations and working as intended. For the Nov 2019 update, that
+# means symlinks pointing to new files for updated packages, old
+# versioned containers removed, backups made, etc.
+
+# The tlpkg/bin/tltestnet script has minimal support for some of the
+# above in the November 2019 change - moving the dev files (to and
+# back), creating a 2019-original-release installation, etc. Take a look
+# for ideas and hacking; don't take it too seriously.
+
+# If you need to make changes to files without a nightly build
+# intervening, as will usually be the case, it is necessary to manually
+# edit the Master/tlpkg/texlive.tlpdb in the test- repo to
+# increment the "revision" number for texlive.infra, or whatever
+# packages need to be updated. Otherwise tl-update-containers will think
+# that nothing needs to be updated.