doc,sync

git-svn-id: svn://tug.org/texlive/trunk@55571 c570f23f-e606-0410-a88d-b1316a301751

2 files changed, 358 insertions, 0 deletions

diff --git a/Master/tlpkg/bin/archive/tl-update-testcow b/Master/tlpkg/bin/archive/tl-update-testcow
new file mode 100755
index 00000000000..29c48f4e04a
--- /dev/null
+++ b/Master/tlpkg/bin/archive/tl-update-testcow
@@ -0,0 +1,354 @@
+#!/bin/sh -e
+# Copyright 2008-2020 Norbert Preining
+# This file is licensed under the GNU General Public License version 2
+# or any later version.
+# 
+# Build a TeX Live tlnet area in a temporary directory, 
+# then (unless -N) do a test installation from it,
+# then (if the test install succeeded), update the live area,
+# all as specified in the options.
+
+vc_id='$Id: tl-update-tlnet 55186 2020-05-17 22:26:21Z karl $'
+renice 20 $$ >/dev/null 2>&1
+unset CDPATH
+unset LS_COLORS
+
+yyyy=2020
+
+check_consistency=true
+chicken=false
+cow_shell=/bin/sh
+critical=
+pretest=false
+recreate=
+scheme=full
+testinstall=true
+tlweb=/home/texlive/tstftp/tstlive/tstnet
+update_install_pkg=true
+verbose=
+gpgcmd=
+
+while test $# -gt 0; do
+  case $1 in
+  --cow)               cow_shell=cow-shell;;     # see comments below
+  --critical)          critical=--all;;
+  --dest)              shift; tlweb=$1;;
+  --dry-run|-n)        chicken=true;;
+  --gpgcmd)            shift; gpgcmd="--gpgcmd \"$1\"";;
+  --master)            shift; Master=$1;;
+  --no-consistency)    check_consistency=false;; # takes a long time.
+  --no-install-pkg)    update_install_pkg=false;;# trunk update after freeze
+  --no-testinstall|-N) testinstall=false;;       # and no updates; quit early.
+  --pretest)           tlweb=/home/ftp/texlive/tlpretest;;
+  --recreate)          recreate=--recreate;;
+  --scheme)            shift; scheme=$1;;        # scheme for test install.
+  --testlocation)      shift; tltrybase=$1;;
+  -v|-vv|-vvv)         verbose=$1;;
+  --help)              echo "ustl. sorry."; exit 0;;
+  --version)           echo "$vc_id"; exit 0;;
+  *) echo "$0: unrecognized option \`$1'; see source." >&2
+     exit 1;;
+  esac
+  shift
+done
+
+if test ! -r "$tlweb/tlpkg/texlive.tlpdb"; then
+  cat <<END_NO_TLPDB >&2
+$0: fatal: no file $tlweb/tlpkg/texlive.tlpdb.
+$0: If you are setting up a new release, touch the file,
+$0: and then use --critical --recreate.
+$0: (Or otherwise set up the tlnet hierarchy manually.)
+$0: Goodbye.
+END_NO_TLPDB
+  # and typically we will fail because there are new messages
+  # in the installer.  move the trial dir by hand to avoid
+  # time-consuming full recreate more than once.
+  exit 1
+fi
+
+if test -z "$Master"; then
+  mydir=`dirname $0`
+  Master=`cd $mydir/../.. && pwd`
+fi
+prg=`basename $0`
+
+# If there are broken symlinks anywhere in the bin directories, give up
+# right away. The nightly cron checks that there no symlinks anywhere else.
+# Below, we check for broken symlinks in the bin/ dir as installed.
+$Master/tlpkg/bin/tl-check-symlinks $Master/bin
+
+# Keep the default out of ~ftp/texlive, which CTAN mirrors much of.
+test -z "$tltrybase" \
+&& tltrybase=`cd $tlweb/../.. && pwd`/tlnet-trial-`date +%y%m%d`
+tltry=$tltrybase/tlsrc.try
+echo "$0: running from Master=$Master"
+echo "$0: working in   tltry=$tltry"
+echo "$0: output to    tlweb=$tlweb"
+
+# Save current tlpdb in case of disaster.
+cp --force --backup $tlweb/tlpkg/texlive.tlpdb* /tmp
+
+# 
+# Be sure we're starting the test cleanly.
+rm -rf $tltrybase
+mkdir -p $tltry
+
+# Keep new directory group-writable to texlive so MacTeX maintainer
+# can update it too.
+chmod g+ws $tltry
+chgrp texlive $tltry || : # in case of strange system
+
+# cp/link files in a loop so we can exclude the mactex files, which is a
+# waste of time, and more importantly can uselessly fail due to permissions.
+for f in $tlweb/*; do
+  if echo "$f" | grep mactex >/dev/null; then
+    : # skip mactex
+  elif test "x$cow_shell" = xcow-shell; then
+    cp -al "$f" $tltry  # assume GNU cp so we can link instead of copy
+    # However, cow-shell is evidently introducing strange errors on the 
+    # tug.org server we switched to in 2020, causing the release (tlnet)
+    # directory to be modified even when changes should be kept to the
+    # test (tltry) directory (ultimately resulting in checksum errors
+    # since the containers and tlpdb don't match, after a failed rebuild).
+    # This can be confirmed with a test hierarchy and build; see tltestnet.
+    # So, do not use cow-shell.
+  else
+    # no cow-shell, straight copy.
+    cp -a "$f" $tltry   # still assuming GNU cp for -a
+  fi
+done
+
+# Update packages in our working dir.
+# These shell assignments have to come outside the cow-shell.
+echo "$prg: Updating $tltry (from $Master) with $cow_shell..."
+containers_prog="$Master/tlpkg/bin/tl-update-containers"
+containers_args=" $verbose -location $tltry $critical $recreate $gpgcmd"
+containers_invoke="$containers_prog $containers_args"
+#
+update_install_prog=$Master/tlpkg/bin/tl-update-install-pkg
+update_install_invoke="$update_install_prog $gpgcmd -o $tltry"
+#
+cd $tltry
+strace_file=$tltry/tlpkg/texlive.tlpdb.md5
+strace -o /tmp/s.log -f -P $strace_file $cow_shell <<END_COW
+echo "$prg: Updating containers with (critical=$critical recreate=$recreate gpgcmd=$gpgcmd)"
+echo "$prg:   $containers_invoke"
+if $containers_invoke; then :; else
+  echo "$prg: tl-update-containers failed, goodbye." >&2
+  exit 1
+fi
+
+# We don't want to update the install packages if we're updating the
+# "frozen" release (tlnet-final) after we've moved on to the pretest.
+# See tlpkg/doc/releng.txt.
+if $update_install_pkg; then
+  # It is scary, but I guess we should update the installer package every
+  # day, partly for the sake of doc.html and partly so it actually gets
+  # tested.  Hopefully we don't break the Perl modules very often.
+  echo "$prg: Updating install pkg with"
+  echo "$prg:   $update_install_invoke"
+  if $update_install_invoke; then :; else
+    echo "$prg: tl-update-install-pkg failed, goodbye." >&2
+    exit 1
+  fi
+else
+  :
+fi
+END_COW
+
+# cow-shell leaves this around, haven't tracked down (or reported).
+#rm -f $tltry/.ilist
+
+# if not doing the test installation, don't push anything out.
+$testinstall || exit 0
+
+# 
+# Now we have an updated tlweb in $tltry where only the changed files
+# are actual files, the rest are hard links.
+# Try to make a test installation.
+cd $tltrybase
+tltryinst=$tltrybase/tlinst.try
+
+zcat $tltry/install-tl-unx.tar.gz | tar -xf -
+cd install-tl-*  # subdir is YYYYMMDD
+
+# create test TL install profile.
+# We set TEXMFVAR for the sake of luaotfload-tool.
+echo "# texlive-profile from $0 `date`
+selected_scheme scheme-$scheme
+TEXDIR $tltryinst/$yyyy
+TEXMFLOCAL $tltryinst/texmf-local
+TEXMFSYSCONFIG $tltryinst/$yyyy/texmf-config
+TEXMFSYSVAR $tltryinst/$yyyy/texmf-var
+TEXMFVAR $tltryinst/$yyyy/texmf-uvar
+TEXMFHOME ~/texmf
+tlpdbopt_install_docfiles 1
+tlpdbopt_install_srcfiles 1
+tlpdbopt_create_formats 1
+instopt_letter 0
+instopt_adjustpath 0
+instopt_adjustrepo 1
+" >texlive.profile
+
+# silence envvar warnings and the welcome message.
+TEXLIVE_INSTALL_ENV_NOCHECK=1; export TEXLIVE_INSTALL_ENV_NOCHECK
+TEXLIVE_INSTALL_NO_WELCOME=1; export TEXLIVE_INSTALL_NO_WELCOME
+
+# Minimal PATH from here on! E.g., we don't want the test install to
+# find its own mktexlsr, but only have the programs in the test
+# environment available.
+PATH=/usr/bin
+
+tlnet_install_log=`pwd`/update-tlnet-install.log
+tlnet_install_cmd="install-tl -location $tltry -profile texlive.profile"
+echo "$prg: Running test install (log: $tlnet_install_log)..."
+echo "$prg:   (in `pwd`)"
+echo "$prg:   $tlnet_install_cmd"
+perl $tlnet_install_cmd >$tlnet_install_log 2>&1 \
+|| true # install-tl can fail, but we test the output, so don't abort.
+
+# the following long grep command should filter away all *normal*
+# installation messages.
+# if there are any other messages they will end up on stdout and 
+# thus be noticed.
+unexpected_output=`cat $tlnet_install_log \
+  | sed '/The following environment variables/,/^ ------/d' \
+  | grep -Ev '^ ------' \
+  | grep -Ev '^$' \
+  | grep -Ev '^cryptographic signature of' \
+  | grep -Ev '^xAutomated TeX Live installation using profile' \
+  | grep -Ev '^Installing from:' \
+  | grep -Ev '^Platform: ' \
+  | grep -Ev '^Distribution: inst' \
+  | grep -Ev '^Directory for temporary files' \
+  | grep -Ev '^Loading ' \
+  | grep -Ev '^Installing ' \
+  | grep -Ev '^(re-)?running mktexlsr' \
+  | grep -Ev '^mktexlsr: Updating ' \
+  | grep -Ev '^mktexlsr: Done' \
+  | grep -Ev '^writing fmtutil.cnf to' \
+  | grep -Ev '^writing updmap.cfg to' \
+  | grep -Ev '^writing language.(dat|def|dat.lua) to' \
+  | grep -Ev '^pre-generating all format file' \
+  | grep -Ev '^making ConTeXt MkIV cache' \
+  | grep -Ev '^running ' \
+  | grep -Ev '^done running ' \
+  | grep -Ev '^finished ' \
+  | grep -Ev '^Welcome to TeX Live' \
+  | grep -Ev '^The TeX Live web site' \
+  | grep -Ev '^contains updates' \
+  | grep -Ev '^TeX Live is a joint project of the TeX user groups' \
+  | grep -Ev '^please consider supporting it by joining the group' \
+  | grep -Ev '^The list of groups is available' \
+  | grep -Ev '^Add ' \
+  | grep -Ev '^Most importantly, add ' \
+  | grep -Ev '^to your PATH for current and future sessions' \
+  | grep -Ev ' \(if not dynamically found\)' \
+  | grep -Ev 'install-tl: done' \
+  | grep -Ev '^Logfile: ' \
+  | grep -Ev '^Time used for installing ' \
+  | grep -Ev '^setting up ConTeXt cache' \
+  | grep -Ev '^resolvers +\|' \
+  | grep -Ev '^system +\|' \
+  | grep -Ev '^mtxrun +\|' \
+  | grep -Ev '^done$' \
+  | cat`
+
+failure=false
+ignore_unexpected_output=false #true, if frustration
+if test -n "$unexpected_output"; then
+  $ignore_unexpected_output || failure=true
+  echo >&2
+  echo "$prg: Test installation failed." >&2
+  echo "$prg: Here is the unexpected output, from $tlnet_install_log:" >&2
+  echo "$unexpected_output" >&2
+  echo "$prg: (end of unexpected output)." >&2
+fi
+
+# more consistency checks.
+if test $failure = false; then
+  for cmd in \
+   "$Master/tlpkg/bin/tlgpg-verify $tltry/tlpkg/texlive.tlpdb.sha512" \
+   "$Master/tlpkg/bin/tl-compare-tlpdbs $critical $tltry/tlpkg/texlive.tlpdb" \
+   "$Master/tlpkg/bin/tl-check-symlinks $tltryinst/$yyyy/bin" \
+   "$tltryinst/$yyyy/bin/*/tlmgr --repository $tltry update --list" \
+   "$tltryinst/$yyyy/bin/*/updmap-sys -n" \
+   "$tltryinst/$yyyy/bin/*/mktexlsr -n --verbose" \
+   "$Master/tlpkg/bin/tl-check-tlnet-consistency --location=$tltry" \
+  ; do
+    cmdname=`echo "$cmd" | awk '{print $1}'`
+    if echo "$cmdname" | grep check-tlnet-consistency >/dev/null; then
+      # skip consistency check if requested (for development only).
+      $check_consistency || continue
+    fi
+    basecmd=`basename $cmdname`
+    echo "$prg: Running $basecmd ($cmd)"
+    outfile=/tmp/tlnet.$basecmd
+    if $cmd >$outfile 2>&1; then :; else
+      echo "$prg: $basecmd failed ($cmd):" >&2
+      sed 8q $outfile >&2
+      echo "... see $outfile for full output ..." >&2
+      echo >&2
+      failure=true
+    fi
+  done
+fi
+
+# Format creation check, in case fmtutil's exit status wasn't right.
+# This might not exist if the test install didn't get this far.
+install_tl_log=$tltryinst/$yyyy/install-tl.log
+
+if test -r $install_tl_log \
+   && grep -i '^fmtutil.*error.*' $install_tl_log >/dev/null; then
+  echo >&2
+  echo "$prg: seems fmtutil failed, check $install_tl_log." >&2
+  failure=true
+else
+  : # appease -e
+fi
+
+# In all cases, make copies in /tmp for inspection in case of
+# undetected failure.
+cp -f $tlnet_install_log /tmp
+test ! -r $install_tl_log || cp -f $install_tl_log /tmp
+
+if $failure || $chicken; then
+  echo >&2
+  echo "$prg: tl-update-tlnet transcript file: $tlnet_install_log" >&2
+  echo "$prg: install-tl log file: $install_tl_log" >&2
+  echo "$prg: Copies of both are in /tmp." >&2
+  echo "$prg: Please rm -rf the trial dir." >&2
+  if $failure; then
+    exit 1
+  elif $chicken; then
+    echo
+    echo "$prg: Chicken mode, not updating anything."
+    exit 0
+  else
+    echo "$prg: impossible non-failure non-chicken." >&2
+    exit 2
+  fi
+fi
+
+# 
+# no unexpected output, so ship the new packages.
+cd $tltrybase
+echo "$prg: Updating $tlweb from $tltry."
+# copy any mactex files since we didn't link them.
+for f in $tlweb/*mactex*; do
+  test ! -r $f || cp -pf $f $tltry
+done
+
+# mv then rm to avoid the mirmon probe failing during the rm.
+mv $tlweb $tltrybase/tlnet.old
+mv $tltry $tlweb
+rm -rf $tltrybase
+
+# We checked this above also, but check again.
+# We've removed the cwd, so cd out of it.
+cd /tmp || exit 1
+$Master/tlpkg/bin/tlgpg-verify $tlweb/tlpkg/texlive.tlpdb.sha512
+
+echo "$0: Done."
+
+exit 0
diff --git a/Master/tlpkg/bin/tltestnet b/Master/tlpkg/bin/tltestnet
index 75ab0544add..3a36fa5c90c 100755
--- a/Master/tlpkg/bin/tltestnet
+++ b/Master/tlpkg/bin/tltestnet
@@ -28,3 +28,7 @@ fi
 
 # run out of PATH, not the reference tree.
 tl-update-tlnet --dest $testnet --scheme $scheme $install_pkg $consistency
+
+# for when we were testing with cow-shell; we have given up on it nowadays.
+# tl-update-testcow is in archive/.
+#tl-update-testcow --cow --dest $testnet --scheme $scheme $install_pkg $consistency