path: root/Master/tlpkg/bin/tl-update-tlnet

#!/bin/sh -e
# Copyright 2008-2021 Norbert Preining
# This file is licensed under the GNU General Public License version 2
# or any later version.
# 
# Build a TeX Live tlnet area in a temporary directory, 
# then (unless -N) do a test installation from it,
# then (if the test install succeeded), update the live area,
# all as specified in the options.

vc_id='$Id$'
renice 20 $$ >/dev/null 2>&1
unset CDPATH
unset LS_COLORS

yyyy=2021

check_consistency=true
chicken=false
cow_shell=/bin/sh
critical=
pretest=false
recreate=
scheme=full
testinstall=true
tlweb=/home/ftp/texlive/tlnet
update_install_pkg=true
verbose=
gpgcmd=

while test $# -gt 0; do
  case $1 in
  --cow)               cow_shell=cow-shell;;     # see comments below
  --critical)          critical=--all;;
  --dest)              shift; tlweb=$1;;
  --dry-run|-n)        chicken=true;;
  --gpgcmd)            shift; gpgcmd="--gpgcmd \"$1\"";;
  --master)            shift; Master=$1;;
  --no-consistency)    check_consistency=false;; # takes a long time.
  --no-install-pkg)    update_install_pkg=false;;# trunk update after freeze
  --no-testinstall|-N) testinstall=false;;       # and no updates; quit early.
  --pretest)           tlweb=/home/ftp/texlive/tlpretest;;
  --recreate)          recreate=--recreate;;
  --scheme)            shift; scheme=$1;;        # scheme for test install.
  --testlocation)      shift; tltrybase=$1;;
  -v|-vv|-vvv)         verbose=$1;;
  --help)              echo "ustl. sorry."; exit 0;;
  --version)           echo "$vc_id"; exit 0;;
  *) echo "$0: unrecognized option \`$1'; see source." >&2
     exit 1;;
  esac
  shift
done

if test ! -r "$tlweb/tlpkg/texlive.tlpdb"; then
  cat <<END_NO_TLPDB >&2
$0: fatal: no file $tlweb/tlpkg/texlive.tlpdb.
$0: If you are setting up a new release, touch the file,
$0: and then use --critical --recreate.
$0: (Or copy in a previous tlnet manually; see releng.txt.)
$0: Goodbye.
END_NO_TLPDB
  # and typically we will fail because there are new messages
  # in the installer.  move the trial dir by hand to avoid
  # time-consuming full recreate more than once.
  exit 1
fi

if test -z "$Master"; then
  mydir=`dirname $0`
  Master=`cd $mydir/../.. && pwd`
fi
prg=`basename $0`

# If there are broken symlinks anywhere in the bin directories, give up
# right away. The nightly cron checks that there no symlinks anywhere else.
# Below, we check for broken symlinks in the bin/ dir as installed.
$Master/tlpkg/bin/tl-check-symlinks $Master/bin

# Keep the default out of ~ftp/texlive, which CTAN mirrors much of.
test -z "$tltrybase" \
&& tltrybase=`cd $tlweb/../.. && pwd`/tlnet-trial-`date +%y%m%d`
tltry=$tltrybase/tlsrc.try
echo "$0: running from Master=$Master"
echo "$0: working in   tltry=$tltry"
echo "$0: output to    tlweb=$tlweb"

# Save current tlpdb in case of disaster.
cp --force --backup $tlweb/tlpkg/texlive.tlpdb* /tmp

# 
# Be sure we're starting the test cleanly.
rm -rf $tltrybase
mkdir -p $tltry

# Keep new directory group-writable to texlive so MacTeX maintainer
# can update it too.
chmod g+ws $tltry
chgrp texlive $tltry || : # in case of strange system

# cp/link files in a loop so we can exclude the mactex files, which is a
# waste of time, and more importantly can uselessly fail due to permissions.
for f in $tlweb/*; do
  if echo "$f" | grep mactex >/dev/null; then
    : # skip mactex
  elif test "x$cow_shell" = xcow-shell; then
    cp -al "$f" $tltry  # assume GNU cp so we can link instead of copy
    # However, cow-shell is evidently introducing strange errors on the 
    # tug.org server we switched to in 2020, causing the release (tlnet)
    # directory to be modified even when changes should be kept to the
    # test (tltry) directory (ultimately resulting in checksum errors
    # since the containers and tlpdb don't match, after a failed rebuild).
    # This can be confirmed with a test hierarchy and build; see tltestnet.
    # So, do not use cow-shell.
  else
    # no cow-shell, straight copy.
    cp -a "$f" $tltry   # still assuming GNU cp for -a
  fi
done

# Update packages in our working dir.
# These shell assignments have to come outside the cow-shell.
echo "$prg: Updating $tltry (from $Master) with $cow_shell..."
containers_prog="$Master/tlpkg/bin/tl-update-containers"
containers_args=" $verbose -location $tltry $critical $recreate $gpgcmd"
containers_invoke="$containers_prog $containers_args"
#
update_install_prog=$Master/tlpkg/bin/tl-update-install-pkg
update_install_invoke="$update_install_prog $gpgcmd -o $tltry"
#
cd $tltry
$cow_shell <<END_COW
echo "$prg: Updating containers with (critical=$critical recreate=$recreate gpgcmd=$gpgcmd)"
echo "$prg:   $containers_invoke"
if $containers_invoke; then :; else
  echo "$prg: tl-update-containers failed, goodbye." >&2
  exit 1
fi

# We don't want to update the install packages if we're updating the
# "frozen" release (tlnet-final) after we've moved on to the pretest.
# See tlpkg/doc/releng.txt.
if $update_install_pkg; then
  # It is scary, but I guess we should update the installer package every
  # day, partly for the sake of doc.html and partly so it actually gets
  # tested.  Hopefully we don't break the Perl modules very often.
  echo "$prg: Updating install pkg with"
  echo "$prg:   $update_install_invoke"
  if $update_install_invoke; then :; else
    echo "$prg: tl-update-install-pkg failed, goodbye." >&2
    exit 1
  fi
else
  :
fi
END_COW

# cow-shell leaves this around, haven't tracked down (or reported).
rm -f $tltry/.ilist

# if not doing the test installation, don't push anything out.
$testinstall || exit 0

# 
# Now we have an updated tlweb in $tltry where only the changed files
# are actual files, the rest are hard links.
# Try to make a test installation.
cd $tltrybase
tltryinst=$tltrybase/tlinst.try

zcat $tltry/install-tl-unx.tar.gz | tar -xf -
cd install-tl-*  # subdir is YYYYMMDD

# create test TL install profile.
# We set TEXMFVAR for the sake of luaotfload-tool.
echo "# texlive-profile from $0 `date`
selected_scheme scheme-$scheme
TEXDIR $tltryinst/$yyyy
TEXMFLOCAL $tltryinst/texmf-local
TEXMFSYSCONFIG $tltryinst/$yyyy/texmf-config
TEXMFSYSVAR $tltryinst/$yyyy/texmf-var
TEXMFVAR $tltryinst/$yyyy/texmf-uvar
TEXMFHOME ~/texmf
tlpdbopt_install_docfiles 1
tlpdbopt_install_srcfiles 1
tlpdbopt_create_formats 1
instopt_letter 0
instopt_adjustpath 0
instopt_adjustrepo 0
" >texlive.profile

# silence envvar warnings and the welcome message.
TEXLIVE_INSTALL_ENV_NOCHECK=1; export TEXLIVE_INSTALL_ENV_NOCHECK
TEXLIVE_INSTALL_NO_WELCOME=1; export TEXLIVE_INSTALL_NO_WELCOME

# Minimal PATH from here on! E.g., we don't want the test install to
# find its own mktexlsr, but only have the programs in the test
# environment available.
PATH=/usr/bin

tlnet_install_log=`pwd`/update-tlnet-install.log
tlnet_install_cmd="install-tl -location $tltry -profile texlive.profile"
echo "$prg: `date`"
echo "$prg: Running test install (log: $tlnet_install_log)..."
echo "$prg:   (in `pwd`)"
echo "$prg:   $tlnet_install_cmd"
perl $tlnet_install_cmd >$tlnet_install_log 2>&1 \
|| true # install-tl can fail, but we test the output, so don't abort.

# the following long grep command should filter away all *normal*
# installation messages.
# if there are any other messages they will end up on stdout and 
# thus be noticed.
unexpected_output=`cat $tlnet_install_log \
  | sed '/The following environment variables/,/^ ------/d' \
  | grep -Ev '^ ------' \
  | grep -Ev '^$' \
  | grep -Ev '^cryptographic signature of' \
  | grep -Ev '^Automated TeX Live installation using profile' \
  | grep -Ev '^Installing from:' \
  | grep -Ev '^Platform: ' \
  | grep -Ev '^Distribution: inst' \
  | grep -Ev '^Directory for temporary files' \
  | grep -Ev '^Loading ' \
  | grep -Ev '^Installing ' \
  | grep -Ev '^(re-)?running mktexlsr' \
  | grep -Ev '^mktexlsr: Updating ' \
  | grep -Ev '^mktexlsr: Done' \
  | grep -Ev '^writing fmtutil.cnf to' \
  | grep -Ev '^writing updmap.cfg to' \
  | grep -Ev '^writing language.(dat|def|dat.lua) to' \
  | grep -Ev '^pre-generating all format file' \
  | grep -Ev '^making ConTeXt MkIV cache' \
  | grep -Ev '^running ' \
  | grep -Ev '^done running ' \
  | grep -Ev '^finished ' \
  | grep -Ev '^Welcome to TeX Live' \
  | grep -Ev '^The TeX Live web site' \
  | grep -Ev '^contains updates' \
  | grep -Ev '^TeX Live is a joint project of the TeX user groups' \
  | grep -Ev '^please consider supporting it by joining the group' \
  | grep -Ev '^The list of groups is available' \
  | grep -Ev '^Add ' \
  | grep -Ev '^Most importantly, add ' \
  | grep -Ev '^to your PATH for current and future sessions' \
  | grep -Ev ' \(if not dynamically found\)' \
  | grep -Ev 'install-tl: done' \
  | grep -Ev '^Logfile: ' \
  | grep -Ev '^Time used for installing ' \
  | grep -Ev '^setting up ConTeXt cache' \
  | grep -Ev '^resolvers +\|' \
  | grep -Ev '^system +\|' \
  | grep -Ev '^mtxrun +\|' \
  | grep -Ev '^done$' \
  | cat`

failure=false
ignore_unexpected_output=false #true, if frustration
if test -n "$unexpected_output"; then
  $ignore_unexpected_output || failure=true
  echo >&2
  echo "$prg: Test installation failed." >&2
  echo "$prg: Here is the unexpected output, from $tlnet_install_log:" >&2
  echo "$unexpected_output" >&2
  echo "$prg: (end of unexpected output)." >&2
fi

# more consistency checks.
if test $failure = false; then
  for cmd in \
   "$Master/tlpkg/bin/tlgpg-verify $tltry/tlpkg/texlive.tlpdb.sha512" \
   "$Master/tlpkg/bin/tl-compare-tlpdbs $critical $tltry/tlpkg/texlive.tlpdb" \
   "$Master/tlpkg/bin/tl-check-symlinks $tltryinst/$yyyy/bin" \
   "$tltryinst/$yyyy/bin/*/tlmgr --repository $tltry update --list" \
   "$tltryinst/$yyyy/bin/*/updmap-sys -n" \
   "$tltryinst/$yyyy/bin/*/mktexlsr -n --verbose" \
   "$Master/tlpkg/bin/tl-check-tlnet-consistency --location=$tltry" \
  ; do
    cmdname=`echo "$cmd" | awk '{print $1}'`
    if echo "$cmdname" | grep check-tlnet-consistency >/dev/null; then
      # skip consistency check if requested (for development only).
      $check_consistency || continue
    fi
    basecmd=`basename $cmdname`
    echo "$prg: `date`"
    echo "$prg: Running $basecmd ($cmd)"
    outfile=/tmp/tlnet.$basecmd
    if $cmd >$outfile 2>&1; then :; else
      echo "$prg: $basecmd failed ($cmd):" >&2
      sed 8q $outfile >&2
      echo "... see $outfile for full output ..." >&2
      echo >&2
      failure=true
    fi
  done
fi

# Format creation check, in case fmtutil's exit status wasn't right.
# This might not exist if the test install didn't get this far.
install_tl_log=$tltryinst/$yyyy/install-tl.log

if test -r $install_tl_log \
   && grep -i '^fmtutil.*error.*' $install_tl_log >/dev/null; then
  echo >&2
  echo "$prg: seems fmtutil failed, check $install_tl_log." >&2
  failure=true
else
  : # appease -e
fi

# In all cases, make copies in /tmp for inspection in case of
# undetected failure.
cp -f $tlnet_install_log /tmp
test ! -r $install_tl_log || cp -f $install_tl_log /tmp

if $failure || $chicken; then
  echo >&2
  echo "$prg: tl-update-tlnet transcript file: $tlnet_install_log" >&2
  echo "$prg: install-tl log file: $install_tl_log" >&2
  echo "$prg: Copies of both are in /tmp." >&2
  echo "$prg: Please rm -rf the trial dir." >&2
  if $failure; then
    exit 1
  elif $chicken; then
    echo
    echo "$prg: Chicken mode, not updating anything."
    exit 0
  else
    echo "$prg: impossible non-failure non-chicken." >&2
    exit 2
  fi
fi

# 
# no unexpected output, so ship the new packages.
cd $tltrybase
echo "$prg: Updating $tlweb from $tltry."
# copy any mactex files since we didn't link them.
for f in $tlweb/*mactex*; do
  test ! -r $f || cp -pf $f $tltry
done

# mv then rm to avoid the mirmon probe failing during the rm.
mv $tlweb $tltrybase/tlnet.old
mv $tltry $tlweb
rm -rf $tltrybase

# We checked this above also, but check again.
# We've removed the cwd, so cd out of it.
cd /tmp || exit 1
$Master/tlpkg/bin/tlgpg-verify $tlweb/tlpkg/texlive.tlpdb.sha512

echo "$0: Done."

exit 0