From 0adaaf2a67e5cb5da54192affc9b5f83ab8ff262 Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Tue, 16 Jun 2020 22:52:20 +0000 Subject: doc,sync git-svn-id: svn://tug.org/texlive/trunk@55571 c570f23f-e606-0410-a88d-b1316a301751 --- Master/tlpkg/bin/archive/tl-update-testcow | 354 +++++++++++++++++++++++++++++ Master/tlpkg/bin/tltestnet | 4 + 2 files changed, 358 insertions(+) create mode 100755 Master/tlpkg/bin/archive/tl-update-testcow (limited to 'Master/tlpkg') diff --git a/Master/tlpkg/bin/archive/tl-update-testcow b/Master/tlpkg/bin/archive/tl-update-testcow new file mode 100755 index 00000000000..29c48f4e04a --- /dev/null +++ b/Master/tlpkg/bin/archive/tl-update-testcow @@ -0,0 +1,354 @@ +#!/bin/sh -e +# Copyright 2008-2020 Norbert Preining +# This file is licensed under the GNU General Public License version 2 +# or any later version. +# +# Build a TeX Live tlnet area in a temporary directory, +# then (unless -N) do a test installation from it, +# then (if the test install succeeded), update the live area, +# all as specified in the options. + +vc_id='$Id: tl-update-tlnet 55186 2020-05-17 22:26:21Z karl $' +renice 20 $$ >/dev/null 2>&1 +unset CDPATH +unset LS_COLORS + +yyyy=2020 + +check_consistency=true +chicken=false +cow_shell=/bin/sh +critical= +pretest=false +recreate= +scheme=full +testinstall=true +tlweb=/home/texlive/tstftp/tstlive/tstnet +update_install_pkg=true +verbose= +gpgcmd= + +while test $# -gt 0; do + case $1 in + --cow) cow_shell=cow-shell;; # see comments below + --critical) critical=--all;; + --dest) shift; tlweb=$1;; + --dry-run|-n) chicken=true;; + --gpgcmd) shift; gpgcmd="--gpgcmd \"$1\"";; + --master) shift; Master=$1;; + --no-consistency) check_consistency=false;; # takes a long time. + --no-install-pkg) update_install_pkg=false;;# trunk update after freeze + --no-testinstall|-N) testinstall=false;; # and no updates; quit early. + --pretest) tlweb=/home/ftp/texlive/tlpretest;; + --recreate) recreate=--recreate;; + --scheme) shift; scheme=$1;; # scheme for test install. + --testlocation) shift; tltrybase=$1;; + -v|-vv|-vvv) verbose=$1;; + --help) echo "ustl. sorry."; exit 0;; + --version) echo "$vc_id"; exit 0;; + *) echo "$0: unrecognized option \`$1'; see source." >&2 + exit 1;; + esac + shift +done + +if test ! -r "$tlweb/tlpkg/texlive.tlpdb"; then + cat <&2 +$0: fatal: no file $tlweb/tlpkg/texlive.tlpdb. +$0: If you are setting up a new release, touch the file, +$0: and then use --critical --recreate. +$0: (Or otherwise set up the tlnet hierarchy manually.) +$0: Goodbye. +END_NO_TLPDB + # and typically we will fail because there are new messages + # in the installer. move the trial dir by hand to avoid + # time-consuming full recreate more than once. + exit 1 +fi + +if test -z "$Master"; then + mydir=`dirname $0` + Master=`cd $mydir/../.. && pwd` +fi +prg=`basename $0` + +# If there are broken symlinks anywhere in the bin directories, give up +# right away. The nightly cron checks that there no symlinks anywhere else. +# Below, we check for broken symlinks in the bin/ dir as installed. +$Master/tlpkg/bin/tl-check-symlinks $Master/bin + +# Keep the default out of ~ftp/texlive, which CTAN mirrors much of. +test -z "$tltrybase" \ +&& tltrybase=`cd $tlweb/../.. && pwd`/tlnet-trial-`date +%y%m%d` +tltry=$tltrybase/tlsrc.try +echo "$0: running from Master=$Master" +echo "$0: working in tltry=$tltry" +echo "$0: output to tlweb=$tlweb" + +# Save current tlpdb in case of disaster. +cp --force --backup $tlweb/tlpkg/texlive.tlpdb* /tmp + +# +# Be sure we're starting the test cleanly. +rm -rf $tltrybase +mkdir -p $tltry + +# Keep new directory group-writable to texlive so MacTeX maintainer +# can update it too. +chmod g+ws $tltry +chgrp texlive $tltry || : # in case of strange system + +# cp/link files in a loop so we can exclude the mactex files, which is a +# waste of time, and more importantly can uselessly fail due to permissions. +for f in $tlweb/*; do + if echo "$f" | grep mactex >/dev/null; then + : # skip mactex + elif test "x$cow_shell" = xcow-shell; then + cp -al "$f" $tltry # assume GNU cp so we can link instead of copy + # However, cow-shell is evidently introducing strange errors on the + # tug.org server we switched to in 2020, causing the release (tlnet) + # directory to be modified even when changes should be kept to the + # test (tltry) directory (ultimately resulting in checksum errors + # since the containers and tlpdb don't match, after a failed rebuild). + # This can be confirmed with a test hierarchy and build; see tltestnet. + # So, do not use cow-shell. + else + # no cow-shell, straight copy. + cp -a "$f" $tltry # still assuming GNU cp for -a + fi +done + +# Update packages in our working dir. +# These shell assignments have to come outside the cow-shell. +echo "$prg: Updating $tltry (from $Master) with $cow_shell..." +containers_prog="$Master/tlpkg/bin/tl-update-containers" +containers_args=" $verbose -location $tltry $critical $recreate $gpgcmd" +containers_invoke="$containers_prog $containers_args" +# +update_install_prog=$Master/tlpkg/bin/tl-update-install-pkg +update_install_invoke="$update_install_prog $gpgcmd -o $tltry" +# +cd $tltry +strace_file=$tltry/tlpkg/texlive.tlpdb.md5 +strace -o /tmp/s.log -f -P $strace_file $cow_shell <&2 + exit 1 +fi + +# We don't want to update the install packages if we're updating the +# "frozen" release (tlnet-final) after we've moved on to the pretest. +# See tlpkg/doc/releng.txt. +if $update_install_pkg; then + # It is scary, but I guess we should update the installer package every + # day, partly for the sake of doc.html and partly so it actually gets + # tested. Hopefully we don't break the Perl modules very often. + echo "$prg: Updating install pkg with" + echo "$prg: $update_install_invoke" + if $update_install_invoke; then :; else + echo "$prg: tl-update-install-pkg failed, goodbye." >&2 + exit 1 + fi +else + : +fi +END_COW + +# cow-shell leaves this around, haven't tracked down (or reported). +#rm -f $tltry/.ilist + +# if not doing the test installation, don't push anything out. +$testinstall || exit 0 + +# +# Now we have an updated tlweb in $tltry where only the changed files +# are actual files, the rest are hard links. +# Try to make a test installation. +cd $tltrybase +tltryinst=$tltrybase/tlinst.try + +zcat $tltry/install-tl-unx.tar.gz | tar -xf - +cd install-tl-* # subdir is YYYYMMDD + +# create test TL install profile. +# We set TEXMFVAR for the sake of luaotfload-tool. +echo "# texlive-profile from $0 `date` +selected_scheme scheme-$scheme +TEXDIR $tltryinst/$yyyy +TEXMFLOCAL $tltryinst/texmf-local +TEXMFSYSCONFIG $tltryinst/$yyyy/texmf-config +TEXMFSYSVAR $tltryinst/$yyyy/texmf-var +TEXMFVAR $tltryinst/$yyyy/texmf-uvar +TEXMFHOME ~/texmf +tlpdbopt_install_docfiles 1 +tlpdbopt_install_srcfiles 1 +tlpdbopt_create_formats 1 +instopt_letter 0 +instopt_adjustpath 0 +instopt_adjustrepo 1 +" >texlive.profile + +# silence envvar warnings and the welcome message. +TEXLIVE_INSTALL_ENV_NOCHECK=1; export TEXLIVE_INSTALL_ENV_NOCHECK +TEXLIVE_INSTALL_NO_WELCOME=1; export TEXLIVE_INSTALL_NO_WELCOME + +# Minimal PATH from here on! E.g., we don't want the test install to +# find its own mktexlsr, but only have the programs in the test +# environment available. +PATH=/usr/bin + +tlnet_install_log=`pwd`/update-tlnet-install.log +tlnet_install_cmd="install-tl -location $tltry -profile texlive.profile" +echo "$prg: Running test install (log: $tlnet_install_log)..." +echo "$prg: (in `pwd`)" +echo "$prg: $tlnet_install_cmd" +perl $tlnet_install_cmd >$tlnet_install_log 2>&1 \ +|| true # install-tl can fail, but we test the output, so don't abort. + +# the following long grep command should filter away all *normal* +# installation messages. +# if there are any other messages they will end up on stdout and +# thus be noticed. +unexpected_output=`cat $tlnet_install_log \ + | sed '/The following environment variables/,/^ ------/d' \ + | grep -Ev '^ ------' \ + | grep -Ev '^$' \ + | grep -Ev '^cryptographic signature of' \ + | grep -Ev '^xAutomated TeX Live installation using profile' \ + | grep -Ev '^Installing from:' \ + | grep -Ev '^Platform: ' \ + | grep -Ev '^Distribution: inst' \ + | grep -Ev '^Directory for temporary files' \ + | grep -Ev '^Loading ' \ + | grep -Ev '^Installing ' \ + | grep -Ev '^(re-)?running mktexlsr' \ + | grep -Ev '^mktexlsr: Updating ' \ + | grep -Ev '^mktexlsr: Done' \ + | grep -Ev '^writing fmtutil.cnf to' \ + | grep -Ev '^writing updmap.cfg to' \ + | grep -Ev '^writing language.(dat|def|dat.lua) to' \ + | grep -Ev '^pre-generating all format file' \ + | grep -Ev '^making ConTeXt MkIV cache' \ + | grep -Ev '^running ' \ + | grep -Ev '^done running ' \ + | grep -Ev '^finished ' \ + | grep -Ev '^Welcome to TeX Live' \ + | grep -Ev '^The TeX Live web site' \ + | grep -Ev '^contains updates' \ + | grep -Ev '^TeX Live is a joint project of the TeX user groups' \ + | grep -Ev '^please consider supporting it by joining the group' \ + | grep -Ev '^The list of groups is available' \ + | grep -Ev '^Add ' \ + | grep -Ev '^Most importantly, add ' \ + | grep -Ev '^to your PATH for current and future sessions' \ + | grep -Ev ' \(if not dynamically found\)' \ + | grep -Ev 'install-tl: done' \ + | grep -Ev '^Logfile: ' \ + | grep -Ev '^Time used for installing ' \ + | grep -Ev '^setting up ConTeXt cache' \ + | grep -Ev '^resolvers +\|' \ + | grep -Ev '^system +\|' \ + | grep -Ev '^mtxrun +\|' \ + | grep -Ev '^done$' \ + | cat` + +failure=false +ignore_unexpected_output=false #true, if frustration +if test -n "$unexpected_output"; then + $ignore_unexpected_output || failure=true + echo >&2 + echo "$prg: Test installation failed." >&2 + echo "$prg: Here is the unexpected output, from $tlnet_install_log:" >&2 + echo "$unexpected_output" >&2 + echo "$prg: (end of unexpected output)." >&2 +fi + +# more consistency checks. +if test $failure = false; then + for cmd in \ + "$Master/tlpkg/bin/tlgpg-verify $tltry/tlpkg/texlive.tlpdb.sha512" \ + "$Master/tlpkg/bin/tl-compare-tlpdbs $critical $tltry/tlpkg/texlive.tlpdb" \ + "$Master/tlpkg/bin/tl-check-symlinks $tltryinst/$yyyy/bin" \ + "$tltryinst/$yyyy/bin/*/tlmgr --repository $tltry update --list" \ + "$tltryinst/$yyyy/bin/*/updmap-sys -n" \ + "$tltryinst/$yyyy/bin/*/mktexlsr -n --verbose" \ + "$Master/tlpkg/bin/tl-check-tlnet-consistency --location=$tltry" \ + ; do + cmdname=`echo "$cmd" | awk '{print $1}'` + if echo "$cmdname" | grep check-tlnet-consistency >/dev/null; then + # skip consistency check if requested (for development only). + $check_consistency || continue + fi + basecmd=`basename $cmdname` + echo "$prg: Running $basecmd ($cmd)" + outfile=/tmp/tlnet.$basecmd + if $cmd >$outfile 2>&1; then :; else + echo "$prg: $basecmd failed ($cmd):" >&2 + sed 8q $outfile >&2 + echo "... see $outfile for full output ..." >&2 + echo >&2 + failure=true + fi + done +fi + +# Format creation check, in case fmtutil's exit status wasn't right. +# This might not exist if the test install didn't get this far. +install_tl_log=$tltryinst/$yyyy/install-tl.log + +if test -r $install_tl_log \ + && grep -i '^fmtutil.*error.*' $install_tl_log >/dev/null; then + echo >&2 + echo "$prg: seems fmtutil failed, check $install_tl_log." >&2 + failure=true +else + : # appease -e +fi + +# In all cases, make copies in /tmp for inspection in case of +# undetected failure. +cp -f $tlnet_install_log /tmp +test ! -r $install_tl_log || cp -f $install_tl_log /tmp + +if $failure || $chicken; then + echo >&2 + echo "$prg: tl-update-tlnet transcript file: $tlnet_install_log" >&2 + echo "$prg: install-tl log file: $install_tl_log" >&2 + echo "$prg: Copies of both are in /tmp." >&2 + echo "$prg: Please rm -rf the trial dir." >&2 + if $failure; then + exit 1 + elif $chicken; then + echo + echo "$prg: Chicken mode, not updating anything." + exit 0 + else + echo "$prg: impossible non-failure non-chicken." >&2 + exit 2 + fi +fi + +# +# no unexpected output, so ship the new packages. +cd $tltrybase +echo "$prg: Updating $tlweb from $tltry." +# copy any mactex files since we didn't link them. +for f in $tlweb/*mactex*; do + test ! -r $f || cp -pf $f $tltry +done + +# mv then rm to avoid the mirmon probe failing during the rm. +mv $tlweb $tltrybase/tlnet.old +mv $tltry $tlweb +rm -rf $tltrybase + +# We checked this above also, but check again. +# We've removed the cwd, so cd out of it. +cd /tmp || exit 1 +$Master/tlpkg/bin/tlgpg-verify $tlweb/tlpkg/texlive.tlpdb.sha512 + +echo "$0: Done." + +exit 0 diff --git a/Master/tlpkg/bin/tltestnet b/Master/tlpkg/bin/tltestnet index 75ab0544add..3a36fa5c90c 100755 --- a/Master/tlpkg/bin/tltestnet +++ b/Master/tlpkg/bin/tltestnet @@ -28,3 +28,7 @@ fi # run out of PATH, not the reference tree. tl-update-tlnet --dest $testnet --scheme $scheme $install_pkg $consistency + +# for when we were testing with cow-shell; we have given up on it nowadays. +# tl-update-testcow is in archive/. +#tl-update-testcow --cow --dest $testnet --scheme $scheme $install_pkg $consistency -- cgit v1.2.3