#!/bin/sh # Public domain. Originally written 2008, Karl Berry. # Update a TeX Live Master/bin/PLATFORM directory. # Requires jq (for github downloads). # # For info about kicking off the github autobuilds, # see x86_64-linux below. vc_id='$Id$' renice 20 $$ >/dev/null 2>&1 unset CDPATH unset LS_COLORS tmpdir=${TMPDIR-/tmp}/tlupbin.$$ trap "rm -rf $tmpdir" 0 1 2 15 test -d $tmpdir || mkdir -p $tmpdir usage="$0 [OPTION]... TLPLATFORM... Update the TeX Live executables for each TLPLATFORM (which must be a TL platform name, e.g., x86_64-linux) from a build directory. The new binaries are taken from the location specified by --bin-loc, either a directory (whose contents are copied), or a tar file (which is unpacked and then copied). The output directory is computed relative to the location of this script by default, or can be explicitly specified via --master. An upper-level directory is specified so multiple platforms can be installed. This checks for broken symlinks, and symlinks with the svn:executable property set (which would mess up Subversion; see the thread around http://svn.haxx.se/users/archive-2007-03/1259.shtml.) It also takes care of doing the explicit remove/add sequence Subversion requires when a symlink is replaced by a file or vice versa. Options: --bin-loc DIR-OR-TAR use binaries from DIR-OR-TAR --cg-branch SUBPATH url component for dl.contextgarden.net [default empty; example: branches/branch2023/] --master DIR install binaries to DIR/bin/TLPLATFORM [default is the bin dir relative to this script] --no-download, -N assume already downloaded, e.g., if this script fails --help display this help and exit --quiet, --silent no progress reports --version output version information and exit For more information about building TeX Live, see http://tug.org/texlive/build.html. Report bugs to tlbuild@tug.org." # parse options. bin_loc= cgbranch= download=wget msg=echo tlplats= while test $# -gt 0; do case $1 in --help|-help|-v) echo "$usage"; exit 0;; --version|-version|-v) echo "$vc_id"; exit 0;; --quiet|-quiet|-q|--silent|-silent|-s) msg=true;; --bin-loc) shift; bin_loc=$1;; --cg-branch) shift; cgbranch=$1;; # e.g., branches/branch2023/ --master) shift; Master=$1;; --no-download|-N) download=true;; --*) echo "$0: unrecognized option \`$1'; try --help if you need it." >&2 exit 1;; *) tlplats="$tlplats $1";; esac shift done if test -z "$tlplats"; then echo "$0: missing TL platform name; try --help if you need it." >&2 exit 1 fi if test -z "$Master"; then mydir=`dirname $0` Master=`cd $mydir/../.. && pwd` fi if test ! -d "$Master/bin"; then echo "$0: Master directory $Master has no bin/ subdir; goodbye." >&2 exit 1 fi cgurl=https://build.contextgarden.net/dl/texlive/ # base url # # We could make this an option but it's better to always mark releases # as "prerelease" on github, so users don't think they can just download # them and have a working system. prerelease=true # # function to return url for the "latest" build on gh; # first argument is TL platform name, # second (optional) argument is github platform name, if different. # github_url () { if test -z "$1"; then echo "$0:github_url: expected platform name (and optional github name)," \ " goodbye." >&2 exit 1 fi tlplat=$1 ghplat=${2-$tlplat} gh_url=https://api.github.com/repos/TeX-Live/texlive-source/releases if $prerelease; then remurl=`curl -s $gh_url \ | jq -r 'map(select(.prerelease)) | first | .assets | .[] | .browser_download_url' \ | grep texlive-bin-$ghplat.tar.gz` else gh_url=$gh_url/latest remurl=`curl -s $gh_url \ | jq -r ' .assets.[].browser_download_url' \ | grep texlive-bin-$ghplat.tar.gz` fi echo "github_url: for $tlplat, got $remurl (prerelease=$prerelease)" >&2 echo "$remurl" } # function to download a url passed as arg, or exit. # do_download () { if test -z "$1"; then echo "$0:do_download: no url given, goodbye." >&2 exit 1 fi $grab "$1" || exit $? } # # loop through tl platform names for tlplat in $tlplats; do destdir=$Master/bin/$tlplat if test ! -d $destdir; then destdir=`echo $destdir | sed 's/-gh$//'` if test ! -d $destdir; then echo "$0: unknown platform name \`$tlplat'" >&2 echo "$0: (no directory $destdir)" >&2 exit 1 fi fi # updating convenience download_loc=$tmpdir/tl.$tlplat.tar.gz default_bin_loc=$download_loc grab="$download -O $download_loc" # case $tlplat in aarch64-linux) # cg build is much much faster than gh, so use it while it works. do_download ${cgurl}/${cgbranch}$tlplat.tar.xz;; aarch64-linux-gh) # but sometimes we end up with the gh build. tlplat=`echo $tlplat | sed s/-gh//` do_download `github_url $tlplat`;; amd64-freebsd) do_download `github_url $tlplat`;; amd64-netbsd) download_loc=$tmpdir/tl.$tlplat.tar.xz default_bin_loc=$download_loc do_download http://www.babafou.eu.org/texlive-netbsd/x86_64-netbsd.tar.xz;; armhf-linux) # as with aarch. do_download ${cgurl}/${cgbranch}$tlplat.tar.xz;; armhf-linux-gh) # as with aarch. tlplat=`echo $tlplat | sed s/-gh//` do_download `github_url $tlplat`;; i386-freebsd) do_download `github_url $tlplat`;; i386-linux) do_download `github_url $tlplat`;; i386-netbsd) download_loc=$tmpdir/tl.$tlplat.tar.xz default_bin_loc=$download_loc grab="$download -O $download_loc" $grab http://www.babafou.eu.org/texlive-netbsd/i386-netbsd.tar.xz;; i386-openbsd) default_bin_loc=$download_loc $grab http://students.dec.bmth.ac.uk/ebarrett/files/tl-bin-20080810.tgz;; i386-solaris) do_download `github_url $tlplat`;; universal-darwin) default_bin_loc=/home/koch/$tlplat.tar.xz;; windows) echo "committed by builder";; x86_64-cygwin) echo "committed by builder";; x86_64-darwinlegacy) do_download ${cgurl}/${cgbranch}$tlplat.tar.xz;; x86_64-linux) # Visit https://github.com/TeX-Live/texlive-source/commits # to check for commits. # Then https://github.com/TeX-Live/texlive-source/releases # "Draft a new release". # "Choose a tag -> "find or create a new tag" -> "svnNNNNN" # "Target:trunk" should already be there # (or set Target to, e.g., "branch2024", if building from branch), # "rNNNNN and perhaps brief msg from commits in "title", # and anything relevant for "description", or leave it blank; # check pre-release box (always), # then "Publish release". # # Can check status at: # https://github.com/TeX-Live/texlive-source/actions # # After ~20min, should have some new release tarballs at: # https://github.com/TeX-Live/texlive-source/releases # (this is a different page than where you're put after creating # the release) # but arm and aarch are slow and will take hours to finish. # Should be able to update the finished ones as they happen. # # If try before ready, no harm, just get error message "missing url". # do_download `github_url $tlplat`;; x86_64-linuxmusl) do_download `github_url $tlplat`;; x86_64-solaris) do_download `github_url $tlplat amd64-solaris`;; *) echo "$0: unknown platform in case: $tlplat" >&2; exit 1;; esac # test -z "$bin_loc" && test -n "$default_bin_loc" \ && bin_loc=$default_bin_loc if test -z "$bin_loc"; then echo "$0: missing binary location, try --help if you need it." >&2 exit 1 fi $msg "installing from $bin_loc to $destdir via $tmpdir" ls -ld "$bin_loc" # if we were given a tar file, unpack it. if test -f "$bin_loc"; then srcdir=$tmpdir/unpacked mkdir $srcdir if echo "$bin_loc" | grep 'zip$' >/dev/null; then (cd $srcdir && unzip -q $bin_loc) || exit 1 else (cd $srcdir && tar xf $bin_loc) || exit 1 fi elif test -d "$bin_loc"; then srcdir=$bin_loc # already have a directory else echo "$0: strange non-file non-directory binary location $bin_loc" >&2 exit 1 fi # in case people mistakenly include .svn dirs in their tars. find "$srcdir" -name .svn | xargs rm -rf # ditto, biber. find "$srcdir" -name biber\* | xargs rm -f # temp cleanup for first pretest24 find "$srcdir" -name htcontext\* | xargs rm -f find "$srcdir" -name memoize-clean | xargs rm -f find "$srcdir" -name memoize-extract | xargs rm -f # may need to cd into a subdirectory, depending on how the tar was made. while test `ls $srcdir | wc -l` -eq 1; do srcdir=$srcdir/* done # destdir is what is in the repo now, srcdir has the new binaries. (cd $destdir && ls) >$tmpdir/now (cd $srcdir && ls) >$tmpdir/new ourdel=$tmpdir/2del ouradd=$tmpdir/2add # looking for deletions, but don't delete (x)asy, biber, context, xindy. comm -23 $tmpdir/now $tmpdir/new \ | egrep -v '^(x?asy(\.exe)?|freeglut\.dll)$' \ | egrep -v '^biber(-ms)?(\.exe)?$' \ | egrep -v '^(luametatex|context(.lua))?|mtxrun$' \ | egrep -v 'xindy' \ >$ourdel # intentionally anything matching xindy comm -13 $tmpdir/now $tmpdir/new >$ouradd # looking for additions # get symlink list. (cd $destdir && find . -type l | sort) >$tmpdir/now.symlink (cd $srcdir && find . -type l | sort) >$tmpdir/new.symlink cd $destdir || exit 1 # svn requires separate delete/add operations when symlinks change to # regular files or vice versa. # # remove symlinks which have become files. comm -23 $tmpdir/now.symlink $tmpdir/new.symlink >$tmpdir/s.now replaced_symlinks= for sl in `cat $tmpdir/s.now`; do test -f $srcdir/$sl && replaced_symlinks="$replaced_symlinks $sl" done test -n "$replaced_symlinks" \ && $msg "removing symlinks which have become files..." \ && svn rm $replaced_symlinks # # remove files which have become symlinks. comm -13 $tmpdir/now.symlink $tmpdir/new.symlink >$tmpdir/s.new replaced_files= for sl in `cat $tmpdir/s.new`; do test -f $destdir/$sl && replaced_files="$replaced_files $sl" done test -n "$replaced_files" \ && $msg "removing files which have become symlinks..." \ && svn rm $replaced_files # the bulk copy. $msg "copying from $srcdir" $msg "to $destdir" (cd $srcdir && tar cf - *) | tar xf - # the normal deletions and additions. $msg "removing old..." test -s $ourdel && svn rm `cat $ourdel` $msg "adding new..." test -s $ouradd && svn add `cat $ouradd` # anything which is no longer a symlink but still exists # needs to be added. test -n "$replaced_symlinks" \ && $msg "adding files that replaced symlinks..." \ && svn add $replaced_symlinks # anything which is now a symlink but didn't used to be # also needs to be added. test -n "$replaced_files" \ && $msg "adding symlinks that replaced files..." \ && svn add $replaced_files # be sure the svn:executable property is not set on any symlink. # there is also a pre-commit hook on the repo, but of course we don't # want to unnecessarily trigger it. # We redirect stderr to /dev/null because as of svn 1.9.3 (or earlier?), # svn gives a useless warning when the property is not set. badlinks=`svn propget svn:executable \`cat $tmpdir/new.symlink\` 2>/dev/null\ | awk '{print $1}'` if test -n "$badlinks"; then $msg "removing svn:executable property from symlinks..." svn propdel svn:executable $badlinks fi # revert xindy.mem (always changes) unless xindy.run changes. if svn status xindy.run | grep '^M' >/dev/null \ || svn status xindy-lisp.exe | grep '^M' >/dev/null; then : elif test -r xindy.run || test -r xindy-lisp.exe; then $msg "reverting xindy.mem..." svn revert xindy*.mem fi # check for broken symlinks. for sl in `cat $tmpdir/new.symlink`; do test ! -r "$sl" && echo "$0: broken new symlink $sl" >&2 done # final results. $msg "final svn status..." svn status | sort echo rm -rf $tmpdir done