From 78591567fc12bcecdf39f8d632d39c12568401ea Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Sun, 18 Jul 2021 17:16:29 +0000 Subject: checking fmt sharability across architectures git-svn-id: svn://tug.org/texlive/trunk@59973 c570f23f-e606-0410-a88d-b1316a301751 --- Master/tlpkg/bin/tl-check-fmtshare | 268 +++++++++++++++++++++++++++++++++++++ Master/tlpkg/doc/releng.txt | 23 +++- 2 files changed, 284 insertions(+), 7 deletions(-) create mode 100755 Master/tlpkg/bin/tl-check-fmtshare (limited to 'Master/tlpkg') diff --git a/Master/tlpkg/bin/tl-check-fmtshare b/Master/tlpkg/bin/tl-check-fmtshare new file mode 100755 index 00000000000..08969ff6a0f --- /dev/null +++ b/Master/tlpkg/bin/tl-check-fmtshare @@ -0,0 +1,268 @@ +#!/bin/sh +# Public domain. Originally written by Karl Berry, 2021. +# Test that a .fmt file built on one system can be used on another. + +version='$Id$' +renice 20 $$ >/dev/null 2>&1 +unset CDPATH +LC_ALL=C; export LC_ALL +umask 077 # since we write in a tmp dir (unsafe still, but oh well) + +fmtutil_path=`cd /tmp && which fmtutil-sys 2>/dev/null` +if test -z "$fmtutil_path"; then + echo "$0: no fmtutil-sys in PATH, goodbye." >&2 + exit 1 +fi +bindir=`dirname "$fmtutil_path"` +mydir=`cd \`dirname $0\` && pwd` # Master/tlpkg/bin +#echo "fmtutil_path=$fmtutil_path, bindir=$bindir, mydir=$mydir" + +Master=`cd "$bindir"/../.. && pwd` +enginedir=`cd "$mydir"/../../../Build/source/Work/texk/web2c 2>/dev/null&& pwd` +outdir=/tmp/fmtshare.`id -u` +default_fmts="tex.fmt pdftex.fmt pdflatex.fmt xetex.fmt xelatex.fmt + luatex.fmt optex.fmt euptex.fmt" # not lualatex, see below +fmts= +remote= + +usage="Usage: $0 [OPTION]... RHOST:RDIR +Build TeX .fmt files locally, then copy them to RHOST:RDIR and try to +load them with the TeX engines there. + +The idea is to support testing for .fmts being sharable among different +system architectures, as intended; for instance, 4-byte long vs. 8-byte +long and BigEndian vs. LittleEndian. + +The further idea is to be able to test new binaries in the Build +directory (since that's where the compiled sources change), while using +support files from a Master directory (since that's where they are +updated). + +fmtutil-sys is run locally (found using PATH) to discern the engine and +full command line used for building the given .fmts. + +The remote execution is done with \"ssh -n RHOST && cd RDIR && ...\", +so ssh has to be working. + +The default for the Build directory is to be relative to where this +script is executed. The default for the Master directory is to be +relative to where fmtutil-sys is found in PATH. These may or may not be +the same. + +The output directory is not cleaned before starting, in case older logs +are helpful in debugging new failures. + +Special case: unfortunately, lualatex.fmt cannot be easily tested, and +it is omitted by default. Loading the fmt requires finding many Lua +files, which are unlikely to be available on the remote machine. The +script does not try to handle this. + +Options (must use --, must use space to separate option from value, sorry): +--Master MDIR use MDIR for local support files + [$Master] +--enginedir EDIR use EDIR for local engine binaries + [$enginedir] +--outdir ODIR build locally in ODIR [$outdir] +--fmt FMT build FMT; can be given more than once + [$default_fmts] + +--help display this help and exit +--version display version information and exit + +Doc: https://tug.org/texinfohtml/web2c.html#Hardware-and-memory-dumps +Bug reports, discussion: tex-live@tug.org +Version: $version" + +while test $# -gt 0; do + case $1 in + --Master) shift; Master=$1;; + --enginedir) shift; enginedir=$1;; + --fmt) shift; fmts="$fmts $1";; + --outdir) shift; outdir=$1;; + # + --help) echo "$usage"; exit 0;; + --version) echo "$version"; exit 0;; + -*) echo "$0: goodbye, unrecognized option: $1" >&2; exit 1;; + # + *) if test -n "$remote"; then + echo "$0: remote argument already seen: $remote" >&2 + echo "$0: now have second: $1" >&2 + exit 1 + fi + remote=$1;; + esac + shift +done + +if test -z "$remote"; then + echo "$0: no remote argument given, goodbye." >&2 + exit 1 +elif echo "$remote" | grep -v : >/dev/null; then + echo "$0: remote argument must have form HOST:DIR, not like: $remote" >&2 + exit 1 +elif test `echo "$remote" | tr -cd : | wc -c` != 1 >/dev/null; then + # not worth the fancier parsing until we need it. + echo "$0: remote argument must have only one colon: $remote" >&2 + exit 1 +fi +remotesys=`echo "$remote" | sed 's/:.*//'` +remotedir=`echo "$remote" | sed 's/.*://'` +# +if test ! -d "$Master/texmf-dist/web2c"; then + echo "$0: no texmf-dist/web2c subdir of Master: $Master" >&2 + ls "$Master" >&2 + exit 1 +fi +# +if test ! -d "$enginedir"; then + echo "$0: enginedir not a directory: $enginedir" >&2 + ls "$enginedir" >&2 + exit 1 +fi + +# create output directory if needed. +test -d "$outdir" || mkdir "$outdir" || exit 1 + +# default fmt list. +test -z "$fmts" && fmts=$default_fmts + +# Our function to create a format locally: +# mkfmt FMT ENGINEDIR MASTERDIR OUTDIR +# Generates FMT using the binary from ENGINEDIR, +# support files from MASTERDIR, and leaving FMT in OUTDIR. +# +# Leaves current directory as OUTDIR. +# +# On success, returns zero and outputs two lines to stdout: +# 1) the full path to the newly-generated fmt file; and +# 2) the basename of the engine binary that was used to build it. +# +# On failure, returns nonzero and outputs nothing to stdout; issues +# diagnostics to stderr. +# +mkfmt () +{ + fmt=$1 + enginedir=$2 + Master=$3 + outdir=$4 + + # Our goal is to build a .fmt using a binary from the Build tree + # while still using the support files from the Master tree. + # The idea being that after changing the source and rebuilding the + # binary, we want to easily test whether the new binary is ok. + # + # First, we run fmtutil-sys (assumed to be in PATH) without generating + # anything (--dry-run), to garner the command line needed to build # FMT. + # + fcmd="fmtutil-sys --dry-run --no-engine-subdir --fmtdir $outdir --byfmt $fmt" + ffot=$outdir/`basename $fmt .fmt`-fmtutil.fot + # + echo "$0: running: $fcmd" >"$ffot" + if $fcmd >>"$ffot" 2>&1; then :; + echo "$0: could not get cmdline to build $fmt" >&2 + echo "$0: fmtutil command failed: $fcmd" >&2 + echo "$0: see transcript: $ffot" >&2 + return 1 + fi + + # Extract the build command from the fmtutil output, which has a line like: + # fmtutil: running `pdftex -ini [more options] *pdfetex.ini' ... + # We want what's between the quotes. Format names and options can't + # contain quote or other special characters. + lq='`' + rq="'" + ecmd=`sed -n "s/^fmtutil.*running $lq\(.*\)$rq.*/\1/p" "$ffot"` + if test -z "$ecmd"; then + echo "$0: could not extract cmdline to build $fmt" >&2 + echo "$0: from fmtutil output; see: $ffot" >&2 + return 1 + fi + + # Generate everything in OUTDIR. + cd "$outdir" || return 1 + + # Set environment variables so the given Master tree is used, and + # prepend the given engine directory to PATH, and run the command. + env="env PATH=$enginedir:$PATH \ + TEXMFCNF=$Master/texmf-dist/web2c \ + TEXMFROOT=$Master" + efot=$outdir/`basename $fmt .fmt`-engine.fot + echo "$0: running: $ecmd" >"$efot" + if $env $ecmd >>"$efot" 2>&1 && test -s "$outdir/$fmt"; then + sed -n '2p;q' "$efot" + echo "$outdir/$fmt" + echo "$ecmd" | awk '{print $1}' # engine that was used + else + echo "$0: could not build $fmt" >&2 + echo "$0: engine command failed: $ecmd" >&2 + echo "$0: see transcript: $efot" >&2 + return 1 + fi +} + +# Loop over all given fmts. +# +for fmt in $fmts; do + if echo "$fmt" | grep '\.fmt$' >/dev/null; then :; else + echo "$0: fmt does not end with .fmt, goodbye: $fmt" >&2 + exit 1 + fi + + # first build fmt on local machine. + set - `mkfmt "$fmt" "$enginedir" "$Master" "$outdir"` + fmtfile=$1 + engine=$2 + # + if test -z "$fmtfile"; then + # error messages already given, but give another just to be sure. + echo "$0: mkfmt failed: $fmt " \ + "(enginedir $enginedir) (Master $Master) (outdir $outdir)" >&2 + exit 1 + elif test -z "$engine"; then + echo "$0: should not happen, mkfmt returned fmtfile: $fmtfile" >&2 + echo "$0: but no engine value?!" >&2 + exit 1 + elif test ! -s "$fmtfile"; then + echo "$0: should not happen, returned fmtfile is empty: $fmtfile" >&2 + exit 1 + fi + echo "$0: (`date`) built fmtfile: `ls -l $fmtfile`" + + # The TeX \command to exit a job immediately. Assume a LaTeX fmt + # if "latex" is in the name, else plain (enough). + if echo "$fmt" | grep latex >/dev/null; then + endcmd='\stop' + else + endcmd='\end' + fi + + # copy to remote machine. + echo "$0: copying $fmtfile to $remotesys:$remotedir..." + scp -pq "$fmtfile" "$remotesys:$remotedir" || exit 1 + + # load on remote machine. + echo "$0: running $engine in $remotesys:$remotedir..." + # + # On the remote side, we need to find a texmf.cnf or pdftex gets the + # mysterious "Must increase the hyph_size"; assume the + # TL source directory structure to find kpathsea/texmf.cnf. + # + # Do not try to dynamically create the .fmt. + # + remoteenv="env MKTEXFMT=0 TEXMFCNF=../../../texk/kpathsea" + remotecmd="cd $remotedir && $remoteenv ./$engine -fmt=./$fmt '$endcmd'" + # + rfot=$outdir/`basename $fmt .fmt`-$remotesys.fot + echo "$0: running on $remotesys: $remotecmd" >$rfot + ssh -n $remotesys "$remotecmd" >$rfot 2>&1 + if test $? -ne 0; then + echo "$0: fmt load failed on $remotesys: $fmt" >&2 + echo "$0: see transcript: $rfot" >&2 + exit 1 + else + echo "$0: fmt load ok on $remotesys: $fmt" + fi +done + +exit 0 diff --git a/Master/tlpkg/doc/releng.txt b/Master/tlpkg/doc/releng.txt index 563ed61953c..a567d28d19b 100644 --- a/Master/tlpkg/doc/releng.txt +++ b/Master/tlpkg/doc/releng.txt @@ -273,11 +273,20 @@ M. As pretest proceeds, update the doc: tl-update-txi Don't forget to commit everything (texmf-dist/doc, Build/source, ...). -N. Really test during pretest (really! don't leave it for Knuth to do!): - .fmt's sharable across endian-different platforms (including luatex). - interactive mf '\drawdot (50,50); showit;' % and from * prompt - -O. Two days before the final pretest (i.e., to-be-published) build: +N. Really test during and at end of pretest that + (really! don't leave it for post-release failure) + .fmt's sharable are across endian/word-different platforms, + including luatex: + tlpkg/bin/tl-check-fmtshare RHOST:RDIR + where RHOST is a host you can ssh to with different specifications + than the local host, and RDIR is a TL:Build/source/Work/texk/web2c/ + build directory. (Must build TL there first before running, naturally.) + Nelson Beebe has a wide variety of hosts available. + +O. Test that basic interactive Metafont is still working: + mf '\drawdot (50,50); showit;' % and from * prompt + +W. Two days before the final pretest (i.e., to-be-published) build: Put date into NEWS and ChangeLog files above. Update Build/source/tardate.ac && reautoconf. tlmgr version number in texmf-dist/scripts/texlive/NEWS. @@ -291,13 +300,13 @@ O. Two days before the final pretest (i.e., to-be-published) build: Tell developers no more commits: tldev + hirata, hironori, luigi, mojca, takuji, -P. The day before the final pretest: +X. The day before the final pretest: In cron.tl, catalogue_compare=false to avoid skew. In tl-update-tlnet, option adjustrepo 1, else install-tl-unx won't go to ctan Update Master/source (if not earlier): xz wget w32tex-src texlive*source mactexdoc install-tl-unx.tar.gz -Q. The day of the final tlnet-from-branch build, which may or may not +Y. The day of the final tlnet-from-branch build, which may or may not also be the day of the final tlpretest-from-trunk build: Set frozen=>1 in branch TLConfig.pm. Freeze of tlnet should happen at that night's rebuild. -- cgit v1.2.3