summaryrefslogtreecommitdiff
path: root/Master/tlpkg/bin/tl-check-fmtshare
blob: c692a6885b94ab3fa4db57d4618ae2d0e2570981 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
#!/bin/sh
# Public domain. Originally written by Karl Berry, 2021.
# Test that a .fmt file built on one system can be used on another.
# 
# Invoked from karl's cron.weekly:
#   tl-check-fmtshare m68:tl/Work/texk/web2c
# where m68 is a host alias for one of Nelson's machines,
# a 32-bit BigEndian m68k architecture.

version='$Id$'
renice 20 $$ >/dev/null 2>&1
unset CDPATH
LC_ALL=C; export LC_ALL
umask 077 # since we write in a tmp dir (unsafe still, but oh well)

fmtutil_path=`cd /tmp && which fmtutil-sys 2>/dev/null`
if test -z "$fmtutil_path"; then
  echo "$0: no fmtutil-sys in PATH, goodbye." >&2
  exit 1
fi
bindir=`dirname "$fmtutil_path"`
mydir=`cd \`dirname $0\` && pwd`	# Master/tlpkg/bin
#echo "fmtutil_path=$fmtutil_path, bindir=$bindir, mydir=$mydir"

Master=`cd "$bindir"/../.. && pwd`
enginedir=`cd "$mydir"/../../../Build/source/Work/texk/web2c 2>/dev/null&& pwd`
outdir=/tmp/fmtshare.`id -u`
default_fmts="tex.fmt pdftex.fmt pdflatex.fmt xetex.fmt xelatex.fmt
		     luatex.fmt euptex.fmt" # not lualatex or optex, see below
fmts=
remote=

usage="Usage: $0 [OPTION]... RHOST:RDIR
Build TeX .fmt files locally, then copy them to RHOST:RDIR and try to
load them with the TeX engines there. That is, RDIR should be a web2c
build directory with binaries; in TeX Live,
/something/like/Build/source/Work/texk/web2c.

The idea is to support testing for .fmts being sharable among different
system architectures, as intended; for instance, 4-byte long vs. 8-byte
long and BigEndian vs. LittleEndian.

The further idea is to be able to test new binaries in the Build
directory (since that's where the compiled sources change), while using
support files from a Master directory (since that's where they are
updated).

fmtutil-sys is run locally (found using PATH) to discern the engine and
full command line used for building the given .fmts.

The remote execution is done with \"ssh -n RHOST && cd RDIR && ...\",
so ssh has to be working.

The default for the Build directory is to be relative to where this
script is executed. The default for the Master directory is to be
relative to where fmtutil-sys is found in PATH. These may or may not be
the same.

The output directory is not cleaned before starting, in case older logs
are helpful in debugging new failures.

Special case #1: unfortunately, lualatex.fmt cannot be easily tested,
and it is omitted by default. Loading the fmt requires finding many Lua
files, which are unlikely to be available on the remote machine. The
script does not try to handle this.

Special case #2: optex.fmt includes precompiled Lua code, which is not
sharable across architectures. Therefore the .fmt isn't either. If
users need this at some future time, the optex maintainer will consider.

Options (must use --, must use space to separate option from value, sorry):
--Master MDIR     use MDIR for local support files
                    [$Master]
--enginedir EDIR  use EDIR for local engine binaries
                    [$enginedir]
--outdir ODIR     build locally in ODIR [$outdir]
--fmt FMT         build FMT; can be given more than once
                    [$default_fmts]

--help            display this help and exit
--version         display version information and exit

Doc: https://tug.org/texinfohtml/web2c.html#Hardware-and-memory-dumps
Bug reports, discussion: tex-live@tug.org
Version: $version"

while test $# -gt 0; do
 case $1 in
  --Master)    shift; Master=$1;;
  --enginedir) shift; enginedir=$1;;
  --fmt)       shift; fmts="$fmts $1";;
  --outdir)    shift; outdir=$1;;
  #
  --help)      echo "$usage"; exit 0;;
  --version)   echo "$version"; exit 0;;
  -*)          echo "$0: goodbye, unrecognized option: $1" >&2; exit 1;;
  #
  *)           if test -n "$remote"; then
                 echo "$0: remote argument already seen: $remote" >&2
                 echo "$0:   now have second: $1" >&2
                 exit 1
               fi
               remote=$1;;
 esac
 shift
done

if test -z "$remote"; then
  echo "$0: no remote argument given, goodbye." >&2
  exit 1
elif echo "$remote" | grep -v : >/dev/null; then
  echo "$0: remote argument must have form HOST:DIR, not like: $remote" >&2
  exit 1
elif test `echo "$remote" | tr -cd : | wc -c` != 1 >/dev/null; then
  # not worth the fancier parsing until we need it.
  echo "$0: remote argument must have only one colon: $remote" >&2
  exit 1
fi
remotesys=`echo "$remote" | sed 's/:.*//'`
remotedir=`echo "$remote" | sed 's/.*://'`
#
if test ! -d "$Master/texmf-dist/web2c"; then
  echo "$0: no texmf-dist/web2c subdir of Master: $Master" >&2
  ls "$Master" >&2
  exit 1
fi
#
if test ! -d "$enginedir"; then
  echo "$0: enginedir not a directory: $enginedir" >&2
  ls "$enginedir" >&2
  exit 1
fi

# create output directory if needed.
test -d "$outdir" || mkdir "$outdir" || exit 1

# default fmt list.
test -z "$fmts" && fmts=$default_fmts

prg=`basename $0`
echo "$prg: PATH=$PATH"
echo "$prg: support files directory: $Master"
echo "$prg:  local engine directory: $enginedir"
echo "$prg:  local output directory: $outdir"
echo "$prg:        formats to build:"$fmts
echo "$prg:                  remote: $remotesys:$remotedir"

# 
# Our function to create a format locally:
#   mkfmt FMT ENGINEDIR MASTERDIR OUTDIR
# Generates FMT using the binary from ENGINEDIR,
# support files from MASTERDIR, and leaving FMT in OUTDIR.
# 
# Leaves current directory as OUTDIR.
#
# On success, returns zero and outputs two lines to stdout:
# 1) the full path to the newly-generated fmt file; and
# 2) the basename of the engine binary that was used to build it.
# 
# On failure, returns nonzero and outputs nothing to stdout; issues
# diagnostics to stderr.
# 
mkfmt ()
{
  fmt=$1
  enginedir=$2
  Master=$3
  outdir=$4

  # Our goal is to build a .fmt using a binary from the Build tree
  # while still using the support files from the Master tree.
  # The idea being that after changing the source and rebuilding the
  # binary, we want to easily test whether the new binary is ok.
  #
  # First, we run fmtutil-sys (assumed to be in PATH) without generating
  # anything (--dry-run), to garner the command line needed to build # FMT.
  # 
  fcmd="fmtutil-sys --dry-run --no-engine-subdir --fmtdir $outdir --byfmt $fmt"
  ffot=$outdir/`basename $fmt .fmt`-fmtutil.fot
  #
  echo "$0: running: $fcmd" >"$ffot"
  if $fcmd >>"$ffot" 2>&1; then :; else
    echo "$0: could not get cmdline to build $fmt" >&2
    echo "$0:  fmtutil command failed: $fcmd" >&2
    echo "$0:  see transcript: $ffot" >&2
    return 1
  fi
  
  # Extract the build command from the fmtutil output, which has a line like:
  #   fmtutil: running `pdftex -ini [more options] *pdfetex.ini' ...
  # We want what's between the quotes. Format names and options can't
  # contain quote or other special characters.
  lq='`'
  rq="'"
  ecmd=`sed -n "s/^fmtutil.*running $lq\(.*\)$rq.*/\1/p" "$ffot"`
  if test -z "$ecmd"; then
    echo "$0: could not extract cmdline to build $fmt" >&2
    echo "$0:  from fmtutil output; see: $ffot" >&2
    return 1
  fi
  
  # the needed engine better be in enginedir.
  engine=`echo "$ecmd" | awk '{print $1}'` # engine that was used
  if test ! -s "$enginedir/$engine"; then
    echo "$0: needed engine not in enginedir: $enginedir/$engine" >&2
    return 1
  fi

  # Generate everything in OUTDIR.
  cd "$outdir" || return 1
  
  # Set environment variables so the given Master tree is used, and
  # prepend the given engine directory to PATH, and run the command.
  env="env PATH=$enginedir:$PATH \
         TEXMFCNF=$Master/texmf-dist/web2c \
         TEXMFROOT=$Master"
  efot=$outdir/`basename $fmt .fmt`-engine.fot
  echo "$0: running: $ecmd" >"$efot"
  if $env $ecmd >>"$efot" 2>&1 && test -s "$outdir/$fmt"; then
    sed -n '2p;q' "$efot"
    echo "$outdir/$fmt"
    echo "$engine"
  else
    echo "$0: could not build $fmt" >&2
    echo "$0: engine command failed: $ecmd" >&2
    echo "$0: see transcript: $efot" >&2
    return 1
  fi
}

#  Loop over all given fmts.
#
for fmt in $fmts; do
  if echo "$fmt" | grep '\.fmt$' >/dev/null; then :; else
    echo "$0: fmt does not end with .fmt, goodbye: $fmt" >&2
    exit 1
  fi
  
  # first build fmt on local machine.
  set - `mkfmt "$fmt" "$enginedir" "$Master" "$outdir"`
  fmtfile=$1
  engine=$2
  #
  if test -z "$fmtfile"; then
    # error messages already given, but give another just to be sure.
    echo "$0: mkfmt failed: $fmt " \
              "(enginedir $enginedir) (Master $Master) (outdir $outdir)" >&2  
    exit 1
  elif test -z "$engine"; then
    echo "$0: should not happen, mkfmt returned fmtfile: $fmtfile" >&2
    echo "$0: but no engine value?!" >&2
    exit 1
  elif test ! -s "$fmtfile"; then
    echo "$0: should not happen, returned fmtfile is empty: $fmtfile" >&2
    exit 1
  fi
  #echo "$0: (`date`)"
  echo "$0: built fmtfile: `ls -l $fmtfile` (on `hostname`)"
  engineversion=`$enginedir/$engine --version | sed 1q`
  echo "$0:   with engine: $enginedir/$engine ($engineversion)"

  # The TeX \command to exit a job immediately. Assume a LaTeX fmt
  # if "latex" is in the name, else plain (enough).
  if echo "$fmt" | grep latex >/dev/null; then
    endcmd='\stop'
  else
    endcmd='\end'
  fi

  # copy to remote machine.
  #echo "$0: copying $fmtfile to $remotesys:$remotedir..."
  scp -pq "$fmtfile" "$remotesys:$remotedir" || exit 1

  # load on remote machine.
  echo "$0: running ./$engine in $remotesys:$remotedir..."
  #
  # On the remote side, we need to find a texmf.cnf or pdftex gets the
  # mysterious "Must increase the hyph_size"; assume the
  # TL source directory structure to find kpathsea/texmf.cnf.
  # 
  # Do not try to dynamically create the .fmt.
  # 
  remoteenv="env MKTEXFMT=0 TEXMFCNF=../../../texk/kpathsea"
  remotecmd="cd $remotedir && $remoteenv ./$engine -fmt=./$fmt '$endcmd'"
  #
  rfot=$outdir/`basename $fmt .fmt`-$remotesys.fot
  echo "$0: running on $remotesys: $remotecmd" >$rfot
  ssh -n $remotesys "$remotecmd" </dev/null >>$rfot 2>&1
  if test $? -ne 0; then
    echo "$0: *** fmt load failed on $remotesys: $fmt" >&2
    echo "$0: *** transcript in: $rfot" >&2
    cat $rfot >&2
    exit 1
  else
    echo "$0: fmt load ok on $remotesys: $fmt"
    echo
  fi
done

exit 0