diff options
author | Karl Berry <karl@freefriends.org> | 2024-01-03 21:18:51 +0000 |
---|---|---|
committer | Karl Berry <karl@freefriends.org> | 2024-01-03 21:18:51 +0000 |
commit | 1e7f7766f2fd93956c4b9bbb7c91ea1f682cef47 (patch) | |
tree | 547c621d1f24f92f9f003cbfe9e4508d44111bba /Build/source | |
parent | 9fe5b3b4175b31a43a984d069b3aaae580e130a7 (diff) |
memoize (3jan24)
git-svn-id: svn://tug.org/texlive/trunk@69285 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Build/source')
4 files changed, 1273 insertions, 301 deletions
diff --git a/Build/source/texk/texlive/linked_scripts/memoize/memoize-clean.pl b/Build/source/texk/texlive/linked_scripts/memoize/memoize-clean.pl index 4e1334c6958..7d7e4f5b433 100755 --- a/Build/source/texk/texlive/linked_scripts/memoize/memoize-clean.pl +++ b/Build/source/texk/texlive/linked_scripts/memoize/memoize-clean.pl @@ -18,18 +18,19 @@ # The files belonging to this work and covered by LPPL are listed in # <texmf>/doc/generic/memoize/FILES. +my $PROG = 'memoize-clean.pl'; +my $VERSION = '2024/01/02 v1.1.0'; + use strict; use Getopt::Long; -# I intend to rewrite this script using Path::Class. use Cwd 'realpath'; use File::Spec; use File::Basename; -my $VERSION = '2023/10/10 v1.0.0'; - -my $usage = "usage: memoize-clean.py [-h] [--yes] [--all] [--quiet] [--prefix PREFIX] [mmz ...]\n"; +my $usage = "usage: $PROG [-h] [--yes] [--all] [--quiet] [--prefix PREFIX] " . + "[mmz ...]\n"; my $Help = <<END; -Remove (stale) memo and extern files. +Remove (stale) memo and extern files produced by package Memoize. positional arguments: mmz .mmz record files @@ -41,7 +42,8 @@ options: --all, -a Remove *all* memos and externs. --quiet, -q --prefix PREFIX, -p PREFIX - A path prefix to clean; this option can be specified multiple times. + A path prefix to clean; + this option can be specified multiple times. For details, see the man page or the Memoize documentation. END @@ -111,8 +113,11 @@ sub populate_tbdeleted { opendir(MD, $dir) or die "Cannot open directory '$dir'"; while( (my $fn = readdir(MD)) ) { my $path = File::Spec->catfile(($dir),$fn); - if ($fn =~ /\Q$basename_prefix\E[0-9A-F]{32}(?:-[0-9A-F]{32})?(?:-[0-9]+)?(\.memo|(?:-[0-9]+)?\.pdf|\.log)/ and ($all || !exists($keep{$path}))) { - push @tbdeleted, $path; + if ($fn =~ + /\Q$basename_prefix\E[0-9A-F]{32}(?:-[0-9A-F]{32})?(?:-[0-9]+)?# + (\.memo|(?:-[0-9]+)?\.pdf|\.log)/x + and ($all || !exists($keep{$path}))) { + push @tbdeleted, $path; } } closedir(MD); @@ -161,3 +166,7 @@ if (scalar(@tbdeleted) != 0) { } elsif (!$quiet) { print "Nothing to do, the directory seems clean.\n"; } + +# Local Variables: +# after-save-hook: pl2dtx +# End: diff --git a/Build/source/texk/texlive/linked_scripts/memoize/memoize-clean.py b/Build/source/texk/texlive/linked_scripts/memoize/memoize-clean.py index dd543327cba..db758bbd352 100755 --- a/Build/source/texk/texlive/linked_scripts/memoize/memoize-clean.py +++ b/Build/source/texk/texlive/linked_scripts/memoize/memoize-clean.py @@ -18,13 +18,14 @@ # The files belonging to this work and covered by LPPL are listed in # <texmf>/doc/generic/memoize/FILES. -__version__ = '2023/10/10 v1.0.0' +__version__ = '2024/01/02 v1.1.0' import argparse, re, sys, pathlib, os parser = argparse.ArgumentParser( description="Remove (stale) memo and extern files.", - epilog = "For details, see the man page or the Memoize documentation." + epilog = "For details, see the man page or the Memoize documentation " + "(https://ctan.org/pkg/memoize)." ) parser.add_argument('--yes', '-y', action = 'store_true', help = 'Do not ask for confirmation.') @@ -63,7 +64,8 @@ for mmz_fn in args.mmz: elif endinput: raise RuntimeError( - r'Bailing out, \endinput is not the last line of file $mmz_fn.') + rf'Bailing out, ' + rf'\endinput is not the last line of file {mmz_fn}.') elif m := re_prefix.match(line): prefix = m[1] @@ -109,7 +111,8 @@ tbdeleted = [] def populate_tbdeleted(folder, basename_prefix): re_aux = re.compile( re.escape(basename_prefix) + - '[0-9A-F]{32}(?:-[0-9A-F]{32})?(?:-[0-9]+)?(?:\.memo|(?:-[0-9]+)?\.pdf|\.log)$') + '[0-9A-F]{32}(?:-[0-9A-F]{32})?' + '(?:-[0-9]+)?(?:\.memo|(?:-[0-9]+)?\.pdf|\.log)$') try: for f in folder.iterdir(): if re_aux.match(f.name) and (args.all or f not in keep): @@ -157,3 +160,8 @@ if tbdeleted: print("Bailing out.") elif not args.quiet: print('Nothing to do, the directory seems clean.') + +# Local Variables: +# fill-column: 79 +# after-save-hook: py2dtx +# End: diff --git a/Build/source/texk/texlive/linked_scripts/memoize/memoize-extract.pl b/Build/source/texk/texlive/linked_scripts/memoize/memoize-extract.pl index 8fba3d356fb..96eec29b97e 100755 --- a/Build/source/texk/texlive/linked_scripts/memoize/memoize-extract.pl +++ b/Build/source/texk/texlive/linked_scripts/memoize/memoize-extract.pl @@ -18,188 +18,757 @@ # The files belonging to this work and covered by LPPL are listed in # <texmf>/doc/generic/memoize/FILES. +my $PROG = 'memoize-extract.pl'; +my $VERSION = '2024/01/02 v1.1.0'; + use strict; +use File::Basename qw/basename/; use Getopt::Long; -use Path::Class; -use File::Spec; -use File::Basename; -use PDF::API2; +use File::Spec::Functions + qw/splitpath catpath splitdir rootdir file_name_is_absolute/; +use File::Path qw(make_path); +# We will only try to import the PDF processing library once we set up the error +# log. + +# Declare variables for command-line arguments and for |kpathsea| +# variables. They are defined here so that they are global in the subs which use +# them. +our ($pdf_file, $prune, $keep, $format, $force, $quiet, + $pdf_library, $print_version, $mkdir, $help, + $openin_any, $openout_any, $texmfoutput, $texmf_output_directory); + +# \paragraph{Messages} + +# The messages are written both to the extraction log and the terminal (we +# output to stdout rather than stderr so that messages on the TeX terminal and +# document |.log| appear in chronological order). Messages are automatically +# adapted to the TeX |--format|. + +# The format of the messages. It depends on the given |--format|; the last +# entry is for t the terminal output. + +my %ERROR = ( + latex => '\PackageError{memoize (perl-based extraction)}{$short}{$long}', + plain => '\errhelp{$long}\errmessage{memoize (perl-based extraction): $short}', + context => '\errhelp{$long}\errmessage{memoize (perl-based extraction): $short}', + '' => '$header$short. $long'); + +my %WARNING = ( + latex => '\PackageWarning{memoize (perl-based extraction)}{$texindent$text}', + plain => '\message{memoize (perl-based extraction) Warning: $texindent$text}', + context => '\message{memoize (perl-based extraction) Warning: $texindent$text}', + '' => '$header$indent$text.'); + +my %INFO = ( + latex => '\PackageInfo{memoize (perl-based extraction)}{$texindent$text}', + plain => '\message{memoize (perl-based extraction): $texindent$text}', + context => '\message{memoize (perl-based extraction): $texindent$text}', + '' => '$header$indent$text.'); -my $VERSION = '2023/10/10 v1.0.0'; +# Some variables used in the message routines; note that |header| will be +# redefined once we parse the arguments. + +my $exit_code = 0; +my $log; +my $header = ''; +my $indent = ''; +my $texindent = ''; + +# The message routines. + +sub error { + my ($short, $long) = @_; + if (! $quiet) { + $_ = $ERROR{''}; + s/\$header/$header/; + s/\$short/$short/; + s/\$long/$long/; + print(STDOUT "$_\n"); + } + if ($log) { + $long =~ s/\\/\\string\\/g; + $_ = $ERROR{$format}; + s/\$short/$short/; + s/\$long/$long/; + print(LOG "$_\n"); + } + $exit_code = 11; + endinput(); +} + +sub warning { + my $text = shift; + if ($log) { + $_ = $WARNING{$format}; + s/\$texindent/$texindent/; + s/\$text/$text/; + print(LOG "$_\n"); + } + if (! $quiet) { + $_ = $WARNING{''}; + s/\$header/$header/; + s/\$indent/$indent/; + s/\$text/$text/; + print(STDOUT "$_\n"); + } + $exit_code = 10; +} + +sub info { + my $text = shift; + if ($text && ! $quiet) { + $_ = $INFO{''}; + s/\$header/$header/; + s/\$indent/$indent/; + s/\$text/$text/; + print(STDOUT "$_\n"); + if ($log) { + $_ = $INFO{$format}; + s/\$texindent/$texindent/; + s/\$text/$text/; + print(LOG "$_\n"); + } + } +} + +# Mark the log as complete and exit. +sub endinput { + if ($log) { + print(LOG "\\endinput\n"); + close(LOG); + } + exit $exit_code; +} + +sub die_handler { + stderr_to_warning(); + my $text = shift; + chomp($text); + error("Perl error: $text", ''); +} + +sub warn_handler { + my $text = shift; + chomp($text); + warning("Perl warning: $text"); +} + +# This is used to print warning messages from PDF::Builder, which are output to +# STDERR. +my $stderr; +sub stderr_to_warning { + if ($stderr) { + my $w = ' Perl info: '; + my $nl = ''; + for (split(/\n/, $stderr)) { + /(^\s*)(.*?)(\s*)$/; + $w .= ($1 ? ' ' : $nl) . $2; + $nl = "\n"; + } + warning("$w"); + $stderr = ''; + } +} + +# \paragraph{Permission-related functions} + +# We will need these variables below. Note that we only support Unix and +# Windows. +my $on_windows = $^O eq 'MSWin32'; +my $dirsep = $on_windows ? '\\' : '/'; + +# |paranoia_in|/|out| should work exactly as +# |kpsewhich -safe-in-name|/|-safe-out-name|. +sub paranoia_in { + my ($f, $remark) = @_; + error("I'm not allowed to read from '$f' (openin_any = $openin_any)", + $remark) unless _paranoia($f, $openin_any); +} + +sub paranoia_out { + my ($f, $remark) = @_; + error("I'm not allowed to write to '$f' (openin_any = $openout_any)", + $remark) unless _paranoia($f, $openout_any); +} + +sub _paranoia { + # |f| is the path to the file (it should not be empty), and |mode| is the + # value of |openin_any| or |openout_any|. + my ($f, $mode) = @_; + return if (! $f); + # We split the filename into the directory and the basename part, and the + # directory into components. + my ($volume, $dir, $basename) = splitpath($f); + my @dir = splitdir($dir); + return ( + # In mode `any' (|a|, |y| or |1|), we may access any file. + $mode =~ /^[ay1]$/ + || ( + # Otherwise, we are at least in the restricted mode, so we should + # not open dot files on Unix-like systems (except file called + # |.tex|). + ! (!$on_windows && $basename =~ /^\./ && !($basename =~ /^\.tex$/)) + && ( + # If we are precisely in the restricted mode (|r|, |n|, |0|), + # then there are no further restrictions. + $mode =~ /^[rn0]$/ + # Otherwise, we are in the paranoid mode (officially |p|, but + # any other value is interpreted as |p| as well). There are + # two further restrictions in the paranoid mode. + || ( + # We're not allowed to go to a parent directory. + ! grep(/^\.\.$/, @dir) && $basename ne '..' + && + # If the given path is absolute, is should be a descendant + # of either |TEXMF_OUTPUT_DIRECTORY| or |TEXMFOUTPUT|. + (!file_name_is_absolute($f) + || + is_ancestor($texmf_output_directory, $f) + || + is_ancestor($texmfoutput, $f) + ))))); +} + +# Only removes final "/"s. This is unlike |File::Spec|'s |canonpath|, which also +# removes |.| components, collapses multiple |/| --- and unfortunately also goes +# up for |..| on Windows. +sub normalize_path { + my $path = shift; + my ($v, $d, $n) = splitpath($path); + if ($n eq '' && $d =~ /[^\Q$dirsep\E]\Q$dirsep\E+$/) { + $path =~ s/\Q$dirsep\E+$//; + } + return $path; +} + +# On Windows, we disallow ``semi-absolute'' paths, i.e.\ paths starting with the +# |\| but lacking the drive. |File::Spec|'s function |file_name_is_absolute| +# returns 2 if the path is absolute with a volume, 1 if it's absolute with no +# volume, and 0 otherwise. After a path was sanitized using this function, +# |file_name_is_absolute| will work as we want it to. +sub sanitize_path { + my $f = normalize_path(shift); + my ($v, $d, $n) = splitpath($f); + if ($on_windows) { + my $a = file_name_is_absolute($f); + if ($a == 1 || ($a == 0 && $v) ) { + error("\"Semi-absolute\" paths are disallowed: " . $f, + "The path must either both contain the drive letter and " . + "start with '\\', or none of these; paths like 'C:foo\\bar' " . + "and '\\foo\\bar' are disallowed"); + } + } +} + +sub access_in { + return -r shift; +} + +sub access_out { + my $f = shift; + my $exists; + eval { $exists = -e $f }; + # Presumably, we get this error when the parent directory is not executable. + return if ($@); + if ($exists) { + # An existing file should be writable, and if it's a directory, it + # should also be executable. + my $rw = -w $f; my $rd = -d $f; my $rx = -x $f; + return -w $f && (! -d $f || -x $f); + } else { + # For a non-existing file, the parent directory should be writable. + # (This is the only place where function |parent| is used, so it's ok + # that it returns the logical parent.) + my $p = parent($f); + return -w $p; + } +} + +# This function finds the location for an input file, respecting +# |TEXMF_OUTPUT_DIRECTORY| and |TEXMFOUTPUT|, and the permissions in the +# filesystem. It returns an absolute file as-is. For a relative file, it tries +# |TEXMF_OUTPUT_DIRECTORY| (if defined), the current directory (always), and +# |TEXMFOUTPUT| directory (if defined), in this order. The first readable file +# found is returned; if no readable file is found, the file in the current +# directory is returned. +sub find_in { + my $f = shift; + sanitize_path($f); + return $f if file_name_is_absolute($f); + for my $df ( + $texmf_output_directory ? join_paths($texmf_output_directory, $f) : undef, + $f, + $texmfoutput ? join_paths($texmfoutput, $f) : undef) { + return $df if $df && -r $df; + } + return $f; +} + +# This function finds the location for an output file, respecting +# |TEXMF_OUTPUT_DIRECTORY| and |TEXMFOUTPUT|, and the permissions in the +# filesystem. It returns an absolute file as-is. For a relative file, it tries +# |TEXMF_OUTPUT_DIRECTORY| (if defined), the current directory (unless +# |TEXMF_OUTPUT_DIRECTORY| is defined), and |TEXMFOUTPUT| directory (if +# defined), in this order. The first writable file found is returned; if no +# writable file is found, the file in either the current or the output directory +# is returned. +sub find_out { + my $f = shift; + sanitize_path($f); + return $f if file_name_is_absolute($f); + for my $df ( + $texmf_output_directory ? join_paths($texmf_output_directory, $f) : undef, + $texmf_output_directory ? undef : $f, + $texmfoutput ? join_paths($texmfoutput, $f) : undef) { + return $df if $df && access_out($df); + } + return $texmf_output_directory ? join_paths($texmf_output_directory, $f) : $f; +} + +# We next define some filename-related utilities matching what Python offers out +# of the box. We avoid using |File::Spec|'s |canonpath|, because on Windows, +# which has no concept of symlinks, this function resolves |..| to the parent. + +sub name { + my $path = shift; + my ($volume, $dir, $filename) = splitpath($path); + return $filename; +} + +sub suffix { + my $path = shift; + my ($volume, $dir, $filename) = splitpath($path); + $filename =~ /\.[^.]*$/; + return $&; +} -my $usage = "usage: memoize-extract.pl [-h] [--pdf PDF] [--prune] [--keep] [--force] [--log LOG] [--warning-template WARNING_TEMPLATE] [--quiet] [--mkdir] mmz\n"; +sub with_suffix { + my ($path, $suffix) = @_; + my ($volume, $dir, $filename) = splitpath($path); + if ($filename =~ s/\.[^.]*$/$suffix/) { + return catpath($volume, $dir, $filename); + } else { + return catpath($volume, $dir, $filename . $suffix); + } +} + +sub with_name { + my ($path, $name) = @_; + my ($volume, $dir, $filename) = splitpath($path); + my ($v,$d,$f) = splitpath($name); + die "Runtime error in with_name: " . + "'$name' should not contain the directory component" + unless $v eq '' && $d eq '' && $f eq $name; + return catpath($volume, $dir, $name); +} + +sub join_paths { + my $path1 = normalize_path(shift); + my $path2 = normalize_path(shift); + return $path2 if !$path1 || file_name_is_absolute($path2); + my ($volume1, $dir1, $filename1) = splitpath($path1, 'no_file'); + my ($volume2, $dir2, $filename2) = splitpath($path2); + die if $volume2; + return catpath($volume1, + join($dirsep, ($dir1 eq $dirsep ? '' : $dir1, $dir2)), + $filename2); +} + +# The logical parent. The same as |pathlib.parent| in Python. +sub parent { + my $f = normalize_path(shift); + my ($v, $dn, $_dummy) = splitpath($f, 1); + my $p_dn = $dn =~ s/[^\Q$dirsep\E]+$//r; + if ($p_dn eq '') { + $p_dn = $dn =~ /^\Q$dirsep\E/ ? $dirsep : '.'; + } + my $p = catpath($v, $p_dn, ''); + $p = normalize_path($p); + return $p; +} + +# This function assumes that both paths are absolute; ancestor may be '', +# signaling a non-path. +sub is_ancestor { + my $ancestor = normalize_path(shift); + my $descendant = normalize_path(shift); + return if ! $ancestor; + $ancestor .= $dirsep unless $ancestor =~ /\Q$dirsep\E$/; + return $descendant =~ /^\Q$ancestor/; +} + +# A paranoid |Path.mkdir|. The given folder is preprocessed by |find_out|. +sub make_directory { + my $folder = find_out(shift); + if (! -d $folder) { + paranoia_out($folder); + # Using |make_path| is fine because we know that + # |TEXMF_OUTPUT_DIRECTORY|/|TEXMFOUTPUT|, if given, exists, and that + # ``folder'' contains no |..|. + make_path($folder); + # This does not get logged when the function is invoked via |--mkdir|, + # as it is not clear what the log name should be. + info("Created directory $folder"); + } +} + +sub unquote { + shift =~ s/"(.*?)"/\1/rg; +} + +# \paragraph{Kpathsea} + +# Get the values of |openin_any|, |openout_any|, |TEXMFOUTPUT| and +# |TEXMF_OUTPUT_DIRECTORY|. + +my $maybe_backslash = $on_windows ? '' : '\\'; +my $query = 'kpsewhich -expand-var=' . + "openin_any=$maybe_backslash\$openin_any," . + "openout_any=$maybe_backslash\$openout_any," . + "TEXMFOUTPUT=$maybe_backslash\$TEXMFOUTPUT"; +my $kpsewhich_output = `$query`; +if (! $kpsewhich_output) { + # No TeX? (Note that |kpsewhich| should exist in MiKTeX as well.) In + # absence of |kpathsea| information, we get very paranoid. + ($openin_any, $openout_any) = ('p', 'p'); + ($texmfoutput, $texmf_output_directory) = ('', ''); + # Unfortunately, this warning can't make it into the log. But then again, + # the chances of a missing |kpsewhich| are very slim, and its absence would + # show all over the place anyway. + warning('I failed to execute "kpsewhich", is there no TeX system installed? ' . + 'Assuming openin_any = openout_any = "p" ' . + '(i.e. restricting all file operations to non-hidden files ' . + 'in the current directory of its subdirectories).'); +} else { + $kpsewhich_output =~ /^openin_any=(.*),openout_any=(.*),TEXMFOUTPUT=(.*)/; + ($openin_any, $openout_any, $texmfoutput) = @{^CAPTURE}; + $texmf_output_directory = $ENV{'TEXMF_OUTPUT_DIRECTORY'}; + if ($openin_any =~ '^\$openin_any') { + # When the |open*_any| variables are not expanded, we assume we're + # running MiKTeX. The two config settings below correspond to TeXLive's + # |openin_any| and |openout_any|; afaik, there is no analogue to + # |TEXMFOUTPUT|. + $query = 'initexmf --show-config-value=[Core]AllowUnsafeInputFiles ' . + '--show-config-value=[Core]AllowUnsafeOutputFiles'; + my $initexmf_output = `$query`; + $initexmf_output =~ /^(.*)\n(.*)\n/m; + $openin_any = $1 eq 'true' ? 'a' : 'p'; + $openout_any = $2 eq 'true' ? 'a' : 'p'; + $texmfoutput = ''; + $texmf_output_directory = ''; + } +} + +# An output directory should exist, and may not point to the root on Linux. On +# Windows, it may point to the root, because being absolute also implies +# containing the drive; see |sanitize_filename|. +sub sanitize_output_dir { + return unless my $d = shift; + sanitize_path($d); + # On Windows, |rootdir| returns |\|, so it cannot possibly match |$d|. + return $d if -d $d && $d ne rootdir(); +} + +$texmfoutput = sanitize_output_dir($texmfoutput); +$texmf_output_directory = sanitize_output_dir($texmf_output_directory); + +# We don't delve into the real script when loaded from the testing code. +return 1 if caller; + +# \paragraph{Arguments} + +my $usage = "usage: $PROG [-h] [-P PDF] [-p] [-k] [-F {latex,plain,context}] [-f] " . + "[-L {PDF::API2,PDF::Builder}] [-q] [-m] [-V] mmz\n"; my $Help = <<END; -Extract extern pages out of the document PDF. +Extract extern pages produced by package Memoize out of the document PDF. positional arguments: - mmz the record file produced by Memoize: doc.mmz when compiling doc.tex + mmz the record file produced by Memoize: + doc.mmz when compiling doc.tex + (doc and doc.tex are accepted as well) options: - --help, -h show this help message and exit - --version, -V show version and exit - --pdf PDF, -P PDF extract from file PDF - --prune, -p remove the extern pages after extraction - --keep, -k do not mark externs as extracted - --force, -f extract even if the size-check fails - --log LOG, -l LOG the log file - --warning-template WARNING_TEMPLATE, -w WARNING_TEMPLATE - \warningtext in the template will be replaced by the warning message - --quiet, -q don't describe what's happening - --embedded, -e prefix all messages to the standard output with the script name - --mkdir, -m create a directory (and exit) + -h, --help show this help message and exit + -P PDF, --pdf PDF extract from file PDF + -p, --prune remove the extern pages after extraction + -k, --keep do not mark externs as extracted + -F, --format {latex,plain,context} + the format of the TeX document invoking extraction + -f, --force extract even if the size-check fails + -q, --quiet describe what's happening + -L, --library {PDF::API2, PDF::Builder} + which PDF library to use for extraction (default: PDF::API2) + -m, --mkdir create a directory (and exit); + mmz argument is interpreted as directory name + -V, --version show program's version number and exit For details, see the man page or the Memoize documentation. END -my ($pdf_arg, $prune, $keep, $log, $quiet, $embedded, $force, $mkdir, $help, $print_version); -my $warning_template = '\warningtext'; +my @valid_libraries = ('PDF::API2', 'PDF::Builder'); Getopt::Long::Configure ("bundling"); GetOptions( - "pdf|P=s" => \$pdf_arg, - "keep|k" => \$keep, + "pdf|P=s" => \$pdf_file, "prune|p" => \$prune, - "log|l=s" => \$log, - "quiet|q" => \$quiet, - "embedded|e" => \$embedded, + "keep|k" => \$keep, + "format|F=s" => \$format, "force|f" => \$force, - "warning-template|w=s" => \$warning_template, + "quiet|q" => \$quiet, + "library|L=s" => \$pdf_library, "mkdir|m" => \$mkdir, - "help|h|?" => \$help, "version|V" => \$print_version, + "help|h|?" => \$help, ) or die $usage; + if ($help) {print("$usage\n$Help"); exit 0} -if ($print_version) { print("memoize-extract.pl of Memoize $VERSION\n"); exit 0 } -die $usage unless @ARGV == 1; -my $message_prefix = $embedded ? basename($0) . ': ' : ''; -print("\n") if ($embedded); +if ($print_version) { print("$PROG of Memoize $VERSION\n"); exit 0 } -my @output_paths = (dir()->absolute->resolve); -my $texmfoutput = `kpsewhich --var-value=TEXMFOUTPUT`; -$texmfoutput =~ s/^\s+|\s+$//g; -if ($texmfoutput) { - my $texmfoutput_dir = dir($texmfoutput)->absolute->resolve; - push(@output_paths, $texmfoutput_dir) unless $texmfoutput_dir->dir_list == 1 && ! $texmfoutput_dir->volume; -} +die "${usage}$PROG: error: the following arguments are required: mmz\n" + unless @ARGV == 1; -sub paranoia { - my $file = $_[0]; - die "${message_prefix}Cannot create a hidden file or dir: $file" - if $file->basename =~ /^\./; - my $parent = $file->parent->absolute->resolve; - die "${message_prefix}Cannot write outside the current working or output directory tree: $file" - unless grep($_->contains($parent), @output_paths); -} +die "${usage}$PROG: error: argument -F/--format: invalid choice: '$format' " . + "(choose from 'latex', 'plain', 'context')\n" + unless grep $_ eq $format, ('', 'latex', 'plain', 'context'); + +die "${usage}$PROG: error: argument -L/--library: invalid choice: '$pdf_library' " . + "(choose from " . join(", ", @valid_libraries) . ")\n" + if $pdf_library && ! grep $_ eq $pdf_library, @valid_libraries; -my $mmz_arg = $ARGV[0]; -my $mmz_file = file($mmz_arg); -my $mmz_dir = $mmz_file->parent; +$header = $format ? basename($0) . ': ' : ''; +# start a new line in the TeX terminal output +print("\n") if $format; + +# \paragraph{Initialization} + +# With |--mkdir|, argument |mmz| is interpreted as the directory to create. if ($mkdir) { - my $dir = dir($mmz_arg)->cleanup; - my $current = dir(File::Spec->catpath($dir->volume, - $dir->is_absolute ? File::Spec->rootdir : File::Spec->curdir, - ''))->absolute; - for my $c ($dir->components) { - $current = $current->subdir($c); - if (-d $current) { - $current = $current->resolve; - } else { - paranoia($current); - mkdir($current); - print("${message_prefix}Created directory $current\n") unless $quiet; - } - } - exit; -} else { - die "${message_prefix}The 'mmz' argument should be a file with suffix '.mmz', not '$mmz_file'\n" unless $mmz_file->basename =~ /\.mmz$/; + make_directory($ARGV[0]); + exit 0; } -# Enable in-place editing (of the .mmz file). -paranoia($mmz_file) unless $keep; -$^I = "" unless $keep; - -my $pdf_file = $pdf_arg ? file($pdf_arg) : $mmz_dir->file($mmz_file->basename =~ s/\.mmz$/\.pdf/r)->cleanup; -paranoia($pdf_file) if $prune; +# Normalize the |mmz| argument into a |.mmz| filename. +my $mmz_file = $ARGV[0]; +$mmz_file = with_suffix($mmz_file, '.mmz') + if suffix($mmz_file) eq '.tex'; +$mmz_file = with_name($mmz_file, name($mmz_file) . '.mmz') + if suffix($mmz_file) ne '.mmz'; -if ($log) { - paranoia(file($log)); +# Once we have the |.mmz| filename, we can open the log. +if ($format) { + my $_log = find_out(with_suffix($mmz_file, '.mmz.log')); + paranoia_out($_log); + info("Logging to '$_log'"); + $log = $_log; open LOG, ">$log"; +} + +# Now that we have opened the log file, we can try loading the PDF processing +# library. +if ($pdf_library) { + eval "use $pdf_library"; + error("Perl module '$pdf_library' was not found", + 'Have you followed the instructions is section 1.1 of the manual?') + if ($@); } else { - *LOG = *STDERR; + for (@valid_libraries) { + eval "use $_"; + if (!$@) { + $pdf_library = $_; + last; + } + } + if (!$pdf_library) { + error("No suitable Perl module for PDF processing was found, options are " . + join(", ", @valid_libraries), + 'Have you followed the instructions is section 1.1 of the manual?'); + } +} + +# Catch any errors in the script and output them to the log. +$SIG{__DIE__} = \&die_handler; +$SIG{__WARN__} = \&warn_handler; +close(STDERR); +open(STDERR, ">", \$stderr); + +# Find the |.mmz| file we will read, but retain the original filename in +# |$given_mmz_file|, as we will still need it. +my $given_mmz_file = $mmz_file; +$mmz_file = find_in($mmz_file, 1); +if (! -e $mmz_file) { + info("File '$given_mmz_file' does not exist, assuming there's nothing to do"); + endinput(); } +paranoia_in($mmz_file); +paranoia_out($mmz_file, + 'I would have to rewrite this file unless option --keep is given.') + unless $keep; -my ($pdf, %extern_pages); -print "${message_prefix}Extracting externs from $pdf_file:\n" unless $quiet; +# Determine the PDF filename: it is either given via |--pdf|, or constructed +# from the |.mmz| filename. +$pdf_file = with_suffix($given_mmz_file, '.pdf') if !$pdf_file; +$pdf_file = find_in($pdf_file); +paranoia_in($pdf_file); +paranoia_out($pdf_file, + 'I would have to rewrite this file because option --prune was given.') + if $prune; +# Various initializations. +my $pdf; +my %extern_pages; +my $new_mmz; my $tolerance = 0.01; -my $done_message = "${message_prefix}Done (there was nothing to extract).\n"; - -while (<>) { - if (/^\\mmzNewExtern *{(?P<extern_path>(?P<prefix>.*?)(?P<code_md5sum>[0-9A-F]{32})-(?P<context_md5sum>[0-9A-F]{32})(?:-[0-9]+)?.pdf)}{(?P<page_n>[0-9]+)}{(?P<expected_width>[0-9.]*)pt}{(?P<expected_height>[0-9.]*)pt}/) { - my $extern_file = file($+{extern_path}); - if (! $extern_file->is_absolute) { - $extern_file = $mmz_dir->file($+{extern_path}); - } - paranoia($extern_file); - my $page = $+{page_n}; - my $expected_width_pt = $+{expected_width}; - my $expected_height_pt = $+{expected_height}; - my $c_memo_file = $mmz_dir->file($+{prefix} . $+{code_md5sum} . '.memo'); - my $cc_memo_file = $mmz_dir->file($+{prefix} . $+{code_md5sum} . '-' . $+{context_md5sum} . '.memo'); - if (!(-e $c_memo_file && -e $cc_memo_file)) { - print LOG ($warning_template =~ s/\\warningtext/Not extracting page $page into extern $extern_file, because the associated (c)c-memo does not exist/gr), "\n\\endinput\n"; - last; - } - eval { $pdf = PDF::API2->open($pdf_file->stringify) unless $pdf; }; - if ($@) { - print LOG ($warning_template =~ s/\\warningtext/Cannot read file "$pdf_file". Perhaps you have to load Memoize earlier in the preamble?/gr), "\n\\endinput\n"; - close LOG; - die "${message_prefix}File '$pdf_file' cannot be read, bailing out.\n"; - } - my $extern = PDF::API2->new(); - $extern->version($pdf->version); - $extern->import_page($pdf, $page); - my $extern_page = $extern->open_page(1); - my ($x0, $y0, $x1, $y1) = $extern_page->get_mediabox(); - my $width_pt = ($x1 - $x0) / 72 * 72.27; - my $height_pt = ($y1 - $y0) / 72 * 72.27; - my $warning = ''; - if (abs($width_pt - $expected_width_pt) > $tolerance - || abs($height_pt - $expected_height_pt) > $tolerance) - { - $warning = "I refuse to extract page $page from $pdf_file, because its size (${width_pt}pt x ${height_pt}pt) is not what I expected (${expected_width_pt}pt x ${expected_height_pt}pt)"; - print LOG ($warning_template =~ s/\\warningtext/$warning/gr), "\n"; - } - if ($warning && !$force) { - unlink $extern_file; - } else { - $extern->saveas($extern_file->stringify); - $done_message = "${message_prefix}Done.\n"; - print STDOUT "${message_prefix} Page $page --> $extern_file\n" unless $quiet; - $extern_pages{$page} = 1 if $prune; - print("%") unless $keep; - } - } - print unless $keep; -} - -print $done_message unless $quiet; - -if ($pdf and $prune) { - paranoia($pdf_file); - my $pruned_pdf = PDF::API2->new(); - for (my $n = 1; $n <= $pdf->page_count(); $n++) { - if (! $extern_pages{$n}) { - $pruned_pdf->import_page($pdf, $n); - } +info("Extracting new externs listed in '$mmz_file' " . + "from '$pdf_file' using Perl module $pdf_library"); +my $done_message = "Done (there was nothing to extract)"; +$indent = ' '; +$texindent = '\space\space '; +my $dir_to_make; + +# \paragraph{Process \texttt{.mmz}} +# +# We cannot process the .mmz file using in-place editing. It would fail when +# the file is writable but its parent directory is not. + +open (MMZ, $mmz_file); +while (<MMZ>) { + my $mmz_line = $_; + if (/^\\mmzPrefix *{(?P<prefix>)}/) { + # Found |\mmzPrefix|: create the extern directory, but only later, if an + # extern file is actually produced. We parse the prefix in two steps + # because we have to unquote the entire prefix. + my $prefix = unquote($+{prefix}); + warning("Cannot parse line '$mmz_line'") unless + $prefix =~ /(?P<dir_prefix>.*\/)?(?P<name_prefix>.*?)/; + $dir_to_make = $+{dir_prefix}; + } elsif (/^\\mmzNewExtern\ *{(?P<extern_path>.*?)}{(?P<page_n>[0-9]+)}# + {(?P<expected_width>[0-9.]*)pt}{(?P<expected_height>[0-9.]*)pt}/x) { + # Found |\mmzNewExtern|: extract the extern page into an extern file. + $done_message = "Done"; + my $ok = 1; + my %m_ne = %+; + # The extern filename, as specified in |.mmz|: + my $extern_file = unquote($m_ne{extern_path}); + # We parse the extern filename in a separate step because we have to + # unquote the entire path. + warning("Cannot parse line '$mmz_line'") unless + $extern_file =~ /(?P<dir_prefix>.*\/)?(?P<name_prefix>.*?)# + (?P<code_md5sum>[0-9A-F]{32})-# + (?P<context_md5sum>[0-9A-F]{32})(?:-[0-9]+)?.pdf/x; + # The actual extern filename: + my $extern_file_out = find_out($extern_file); + paranoia_out($extern_file_out); + my $page = $m_ne{page_n}; + # Check whether c-memo and cc-memo exist (in any input directory). + my $c_memo = with_name($extern_file, + $+{name_prefix} . $+{code_md5sum} . '.memo'); + my $cc_memo = with_name($extern_file, + $+{name_prefix} . $+{code_md5sum} . + '-' . $+{context_md5sum} . '.memo'); + my $c_memo_in = find_in($c_memo); + my $cc_memo_in = find_in($cc_memo); + if ((! access_in($c_memo_in) || ! access_in($cc_memo_in)) && !$force) { + warning("I refuse to extract page $page into extern '$extern_file', " . + "because the associated c-memo '$c_memo' and/or " . + "cc-memo '$cc_memo' does not exist"); + $ok = ''; + } + # Load the PDF. We only do this now so that we don't load it if there + # is nothing to extract. + if ($ok && ! $pdf) { + if (!access_in($pdf_file)) { + warning("Cannot open '$pdf_file'", ''); + endinput(); + } + # Temporarily disable error handling, so that we can catch the error + # ourselves. + $SIG{__DIE__} = undef; $SIG{__WARN__} = undef; + # All safe, |paranoia_in| was already called above. + eval { $pdf = $pdf_library->open($pdf_file, msgver => 0) }; + $SIG{__DIE__} = \&die_handler; $SIG{__WARN__} = \&warn_handler; + error("File '$pdf_file' seems corrupted. " . + "Perhaps you have to load Memoize earlier in the preamble", + "In particular, Memoize must be loaded before TikZ library " . + "'fadings' and any package deploying it, and in Beamer, " . + "load Memoize by writing \\RequirePackage{memoize} before " . + "\\documentclass{beamer}. " . + "This was the error thrown by Perl:" . "\n$@") if $@; + } + # Does the page exist? + if ($ok && $page > (my $n_pages = $pdf->page_count())) { + error("I cannot extract page $page from '$pdf_file', " . + "as it contains only $n_pages page" . + ($n_pages > 1 ? 's' : ''), ''); + } + if ($ok) { + # Import the page into the extern PDF (no disk access yet). + my $extern = $pdf_library->new(outver => $pdf->version); + $extern->import_page($pdf, $page); + my $extern_page = $extern->open_page(1); + # Check whether the page size matches the |.mmz| expectations. + my ($x0, $y0, $x1, $y1) = $extern_page->get_mediabox(); + my $width_pt = ($x1 - $x0) / 72 * 72.27; + my $height_pt = ($y1 - $y0) / 72 * 72.27; + my $expected_width_pt = $m_ne{expected_width}; + my $expected_height_pt = $m_ne{expected_height}; + if ((abs($width_pt - $expected_width_pt) > $tolerance + || abs($height_pt - $expected_height_pt) > $tolerance) && !$force) { + warning("I refuse to extract page $page from $pdf_file, " . + "because its size (${width_pt}pt x ${height_pt}pt) " . + "is not what I expected " . + "(${expected_width_pt}pt x ${expected_height_pt}pt)"); + } else { + # All tests were successful, let's create the extern file. + # First, the containing directory, if necessary. + if ($dir_to_make) { + make_directory($dir_to_make); + $dir_to_make = undef; + } + # Now the extern file. Note that |paranoia_out| was already + # called above. + info("Page $page --> $extern_file_out"); + $extern->saveas($extern_file_out); + # This page will get pruned. + $extern_pages{$page} = 1 if $prune; + # Comment out this |\mmzNewExtern|. + $new_mmz .= '%' unless $keep; + } + } } - $pruned_pdf->save($pdf_file->stringify); - print("${message_prefix}The following extern pages were pruned out of the PDF: ", - join(",", sort(keys(%extern_pages))) . "\n") unless $quiet; + $new_mmz .= $mmz_line unless $keep; + stderr_to_warning(); } +close(MMZ); +$indent = ''; +$texindent = ''; +info($done_message); -if ($log) { - print LOG "\\endinput\n"; - close LOG; +# Write out the |.mmz| file with |\mmzNewExtern| lines commented out. (All safe, +# |paranoia_out| was already called above.) +if (!$keep) { + open(MMZ, ">", $mmz_file); + print MMZ $new_mmz; + close(MMZ); } + +# Remove the extracted pages from the original PDF. (All safe, |paranoia_out| +# was already called above.) +if ($prune and keys(%extern_pages) != 0) { + my $pruned_pdf = $pdf_library->new(); + for (my $n = 1; $n <= $pdf->page_count(); $n++) { + if (! $extern_pages{$n}) { + $pruned_pdf->import_page($pdf, $n); + } + } + $pruned_pdf->save($pdf_file); + info("The following extern pages were pruned out of the PDF: " . + join(",", sort(keys(%extern_pages)))); +} + +endinput(); + +# Local Variables: +# fill-column: 79 +# after-save-hook: pl2dtx +# End: diff --git a/Build/source/texk/texlive/linked_scripts/memoize/memoize-extract.py b/Build/source/texk/texlive/linked_scripts/memoize/memoize-extract.py index 54d8870e7b4..29ea58192c9 100755 --- a/Build/source/texk/texlive/linked_scripts/memoize/memoize-extract.py +++ b/Build/source/texk/texlive/linked_scripts/memoize/memoize-extract.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 - +# # This file is a part of Memoize, a TeX package for externalization of # graphics and memoization of compilation results in general, available at # https://ctan.org/pkg/memoize and https://github.com/sasozivanovic/memoize. @@ -18,166 +18,552 @@ # The files belonging to this work and covered by LPPL are listed in # <texmf>/doc/generic/memoize/FILES. -__version__ = '2023/10/10 v1.0.0' - -import argparse, re, sys, os, pdfrw, subprocess, itertools -from pathlib import Path - -parser = argparse.ArgumentParser( - description = "Extract extern pages out of the document PDF.", - epilog = "For details, see the man page or the Memoize documentation.", - prog = 'memoize-extract.py', -) -parser.add_argument('--pdf', '-P', help = 'extract from file PDF') -parser.add_argument('--prune', '-p', action = 'store_true', - help = 'remove the extern pages after extraction') -parser.add_argument('--keep', '-k', action = 'store_true', - help = 'do not mark externs as extracted') -parser.add_argument('--force', '-f', action = 'store_true', - help = 'extract even if the size-check fails') -parser.add_argument('--log', '-l', default = os.devnull, help = 'the log file') -parser.add_argument('--warning-template', '-w', default = '\warningtext', - help = '\warningtext in the template will be replaced by the warning message') -parser.add_argument('--quiet', '-q', action = 'store_true', - help = "describe what's happening") -parser.add_argument('--embedded', '-e', action = 'store_true', - help = "prefix all messages to the standard output with the script name") -parser.add_argument('--mkdir', '-m', action = 'store_true', - help = 'create a directory (and exit)') -parser.add_argument('mmz', - help = 'the record file produced by Memoize: doc.mmz when compiling doc.tex') -parser.add_argument('--version', '-V', action = 'version', - version = f"%(prog)s of Memoize " + __version__) - -args = parser.parse_args() - -message_prefix = parser.prog + ': ' if args.embedded else '' -if args.embedded: - print() - -# Even a bit more paranoid than required: -# (a) disallowing TEXMFOUTPUT=/ (while allowing C:\ on Windows) -# (b) waiting for access to '-output-directory'. -output_paths = [Path.cwd()] -if texmfoutput := subprocess.run( - ['kpsewhich', '--var-value=TEXMFOUTPUT'], - capture_output = True).stdout.decode().strip(): - texmfoutput_dir = Path(texmfoutput).resolve() - if len(texmfoutput_dir.parts) != 1 or texmfoutput_dir.drive: - output_paths.append(texmfoutput_dir) - -def paranoia(f): - p = Path(f).resolve() - if p.stem.startswith('.'): - raise RuntimeError(f"{message_prefix}Cannot create a hidden file or dir: {f}") - if not any(p.is_relative_to(op) for op in output_paths): - raise RuntimeError(f"{message_prefix}Cannot write outside the current working or output directory tree: {f}") - -mmz_file = Path(args.mmz) +__version__ = '2024/01/02 v1.1.0' + +import argparse, re, sys, os, subprocess, itertools, traceback, platform +from pathlib import Path, PurePath +# We will only try to import the PDF processing library once we set up the +# error log. + +# \paragraph{Messages} + +# The messages are written both to the extraction log and the terminal (we +# output to stdout rather than stderr so that messages on the TeX terminal and +# document |.log| appear in chronological order). Messages are automatically +# adapted to the TeX |--format|. + +# The format of the messages. It depends on the given |--format|; the last +# entry is for t the terminal output. + +ERROR = { + 'latex': r'\PackageError{{{package_name}}}{{{short}}}{{{long}}}', + 'plain': r'\errhelp{{{long}}}\errmessage{{{package_name}: {short}}}', + 'context': r'\errhelp{{{long}}}\errmessage{{{package_name}: {short}}}', + None: '{header}{short}.\n{long}', +} + +WARNING = { + 'latex': r'\PackageWarning{{{package_name}}}{{{texindent}{text}}}', + 'plain': r'\message{{{package_name}: {texindent}{text}}}', + 'context': r'\message{{{package_name}: {texindent}{text}}}', + None: r'{header}{indent}{text}.', +} + +INFO = { + 'latex': r'\PackageInfo{{{package_name}}}{{{texindent}{text}}}', + 'plain': r'\message{{{package_name}: {texindent}{text}}}', + 'context': r'\message{{{package_name}: {texindent}{text}}}', + None: r'{header}{indent}{text}.', +} + +# Some variables used in the message routines; note that |header| will be +# redefined once we parse the arguments. + +package_name = 'memoize (python-based extraction)' +exit_code = 0 +log = None +header = '' +indent = '' +texindent = '' + +# The message routines. + +def error(short, long): + if not args.quiet: + print(ERROR[None].format(short = short, long = long, header = header)) + if log: + long = long.replace('\\', '\\string\\') + print( + ERROR[args.format].format( + short = short, long = long, package_name = package_name), + file = log) + global exit_code + exit_code = 11 + endinput() + +def warning(text): + if log: + print( + WARNING[args.format].format( + text = text, texindent = texindent, package_name = package_name), + file = log) + if not args.quiet: + print(WARNING[None].format(text = text, header = header, indent = indent)) + global exit_code + exit_code = 10 -if args.mkdir: # Here, argument "mmz" is interpreted as the directory to create. - # We cannot simply say - # paranoia(mmz_file) - # mmz_file.mkdir(parents = True, exist_ok = True) - # because have be paranoid about the intermediate directories as well: - # memoize-extract.py -m a/../../BAD/../<cwd-name>/b - # Note that paranoia might kick in only after a part of the given path was - # already created. This is in accord to how "mkdir" behaves wrt existing - # files. - for folder in itertools.chain(reversed(mmz_file.parents), (mmz_file,)): - if not folder.is_dir(): - paranoia(folder) - folder.mkdir(exist_ok = True) - if not args.quiet: - print(f"{message_prefix}Created directory {folder}") - sys.exit() -elif mmz_file.suffix != '.mmz': - raise RuntimeError(f"{message_prefix}The 'mmz' argument should be a file with suffix '.mmz', not '{mmz_file}'") - -mmz_dir = mmz_file.parent -pdf_file = Path(args.pdf) if args.pdf else mmz_file.with_suffix('.pdf') -paranoia(pdf_file) -pdf = None -extern_pages = [] -new_mmz = [] -args.log is os.devnull or paranoia(Path(args.log)) -re_newextern = re.compile(r'\\mmzNewExtern *{(?P<extern_fn>(?P<prefix>.*?)(?P<code_md5sum>[0-9A-F]{32})-(?P<context_md5sum>[0-9A-F]{32})(?:-[0-9]+)?.pdf)}{(?P<page_n>[0-9]+)}{(?P<expected_width>[0-9.]*)pt}{(?P<expected_height>[0-9.]*)pt}') -tolerance = 0.01 -done_message = f"{message_prefix}Done (there was nothing to extract)." - -# Complication: I want to use 'with', but don't have to open stderr. -with open(args.log, 'w') as log: - log = sys.stderr if args.log is os.devnull else log +def info(text): + if text and not args.quiet: + print(INFO[None].format(text = text, header = header, indent = indent)) + if log: + print( + INFO[args.format].format( + text = text, texindent = texindent, package_name = package_name), + file = log) + +# Mark the log as complete and exit. +def endinput(): + if log: + print(r'\endinput', file = log) + log.close() + sys.exit(exit_code) + +# \paragraph{Permission-related functions} + +# |paranoia_in|/|out| should work exactly as +# |kpsewhich -safe-in-name|/|-safe-out-name|. +def paranoia_in(f, remark = ''): + if f and not _paranoia(f, openin_any): + error(f"I'm not allowed to read from '{f}' (openin_any = {openin_any})", + remark) + +def paranoia_out(f, remark = ''): + if f and not _paranoia(f, openout_any): + error(f"I'm not allowed to write to '{f}' (openout_any = {openout_any})", + remark) + +def _paranoia(f, mode): + # |mode| is the value of |openin_any| or |openout_any|. |f| is a + # |pathlib.Path| object. + return ( + # In mode `any' (|a|, |y| or |1|), we may access any file. + mode in 'ay1' + or ( + # Otherwise, we are at least in the restricted mode, so we should not + # open dot files on Unix-like systems (except file called |.tex|). + not (os.name == 'posix' and f.stem.startswith('.') and f.stem != '.tex') + and ( + # If we are precisely in the restricted mode (|r|, |n|, |0|), + # then there are no further restrictions. + mode in 'rn0' + # Otherwise, we are in the paranoid mode (officially |p|, but + # any other value is interpreted as |p| as well). There are + # two further restrictions in the paranoid mode. + or ( + # We're not allowed to go to a parent directory. + '..' not in f.parts + and + # If the given path is absolute, is should be a descendant + # of either |TEXMF_OUTPUT_DIRECTORY| or |TEXMFOUTPUT|. + (not f.is_absolute() + or + is_ancestor(texmf_output_directory, f) + or + is_ancestor(texmfoutput, f) + ))))) + +# On Windows, we disallow ``semi-absolute'' paths, i.e.\ paths starting with +# the |\| but lacking the drive. On Windows, |pathlib|'s |is_absolute| returns +# |True| only for paths starting with |\| and containing the drive. +def sanitize_filename(f): + if f and platform.system() == 'Windows' and not (f.is_absolute() or not f.drive): + error(f"\"Semi-absolute\" paths are disallowed: '{f}'", r"The path must " + r"either contain both the drive letter and start with '\', " + r"or none of these; paths like 'C:foo' and '\foo' are disallowed") + +def access_in(f): + return os.access(f, os.R_OK) + +# This function can fail on Windows, reporting a non-writable file or dir as +# writable, because |os.access| does not work with Windows' |icacls| +# permissions. Consequence: we might try to write to a read-only current or +# output directory instead of switching to the temporary directory. Paranoia +# is unaffected, as it doesn't use |access_*| functions. +def access_out(f): try: - mmz = mmz_file.open() - except FileNotFoundError: - print(f'''{message_prefix}File "{mmz_file}" does not exist, assuming there's nothing to do.''', - file = sys.stderr) + exists = f.exists() + # Presumably, we get this error when the parent directory is not + # executable. + except PermissionError: + return + if exists: + # An existing file should be writable, and if it's a directory, it + # should also be executable. + return os.access(f, os.W_OK) and (not f.is_dir() or os.access(f, os.X_OK)) else: - if not args.quiet: - print(f"{message_prefix}Extracting externs from {pdf_file}") + # For a non-existing file, the parent directory should be writable. + # (This is the only place where function |pathlib.parent| is used, so + # it's ok that it returns the logical parent.) + return os.access(f.parent, os.W_OK) + +# This function finds the location for an input file, respecting +# |TEXMF_OUTPUT_DIRECTORY| and |TEXMFOUTPUT|, and the permissions in the +# filesystem. It returns an absolute file as-is. For a relative file, it +# tries |TEXMF_OUTPUT_DIRECTORY| (if defined), the current directory (always), +# and |TEXMFOUTPUT| directory (if defined), in this order. The first readable +# file found is returned; if no readable file is found, the file in the current +# directory is returned. +def find_in(f): + sanitize_filename(f) + if f.is_absolute(): + return f + for df in (texmf_output_directory / f if texmf_output_directory else None, + f, + texmfoutput / f if texmfoutput else None): + if df and access_in(df): + return df + return f + +# This function finds the location for an output file, respecting +# |TEXMF_OUTPUT_DIRECTORY| and |TEXMFOUTPUT|, and the permissions in the +# filesystem. It returns an absolute file as-is. For a relative file, it +# tries |TEXMF_OUTPUT_DIRECTORY| (if defined), the current directory (unless +# |TEXMF_OUTPUT_DIRECTORY| is defined), and |TEXMFOUTPUT| directory (if +# defined), in this order. The first writable file found is returned; if no +# writable file is found, the file in either the current or the output +# directory is returned. +def find_out(f): + sanitize_filename(f) + if f.is_absolute(): + return f + for df in (texmf_output_directory / f if texmf_output_directory else None, + f if not texmf_output_directory else None, + texmfoutput / f if texmfoutput else None): + if df and access_out(df): + return df + return texmf_output_directory / f if texmf_output_directory else f + +# This function assumes that both paths are absolute; ancestor may be |None|, +# signaling a non-path. +def is_ancestor(ancestor, descendant): + if not ancestor: + return + a = ancestor.parts + d = descendant.parts + return len(a) < len(d) and a == d[0:len(a)] + +# A paranoid |Path.mkdir|. The given folder is preprocessed by |find_out|. +def mkdir(folder): + folder = find_out(Path(folder)) + if not folder.exists(): + paranoia_out(folder) + # Using |folder.mkdir| is fine because we know that + # |TEXMF_OUTPUT_DIRECTORY|/|TEXMFOUTPUT|, if given, exists, and that + # ``folder'' contains no |..|. + folder.mkdir(parents = True, exist_ok = True) + # This does not get logged when the function is invoked via |--mkdir|, + # as it is not clear what the log name should be. + info(f"Created directory {folder}") + +_re_unquote = re.compile(r'"(.*?)"') +def unquote(fn): + return _re_unquote.sub(r'\1', fn) + +# \paragraph{Kpathsea} + +# Get the values of |openin_any|, |openout_any|, |TEXMFOUTPUT| and +# |TEXMF_OUTPUT_DIRECTORY|. + +kpsewhich_output = subprocess.run(['kpsewhich', + f'-expand-var=' + f'openin_any=$openin_any,' + f'openout_any=$openout_any,' + f'TEXMFOUTPUT=$TEXMFOUTPUT'], + capture_output = True + ).stdout.decode().strip() +if not kpsewhich_output: + # No TeX? (Note that |kpsewhich| should exist in MiKTeX as well.) In + # absence of |kpathsea| information, we get very paranoid, but still try to + # get |TEXMFOUTPUT| from an environment variable. + openin_any, openout_any = 'p', 'p' + texmfoutput, texmf_output_directory = None, None + # Unfortunately, this warning can't make it into the log. But then again, + # the chances of a missing |kpsewhich| are very slim, and its absence would + # show all over the place anyway. + warning('I failed to execute "kpsewhich"; , is there no TeX system installed? ' + 'Assuming openin_any = openout_any = "p" ' + '(i.e. restricting all file operations to non-hidden files ' + 'in the current directory of its subdirectories).') +else: + m = re.fullmatch(r'openin_any=(.*),openout_any=(.*),TEXMFOUTPUT=(.*)', + kpsewhich_output) + openin_any, openout_any, texmfoutput = m.groups() + texmf_output_directory = os.environ.get('TEXMF_OUTPUT_DIRECTORY', None) + if openin_any == '$openin_any': + # When the |open*_any| variables are not expanded, we assume we're + # running MiKTeX. The two config settings below correspond to TeXLive's + # |openin_any| and |openout_any|; afaik, there is no analogue to + # |TEXMFOUTPUT|. + initexmf_output = subprocess.run( + ['initexmf', '--show-config-value=[Core]AllowUnsafeInputFiles', + '--show-config-value=[Core]AllowUnsafeOutputFiles'], + capture_output = True).stdout.decode().strip() + openin_any, openout_any = initexmf_output.split() + openin_any = 'a' if openin_any == 'true' else 'p' + openout_any = 'a' if openout_any == 'true' else 'p' + texmfoutput = None + texmf_output_directory = None + +# An output directory should exist, and may not point to the root on Linux. On +# Windows, it may point to the root, because we only allow absolute filenames +# containing the drive, e.g.\ |F:\|; see |is_absolute|. +def sanitize_output_dir(d_str): + d = Path(d_str) if d_str else None + sanitize_filename(d) + return d if d and d.is_dir() and \ + (not d.is_absolute() or len(d.parts) != 1 or d.drive) else None + +texmfoutput = sanitize_output_dir(texmfoutput) +texmf_output_directory = sanitize_output_dir(texmf_output_directory) + +class NotExtracted(UserWarning): + pass + +# We don't delve into the real script when loaded from the testing code. +if __name__ == '__main__': + + # \paragraph{Arguments} + + parser = argparse.ArgumentParser( + description = "Extract extern pages produced by package Memoize " + "out of the document PDF.", + epilog = "For details, see the man page or the Memoize documentation.", + prog = 'memoize-extract.py', + ) + parser.add_argument('-P', '--pdf', help = 'extract from file PDF') + parser.add_argument('-p', '--prune', action = 'store_true', + help = 'remove the extern pages after extraction') + parser.add_argument('-k', '--keep', action = 'store_true', + help = 'do not mark externs as extracted') + parser.add_argument('-F', '--format', choices = ['latex', 'plain', 'context'], + help = 'the format of the TeX document invoking extraction') + parser.add_argument('-f', '--force', action = 'store_true', + help = 'extract even if the size-check fails') + parser.add_argument('-q', '--quiet', action = 'store_true', + help = "describe what's happening") + parser.add_argument('-m', '--mkdir', action = 'store_true', + help = 'create a directory (and exit); ' + 'mmz argument is interpreted as directory name') + parser.add_argument('-V', '--version', action = 'version', + version = f"%(prog)s of Memoize " + __version__) + parser.add_argument('mmz', help = 'the record file produced by Memoize: ' + 'doc.mmz when compiling doc.tex ' + '(doc and doc.tex are accepted as well)') + + args = parser.parse_args() + + header = parser.prog + ': ' if args.format else '' + + # Start a new line in the TeX terminal output. + if args.format: + print() + + # \paragraph{Initialization} + + # With |--mkdir|, argument |mmz| is interpreted as the directory to create. + if args.mkdir: + mkdir(args.mmz) + sys.exit() + + # Normalize the |mmz| argument into a |.mmz| filename. + mmz_file = Path(args.mmz) + if mmz_file.suffix == '.tex': + mmz_file = mmz_file.with_suffix('.mmz') + elif mmz_file.suffix != '.mmz': + mmz_file = mmz_file.with_name(mmz_file.name + '.mmz') + + # Once we have the |.mmz| filename, we can open the log. + if args.format: + log_file = find_out(mmz_file.with_suffix('.mmz.log')) + paranoia_out(log_file) + info(f"Logging to '{log_file}'"); + log = open(log_file, 'w') + + # Now that we have opened the log file, we can try loading the PDF + # processing library. + try: + import pdfrw + except ModuleNotFoundError: + error("Python module 'pdfrw' was not found", + 'Have you followed the instructions is section 1.1 of the manual?') + + # Catch any errors in the script and output them to the log. + try: + + # Find the |.mmz| file we will read, but retain the original filename + # in |given_mmz_file|, as we will still need it. + given_mmz_file = mmz_file + mmz_file = find_in(mmz_file) + paranoia_in(mmz_file) + if not args.keep: + paranoia_out(mmz_file, + remark = 'This file is rewritten unless option --keep is given.') + try: + mmz = open(mmz_file) + except FileNotFoundError: + info(f"File '{given_mmz_file}' does not exist, " + f"assuming there's nothing to do") + endinput() + + # Determine the PDF filename: it is either given via |--pdf|, or + # constructed from the |.mmz| filename. + pdf_file = find_in(Path(args.pdf) + if args.pdf else given_mmz_file.with_suffix('.pdf')) + paranoia_in(pdf_file) + if args.prune: + paranoia_out(pdf_file, + remark = 'I would have to rewrite this file ' + 'because option --prune was given.') + + # Various initializations. + + re_prefix = re.compile(r'\\mmzPrefix *{(?P<prefix>.*?)}') + re_split_prefix = re.compile(r'(?P<dir_prefix>.*/)?(?P<name_prefix>.*?)') + re_newextern = re.compile( + r'\\mmzNewExtern *{(?P<extern_path>.*?)}{(?P<page_n>[0-9]+)}' + r'{(?P<expected_width>[0-9.]*)pt}{(?P<expected_height>[0-9.]*)pt}') + re_extern_path = re.compile( + r'(?P<dir_prefix>.*/)?(?P<name_prefix>.*?)' + r'(?P<code_md5sum>[0-9A-F]{32})-' + r'(?P<context_md5sum>[0-9A-F]{32})(?:-[0-9]+)?.pdf') + pdf = None + extern_pages = [] + new_mmz = [] + tolerance = 0.01 + dir_to_make = None + info(f"Extracting new externs listed in '{mmz_file}' from '{pdf_file}'") + done_message = "Done (there was nothing to extract)" + indent = ' ' + texindent = '\space\space ' + + # \paragraph{Process \texttt{.mmz}} + for line in mmz: - if m := re_newextern.match(line): - extern_file = mmz_dir / m['extern_fn'] - paranoia(extern_file) - page_n = int(m['page_n'])-1 - c_memo = mmz_dir / (m['prefix'] + m['code_md5sum'] + '.memo') - cc_memo = mmz_dir / (m['prefix'] + m['code_md5sum'] + '-' + m['context_md5sum'] + '.memo') - if not (c_memo.exists() and cc_memo.exists()): - print(args.warning_template.replace('\warningtext', f'Not extracting page {page_n} into extern {extern_file}, because the associated (c)c-memo does not exist'), file = log) - continue - if not pdf: - try: - pdf = pdfrw.PdfReader(pdf_file) - except pdfrw.errors.PdfParseError: - print(f'{message_prefix}File "{pdf_file}" cannot be read, bailing out.', file = sys.stderr) - print(args.warning_template.replace('\warningtext', f'Cannot read file "{pdf_file}". Perhaps you have to load Memoize earlier in the preamble?'), file = log) - args.keep = True - break - extern = pdfrw.PdfWriter(extern_file) - page = pdf.pages[page_n] - expected_width_pt, expected_height_pt = float(m['expected_width']), float(m['expected_height']) - mb = page['/MediaBox'] - width_bp, height_bp = float(mb[2]) - float(mb[0]), float(mb[3]) - float(mb[1]) - width_pt = width_bp / 72 * 72.27 - height_pt = height_bp / 72 * 72.27 - warning = None - if abs(width_pt - expected_width_pt) > tolerance \ - or abs(height_pt - expected_height_pt) > tolerance: - warning = ( - f'I refuse to extract page {page_n+1} from "{pdf_file}", ' - f'because its size ({width_pt}pt x {height_pt}pt) is not ' - f'what I expected ({expected_width_pt}pt x {expected_height_pt}pt)') - print(args.warning_template.replace('\warningtext', warning), file = log) - if warning and not args.force: - extern_file.unlink(missing_ok = True) - else: + try: + if m_p := re_prefix.match(line): + # Found |\mmzPrefix|: create the extern directory, but only + # later, if an extern file is actually produced. We parse + # the prefix in two steps because we have to unquote the + # entire prefix. + prefix = unquote(m_p['prefix']) + if not (m_sp := re_split_prefix.match(prefix)): + warning(f"Cannot parse line {line.strip()}") + dir_to_make = m_sp['dir_prefix'] + elif m_ne := re_newextern.match(line): + # Found |\mmzNewExtern|: extract the extern page into an + # extern file. + done_message = "Done" + # The extern filename, as specified in |.mmz|: + unquoted_extern_path = unquote(m_ne['extern_path']) + extern_file = Path(unquoted_extern_path) + # We parse the extern filename in a separate step because + # we have to unquote the entire path. + if not (m_ep := re_extern_path.match(unquoted_extern_path)): + warning(f"Cannot parse line {line.strip()}") + # The actual extern filename: + extern_file_out = find_out(extern_file) + paranoia_out(extern_file_out) + page_n = int(m_ne['page_n'])-1 + # Check whether c-memo and cc-memo exist (in any input + # directory). + c_memo = extern_file.with_name( + m_ep['name_prefix'] + m_ep['code_md5sum'] + '.memo') + cc_memo = extern_file.with_name( + m_ep['name_prefix'] + m_ep['code_md5sum'] + + '-' + m_ep['context_md5sum'] + '.memo') + c_memo_in = find_in(c_memo) + cc_memo_in = find_in(cc_memo) + if not (access_in(c_memo_in) and access_in(cc_memo_in)) \ + and not args.force: + warning(f"I refuse to extract page {page_n+1} into extern " + f"'{extern_file}', because the associated c-memo " + f"'{c_memo}' and/or cc-memo '{cc_memo}' " + f"does not exist") + raise NotExtracted() + # Load the PDF. We only do this now so that we don't load + # it if there is nothing to extract. + if not pdf: + if not access_in(pdf_file): + warning(f"Cannot open '{pdf_file}'") + endinput() + try: + # All safe, |paranoia_in| was already called above. + pdf = pdfrw.PdfReader(pdf_file) + except pdfrw.errors.PdfParseError as err: + error(rf"File '{pdf_file}' seems corrupted. Perhaps you " + rf"have to load Memoize earlier in the preamble", + f"In particular, Memoize must be loaded before " + f"TikZ library 'fadings' and any package " + f"deploying it, and in Beamer, load Memoize " + f"by writing \RequirePackage{{memoize}} before " + f"\documentclass{{beamer}}. " + f"This was the error thrown by Python: \n{err}") + # Does the page exist? + if page_n >= len(pdf.pages): + error(rf"I cannot extract page {page_n} from '{pdf_file}', " + rf"as it contains only {len(pdf.pages)} page" + + ('s' if len(pdf.pages) > 1 else ''), '') + # Check whether the page size matches the |.mmz| + # expectations. + page = pdf.pages[page_n] + expected_width_pt = float(m_ne['expected_width']) + expected_height_pt = float(m_ne['expected_height']) + mb = page['/MediaBox'] + width_bp = float(mb[2]) - float(mb[0]) + height_bp = float(mb[3]) - float(mb[1]) + width_pt = width_bp / 72 * 72.27 + height_pt = height_bp / 72 * 72.27 + if (abs(width_pt - expected_width_pt) > tolerance + or abs(height_pt - expected_height_pt) > tolerance) \ + and not args.force: + warning( + f"I refuse to extract page {page_n+1} from '{pdf_file}' " + f"because its size ({width_pt}pt x {height_pt}pt) " + f"is not what I expected " + f"({expected_width_pt}pt x {expected_height_pt}pt)") + raise NotExtracted() + # All tests were successful, let's create the extern file. + # First, the containing directory, if necessary. + if dir_to_make: + mkdir(dir_to_make) + dir_to_make = None + # Now the extern file. Note that |paranoia_out| was + # already called above. + info(f"Page {page_n+1} --> {extern_file_out}") + extern = pdfrw.PdfWriter(extern_file_out) extern.addpage(page) - if not args.quiet: - print(f"{message_prefix} Page {page_n+1} --> {extern_file}", file = sys.__stdout__) extern.write() - done_message = f"{message_prefix}Done." + # This page will get pruned. if args.prune: extern_pages.append(page_n) + # Comment out this |\mmzNewExtern|. if not args.keep: line = '%' + line - if not args.keep: - new_mmz.append(line) + except NotExtracted: + pass + finally: + if not args.keep: + new_mmz.append(line) mmz.close() - if not args.quiet: - print(done_message) + indent = '' + texindent = '' + info(done_message) + + # Write out the |.mmz| file with |\mmzNewExtern| lines commented + # out. (All safe, |paranoia_out| was already called above.) if not args.keep: - paranoia(mmz_file) with open(mmz_file, 'w') as mmz: for line in new_mmz: print(line, file = mmz, end = '') + + # Remove the extracted pages from the original PDF. (All safe, + # |paranoia_out| was already called above.) if args.prune and extern_pages: pruned_pdf = pdfrw.PdfWriter(pdf_file) pruned_pdf.addpages( page for n, page in enumerate(pdf.pages) if n not in extern_pages) pruned_pdf.write() - if not args.quiet: - print(f"{message_prefix}The following extern pages were pruned out of the PDF:", - ",".join(str(page+1) for page in extern_pages)) - if args.log is not os.devnull: - print(r'\endinput', file = log) + info(f"The following extern pages were pruned out of the PDF: " + + ",".join(str(page+1) for page in extern_pages)) + + # Report that extraction was successful. + endinput() + + # Catch any errors in the script and output them to the log. + except Exception as err: + error(f'Python error: {err}', traceback.format_exc()) + +# Local Variables: +# fill-column: 79 +# after-save-hook: py2dtx +# End: |