summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/scripts/memoize
diff options
context:
space:
mode:
authorKarl Berry <karl@freefriends.org>2023-10-11 20:46:52 +0000
committerKarl Berry <karl@freefriends.org>2023-10-11 20:46:52 +0000
commitf52476c6ec7e3d1a646e83baf4b64a9999c5ecd1 (patch)
tree90b962392e9c6bb7fb2d4ffde2440c0690b9f7a3 /Master/texmf-dist/scripts/memoize
parent9a9196eacd240c7126b5660aaf6d11c4ce30b111 (diff)
memoize (11oct23)
git-svn-id: svn://tug.org/texlive/trunk@68515 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/scripts/memoize')
-rwxr-xr-xMaster/texmf-dist/scripts/memoize/memoize-clean.pl163
-rwxr-xr-xMaster/texmf-dist/scripts/memoize/memoize-clean.py159
-rwxr-xr-xMaster/texmf-dist/scripts/memoize/memoize-extract.pl205
-rwxr-xr-xMaster/texmf-dist/scripts/memoize/memoize-extract.py183
4 files changed, 710 insertions, 0 deletions
diff --git a/Master/texmf-dist/scripts/memoize/memoize-clean.pl b/Master/texmf-dist/scripts/memoize/memoize-clean.pl
new file mode 100755
index 00000000000..4e1334c6958
--- /dev/null
+++ b/Master/texmf-dist/scripts/memoize/memoize-clean.pl
@@ -0,0 +1,163 @@
+#!/usr/bin/env perl
+
+# This file is a part of Memoize, a TeX package for externalization of
+# graphics and memoization of compilation results in general, available at
+# https://ctan.org/pkg/memoize and https://github.com/sasozivanovic/memoize.
+#
+# Copyright (c) 2020- Saso Zivanovic <saso.zivanovic@guest.arnes.si>
+#
+# This work may be distributed and/or modified under the conditions of the
+# LaTeX Project Public License, either version 1.3c of this license or (at
+# your option) any later version. The latest version of this license is in
+# https://www.latex-project.org/lppl.txt and version 1.3c or later is part of
+# all distributions of LaTeX version 2008 or later.
+#
+# This work has the LPPL maintenance status `maintained'.
+# The Current Maintainer of this work is Saso Zivanovic.
+#
+# The files belonging to this work and covered by LPPL are listed in
+# <texmf>/doc/generic/memoize/FILES.
+
+use strict;
+use Getopt::Long;
+# I intend to rewrite this script using Path::Class.
+use Cwd 'realpath';
+use File::Spec;
+use File::Basename;
+
+my $VERSION = '2023/10/10 v1.0.0';
+
+my $usage = "usage: memoize-clean.py [-h] [--yes] [--all] [--quiet] [--prefix PREFIX] [mmz ...]\n";
+my $Help = <<END;
+Remove (stale) memo and extern files.
+
+positional arguments:
+ mmz .mmz record files
+
+options:
+ -h, --help show this help message and exit
+ --version, -V show version and exit
+ --yes, -y Do not ask for confirmation.
+ --all, -a Remove *all* memos and externs.
+ --quiet, -q
+ --prefix PREFIX, -p PREFIX
+ A path prefix to clean; this option can be specified multiple times.
+
+For details, see the man page or the Memoize documentation.
+END
+
+my ($yes, $all, @prefixes, $quiet, $help, $print_version);
+GetOptions(
+ "yes|y" => \$yes,
+ "all|a" => \$all,
+ "prefix|p=s" => \@prefixes,
+ "quiet|q|?" => \$quiet,
+ "help|h|?" => \$help,
+ "version|V" => \$print_version,
+ ) or die $usage;
+$help and die "$usage\n$Help";
+if ($print_version) { print("memoize-clean.pl of Memoize $VERSION\n"); exit 0 }
+
+my (%keep, %prefixes);
+
+my $curdir = Cwd::getcwd();
+
+for my $prefix (@prefixes) {
+ $prefixes{Cwd::realpath(File::Spec->catfile(($curdir), $prefix))} = '';
+}
+
+my @mmzs = @ARGV;
+
+for my $mmz (@mmzs) {
+ my ($mmz_filename, $mmz_dir) = File::Basename::fileparse($mmz);
+ @ARGV = ($mmz);
+ my $endinput = 0;
+ my $empty = -1;
+ my $prefix = "";
+ while (<>) {
+ if (/^ *$/) {
+ } elsif ($endinput) {
+ die "Bailing out, \\endinput is not the last line of file $mmz.\n";
+ } elsif (/^ *\\mmzPrefix *{(.*?)}/) {
+ $prefix = $1;
+ $prefixes{Cwd::realpath(File::Spec->catfile(($curdir,$mmz_dir), $prefix))} = '';
+ $empty = 1 if $empty == -1;
+ } elsif (/^%? *\\mmz(?:New|Used)(?:CC?Memo|Extern) *{(.*?)}/) {
+ my $fn = $1;
+ if ($prefix eq '') {
+ die "Bailing out, no prefix announced before file $fn.\n";
+ }
+ $keep{Cwd::realpath(File::Spec->catfile(($mmz_dir), $fn))} = 1;
+ $empty = 0;
+ if (rindex($fn, $prefix, 0) != 0) {
+ die "Bailing out, prefix of file $fn does not match " .
+ "the last announced prefix ($prefix).\n";
+ }
+ } elsif (/^ *\\endinput *$/) {
+ $endinput = 1;
+ } else {
+ die "Bailing out, file $mmz contains an unrecognized line: $_\n";
+ }
+ }
+ die "Bailing out, file $mmz is empty.\n" if $empty && !$all;
+ die "Bailing out, file $mmz does not end with \\endinput; this could mean that " .
+ "the compilation did not finish properly. You can only clean with --all.\n"
+ if $endinput == 0 && !$all;
+}
+
+my @tbdeleted;
+sub populate_tbdeleted {
+ my ($basename_prefix, $dir, $suffix_dummy) = @_;
+ opendir(MD, $dir) or die "Cannot open directory '$dir'";
+ while( (my $fn = readdir(MD)) ) {
+ my $path = File::Spec->catfile(($dir),$fn);
+ if ($fn =~ /\Q$basename_prefix\E[0-9A-F]{32}(?:-[0-9A-F]{32})?(?:-[0-9]+)?(\.memo|(?:-[0-9]+)?\.pdf|\.log)/ and ($all || !exists($keep{$path}))) {
+ push @tbdeleted, $path;
+ }
+ }
+ closedir(MD);
+}
+for my $prefix (keys %prefixes) {
+ my ($basename_prefix, $dir, $suffix);
+ if (-d $prefix) {
+ populate_tbdeleted('', $prefix, '');
+ }
+ populate_tbdeleted(File::Basename::fileparse($prefix));
+}
+@tbdeleted = sort(@tbdeleted);
+
+my @allowed_dirs = ($curdir);
+my @deletion_not_allowed;
+for my $f (@tbdeleted) {
+ my $f_allowed = 0;
+ for my $dir (@allowed_dirs) {
+ if ($f =~ /^\Q$dir\E/) {
+ $f_allowed = 1;
+ last;
+ }
+ }
+ push(@deletion_not_allowed, $f) if ! $f_allowed;
+}
+die "Bailing out, I was asked to delete these files outside the current directory:\n" .
+ join("\n", @deletion_not_allowed) if (@deletion_not_allowed);
+
+if (scalar(@tbdeleted) != 0) {
+ my $a;
+ unless ($yes) {
+ print("I will delete the following files:\n" .
+ join("\n",@tbdeleted) . "\n" .
+ "Proceed (y/n)? ");
+ $a = lc(<>);
+ chomp $a;
+ }
+ if ($yes || $a eq 'y' || $a eq 'yes') {
+ foreach my $fn (@tbdeleted) {
+ print "Deleting ", $fn, "\n" unless $quiet;
+ unlink $fn;
+ }
+ } else {
+ die "Bailing out.\n";
+ }
+} elsif (!$quiet) {
+ print "Nothing to do, the directory seems clean.\n";
+}
diff --git a/Master/texmf-dist/scripts/memoize/memoize-clean.py b/Master/texmf-dist/scripts/memoize/memoize-clean.py
new file mode 100755
index 00000000000..dd543327cba
--- /dev/null
+++ b/Master/texmf-dist/scripts/memoize/memoize-clean.py
@@ -0,0 +1,159 @@
+#!/usr/bin/env python
+
+# This file is a part of Memoize, a TeX package for externalization of
+# graphics and memoization of compilation results in general, available at
+# https://ctan.org/pkg/memoize and https://github.com/sasozivanovic/memoize.
+#
+# Copyright (c) 2020- Saso Zivanovic <saso.zivanovic@guest.arnes.si>
+#
+# This work may be distributed and/or modified under the conditions of the
+# LaTeX Project Public License, either version 1.3c of this license or (at
+# your option) any later version. The latest version of this license is in
+# https://www.latex-project.org/lppl.txt and version 1.3c or later is part of
+# all distributions of LaTeX version 2008 or later.
+#
+# This work has the LPPL maintenance status `maintained'.
+# The Current Maintainer of this work is Saso Zivanovic.
+#
+# The files belonging to this work and covered by LPPL are listed in
+# <texmf>/doc/generic/memoize/FILES.
+
+__version__ = '2023/10/10 v1.0.0'
+
+import argparse, re, sys, pathlib, os
+
+parser = argparse.ArgumentParser(
+ description="Remove (stale) memo and extern files.",
+ epilog = "For details, see the man page or the Memoize documentation."
+)
+parser.add_argument('--yes', '-y', action = 'store_true',
+ help = 'Do not ask for confirmation.')
+parser.add_argument('--all', '-a', action = 'store_true',
+ help = 'Remove *all* memos and externs.')
+parser.add_argument('--quiet', '-q', action = 'store_true')
+parser.add_argument('--prefix', '-p', action = 'append', default = [],
+ help = 'A path prefix to clean; this option can be specified multiple times.')
+parser.add_argument('mmz', nargs= '*', help='.mmz record files')
+parser.add_argument('--version', '-V', action = 'version',
+ version = f"%(prog)s of Memoize " + __version__)
+args = parser.parse_args()
+
+re_prefix = re.compile(r'\\mmzPrefix *{(.*?)}')
+re_memo = re.compile(r'%? *\\mmz(?:New|Used)(?:CC?Memo|Extern) *{(.*?)}')
+re_endinput = re.compile(r' *\\endinput *$')
+
+prefixes = set(pathlib.Path(prefix).resolve() for prefix in args.prefix)
+keep = set()
+
+# We loop through the given .mmz files, adding prefixes to whatever manually
+# specified by the user, and collecting the files to keep.
+for mmz_fn in args.mmz:
+ mmz = pathlib.Path(mmz_fn)
+ mmz_parent = mmz.parent.resolve()
+ try:
+ with open(mmz) as mmz_fh:
+ prefix = ''
+ endinput = False
+ empty = None
+ for line in mmz_fh:
+ line = line.strip()
+
+ if not line:
+ pass
+
+ elif endinput:
+ raise RuntimeError(
+ r'Bailing out, \endinput is not the last line of file $mmz_fn.')
+
+ elif m := re_prefix.match(line):
+ prefix = m[1]
+ prefixes.add( (mmz_parent/prefix).resolve() )
+ if empty is None:
+ empty = True
+
+ elif m := re_memo.match(line):
+ if not prefix:
+ raise RuntimeError(
+ f'Bailing out, no prefix announced before file "{m[1]}".')
+ if not m[1].startswith(prefix):
+ raise RuntimeError(
+ f'Bailing out, prefix of file "{m[1]}" does not match '
+ f'the last announced prefix ({prefix}).')
+ keep.add((mmz_parent / m[1]))
+ empty = False
+
+ elif re_endinput.match(line):
+ endinput = True
+ continue
+
+ else:
+ raise RuntimeError(fr"Bailing out, "
+ fr"file {mmz_fn} contains an unrecognized line: {line}")
+
+ if empty and not args.all:
+ raise RuntimeError(fr'Bailing out, file {mmz_fn} is empty.')
+
+ if not endinput and empty is not None and not args.all:
+ raise RuntimeError(
+ fr'Bailing out, file {mmz_fn} does not end with \endinput; '
+ fr'this could mean that the compilation did not finish properly. '
+ fr'You can only clean with --all.'
+ )
+
+ # It is not an error if the file doesn't exist.
+ # Otherwise, cleaning from scripts would be cumbersome.
+ except FileNotFoundError:
+ pass
+
+tbdeleted = []
+def populate_tbdeleted(folder, basename_prefix):
+ re_aux = re.compile(
+ re.escape(basename_prefix) +
+ '[0-9A-F]{32}(?:-[0-9A-F]{32})?(?:-[0-9]+)?(?:\.memo|(?:-[0-9]+)?\.pdf|\.log)$')
+ try:
+ for f in folder.iterdir():
+ if re_aux.match(f.name) and (args.all or f not in keep):
+ tbdeleted.append(f)
+ except FileNotFoundError:
+ pass
+
+for prefix in prefixes:
+ # "prefix" is interpreted both as a directory (if it exists) and a basename prefix.
+ if prefix.is_dir():
+ populate_tbdeleted(prefix, '')
+ populate_tbdeleted(prefix.parent, prefix.name)
+
+allowed_dirs = [pathlib.Path().absolute()] # todo: output directory
+deletion_not_allowed = [f for f in tbdeleted if not f.is_relative_to(*allowed_dirs)]
+if deletion_not_allowed:
+ raise RuntimeError("Bailing out, "
+ "I was asked to delete these files outside the current directory:\n" +
+ "\n".join(str(f) for f in deletion_not_allowed))
+
+_cwd_absolute = pathlib.Path().absolute()
+def relativize(path):
+ try:
+ return path.relative_to(_cwd_absolute)
+ except ValueError:
+ return path
+
+if tbdeleted:
+ tbdeleted.sort()
+ if not args.yes:
+ print('I will delete the following files:')
+ for f in tbdeleted:
+ print(relativize(f))
+ print("Proceed (y/n)? ")
+ a = input()
+ if args.yes or a == 'y' or a == 'yes':
+ for f in tbdeleted:
+ if not args.quiet:
+ print("Deleting", relativize(f))
+ try:
+ f.unlink()
+ except FileNotFoundError:
+ print(f"Cannot delete {f}")
+ else:
+ print("Bailing out.")
+elif not args.quiet:
+ print('Nothing to do, the directory seems clean.')
diff --git a/Master/texmf-dist/scripts/memoize/memoize-extract.pl b/Master/texmf-dist/scripts/memoize/memoize-extract.pl
new file mode 100755
index 00000000000..8fba3d356fb
--- /dev/null
+++ b/Master/texmf-dist/scripts/memoize/memoize-extract.pl
@@ -0,0 +1,205 @@
+#!/usr/bin/env perl
+
+# This file is a part of Memoize, a TeX package for externalization of
+# graphics and memoization of compilation results in general, available at
+# https://ctan.org/pkg/memoize and https://github.com/sasozivanovic/memoize.
+#
+# Copyright (c) 2020- Saso Zivanovic <saso.zivanovic@guest.arnes.si>
+#
+# This work may be distributed and/or modified under the conditions of the
+# LaTeX Project Public License, either version 1.3c of this license or (at
+# your option) any later version. The latest version of this license is in
+# https://www.latex-project.org/lppl.txt and version 1.3c or later is part of
+# all distributions of LaTeX version 2008 or later.
+#
+# This work has the LPPL maintenance status `maintained'.
+# The Current Maintainer of this work is Saso Zivanovic.
+#
+# The files belonging to this work and covered by LPPL are listed in
+# <texmf>/doc/generic/memoize/FILES.
+
+use strict;
+use Getopt::Long;
+use Path::Class;
+use File::Spec;
+use File::Basename;
+use PDF::API2;
+
+my $VERSION = '2023/10/10 v1.0.0';
+
+my $usage = "usage: memoize-extract.pl [-h] [--pdf PDF] [--prune] [--keep] [--force] [--log LOG] [--warning-template WARNING_TEMPLATE] [--quiet] [--mkdir] mmz\n";
+my $Help = <<END;
+Extract extern pages out of the document PDF.
+
+positional arguments:
+ mmz the record file produced by Memoize: doc.mmz when compiling doc.tex
+
+options:
+ --help, -h show this help message and exit
+ --version, -V show version and exit
+ --pdf PDF, -P PDF extract from file PDF
+ --prune, -p remove the extern pages after extraction
+ --keep, -k do not mark externs as extracted
+ --force, -f extract even if the size-check fails
+ --log LOG, -l LOG the log file
+ --warning-template WARNING_TEMPLATE, -w WARNING_TEMPLATE
+ \warningtext in the template will be replaced by the warning message
+ --quiet, -q don't describe what's happening
+ --embedded, -e prefix all messages to the standard output with the script name
+ --mkdir, -m create a directory (and exit)
+
+For details, see the man page or the Memoize documentation.
+END
+
+my ($pdf_arg, $prune, $keep, $log, $quiet, $embedded, $force, $mkdir, $help, $print_version);
+my $warning_template = '\warningtext';
+Getopt::Long::Configure ("bundling");
+GetOptions(
+ "pdf|P=s" => \$pdf_arg,
+ "keep|k" => \$keep,
+ "prune|p" => \$prune,
+ "log|l=s" => \$log,
+ "quiet|q" => \$quiet,
+ "embedded|e" => \$embedded,
+ "force|f" => \$force,
+ "warning-template|w=s" => \$warning_template,
+ "mkdir|m" => \$mkdir,
+ "help|h|?" => \$help,
+ "version|V" => \$print_version,
+ ) or die $usage;
+if ($help) {print("$usage\n$Help"); exit 0}
+if ($print_version) { print("memoize-extract.pl of Memoize $VERSION\n"); exit 0 }
+die $usage unless @ARGV == 1;
+
+my $message_prefix = $embedded ? basename($0) . ': ' : '';
+print("\n") if ($embedded);
+
+my @output_paths = (dir()->absolute->resolve);
+my $texmfoutput = `kpsewhich --var-value=TEXMFOUTPUT`;
+$texmfoutput =~ s/^\s+|\s+$//g;
+if ($texmfoutput) {
+ my $texmfoutput_dir = dir($texmfoutput)->absolute->resolve;
+ push(@output_paths, $texmfoutput_dir) unless $texmfoutput_dir->dir_list == 1 && ! $texmfoutput_dir->volume;
+}
+
+sub paranoia {
+ my $file = $_[0];
+ die "${message_prefix}Cannot create a hidden file or dir: $file"
+ if $file->basename =~ /^\./;
+ my $parent = $file->parent->absolute->resolve;
+ die "${message_prefix}Cannot write outside the current working or output directory tree: $file"
+ unless grep($_->contains($parent), @output_paths);
+}
+
+my $mmz_arg = $ARGV[0];
+my $mmz_file = file($mmz_arg);
+my $mmz_dir = $mmz_file->parent;
+
+if ($mkdir) {
+ my $dir = dir($mmz_arg)->cleanup;
+ my $current = dir(File::Spec->catpath($dir->volume,
+ $dir->is_absolute ? File::Spec->rootdir : File::Spec->curdir,
+ ''))->absolute;
+ for my $c ($dir->components) {
+ $current = $current->subdir($c);
+ if (-d $current) {
+ $current = $current->resolve;
+ } else {
+ paranoia($current);
+ mkdir($current);
+ print("${message_prefix}Created directory $current\n") unless $quiet;
+ }
+ }
+ exit;
+} else {
+ die "${message_prefix}The 'mmz' argument should be a file with suffix '.mmz', not '$mmz_file'\n" unless $mmz_file->basename =~ /\.mmz$/;
+}
+
+# Enable in-place editing (of the .mmz file).
+paranoia($mmz_file) unless $keep;
+$^I = "" unless $keep;
+
+my $pdf_file = $pdf_arg ? file($pdf_arg) : $mmz_dir->file($mmz_file->basename =~ s/\.mmz$/\.pdf/r)->cleanup;
+paranoia($pdf_file) if $prune;
+
+if ($log) {
+ paranoia(file($log));
+ open LOG, ">$log";
+} else {
+ *LOG = *STDERR;
+}
+
+my ($pdf, %extern_pages);
+print "${message_prefix}Extracting externs from $pdf_file:\n" unless $quiet;
+
+my $tolerance = 0.01;
+my $done_message = "${message_prefix}Done (there was nothing to extract).\n";
+
+while (<>) {
+ if (/^\\mmzNewExtern *{(?P<extern_path>(?P<prefix>.*?)(?P<code_md5sum>[0-9A-F]{32})-(?P<context_md5sum>[0-9A-F]{32})(?:-[0-9]+)?.pdf)}{(?P<page_n>[0-9]+)}{(?P<expected_width>[0-9.]*)pt}{(?P<expected_height>[0-9.]*)pt}/) {
+ my $extern_file = file($+{extern_path});
+ if (! $extern_file->is_absolute) {
+ $extern_file = $mmz_dir->file($+{extern_path});
+ }
+ paranoia($extern_file);
+ my $page = $+{page_n};
+ my $expected_width_pt = $+{expected_width};
+ my $expected_height_pt = $+{expected_height};
+ my $c_memo_file = $mmz_dir->file($+{prefix} . $+{code_md5sum} . '.memo');
+ my $cc_memo_file = $mmz_dir->file($+{prefix} . $+{code_md5sum} . '-' . $+{context_md5sum} . '.memo');
+ if (!(-e $c_memo_file && -e $cc_memo_file)) {
+ print LOG ($warning_template =~ s/\\warningtext/Not extracting page $page into extern $extern_file, because the associated (c)c-memo does not exist/gr), "\n\\endinput\n";
+ last;
+ }
+ eval { $pdf = PDF::API2->open($pdf_file->stringify) unless $pdf; };
+ if ($@) {
+ print LOG ($warning_template =~ s/\\warningtext/Cannot read file "$pdf_file". Perhaps you have to load Memoize earlier in the preamble?/gr), "\n\\endinput\n";
+ close LOG;
+ die "${message_prefix}File '$pdf_file' cannot be read, bailing out.\n";
+ }
+ my $extern = PDF::API2->new();
+ $extern->version($pdf->version);
+ $extern->import_page($pdf, $page);
+ my $extern_page = $extern->open_page(1);
+ my ($x0, $y0, $x1, $y1) = $extern_page->get_mediabox();
+ my $width_pt = ($x1 - $x0) / 72 * 72.27;
+ my $height_pt = ($y1 - $y0) / 72 * 72.27;
+ my $warning = '';
+ if (abs($width_pt - $expected_width_pt) > $tolerance
+ || abs($height_pt - $expected_height_pt) > $tolerance)
+ {
+ $warning = "I refuse to extract page $page from $pdf_file, because its size (${width_pt}pt x ${height_pt}pt) is not what I expected (${expected_width_pt}pt x ${expected_height_pt}pt)";
+ print LOG ($warning_template =~ s/\\warningtext/$warning/gr), "\n";
+ }
+ if ($warning && !$force) {
+ unlink $extern_file;
+ } else {
+ $extern->saveas($extern_file->stringify);
+ $done_message = "${message_prefix}Done.\n";
+ print STDOUT "${message_prefix} Page $page --> $extern_file\n" unless $quiet;
+ $extern_pages{$page} = 1 if $prune;
+ print("%") unless $keep;
+ }
+ }
+ print unless $keep;
+}
+
+print $done_message unless $quiet;
+
+if ($pdf and $prune) {
+ paranoia($pdf_file);
+ my $pruned_pdf = PDF::API2->new();
+ for (my $n = 1; $n <= $pdf->page_count(); $n++) {
+ if (! $extern_pages{$n}) {
+ $pruned_pdf->import_page($pdf, $n);
+ }
+ }
+ $pruned_pdf->save($pdf_file->stringify);
+ print("${message_prefix}The following extern pages were pruned out of the PDF: ",
+ join(",", sort(keys(%extern_pages))) . "\n") unless $quiet;
+}
+
+if ($log) {
+ print LOG "\\endinput\n";
+ close LOG;
+}
diff --git a/Master/texmf-dist/scripts/memoize/memoize-extract.py b/Master/texmf-dist/scripts/memoize/memoize-extract.py
new file mode 100755
index 00000000000..54d8870e7b4
--- /dev/null
+++ b/Master/texmf-dist/scripts/memoize/memoize-extract.py
@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+
+# This file is a part of Memoize, a TeX package for externalization of
+# graphics and memoization of compilation results in general, available at
+# https://ctan.org/pkg/memoize and https://github.com/sasozivanovic/memoize.
+#
+# Copyright (c) 2020- Saso Zivanovic <saso.zivanovic@guest.arnes.si>
+#
+# This work may be distributed and/or modified under the conditions of the
+# LaTeX Project Public License, either version 1.3c of this license or (at
+# your option) any later version. The latest version of this license is in
+# https://www.latex-project.org/lppl.txt and version 1.3c or later is part of
+# all distributions of LaTeX version 2008 or later.
+#
+# This work has the LPPL maintenance status `maintained'.
+# The Current Maintainer of this work is Saso Zivanovic.
+#
+# The files belonging to this work and covered by LPPL are listed in
+# <texmf>/doc/generic/memoize/FILES.
+
+__version__ = '2023/10/10 v1.0.0'
+
+import argparse, re, sys, os, pdfrw, subprocess, itertools
+from pathlib import Path
+
+parser = argparse.ArgumentParser(
+ description = "Extract extern pages out of the document PDF.",
+ epilog = "For details, see the man page or the Memoize documentation.",
+ prog = 'memoize-extract.py',
+)
+parser.add_argument('--pdf', '-P', help = 'extract from file PDF')
+parser.add_argument('--prune', '-p', action = 'store_true',
+ help = 'remove the extern pages after extraction')
+parser.add_argument('--keep', '-k', action = 'store_true',
+ help = 'do not mark externs as extracted')
+parser.add_argument('--force', '-f', action = 'store_true',
+ help = 'extract even if the size-check fails')
+parser.add_argument('--log', '-l', default = os.devnull, help = 'the log file')
+parser.add_argument('--warning-template', '-w', default = '\warningtext',
+ help = '\warningtext in the template will be replaced by the warning message')
+parser.add_argument('--quiet', '-q', action = 'store_true',
+ help = "describe what's happening")
+parser.add_argument('--embedded', '-e', action = 'store_true',
+ help = "prefix all messages to the standard output with the script name")
+parser.add_argument('--mkdir', '-m', action = 'store_true',
+ help = 'create a directory (and exit)')
+parser.add_argument('mmz',
+ help = 'the record file produced by Memoize: doc.mmz when compiling doc.tex')
+parser.add_argument('--version', '-V', action = 'version',
+ version = f"%(prog)s of Memoize " + __version__)
+
+args = parser.parse_args()
+
+message_prefix = parser.prog + ': ' if args.embedded else ''
+if args.embedded:
+ print()
+
+# Even a bit more paranoid than required:
+# (a) disallowing TEXMFOUTPUT=/ (while allowing C:\ on Windows)
+# (b) waiting for access to '-output-directory'.
+output_paths = [Path.cwd()]
+if texmfoutput := subprocess.run(
+ ['kpsewhich', '--var-value=TEXMFOUTPUT'],
+ capture_output = True).stdout.decode().strip():
+ texmfoutput_dir = Path(texmfoutput).resolve()
+ if len(texmfoutput_dir.parts) != 1 or texmfoutput_dir.drive:
+ output_paths.append(texmfoutput_dir)
+
+def paranoia(f):
+ p = Path(f).resolve()
+ if p.stem.startswith('.'):
+ raise RuntimeError(f"{message_prefix}Cannot create a hidden file or dir: {f}")
+ if not any(p.is_relative_to(op) for op in output_paths):
+ raise RuntimeError(f"{message_prefix}Cannot write outside the current working or output directory tree: {f}")
+
+mmz_file = Path(args.mmz)
+
+if args.mkdir: # Here, argument "mmz" is interpreted as the directory to create.
+ # We cannot simply say
+ # paranoia(mmz_file)
+ # mmz_file.mkdir(parents = True, exist_ok = True)
+ # because have be paranoid about the intermediate directories as well:
+ # memoize-extract.py -m a/../../BAD/../<cwd-name>/b
+ # Note that paranoia might kick in only after a part of the given path was
+ # already created. This is in accord to how "mkdir" behaves wrt existing
+ # files.
+ for folder in itertools.chain(reversed(mmz_file.parents), (mmz_file,)):
+ if not folder.is_dir():
+ paranoia(folder)
+ folder.mkdir(exist_ok = True)
+ if not args.quiet:
+ print(f"{message_prefix}Created directory {folder}")
+ sys.exit()
+elif mmz_file.suffix != '.mmz':
+ raise RuntimeError(f"{message_prefix}The 'mmz' argument should be a file with suffix '.mmz', not '{mmz_file}'")
+
+mmz_dir = mmz_file.parent
+pdf_file = Path(args.pdf) if args.pdf else mmz_file.with_suffix('.pdf')
+paranoia(pdf_file)
+pdf = None
+extern_pages = []
+new_mmz = []
+args.log is os.devnull or paranoia(Path(args.log))
+re_newextern = re.compile(r'\\mmzNewExtern *{(?P<extern_fn>(?P<prefix>.*?)(?P<code_md5sum>[0-9A-F]{32})-(?P<context_md5sum>[0-9A-F]{32})(?:-[0-9]+)?.pdf)}{(?P<page_n>[0-9]+)}{(?P<expected_width>[0-9.]*)pt}{(?P<expected_height>[0-9.]*)pt}')
+tolerance = 0.01
+done_message = f"{message_prefix}Done (there was nothing to extract)."
+
+# Complication: I want to use 'with', but don't have to open stderr.
+with open(args.log, 'w') as log:
+ log = sys.stderr if args.log is os.devnull else log
+ try:
+ mmz = mmz_file.open()
+ except FileNotFoundError:
+ print(f'''{message_prefix}File "{mmz_file}" does not exist, assuming there's nothing to do.''',
+ file = sys.stderr)
+ else:
+ if not args.quiet:
+ print(f"{message_prefix}Extracting externs from {pdf_file}")
+ for line in mmz:
+ if m := re_newextern.match(line):
+ extern_file = mmz_dir / m['extern_fn']
+ paranoia(extern_file)
+ page_n = int(m['page_n'])-1
+ c_memo = mmz_dir / (m['prefix'] + m['code_md5sum'] + '.memo')
+ cc_memo = mmz_dir / (m['prefix'] + m['code_md5sum'] + '-' + m['context_md5sum'] + '.memo')
+ if not (c_memo.exists() and cc_memo.exists()):
+ print(args.warning_template.replace('\warningtext', f'Not extracting page {page_n} into extern {extern_file}, because the associated (c)c-memo does not exist'), file = log)
+ continue
+ if not pdf:
+ try:
+ pdf = pdfrw.PdfReader(pdf_file)
+ except pdfrw.errors.PdfParseError:
+ print(f'{message_prefix}File "{pdf_file}" cannot be read, bailing out.', file = sys.stderr)
+ print(args.warning_template.replace('\warningtext', f'Cannot read file "{pdf_file}". Perhaps you have to load Memoize earlier in the preamble?'), file = log)
+ args.keep = True
+ break
+ extern = pdfrw.PdfWriter(extern_file)
+ page = pdf.pages[page_n]
+ expected_width_pt, expected_height_pt = float(m['expected_width']), float(m['expected_height'])
+ mb = page['/MediaBox']
+ width_bp, height_bp = float(mb[2]) - float(mb[0]), float(mb[3]) - float(mb[1])
+ width_pt = width_bp / 72 * 72.27
+ height_pt = height_bp / 72 * 72.27
+ warning = None
+ if abs(width_pt - expected_width_pt) > tolerance \
+ or abs(height_pt - expected_height_pt) > tolerance:
+ warning = (
+ f'I refuse to extract page {page_n+1} from "{pdf_file}", '
+ f'because its size ({width_pt}pt x {height_pt}pt) is not '
+ f'what I expected ({expected_width_pt}pt x {expected_height_pt}pt)')
+ print(args.warning_template.replace('\warningtext', warning), file = log)
+ if warning and not args.force:
+ extern_file.unlink(missing_ok = True)
+ else:
+ extern.addpage(page)
+ if not args.quiet:
+ print(f"{message_prefix} Page {page_n+1} --> {extern_file}", file = sys.__stdout__)
+ extern.write()
+ done_message = f"{message_prefix}Done."
+ if args.prune:
+ extern_pages.append(page_n)
+ if not args.keep:
+ line = '%' + line
+ if not args.keep:
+ new_mmz.append(line)
+ mmz.close()
+ if not args.quiet:
+ print(done_message)
+ if not args.keep:
+ paranoia(mmz_file)
+ with open(mmz_file, 'w') as mmz:
+ for line in new_mmz:
+ print(line, file = mmz, end = '')
+ if args.prune and extern_pages:
+ pruned_pdf = pdfrw.PdfWriter(pdf_file)
+ pruned_pdf.addpages(
+ page for n, page in enumerate(pdf.pages) if n not in extern_pages)
+ pruned_pdf.write()
+ if not args.quiet:
+ print(f"{message_prefix}The following extern pages were pruned out of the PDF:",
+ ",".join(str(page+1) for page in extern_pages))
+ if args.log is not os.devnull:
+ print(r'\endinput', file = log)