From 1060676aa889ea90c442c657b4048f6417d6f83b Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Wed, 4 Mar 2020 22:37:58 +0000 Subject: TLPSRC.pm (from_file): try harder to return the same shortdesc and longdesc strings from *.tlpsrc parsing as from tlpdb. tl-update-tlpdb (main): clear $opt_catalogue_compare in addition to $opt_fix_reverse_revisions when checking for the buggy tlpdb, since the comparison tlpdb with old data (tlnet/tlpretest) can't get updated. Also, insert commented-out kludge to dump and reread Catalogue data. (compare_and_fix_tlpdbs): compare newtlpdb to the catalogue_compare TLPDB, not oldtlpdb, although it's unlikely to make much difference. (tlpobj_catalogue_equal): show shortdesc and longdesc strings. git-svn-id: svn://tug.org/texlive/trunk@54079 c570f23f-e606-0410-a88d-b1316a301751 --- Master/tlpkg/bin/tl-update-tlpdb | 125 +++++++++++++++++++++++++-------------- 1 file changed, 82 insertions(+), 43 deletions(-) (limited to 'Master/tlpkg/bin') diff --git a/Master/tlpkg/bin/tl-update-tlpdb b/Master/tlpkg/bin/tl-update-tlpdb index 02a7a7d08a2..5c087db4c26 100755 --- a/Master/tlpkg/bin/tl-update-tlpdb +++ b/Master/tlpkg/bin/tl-update-tlpdb @@ -2,10 +2,11 @@ # $Id$ # Convert a TL tree and *.tlpsrc files into a (possibly updated) texlive.tlpdb. # -# Copyright 2007-2019 Norbert Preining +# Copyright 2007-2020 Norbert Preining # This file is licensed under the GNU General Public License version 2 # or any later version. +#use warnings FATAL => 'all'; # can help in development BEGIN { $^W = 1; chomp($mydir = `dirname $0`); @@ -97,6 +98,20 @@ sub main { if (-r $opt_catalogue) { info("$prg: reading TeX Catalogue $opt_catalogue ...\n"); $tlc = TeXLive::TeXCatalogue->new("location" => $opt_catalogue); + # way to dump the catalogue and then use it instead of reparsing. + # Maybe someday have the energy to make it a real option, + # and merge this with the same code in TLUtils as a common function. + #require "/home/karl/cat.pl"; + #if (0) { + # require Data::Dumper; + # $Data::Dumper::Indent = 1; + # $Data::Dumper::Sortkeys = 1; # stable output + # $Data::Dumper::Purity = 1; # recursive structures must be safe + # print STDERR "DDdumping tlc(atalogue)\n"; + # open (my $fh, ">/home/karl/cat.pl") || die "open(cat) failed: $!\n"; + # print $fh Data::Dumper->Dump([$tlc], [qw(::tlc)]); + # close ($fh) || die "close(cat) failed: $!\n"; + #} } else { tlwarn("$prg: reusing Catalogue data, since source is not readable: " . "$opt_catalogue\n"); @@ -106,7 +121,7 @@ sub main { } } - # Create our working tlpdb with Catalogue data. + # Create our new working tlpdb with Catalogue data. my ($newtlpdb,$pkgcount) = create_tlpdb($tlc); my $oldtlpdb = undef; @@ -115,23 +130,23 @@ sub main { $oldtlpdb = TeXLive::TLPDB->new("root" => $opt_master); } if (!defined($oldtlpdb)) { - tlwarn("$prg: No revision check since no old tlpdb: $OLDTLPDB\n"); + tlwarn("$prg: no revision check since no old tlpdb: $OLDTLPDB\n"); finish($newtlpdb, $pkgcount); } if ($opt_no_revision_check) { if ($opt_keep_revisions) { copy_revisions_over($oldtlpdb, $newtlpdb); - info("$prg: Keeping old revisions due to --keep-revisions\n"); + info("$prg: keeping old revisions due to --keep-revisions\n"); } else { - info("$prg: Not checking revisions due to --no-reverse-revision-check\n"); + info("$prg: not checking revisions due to --no-reverse-revision-check\n"); } finish($newtlpdb, $pkgcount); } - # compare_and_fix_tlpdbs returns true if any reversed revisions were found. - # If the option --fix-reverse-revisions (etc.) specify, it also tried - # to fix them, by committing changes. + # compare_and_fix_tlpdbs returns true if any reversed revisions were + # found. If the option --fix-reverse-revisions (etc.) was specified, + # it also tried to fix them, by committing changes. my $buggy_tlpdb = 0; if (compare_and_fix_tlpdbs($oldtlpdb, $newtlpdb)) { $buggy_tlpdb = 1; @@ -141,21 +156,30 @@ sub main { # # If we are running from svn (--from-files is not given), # and we tried to fix revisions (--fix-reverse-revisions is given), - # and we are committing the changes (--no_commit is not given), + # and we are committing the changes (--no-commit is not given), # then we recreate the tlpdb, else we will report as buggy. # if (!$opt_fromfiles && $opt_fix_reverse_revisions && !$opt_no_commit) { - info("$prg: Regenerating tlpdb after fixing, " - . "without -fix-reverse-revisions\n"); + info("$prg: regenerating tlpdb after fixing\n"); ($newtlpdb,$pkgcount) = create_tlpdb($tlc); # # now recheck, but do NOT try to check in changes again, since it # already failed: forcibly unset $opt_fix_reverse_revisions. + info("$prg: unsetting --fix-reverse-revisions for second check\n"); $opt_fix_reverse_revisions = 0; + # + # Also do not do catalogue checks, since the Catalogue comparison + # tree, which has old data, has not changed, so they would just + # fail again. (We should save and restore these values, or pass + # them as arguments, instead of just eradicating them, but in + # practice we do not use them again.) + info("$prg: unsetting --catalogue-compare for second check\n"); + $opt_catalogue_compare = ""; + # if (compare_and_fix_tlpdbs($oldtlpdb, $newtlpdb)) { - tlwarn("$prg: *** oops, should not happen!\n" + tlwarn("$prg: *** OOPS, should not happen!\n" . " After one iteration of compare_tlpdbs\n" - . " the new tlpdb is still going backwards in time;\n" + . " the new tlpdb still has changes;\n" . " check output above and fix!\n"); # $buggy_tlpdb is already set to 1 } else { @@ -175,7 +199,8 @@ sub main { tlwarn("$prg: saving it anyway due to --save-anyway\n"); $buggy_tlpdb = 0; } else { - tlwarn("$prg: not saving it, use --save-anyway if you want it\n"); + tlwarn("$prg: not saving it, use --save-anyway or --output " + . "if you want it\n"); } } if (!$buggy_tlpdb) { @@ -197,7 +222,7 @@ sub copy_revisions_over { if (defined($oldtlp)) { my $oldrev = $oldtlp->revision; my $newrev = $newtlpdb->get_package($pkg)->revision; - warn "xxxpkg $pkg: changing revision from $newrev to $oldrev\n"; + debug("pkg $pkg: changing revision from $newrev to $oldrev\n"); $newtlpdb->get_package($pkg)->revision($oldtlp->revision); } } @@ -206,9 +231,9 @@ sub copy_revisions_over { # Compare OLDTLPDB and NEWTLPDB, returning whether any packages needed # special fixing. Within the list of packages that differ, the -# problematic ones are those where either files were added or removed, -# or there were Catalogue changes (against --catalogue-compare) but the -# revision did not increase. +# problematic ones are those where either (a) files were added or +# removed, or (b) NEWTLPDB has Catalogue data changes compared to the +# --catalogue-compare tree, but the revision did not increase. # # If a package has been removed or added (or not changed at all) that is # fine for here, as it will be dealt with in tl-update-containers. @@ -246,7 +271,7 @@ sub compare_and_fix_tlpdbs { # since we only register 'revision' key when there is a # change, we have either rA < rB or rA > rB if ($rA > $rB) { - tlwarn("$prg: big warning for $p: file lists didn't change " + tlwarn("$prg: BIG WARNING for $p: file lists didn't change " . "but revision is going backward! Very strange!\n"); $packages_needing_fixup{$p} = 1; } # if strange revision change @@ -257,7 +282,7 @@ sub compare_and_fix_tlpdbs { # if requested, also may need to fix up packages with catalogue changs. if ($opt_catalogue_compare) { - my %cat_fixup = tlpdb_catalogue_compare($oldtlpdb, $opt_catalogue_compare); + my %cat_fixup = tlpdb_catalogue_compare($newtlpdb, $opt_catalogue_compare); # hash slice to merge %cat_fixup into %packages_needing_fixup; # all the values are 1, so that doesn't matter. @packages_needing_fixup{ keys %cat_fixup } = values %cat_fixup; @@ -270,7 +295,7 @@ sub compare_and_fix_tlpdbs { tlwarn("$prg: package $p has different files but " . "same/decreasing revision number", $opt_fix_reverse_revisions ? "\n" - : " [skipping fix since no fix_reverse_revisions]\n"); + : " [skipping fix due to no fix_reverse_revisions]\n"); if ($opt_fix_reverse_revisions) { my $fixfile = "$opt_master/tlpkg/tlpsrc/$p.tlpsrc"; # make sure that we actually fix a file of the right package @@ -281,15 +306,16 @@ sub compare_and_fix_tlpdbs { # in the packages my $tlp = $newtlpdb->get_package($p); if (!defined($tlp)) { - tlwarn("Very strange, $p is mentioned as changed files,\n"); - tlwarn("but it is not found in new tlpdb.\n"); - tlwarn("Not trying to fix anything, don't know how!\n"); + tlwarn("$prg: Very strange, $p is said to have changed files,\n"); + tlwarn("but it is not found in new tlpdb;\n"); + tlwarn("skipping $p.\n"); next; } my @allf = $tlp->all_files; if ($#allf < 0) { - tlwarn("No files found in $p, and no tlpsrc file present.\n"); - tlwarn("Don't know how to fix decreasing version in this case.\n"); + tlwarn("$prg: no files found in $p, and no tlpsrc file present.\n"); + tlwarn("don't know how to fix decreasing version in this case;\n"); + tlwarn("skipping $p.\n"); next; } # still here, so use the first file in the list. @@ -343,7 +369,7 @@ sub create_tlpdb { if (@ARGV) { # if we have cmd line arguments act as if --tlpsrc-from-cmdline was given. if (!$opt_tlpsrc_from_cmdline) { - debug("$prg: generating tlpdb in /tmp/tlut for @ARGV.\n"); + debug("$prg: generating tlpdb in /tmp/tlut from tlpsrc args: @ARGV\n"); $opt_tlpsrc_from_cmdline = 1; } } @@ -478,20 +504,20 @@ sub create_tlpdb { # of packages that so differ. # # We do not worry about packages found in one tlpdb and not the other; -# those will get found in other places. Our goal here is just to compare +# those will get dealt with elsewhere. Our goal here is just to compare # the Catalogue data in packages common to the two tlpdbs. # -# In practice, CMP_TLPDB is tlnet. This whole check is necessary because -# we intentionally do not (normally) update packages in tlnet when only -# Catalogue data has changed; that can easily lead to extra downloads -# for a single real update of the package, since the metadata often -# needs tweaking after an upload. +# In practice, CMP_TLPDB is tlnet (or tlpretest). This whole check is +# necessary because we intentionally do not (normally) update packages +# in tlnet when only Catalogue data has changed; that can easily lead to +# extra downloads for a single real update of the package, since the +# metadata often needs tweaking after an upload. # # On the other hand, we do want to get current Catalogue data into TL # sometimes, or it would grow stale forever; so that's what this # comparison is for. We enable it once a month or so from cron.tl. # -# We do insert Catalogue updates into the texlive.tlpdb checked into the +# We do update Catalogue data in the texlive.tlpdb checked into the # repository (under Master/) nightly, but that is not what users # (typically) use. # @@ -505,7 +531,7 @@ sub tlpdb_catalogue_compare { my %ret; my $cmp_tlpdb = TeXLive::TLPDB->new("root" => $cmp_tlpdb_str); - die "$prg: no tlpdb to compare under: $cmp_tlpdb_str\n" + die "$prg: could not get comparison tlpdb from: $cmp_tlpdb_str\n" if ! defined $cmp_tlpdb; for my $p ($master_tlpdb->list_packages()) { @@ -539,13 +565,19 @@ sub tlpobj_catalogue_equal { my $pkg = $tlpA->name; # just for debugging output debug("$pkg: comparing catalogue info:\n"); - # the shortdesc and longdesc usually come from the Catalogue, - # so are not in the tlpsrc. + # The shortdesc and longdesc usually come from the Catalogue, + # but sometimes are in the tlpsrc. if (! &equal_strings($tlpA->shortdesc, $tlpB->shortdesc)) { - debug("$pkg: shortdesc changed\n"); + # Maybe someday combine this silly code to avoid redundancy, + # and show strings (hashes) for all. + debug("$pkg: shortdesc changed: ", $tlpA->shortdesc, "\n", + " " x length($pkg), + " vs. ", $tlpB->shortdesc, "\n"); } elsif (! &equal_strings($tlpA->longdesc, $tlpB->longdesc)) { - debug("$pkg: longdesc changed\n"); + debug("$pkg: longdesc changed: ", $tlpA->longdesc, "\n", + " " x length($pkg), + " vs. ", $tlpB->longdesc, "\n"); } elsif (! &equal_strings($tlpA->catalogue, $tlpB->catalogue)) { debug("$pkg: catalogue value changed\n"); @@ -688,10 +720,17 @@ C<--catalogue=/nonesuch>. By default, packages for which the only change is in the Catalogue data are not marked as updated and written with new revisions. This option -changes that (except that the catalogue-date field is ignored). We -enable this from time to time to re-sync package information. This -implies C<--reverse-revision-check>, since the same mechanism is used to -force the new version numbers. +checks the Catalogue data against that in I (typically tlnet or +tlpretest), and forces the package to be updated if there are +differences. We enable this from time to time to re-sync the package +information even if the package contents have not changed. + +As a consequence, after this has been invoked the first time, the same +packages will continue to be found as different until I is +updated. Beware of confusion when testing/developing. + +This implies C<--reverse-revision-check>, since the same mechanism is +used to force the new version numbers. =item B<--fix-reverse-revisions> -- cgit v1.2.3