latexmk 4.11 (2dec09)

git-svn-id: svn://tug.org/texlive/trunk@16285 c570f23f-e606-0410-a88d-b1316a301751

1 files changed, 219 insertions, 50 deletions

diff --git a/Master/texmf-dist/scripts/latexmk/latexmk.pl b/Master/texmf-dist/scripts/latexmk/latexmk.pl
index 32db4dfbc95..0588b2efe9c 100755
--- a/Master/texmf-dist/scripts/latexmk/latexmk.pl
+++ b/Master/texmf-dist/scripts/latexmk/latexmk.pl
@@ -107,8 +107,8 @@ use warnings;
 
 $my_name = 'latexmk';
 $My_name = 'Latexmk';
-$version_num = '4.10';
-$version_details = "$My_name, John Collins, 21 August 2009";
+$version_num = '4.11';
+$version_details = "$My_name, John Collins, 27 November 2009";
 
 
 use Config;
@@ -177,6 +177,47 @@ else {
 ##
 ##   Modification log from 21 Aug 2009 onwards in detail
 ##
+##     27 Nov 2009, John Collins  Solve problem that changes in \include 
+##                                   files weren't reflected in changes
+##                                   in bibtex source files.
+##                                Remove excess recalculations of md5 in 
+##                                   startup and when initializing rule 
+##                                   from fdb_latexmk file.
+##                                Fdb_latexmk now has data for all rules
+##                                   need for making target files,
+##                                   not just rules accessible from primary
+##                                   rules.
+##     26 Nov 2009, John Collins  In testing for changes, when filetime
+##                                   has changed, but file is unchanged,
+##                                   update filetime in rule.
+##                                   This saves excessive md5 calculations.
+##     25 Nov 2009, John Collins  Fixed issue with fdb_get and 1-second
+##                                   granularity of file timestamps.
+##                                   The issue was that if runs of
+##                                   latex (etc) occur within the
+##                                   1-second time granularity, file
+##                                   contents can change even though
+##                                   the file's time and size are
+##                                   unchanged, falsifying an
+##                                   assumption made by fdb_get, but
+##                                   only when the old time is the
+##                                   same as the run time of a command
+##                                   that uses the file.
+##                                Avoided excess md5 calculations at
+##                                   startup.
+##     24 Nov 2009, John Collins  Start 4.11
+##     22 Sep 2009, John Collins  Bibtex error => message see blg file.
+##     10 Sep 2009, John Collins  When tmp view file is moved to view file,
+##                                  delete original file first.  Avoids
+##                                  bug in kpdf
+##      4 Sep 2009, John Collins  A missing file message is now suppressed 
+##                                  when -silent option used
+##                                kpsewhich is used to find non-existent 
+##                                   converted files (as with epstopdf)
+##      1 Sep 2009, John Collins  Change "last FILE" to "next FILE"
+##                                   on failure in main loop.
+##                    I had earlier changed this from "next FILE" to 
+##                    "last FILE" (for v. 4.08), but I don't see why.
 ##     21 Aug 2009, John Collins  V. 4.10 for CTAN release.  
 ##
 ##   1998-2009, John Collins.  Many improvements and fixes.
@@ -1093,6 +1134,23 @@ if (!$TEXINPUTS) { $TEXINPUTS = '.'; }
                     #      the run, so that even though the file changes during
                     #      a primary run, there is no need to trigger another 
                     #      run because of this.
+                    #  Size and md5 correspond to the values at the last run.
+                    #  But time may be updated to correspond to the time
+                    #  for the file, if the file is otherwise unchanged.
+                    #  This saves excessive md5 calculations, which would
+                    #  otherwise be done everytime the file is checked, 
+                    #  in the following situation:
+                    #     When the file has been rewritten after a run
+                    #     has started (commonly aux, bbl files etc),
+                    #     but the actual file contents haven't
+                    #     changed.  Then because the filetime has
+                    #     changed, on every file-change check latexmk
+                    #     would normally redo the md5 calculation to
+                    #     test for actual changes.  Once one such
+                    #     check is done, and the contents are
+                    #     unchanged, later checks are superfluous, and
+                    #     can be avoided by changing the file's time
+                    #     in the source-file list.
 
 %fdb_current = ();  # Fdb-hash for all files used.
 
@@ -1665,6 +1723,15 @@ foreach $filename ( @file_list )
 	    rdb_for_some( [keys %possible_primaries], \&rdb_set_from_logB );
 	}
     }
+    foreach $rule ( rdb_accessible( uniq1( keys %requested_filerules  )  ) ){
+        # For all source files of all accessible rules, 
+        #    if the file data are not already set (e.g., from fdb_latexmk
+        #    file, set them from disk.
+        rdb_one_rule ($rule, undef, 
+                      sub{ if ( $$Ptime == 0) { &rdb_update1; } }
+	);
+    }
+
     if ($go_mode) {
         # Force everything to be remade.
 	rdb_recurseA( [keys %requested_filerules], sub{$$Pout_of_date=1;}  );
@@ -1701,7 +1768,7 @@ foreach $filename ( @file_list )
     $failure = 0;
     $failure_msg = '';
     $failure = rdb_makeB( keys %requested_filerules );
-    if ($failure > 0) { last FILE; }
+    if ($failure > 0) { next FILE; }
     rdb_for_some( [keys %one_time], \&rdb_run1 );
 } # end FILE
 continue {
@@ -1898,7 +1965,9 @@ sub rdb_set_rules {
     #              "    S='$source', D='$dest', B='$base' $needs_making\n";
 	    rdb_create_rule( $rule, $cmd_type, $ext_cmd, $int_cmd, $test_kind, 
 			     $source, $dest, $base,
-			     $needs_making );
+			     $needs_making, undef, 1 );
+# !! ?? Last line was
+#			     $needs_making, undef, ($test_kind==1) );
 	}
     } # End arguments of subroutine
     &rdb_make_links;
@@ -2027,6 +2096,7 @@ sub do_viewfile {
     if ( &view_file_via_temporary ) {
         my $tmpfile = tempfile1( "${root_filename}_tmp", $ext );
         $return = &rdb_ext_cmd1( '', '', $tmpfile );
+        unlink $$Pdest;
         move( $tmpfile, $$Pdest );
     }
     else {
@@ -2323,7 +2393,7 @@ CHANGE:
   WAIT: while (1) {
            sleep($sleep_time);
 	   if ($have_break) { last WAIT; }
-           if ( rdb_have_changes(@targets) ) { 
+           if ( rdb_new_changes(@targets) ) { 
   	       if (!$silent) {
                    warn "$My_name: Need to remake files.\n";
  	           &rdb_diagnose_changes( '  ' );
@@ -2792,7 +2862,7 @@ sub parse_logB {
     my $delegated_source = "";   # If it is a file conversion, specify source
     my $delegated_output = "";   #    and output file.  (Don't put in
                                  #    data structure until block is ended.)
-    my @converted_pairs = ();
+    my %new_conversions = ();
 LINE:
     while( $line <= $#lines ) { 
         $_ = $lines[$line];
@@ -2822,8 +2892,7 @@ LINE:
 	    }
             # Block has ended.
             if ($block_type eq 'conversion') {
-                 push @converted_pairs, 
-                      [ $delegated_source, $delegated_output ];
+                 $new_conversions{$delegated_source} =  $delegated_output;
 	    }
             $current_pkg = $block_type 
                  = $delegated_source = $delegated_output = "";
@@ -3167,11 +3236,21 @@ CANDIDATE:
 	}
     }
 CANDIDATE_PAIR:
-    foreach $_ (@converted_pairs) {
-	my ($delegated_source, $delegated_output) = @$_;
+    foreach my $delegated_source (keys %new_conversions) {
+	my $delegated_output = $new_conversions{$delegated_source};
         my $rule = "Delegated $delegated_source, $delegated_output";
         # N.B. $delegated_source eq '' means the output file
         #      was created without a named input file.
+        foreach my $candidate ($delegated_source, $delegated_output) {
+            if (! -e $candidate ) {
+                # The file might be somewhere that can be found
+                #   in the search path of kpathsea:
+                my @kpse_result = kpsewhich( $candidate,);
+                if ($#kpse_result > -1) {
+                    $candidate = $kpse_result[0];
+	        }
+	    }
+	}
         if ( ( (-e $delegated_source) || ($delegated_source eq '') )
               && (-e $delegated_output) )
         {
@@ -3181,13 +3260,13 @@ CANDIDATE_PAIR:
                 $dependents{$delegated_source} = 4;
 	    }
 	}
-	else {
+	elsif (!$silent) {
             print "Logfile claimed conversion from '$delegated_source' ",
-      	          "to '$delegated_output'.  But:\n";
+        	  "to '$delegated_output'.  But:\n";
             if (! -e $delegated_output) {
 		print  "   Output file does not exist\n";
 	    }
-            if ( ($delegated_source ne '') && (! -e $delegated_output) ) {
+            if ( ($delegated_source ne '') && (! -e $delegated_source) ) {
 		print  "   Input file does not exist\n";
 	    }
 	}
@@ -3342,11 +3421,33 @@ AUX_LINE:
 #************************************************************
 
 sub fdb_get {
-    # Call: fdb_get(filename)
+    # Call: fdb_get(filename [, runtime])
     # Returns an array (time, size, md5) for the current state of the
     #    named file.
-    # For non-existent file, deletes entry in fdb_current, and returns (0,-1,0) 
-    my $file = shift;
+    # The optional argument runtime is the runtime of some command
+    #    associated with the use of the file --- see below.
+    # For non-existent file, deletes its entry in fdb_current, 
+    #    and returns (0,-1,0)  
+    # As an optimization, the md5 value is taken from the cache in 
+    #    fdb_current, if the time and size stamp indicate that the 
+    #    file has not changed.
+    # The md5 value is recalculated if
+    #    the current filetime differs from the cached value: 
+    #               file has been written
+    #    the current filesize differs from the cached value: 
+    #               file has definitely changed
+    # But the file can also be rewritten without change in filetime when 
+    #    file processing happens within the 1-second granularity of the 
+    #    timestamp (notably for aux files from latex on a short source file).
+    # The only case that concerns us is when the file is an input to a program
+    #    at some runtime t, the file is rewritten later by the same or another
+    #    program, with timestamp t, and when the initial file also has 
+    #    timestamp t.
+    # A test is applied for this situation if the runtime argument is 
+    #    supplied and is nonzero.
+
+    my ($file, $run_time) = @_;
+    if ( ! defined $run_time ) { $run_time = 0;}
     my ($new_time, $new_size) = get_time_size($file);
     my @nofile =  (0,-1,0);     # What we use for initializing
 				# a new entry in fdb or flagging
@@ -3363,19 +3464,41 @@ sub fdb_get {
     }
     my $file_data = $fdb_current{$file};
     my ( $time, $size, $md5 ) = @$file_data;
-    if ( ($new_time != $time) || ($new_size != $size) ) {
+#warn "--------- Getting MD5: $file: (N,O,R) = $new_time, $time, $run_time\n";
+#warn "--------- $file: (OT,OS,OM) = @$file_data\n";
+    if ( ($new_time != $time) || ($new_size != $size) 
+         || ( $run_time && ($run_time == $time ) )
+       ) {
         # Only force recalculation of md5 if time or size changed
+        # Or if the time equals runtime, so that the file may
+        #   have changed within the 1-second granularity of the time
         # Else we assume file is really unchanged.
         $recalculate_md5 = 1;
     }
     if ($recalculate_md5) {
+#warn "--------- RECALC MD5: $rule $file: (N,O,R) = $new_time, $time, $run_time\n";
+#warn "  ------- $file: (OT,OS,OM) = @$file_data\n";
+#&traceback;
         @$file_data = ( $new_time, $new_size, get_checksum_md5( $file ) );
+#warn "  ------- $file: (NT,NS,NM) = @$file_data\n";
     }
     return @$file_data;;
 } #END fdb_get
 
 #************************************************************
 
+sub fdb_set {
+    # Call: fdb_set(filename, $time, $size, $md5 )
+    # Set data in file data cache, i.e., %fdb_current
+    my ($file, $time, $size, $md5 ) = @_;
+    if ( ! exists $fdb_current{$file} ) {
+        $fdb_current{$file} = [0, -1, 0];
+    }
+    @{$fdb_current{$file}} = ( $time, $size, $md5 );
+} #END fdb_set
+
+#************************************************************
+
 sub fdb_show {
     # Displays contents of fdb
     foreach my $file ( sort keys %fdb_current ) {
@@ -3397,7 +3520,6 @@ sub rdb_read {
     # Returns -1 if file could not be read else number of errors.
     # Thus return value on success is 0
     my $in_name = $_[0];
-
     my $in_handle = new FileHandle;
     $in_handle->open( $in_name, '<' )
        or return ();
@@ -3458,8 +3580,11 @@ LINE:
                        $$PAnew_cmd[1] = $func_name;
                     }
                 }
+                # Set source file as non-existent.  
+                # If it existed on last run, it will be in later 
+                #    lines of the fdb file
                 rdb_create_rule( $rule, 'cusdep', '', $PAnew_cmd, 1, 
-                                 $source, $dest, $base, 0, $run_time );
+                                 $source, $dest, $base, 0, $run_time, 1 );
 	    }
             elsif ( $rule =~ /^(makeindex|bibtex)\s*(.*)$/ ) {
                 my $rule_generic = $1;
@@ -3485,8 +3610,11 @@ LINE:
 		warn "  Rule kind = '$rule_generic'; ext_cmd = '$ext_cmd';\n",
 		     "  source = '$source'; dest = '$dest'; base = '$base';\n"
                    if $diagnostics;
+                # Set source file as non-existent.  
+                # If it existed on last run, it will be in later 
+                #    lines of the fdb file
                 rdb_create_rule( $rule, $cmd_type, $ext_cmd, '', 1, 
-                                 $source, $dest, $base, 0, $run_time);
+                                 $source, $dest, $base, 0, $run_time, 1);
 	    }
             else {
   	        warn "$My_name: In file-database '$in_name' rule '$rule'\n",
@@ -3517,8 +3645,13 @@ LINE:
 		$errors++;
 		next LINE;
 	    }
-	    rdb_ensure_file( $rule, $file );
+            # Set file in database.  But ensure we don't do an unnecessary 
+            #    fdb_get, which can trigger a new MD5 calculation, which is
+            #    lengthy for a big file.  Ininitially flagging the file
+            #    as non-existent solves the problem:
+	    rdb_ensure_file( $rule, $file, undef, 1 ); 
             rdb_set_file1( $rule, $file, $time, $size, $md5 );
+            fdb_set( $file, $time, $size, $md5 );
             # Save the rest of the data, especially the from_fule until we know all 
             #   the rules, otherwise the from_rule may not exist.
             # Also we'll have a better chance of looping through files.
@@ -3636,7 +3769,10 @@ LINE:
 sub rdb_write {
     # Call: rdb_write( $out_name )
     # Writes to the given file name the database of file and rule data
-    #   accessible from the primary rules.
+    #   for all rules needed to make final output
+    # !!?? Previously was:
+    # OLD Writes to the given file name the database of file and rule data
+    # OLD   accessible from the primary rules.
     # Returns 1 on success, 0 if file couldn't be opened.
     local $out_name = $_[0];
     local $out_handle = new FileHandle;
@@ -3658,7 +3794,9 @@ sub rdb_write {
     &rdb_classify_rules( \%possible_primaries, keys %requested_filerules );
 
     print $out_handle "# Fdb version $fdb_ver\n";
-    my @rules = rdb_accessible( uniq1( keys %possible_primaries )  ) ;
+# !!??   Rules or rules accessible from primary
+#    my @rules = rdb_accessible( uniq1( keys %possible_primaries )  ) ;
+    my @rules = rdb_accessible( uniq1( keys %possible_primaries, keys %requested_filerules  )  ) ;
     # Separate call to sort.  Otherwise rdb_accessible seems to get wrong argument.
     @rules = sort( @rules );
     rdb_for_some(
@@ -3767,24 +3905,35 @@ sub rdb_set_from_logB {
         my @new_aux_files;
         &parse_aux( "$bbl_base.aux", \@new_bib_files, \@new_aux_files );
         my $from_rule = "bibtex $bbl_base";
+        print "!!!===Dealing with rule '$from_rule'\n"
+            if ($diagnostics);
         if ( ! rdb_rule_exists( $from_rule ) ){
-            print "!!!===Creating rule '$from_rule'\n"
+            print "   ===Creating rule '$from_rule'\n"
               if ($diagnostics);
             rdb_create_rule( $from_rule, 'external', $bibtex, '', 1, 
                              "$bbl_base.aux", $bbl_file, $bbl_base, 1, 0);
-            foreach my $source ( @new_bib_files, @new_aux_files ) {
-                print "  ===Source file '$source'\n"
-                  if ($diagnostics);
-                rdb_ensure_file( $from_rule, $source );
-	    }
-            print "  ===Source file '$bbl_file' for '$rule'\n"
-                  if ($diagnostics);
-            rdb_ensure_file( $rule, $bbl_file, $from_rule );
-            if ( ! -e $bbl_file ) { 
-                # Failure was non-existence of makable file
-                # Leave failure issue to other rules.
-               $failure = 0;
-	    }
+	}
+        local %old_sources = ();
+        rdb_one_rule( $from_rule, sub { %old_sources = %$PHsource; } );
+        foreach my $source ( @new_bib_files, @new_aux_files ) {
+            print "  === Source file '$source' for '$from_rule'\n"
+               if ($diagnostics);
+            rdb_ensure_file( $from_rule, $source );
+            delete $old_sources{$source};
+        }
+        if ($diagnostics>-1 ) {
+            foreach ( keys %old_sources ) {
+                print "Removing no-longer-needed dependent '$_' from rule '$from_rule'\n";
+            }
+        }        
+        rdb_remove_files( $from_rule, keys %old_sources );
+        print "  ===Source file '$bbl_file' for '$rule'\n"
+            if ($diagnostics);
+        rdb_ensure_file( $rule, $bbl_file, $from_rule );
+        if ( ! -e $bbl_file ) { 
+            # Failure was non-existence of makable file
+            # Leave failure issue to other rules.
+            $failure = 0;
 	}
     }
 
@@ -4589,7 +4738,11 @@ sub rdb_makeB1 {
     if ($return != 0) {
         $failure = 1; 
 	$$Plast_result = 2;
-        $$Plast_message = $failure_msg = "Run of rule '$rule' gave a non-zero error code";
+        if ( !$$Plast_message ) {
+            $$Plast_message = "Run of rule '$rule' gave a non-zero error code";
+	}
+# !!??        $failure_msg = $$Plast_message;
+        
     }
 }  #END rdb_makeB1
 
@@ -4759,13 +4912,15 @@ sub rdb_run1 {
         $$Plast_message = "Bug or configuration error; incorrect command type";
     }
     if ( $rule =~ /^bibtex/ ) {
-        my $retcode = &check_bibtex_log($$Pbase);
+        my $retcode = check_bibtex_log($$Pbase);
         if ($retcode == 3) {
             $$Plast_result = 2;
             $$Plast_message = "Could not open bibtex log file for '$$Pbase'";
             push @warnings, $$Plast_message;
         }
         elsif ($retcode == 2) {
+            $$Plast_message = "Bibtex errors: See file '$$Pbase.blg'";
+            push @warnings, $$Plast_message;
         }
         elsif ($retcode == 1) {
             push @warnings, "Bibtex warnings for '$$Pbase'";
@@ -4783,7 +4938,7 @@ sub rdb_run1 {
     if ( ($$Plast_result == 0) && ($return != 0) ) {
         $$Plast_result = 2;
         if ($$Plast_message eq '') {
-	    $$Plast_message = "Command gave return code $return";
+	    $$Plast_message = "Command for '$rule' gave return code $return";
 	}
     }
     elsif ( $$Pdest && (! -e $$Pdest) ) {
@@ -4940,6 +5095,10 @@ sub rdb_clear_change_record {
 sub rdb_flag_changes_here {
     # Flag changes in current rule.  
     # Assumes rule context.
+    # Optional argument: if true then fdb_get shouldn't do runtime test
+    #                    for recalculation of md5
+    local $ignore_run_time = $_[0];
+    if ( ! defined $ignore_run_time ) { $ignore_run_time = 0; }
     local $dest_mtime = 0;
     $dest_mtime = get_mtime($$Pdest) if ($$Pdest);
     rdb_do_files( \&rdb_file_change1);
@@ -4956,7 +5115,8 @@ sub rdb_file_change1 {
     # Assumes rule and file context.  Assumes $dest_mtime set.
     # Flag whether $file in $rule has changed or disappeared.
     # Set rule's make flag if there's a change.
-    my ($new_time, $new_size, $new_md5) = fdb_get($file);
+    my $run_time_argument = $ignore_run_time ? 0 : $$Prun_time;
+    my ($new_time, $new_size, $new_md5) = fdb_get($file, $run_time_argument);
 #??    print "FC1 '$rule':$file $$Pout_of_date TK=$$Ptest_kind\n"; 
 #??    print "    OLD $$Ptime, $$Psize, $$Pmd5\n",
 #??          "    New $new_time, $new_size, $new_md5\n";
@@ -4988,6 +5148,10 @@ sub rdb_file_change1 {
             $$Pout_of_date_user = 1;
 	}
     }
+    elsif ( $new_time != $$Ptime ) {
+#warn "--==-- Unchanged $file, changed time, update filetime in $rule\n";
+	$$Ptime = $new_time;
+    }
     if ( ( ($$Ptest_kind == 2) || ($$Ptest_kind == 3) )
          && (! exists $generated_exts_all{$ext} )
          && ( $new_time > $dest_mtime )
@@ -5000,11 +5164,11 @@ sub rdb_file_change1 {
 
 #************************************************************
 
-sub rdb_have_changes {
+sub rdb_new_changes {
     &rdb_clear_change_record;
-    rdb_recurseA( [@_], \&rdb_flag_changes_here );
+    rdb_recurseA( [@_], sub{ &rdb_flag_changes_here(1); } );
     return ($#changed >= 0) || ($#no_dest >= 0) || ($#rules_to_apply >= 0);
-} #END rdb_have_changes
+} #END rdb_new_changes
 
 #************************************************************
 
@@ -5334,7 +5498,7 @@ sub rdb_one_file {
 sub rdb_create_rule {
     # rdb_create_rule( rule, command_type, ext_cmd, int_cmd, test_kind,
     #                  source, dest, base, 
-    #                  needs_making, run_time )
+    #                  needs_making, run_time, set_file_not_exists )
     # int_cmd is either a string naming a perl subroutine or it is a
     # reference to an array containing the subroutine name and its
     # arguments. 
@@ -5343,10 +5507,11 @@ sub rdb_create_rule {
 # ==== Sets rule data ====
     my ( $rule, $cmd_type, $int_cmd, $PAext_cmd, $test_kind, 
          $source, $dest, $base, 
-         $needs_making, $run_time ) = @_;
+         $needs_making, $run_time, $set_file_not_exists ) = @_;
     my $changed = 0;
     # Set defaults, and normalize parameters:
-    foreach ( $cmd_type, $int_cmd, $PAext_cmd, $source, $dest, $base ) {
+    foreach ( $cmd_type, $int_cmd, $PAext_cmd, $source, $dest, $base, 
+              $set_file_not_exists ) {
         if (! defined $_) { $_ = ''; }
     }
     foreach ( $needs_making, $run_time, $test_kind ) {
@@ -5371,7 +5536,9 @@ sub rdb_create_rule {
             $changed, -1, '' ],
            {}
 	];
-    if ($source) { rdb_ensure_file( $rule, $source );  }
+    if ($source) { 
+       rdb_ensure_file( $rule, $source, undef, $set_file_not_exists );  
+    }
 } #END rdb_create_rule
 
 #************************************************************
@@ -5404,7 +5571,8 @@ sub rdb_ensure_file {
                               ${$PHsource}{$new_file} = [0, -1, 0, '', 0];
 		          }
 		          else {
-                              ${$PHsource}{$new_file} = [fdb_get($new_file), '', 0];
+                              ${$PHsource}{$new_file} 
+                              = [fdb_get($new_file, $$Prun_time), '', 0];
                	          }
 		      }
 		  }
@@ -5472,7 +5640,7 @@ sub rdb_update_filesA {
 #************************************************************
 
 sub rdb_update1 {
-    # Call: fdb_update1.  
+    # Call: rdb_update1.  
     # Assumes file context.  Updates file data to correspond to
     # current file state on disk
     ($$Ptime, $$Psize, $$Pmd5) = fdb_get($file);
@@ -5742,6 +5910,7 @@ sub get_checksum_md5 {
     my $md5 = Digest->MD5;
     my $ignore_pattern = '';
 
+#warn "======= GETTING MD5: $source\n";
     if ( $source eq "" ) { 
        # STDIN:
        open( $input, '-' );