summaryrefslogtreecommitdiff
path: root/support
diff options
context:
space:
mode:
Diffstat (limited to 'support')
-rw-r--r--support/pkgcheck/CHANGES.md9
-rw-r--r--support/pkgcheck/Cargo.toml15
-rwxr-xr-xsupport/pkgcheck/bin/pkgcheckbin6203840 -> 6057000 bytes
-rw-r--r--support/pkgcheck/docs/errorsd.tex36
-rw-r--r--support/pkgcheck/docs/informationd.tex6
-rw-r--r--support/pkgcheck/docs/pkgcheck.pdfbin77796 -> 77953 bytes
-rw-r--r--support/pkgcheck/docs/title.tex2
-rw-r--r--support/pkgcheck/docs/warningsd.tex2
-rw-r--r--support/pkgcheck/src/filemagic.rs53
-rw-r--r--support/pkgcheck/src/linkcheck.rs24
-rw-r--r--support/pkgcheck/src/main.rs191
-rw-r--r--support/pkgcheck/src/messages/errorsd.rs2
-rw-r--r--support/pkgcheck/src/messages/mod.rs28
-rw-r--r--support/pkgcheck/src/utils.rs39
14 files changed, 214 insertions, 193 deletions
diff --git a/support/pkgcheck/CHANGES.md b/support/pkgcheck/CHANGES.md
index 9ef9e83474..98c911348a 100644
--- a/support/pkgcheck/CHANGES.md
+++ b/support/pkgcheck/CHANGES.md
@@ -152,3 +152,12 @@
- new error message e0040: too few top level directories in tds zip archive
- new error message e0041: map file found in path not starting with `fonts/map/dvips`
- add more test cases
+2021-10-03 (2.3.0)
+ - if a URL ends with '。', which is an ideographic full stop (U+3002), remove that character
+ - add temporary file endings .hd and tpt (pointed to by Karl Berry)
+ - add option `--ignore-same-named` to ignore same named files
+ - add `.lua` to file endings recognized as scripts
+ - add latex-firstaid-dev to exceptions when checking paths in the TDS archive
+ - ignore if generated files with ending .html or .css exist (merge request from Vít Novotný).
+ Reason is that documentation could also exist as html.
+ - some code simplifications most of them by clippy recommendations
diff --git a/support/pkgcheck/Cargo.toml b/support/pkgcheck/Cargo.toml
index 870304d75c..fa6fcc58ae 100644
--- a/support/pkgcheck/Cargo.toml
+++ b/support/pkgcheck/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "pkgcheck"
-version = "2.2.0"
+version = "2.3.0"
authors = ["Manfred Lotz <manfred@ctan.org>"]
description = "Command-line tool to check packages uploaded to CTAN."
license = "MIT/Apache-2.0"
@@ -18,20 +18,21 @@ walkdir = "2.3"
scoped_threadpool = "0.1"
num_cpus = "1"
blake2 = "0.9"
-fnv = "1.0.6"
+rustc-hash = "1.1"
unbytify = "0.2"
regex = "1"
glob = "0.3"
colored = "2"
-linkify = "0.4"
+linkify = "0.7"
-reqwest = { version = "0.10", features = ["blocking", "gzip"] }
+reqwest = { version = "0.11", features = ["blocking", "gzip"] }
+tokio = { version = "1", features = ["full"] }
lazy_static = "1"
tempfile = "3.1"
-assert_cmd = "1"
-predicates = "1"
-openssl-probe = "0.1.2"
+assert_cmd = "2"
+predicates = "2"
+openssl-probe = "0.1.4"
pest = "2.1"
pest_derive = "2.1"
escargot = "0.5"
diff --git a/support/pkgcheck/bin/pkgcheck b/support/pkgcheck/bin/pkgcheck
index 38a8dd0ae4..dbe8182c07 100755
--- a/support/pkgcheck/bin/pkgcheck
+++ b/support/pkgcheck/bin/pkgcheck
Binary files differ
diff --git a/support/pkgcheck/docs/errorsd.tex b/support/pkgcheck/docs/errorsd.tex
index db76736db0..e9b689610f 100644
--- a/support/pkgcheck/docs/errorsd.tex
+++ b/support/pkgcheck/docs/errorsd.tex
@@ -7,7 +7,7 @@ names should not contain control characters or other characters which
may have a special meaning for UNIX shells.
For more details refer to:
-\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#nounixspecialcharacters}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#nounixspecialcharacters}
+\url{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#nounixspecialcharacters}
\hypertarget{e0002----file-permissions}{%
\subsection{E0002 -\/- File
@@ -19,7 +19,7 @@ Only files that are truly executable (like scripts and binaries) should
be marked as such.
For more details refer to:
-\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#filepermissions}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#filepermissions}
+\url{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#filepermissions}
\hypertarget{e0003----readme-is-not-a-text-file}{%
\subsection{E0003 -\/- README is not a text
@@ -37,7 +37,7 @@ accepted as part of a package, neither in the package tree nor in the
TDS zip archive.
For more details refer to:
-\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noemptyfiles}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noemptyfiles}
+\url{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noemptyfiles}
\hypertarget{e0005----empty-files-not-allowed}{%
\subsection{E0005 -\/- Empty files not
@@ -47,7 +47,7 @@ Empty files are considered as rubbish, and are usually not accepted as
part of a package.
For more details refer to:
-\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noemptyfiles}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noemptyfiles}
+\url{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noemptyfiles}
\hypertarget{e0006----hidden-directories-not-allowed}{%
\subsection{E0006 -\/- Hidden directories not
@@ -57,7 +57,7 @@ A package should not contain hidden directories, neither in the package
tree nor in the TDS zip archive.
For more details refer to:
-\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles}
+\url{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles}
\hypertarget{e0007----hidden-files-not-allowed}{%
\subsection{E0007 -\/- Hidden files not
@@ -67,7 +67,7 @@ A package should not contain hidden files, neither in the package tree
nor in the TDS zip archive.
For more details refer to:
-\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles}
+\url{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles}
\hypertarget{e0008----temporary-file-detected}{%
\subsection{E0008 -\/- Temporary file
@@ -79,7 +79,7 @@ A temporary file was detected. These are typically files created by TeX
Temporary files will also be detected in a TDS zip archive.
For more details refer to:
-\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles}
+\url{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles}
\hypertarget{e0009----package-doesnt-contain-a-readme-file}{%
\subsection{E0009 -\/- Package doesn't contain a README
@@ -89,7 +89,7 @@ A package must contain at least one of \texttt{README},
\texttt{README.md} or \texttt{README.txt} file.
For more details refer to:
-\href{http://mirrors.ibiblio.org/CTAN/help/ctan/CTAN-upload-addendum.html\#readme}{http://mirrors.ibiblio.org/CTAN/help/ctan/CTAN-upload-addendum.html\#readme}
+\url{http://mirrors.ibiblio.org/CTAN/help/ctan/CTAN-upload-addendum.html\#readme}
\hypertarget{e0010----broken-symlink-detected}{%
\subsection{E0010 -\/- Broken symlink
@@ -105,7 +105,7 @@ Directories should have rwx for the owner and at least \texttt{r-x} for
others (i.e. world readable).
For more details refer to:
-\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#filepermissions}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#filepermissions}
+\url{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#filepermissions}
\hypertarget{e0012----crlf-line-endings-detected}{%
\subsection{E0012 -\/- CRLF line endings
@@ -115,7 +115,7 @@ The file specified in the error message contains CRLF line endings. Text
files should have UNIX style line endings.
For more details refer to:
-\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#crlf}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#crlf}
+\url{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#crlf}
\hypertarget{e0013----socket-special-fie-detected}{%
\subsection{E0013 -\/- Socket special fie
@@ -183,14 +183,14 @@ package which easily can be generated from other files in the
submission.
Exceptions are the \texttt{README} files of the package, i.e.
-\texttt{README}, \texttt{README.md} or \texttt{README.txt}, or
-\texttt{.pdf} files.
+\texttt{README}, \texttt{README.md} or \texttt{README.txt},
+\texttt{.pdf}, \texttt{.html}, or \texttt{.css} files.
\texttt{pkgcheck} detects generated files anywhere in the package
directory tree.
For more details refer to:
-\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#nogeneratedfiles}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#nogeneratedfiles}
+\url{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#nogeneratedfiles}
\hypertarget{e0020----unwanted-directory-detected-in-the-top-level-directory-in-tds-zip-archive}{%
\subsection{E0020 -\/- Unwanted directory detected in the top level
@@ -237,7 +237,7 @@ The TDS zip archive should have at least \texttt{r-\/-} for the owner
and at least \texttt{r-\/-} for others (i.e. world readable).
For more details refer to:
-\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#filepermissions}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#filepermissions}
+\url{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#filepermissions}
\hypertarget{e0025----duplicate-names-when-ignoring-letter-case-for-files-or-directories}{%
\subsection{E0025 -\/- Duplicate names when ignoring letter case for
@@ -249,7 +249,7 @@ As there are operating systems which do not distinguish between
a directory which are the same after converting to lower case.
For more details refer to:
-\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#filenames}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#filenames}
+\url{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#filenames}
\hypertarget{e0026----files-not-in-tds-or-different-in-tds-and-non-install-tree}{%
\subsection{E0026 -\/- Files not in TDS or different in TDS and
@@ -291,7 +291,7 @@ A README file should be either ASCII or UTF-8 without BOM(byte order
mark)
For more details refer to:
-\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#readme}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#readme}
+\url{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#readme}
\hypertarget{e0030----a-symlink-was-found-which-points-outside-of-the-package-directory-tree}{%
\subsection{E0030 -\/- A symlink was found which points outside of the
@@ -347,7 +347,7 @@ The file specified in the error message contains CR line endings. Text
files should have UNIX style line endings.
For more details refer to:
-\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#crlf}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#crlf}
+\url{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#crlf}
\hypertarget{e0038----file-has-inconsistent-line-endings-cr-x-lf-y-crlf-z}{%
\subsection{E0038 -\/- File has inconsistent line endings: CR: x, LF: y,
@@ -358,7 +358,7 @@ The file specified in the error message contains CR line endings. Text
files should have UNIX style line endings.
For more details refer to:
-\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#crlf}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#crlf}
+\url{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#crlf}
\hypertarget{e0039----no-doc-directory-found-in-the-top-level-directory-of-the-tds-zip-archive}{%
\subsection{E0039 -\/- No doc/ directory found in the top level
diff --git a/support/pkgcheck/docs/informationd.tex b/support/pkgcheck/docs/informationd.tex
index b1a870ebba..cc53641fc7 100644
--- a/support/pkgcheck/docs/informationd.tex
+++ b/support/pkgcheck/docs/informationd.tex
@@ -27,7 +27,7 @@ The file had CRLF line ending and will be corrected to have LF (Unix
like) line endings.
For more details refer to:
-\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#crlf}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#crlf}
+\url{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#crlf}
\hypertarget{i0005----corrections-permissions-for-file-or-directory}{%
\subsection{I0005 -\/- Corrections permissions for file or
@@ -37,7 +37,7 @@ directory}\label{i0005----corrections-permissions-for-file-or-directory}}
directories. It runs the \texttt{chmod} command in verbose mode.
For more details refer to:
-\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#filepermissions}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#filepermissions}
+\url{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#filepermissions}
\hypertarget{i0006----files-having-one-of-the-following-file-name-endings-are-regarded-as-temporary}{%
\subsection{I0006 -\/- Files having one of the following file name
@@ -48,7 +48,7 @@ Option -\/-show-temp-endings was used, and pkgcheck prints a list of
temporary file endings and their meanings.
For more details refer to:
-\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles}
+\url{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles}
\hypertarget{i0007----successfully-corrected-wrong-line-endings-to-lf-resp-crlf}{%
\subsection{I0007 -\/- Successfully corrected wrong line endings to LF
diff --git a/support/pkgcheck/docs/pkgcheck.pdf b/support/pkgcheck/docs/pkgcheck.pdf
index 9b8a0d7468..ede6f9fd41 100644
--- a/support/pkgcheck/docs/pkgcheck.pdf
+++ b/support/pkgcheck/docs/pkgcheck.pdf
Binary files differ
diff --git a/support/pkgcheck/docs/title.tex b/support/pkgcheck/docs/title.tex
index 0ee70ed152..86ff2b23cc 100644
--- a/support/pkgcheck/docs/title.tex
+++ b/support/pkgcheck/docs/title.tex
@@ -1 +1 @@
-\title{pkgcheck Utility, v2.2.0}
+\title{pkgcheck Utility, v2.3.0}
diff --git a/support/pkgcheck/docs/warningsd.tex b/support/pkgcheck/docs/warningsd.tex
index 416661ff00..cf21cb63d4 100644
--- a/support/pkgcheck/docs/warningsd.tex
+++ b/support/pkgcheck/docs/warningsd.tex
@@ -26,7 +26,7 @@ names like \texttt{README}, \texttt{README.txt}, \texttt{README.md},
\texttt{makefile} are ignored when checking.
For more details refer to:
-\href{http://mirror.utexas.edu/ctan/help/ctan/CTAN-upload-addendum.html\#uniquefilenames}{http://mirror.utexas.edu/ctan/help/ctan/CTAN-upload-addendum.html\#uniquefilenames}
+\url{http://mirror.utexas.edu/ctan/help/ctan/CTAN-upload-addendum.html\#uniquefilenames}
\hypertarget{w0004-----encoding-with-bom-detected}{%
\subsection{\texorpdfstring{W0004 -\/- encoding with BOM
diff --git a/support/pkgcheck/src/filemagic.rs b/support/pkgcheck/src/filemagic.rs
index aa9c50feda..f3a4bcc391 100644
--- a/support/pkgcheck/src/filemagic.rs
+++ b/support/pkgcheck/src/filemagic.rs
@@ -21,10 +21,10 @@ use unicode_bom::Bom;
#[derive(Debug, PartialEq)]
pub enum LineEnding {
- LF,
- CR,
- CRLF,
- MIXED(usize, usize, usize),
+ Lf,
+ Cr,
+ Crlf,
+ Mixed(usize, usize, usize),
}
#[derive(Debug, PartialEq)]
@@ -45,7 +45,7 @@ pub enum Mimetype {
Socket,
Zerofile,
VeryShort,
- BOM(Bom),
+ Bom(Bom),
}
pub struct Filetype {
@@ -83,7 +83,7 @@ fn _is_crlf(buffer: &[u8], len: usize) -> bool {
}
//println!("cr: {}, lf: {}", cr, lf);
- // Heuristics: we accept if only a few lines are not CRLF
+ // Heuristics: we accept if only a few lines are not Crlf
match (cr, lf) {
(0, _lf) => return false,
(_cr, 0) => return true,
@@ -120,17 +120,17 @@ fn is_crlf(buffer: &[u8], len: usize) -> LineEnding {
seen_cr = *c == CR;
}
- // println!("LF / CR / CRLF: {} / {} / {}", n_lf, n_cr, n_crlf);
+ // println!("Lf / Cr / Crlf: {} / {} / {}", n_lf, n_cr, n_crlf);
// println!("cr: {}, lf: {}, crlf: {}", n_cr, n_lf, n_crlf);
// if (n_crlf == 0 && n_cr == 0 && n_nel == 0 && n_lf == 0)
// --> no line terminators
match (n_cr, n_lf, n_crlf) {
- (0, 0, z) if z > 0 => LineEnding::CRLF,
- (x, 0, 0) if x > 0 => LineEnding::CR,
- (0, y, 0) if y > 0 => LineEnding::LF,
- (x, y, z) => LineEnding::MIXED(x, y, z),
+ (0, 0, z) if z > 0 => LineEnding::Crlf,
+ (x, 0, 0) if x > 0 => LineEnding::Cr,
+ (0, y, 0) if y > 0 => LineEnding::Lf,
+ (x, y, z) => LineEnding::Mixed(x, y, z),
}
}
@@ -194,7 +194,7 @@ impl Filetype {
let bom: Bom = Bom::from(&self.buffer[0..]);
if bom != Bom::Null {
- return Ok(Mimetype::BOM(bom));
+ return Ok(Mimetype::Bom(bom));
}
if is_binary_data(&self.buffer, bytes_read) {
@@ -231,13 +231,16 @@ impl Filetype {
let crlf = is_crlf(&self.buffer, bytes_read);
//println!("{:?}", crlf);
+ // checks for
+ // - shebang
+ // - php indicator
if bytes_read >= 5 && (self.buffer.starts_with(b"#!") || self.buffer.starts_with(b"<?php"))
{
return Ok(Mimetype::Script(crlf));
}
// recognize files as script files by file endings
- let is_script = Regex::new(r"\.(csh|ksh|py|pl|sh|tcsh)$")
+ let is_script = Regex::new(r"\.(csh|ksh|py|pl|sh|tcsh|lua)$")
.unwrap()
.is_match(fname.as_bytes());
@@ -248,13 +251,13 @@ impl Filetype {
Ok(Mimetype::Text(crlf))
}
// match (crlf, is_script) {
- // (LineEnding::LF, false) => Ok(Mimetype::Text(LineEnding::LF)),
- // (LineEnding::CR, false) => Ok(Mimetype::Text(LineEnding::CR)),
- // (LineEnding::CRLF, false) => Ok(Mimetype::Text(LineEnding::CRLF)),
- // (LineEnding::LF, true) => Ok(Mimetype::Script(LineEnding::LF)),
- // (LineEnding::CR, true) => Ok(Mimetype::Script(LineEnding::CR)),
- // (LineEnding::CRLF, true) => Ok(Mimetype::Script(LineEnding::CRLF)),
- // (_, _) => Ok(Mimetype::Text(LineEnding::LF)),
+ // (LineEnding::Lf, false) => Ok(Mimetype::Text(LineEnding::Lf)),
+ // (LineEnding::Cr, false) => Ok(Mimetype::Text(LineEnding::Cr)),
+ // (LineEnding::Crlf, false) => Ok(Mimetype::Text(LineEnding::Crlf)),
+ // (LineEnding::Lf, true) => Ok(Mimetype::Script(LineEnding::Lf)),
+ // (LineEnding::Cr, true) => Ok(Mimetype::Script(LineEnding::Cr)),
+ // (LineEnding::Crlf, true) => Ok(Mimetype::Script(LineEnding::Crlf)),
+ // (_, _) => Ok(Mimetype::Text(LineEnding::Lf)),
// }
}
}
@@ -396,11 +399,11 @@ fn test_filetype() {
// This file is a pdf but has lines starting with % before the pdf signature shows up
// The unix `file` command) says: data
- // analyze() says TextCRLF
+ // analyze() says TextCrlf
//assert!(ft.analyze("tests_filemagic/musterlogo.pdf").ok() == Some(Mimetype::Script));
- assert!(ft.analyze("tests_filemagic/x.pl").ok() == Some(Mimetype::Script(LineEnding::LF)));
- assert!(ft.analyze("tests_filemagic/main.php").ok() == Some(Mimetype::Script(LineEnding::LF)));
+ assert!(ft.analyze("tests_filemagic/x.pl").ok() == Some(Mimetype::Script(LineEnding::Lf)));
+ assert!(ft.analyze("tests_filemagic/main.php").ok() == Some(Mimetype::Script(LineEnding::Lf)));
assert!(ft.analyze("tests_filemagic/test.7z").ok() == Some(Mimetype::Archive));
assert!(ft.analyze("tests_filemagic/x.tgz").ok() == Some(Mimetype::Archive));
@@ -417,7 +420,7 @@ fn test_filetype() {
assert!(
ft.analyze("tests_filemagic/README").ok()
- == Some(Mimetype::Text(LineEnding::MIXED(0, 0, 0)))
+ == Some(Mimetype::Text(LineEnding::Mixed(0, 0, 0)))
);
// assert!(ft.analyze("tests_filemagic/README1").ok() == Some(Mimetype::Text));
@@ -434,6 +437,6 @@ fn test_filetype() {
assert!(
ft.analyze("tests_filemagic/8stbu11h.htm").ok()
- == Some(Mimetype::Text(LineEnding::MIXED(0, 1, 8710)))
+ == Some(Mimetype::Text(LineEnding::Mixed(0, 1, 8710)))
);
}
diff --git a/support/pkgcheck/src/linkcheck.rs b/support/pkgcheck/src/linkcheck.rs
index d7d1671dcf..cc7aa36e31 100644
--- a/support/pkgcheck/src/linkcheck.rs
+++ b/support/pkgcheck/src/linkcheck.rs
@@ -15,8 +15,7 @@ use std::sync::Mutex;
use std::sync::atomic::Ordering;
-use std::collections::HashMap;
-use std::collections::HashSet;
+use rustc_hash::{FxHashMap, FxHashSet};
enum UrlStatus {
Unknown,
@@ -25,11 +24,11 @@ enum UrlStatus {
}
struct HashVal {
- paths: HashSet<String>,
+ paths: FxHashSet<String>,
status: UrlStatus,
}
-type UrlHash = HashMap<String, HashVal>;
+type UrlHash = FxHashMap<String, HashVal>;
pub struct LinkCheck {
pool: Mutex<ThreadPool>,
@@ -43,7 +42,7 @@ impl LinkCheck {
let pool = Mutex::new(ThreadPool::new(num_threads));
LinkCheck {
pool,
- urlhash: Arc::new(Mutex::new(HashMap::default())),
+ urlhash: Arc::new(Mutex::new(FxHashMap::default())),
print_all,
}
}
@@ -86,7 +85,7 @@ fn check_link(url: &str, fname: &str, urlhash: &Arc<Mutex<UrlHash>>, print_all:
let mut urlhash = urlhash.lock().unwrap();
if !urlhash.contains_key(&url) {
- let mut hs = HashSet::default();
+ let mut hs = FxHashSet::default();
hs.insert(f);
let url1 = url.clone();
@@ -166,6 +165,8 @@ fn get_links(fname: &str) -> Option<Vec<String>> {
fn get_links_inner(s: &str) -> Option<Vec<String>> {
let mut finder = LinkFinder::new();
finder.kinds(&[LinkKind::Url]);
+
+ // finder.links() does the actual search for URLs
let links: Vec<_> = finder.links(s).collect();
let result: Vec<&str> = links.iter().map(|e| e.as_str()).collect();
@@ -174,12 +175,11 @@ fn get_links_inner(s: &str) -> Option<Vec<String>> {
if !r.starts_with("http://") && !r.starts_with("https://") && !r.starts_with("ftp://") {
continue;
}
- // This is a workaround to prevent URLs ending with `
- if r.ends_with('`') {
- links.push(String::from(&r[..r.len() - 1]));
- } else {
- links.push(String::from(r));
- }
+
+ // This is a workaround to prevent URLs ending with certain characters
+ let url = r.trim_end_matches(|c| c == '。' || c == '`');
+
+ links.push(String::from(url));
}
if !links.is_empty() {
Some(links)
diff --git a/support/pkgcheck/src/main.rs b/support/pkgcheck/src/main.rs
index d4ef7d6db2..bb8e90b76e 100644
--- a/support/pkgcheck/src/main.rs
+++ b/support/pkgcheck/src/main.rs
@@ -50,7 +50,8 @@ use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering};
use blake2::{Blake2b, Digest};
-use fnv::{FnvHashMap as HashMap, FnvHashSet as HashSet};
+use rustc_hash::{FxHashMap, FxHashSet};
+
use std::sync::mpsc::{channel, Sender};
use structopt::clap::Shell;
@@ -58,7 +59,7 @@ use structopt::StructOpt;
#[cfg(unix)]
use walkdir::{DirEntry, WalkDir};
-fn err(path: &PathBuf, err: &io::Error) {
+fn err(path: &Path, err: &io::Error) {
e0027!(path.display(), err);
}
@@ -68,15 +69,15 @@ type HashSender = Sender<(u64, PathBuf, Vec<u8>)>;
// SizesHashMap contains
// - file sizes
// - and a vector of file names having that size
-type SizesHashMap = HashMap<u64, Vec<PathBuf>>;
+type SizesHashMap = FxHashMap<u64, Vec<PathBuf>>;
-type GeneratedHashMap = HashMap<String, String>;
+type GeneratedHashMap = FxHashMap<String, String>;
-type FileNamesHashMap = HashMap<PathBuf, (Metadata, String, ReadmeKind)>;
+type FileNamesHashMap = FxHashMap<PathBuf, (Metadata, String, ReadmeKind)>;
const BLOCKSIZE: usize = 4096;
-fn hash_file_inner(path: &PathBuf) -> io::Result<Vec<u8>> {
+fn hash_file_inner(path: &Path) -> io::Result<Vec<u8>> {
let mut buf = [0u8; BLOCKSIZE];
let mut fp = File::open(&path)?;
let mut digest = Blake2b::default();
@@ -141,7 +142,7 @@ fn check_readme(dir_entry: &str, is_readme: &ReadmeKind, ft: &filemagic::Mimetyp
e0003!(msg_name);
return false;
}
- filemagic::Mimetype::BOM(b) => {
+ filemagic::Mimetype::Bom(b) => {
e0029!(msg_name, b.as_ref());
return false;
}
@@ -177,10 +178,7 @@ fn check_readme_inner(fname: &str, f: &std::fs::File) -> bool {
}
fn is_readme(entry: &str) -> bool {
- match entry {
- "README" | "README.txt" | "README.md" => true,
- _ => false,
- }
+ matches!(entry, "README" | "README.txt" | "README.md")
}
fn get_devno(meta: &Metadata) -> u64 {
@@ -200,6 +198,8 @@ fn _get_devno(entry: &DirEntry) -> u64 {
struct Args {
#[structopt(short = "I", long = "ignore-dupes", help = "Ignore dupes")]
ignore_dupes: bool,
+ #[structopt(long = "ignore-same-named", help = "Ignore same-named files")]
+ ignore_same_named: bool,
#[structopt(short = "v", long = "verbose", help = "Verbose operation?")]
verbose: bool,
#[structopt(short = "L", long = "correct-le", help = "Correct line endings")]
@@ -240,12 +240,12 @@ struct Args {
// We take care to avoid visiting a single inode twice,
// which takes care of (false positive) hardlinks.
#[cfg(unix)]
-fn check_inode(set: &mut HashSet<(u64, u64)>, meta: &Metadata) -> bool {
+fn check_inode(set: &mut FxHashSet<(u64, u64)>, meta: &Metadata) -> bool {
set.insert((get_devno(meta), meta.ino()))
}
#[cfg(not(unix))]
-fn check_inode(_: &mut HashSet<u64>, _: &Metadata) -> bool {
+fn check_inode(_: &mut FxHashSet<u64>, _: &Metadata) -> bool {
true
}
@@ -309,13 +309,13 @@ impl DupPath {
// Args::clap().gen_completions(env!("CARGO_PKG_NAME"), Shell::Fish, "target");
// }
-type DupHashes = HashMap<(u64, Vec<u8>), DupPath>;
+type DupHashes = FxHashMap<(u64, Vec<u8>), DupPath>;
fn main() {
match &ARGS.explain {
None => (),
Some(e) => {
- explains(&e);
+ explains(e);
process::exit(0);
}
}
@@ -374,12 +374,12 @@ fn main() {
None => None,
Some(tz) => {
let pn = check_tds_archive_name(tds_zip);
- if !exists_file(&tz) {
+ if !exists_file(tz) {
f0003!(&tz);
process::exit(1);
}
let mut fmagic = filemagic::Filetype::new();
- match fmagic.analyze(&tz) {
+ match fmagic.analyze(tz) {
Ok(filemagic::Mimetype::Zip) => (),
_ => {
f0004!(&tz);
@@ -528,12 +528,12 @@ fn check_generated_files(entry: &str, generated: &mut GeneratedHashMap) {
// a package we ignore if a README was generated by an
// .ins or .dtx file
// CAVEAT: If this happens in a subdirectory it could be an error!!!!
- if is_readme(&filename) {
+ if is_readme(filename) {
continue;
}
- // Ignore generated pdf files
- if fname.ends_with(".pdf") {
+ // Ignore generated pdf, html, and css files
+ if fname.ends_with(".pdf") || fname.ends_with(".html") || fname.ends_with(".css") {
continue;
}
@@ -568,7 +568,7 @@ fn check_tds_archive_name(tds_zip: &Option<String>) -> Option<String> {
f0005!(tz);
process::exit(1);
}
- let mut pname = String::from(utils::basename(&tz));
+ let mut pname = String::from(utils::basename(tz));
let plen = pname.len();
pname.truncate(plen - 8);
Some(pname)
@@ -577,7 +577,7 @@ fn check_tds_archive_name(tds_zip: &Option<String>) -> Option<String> {
}
fn unzip_tds_archive(tds_zip: &str, tmp_dir: &str) {
- match run_cmd("unzip", &["-q", "-d", tmp_dir, &tds_zip]) {
+ match run_cmd("unzip", &["-q", "-d", tmp_dir, tds_zip]) {
CmdReturn { status: true, .. } => (),
CmdReturn {
status: false,
@@ -597,16 +597,16 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) {
i0003!(tds_zip);
let dir_entry = Path::new(tds_zip);
- let p = get_perms(&dir_entry);
+ let p = get_perms(dir_entry);
if !owner_has(p, 4) || !others_have(p, 4) || x_bit_set(p) {
e0024!(tds_zip, perms_to_string(p));
if ARGS.correct_perms {
i0005!(&tds_zip);
- set_perms(&tds_zip, 0o664);
+ set_perms(tds_zip, 0o664);
}
};
- let ut = Utils::new(utils::CheckType::TDS);
+ let ut = Utils::new(utils::CheckType::Tds);
// We have a discrepancy between
// TDS zip archive name: babel-base.tds.zip
@@ -616,17 +616,18 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) {
// Therefore, we correct here.
// latex-tools, latex-graphics, latex-amsmath, latex-base => latex
// latex-tools-dev, latex-graphics-dev, latex-amsmath-dev, latex-base-dev => latex-dev
- let mut pkg_replacements: HashMap<&str, &str> = HashMap::default();
+ let mut pkg_replacements: FxHashMap<&str, &str> = FxHashMap::default();
pkg_replacements.insert("babel-base", "babel");
pkg_replacements.insert("latex-base", "latex");
pkg_replacements.insert("latex-tools", "latex");
pkg_replacements.insert("latex-graphics", "latex");
pkg_replacements.insert("latex-amsmath", "latex");
+ pkg_replacements.insert("latex-firstaid", "latex/firstaid");
pkg_replacements.insert("latex-base-dev", "latex-dev");
pkg_replacements.insert("latex-tools-dev", "latex-dev");
pkg_replacements.insert("latex-graphics-dev", "latex-dev");
pkg_replacements.insert("latex-amsmath-dev", "latex-dev");
- pkg_replacements.insert("latex-firstaid", "latex/firstaid");
+ pkg_replacements.insert("latex-firstaid-dev", "latex-dev/firstaid");
let real_pkg_name = if let Some(real_name) = pkg_replacements.get(pkg_name) {
real_name
} else {
@@ -648,7 +649,7 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) {
// tds zip archive we need to checksum the files in the tds zip
// archive.
- let mut sizes: SizesHashMap = HashMap::default();
+ let mut sizes: SizesHashMap = FxHashMap::default();
let mut pool = Pool::new(num_cpus::get() as u32 + 1);
{
// Processing a single file entry, with the "sizes" hashmap collecting
@@ -668,7 +669,7 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) {
// those top level directories are the directories found in the
// texmf-dist/ directory of a texlive installation
- let tds_toplevel_dirs: HashSet<String> = [
+ let tds_toplevel_dirs: FxHashSet<String> = [
"asymptote",
"bibtex",
"chktex",
@@ -742,7 +743,7 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) {
&dir_entry_str[tmp_dir_offset..]
};
- ut.check_for_temporary_file(&dir_entry_display);
+ ut.check_for_temporary_file(dir_entry_display);
// In the top level directory of a TDS zip archive
// ... no files are allowed
@@ -764,8 +765,8 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) {
}
if ft == FType::Directory {
- ut.check_for_empty_directory(&dir_entry_str, &dir_entry_display);
- ut.check_for_hidden_directory(&file_name, &dir_entry_display);
+ ut.check_for_empty_directory(dir_entry_str, dir_entry_display);
+ ut.check_for_hidden_directory(&file_name, dir_entry_display);
ut.is_unwanted_directory(&file_name, dir_entry_str);
continue;
@@ -777,12 +778,12 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) {
match (pkg_name, dir_entry_display) {
("latex-tools", "tex/latex/tools/.tex") => (),
("latex-tools-dev", "tex/latex-dev/tools/.tex") => (),
- (_, _) => ut.check_for_hidden_file(&file_name, &dir_entry_display),
+ (_, _) => ut.check_for_hidden_file(&file_name, dir_entry_display),
};
let fsize = meta.len();
process(fsize, &dir_entry);
- ut.check_filesize(fsize, &dir_entry_display);
+ ut.check_filesize(fsize, dir_entry_display);
// if we encounter a .dtx or .ins file we check
// that it is in a subdirectory of either source/ or doc/
@@ -795,10 +796,10 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) {
}
// if the path doesn't contain a man page...
- if dir_entry_str.find("/man/").is_none() {
+ if !dir_entry_str.contains("/man/") {
let pkg_name_s = format!("/{}/", real_pkg_name);
// ...then we want to have the package name in the path
- if dir_entry_str.find(&pkg_name_s).is_none() {
+ if !dir_entry_str.contains(&pkg_name_s) {
e0028!(real_pkg_name, dir_entry_display);
}
}
@@ -828,7 +829,7 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) {
}
};
- let mut tds_hashes: HashMap<(u64, Vec<u8>), Vec<PathBuf>> = HashMap::default();
+ let mut tds_hashes: FxHashMap<(u64, Vec<u8>), Vec<PathBuf>> = FxHashMap::default();
pool.scoped(|scope| {
let (tx, rx) = channel();
@@ -909,7 +910,7 @@ fn check_and_correct_perms(dir_entry: &str, p: u32) {
e0002!(dir_entry, perms_to_string(p));
if ARGS.correct_perms {
i0005!(&dir_entry);
- set_perms(&dir_entry, 0o664);
+ set_perms(dir_entry, 0o664);
}
};
}
@@ -955,13 +956,13 @@ enum ReadmeKind {
}
fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> {
- let mut lcnames: HashMap<PathBuf, Vec<(PathBuf, FileKind)>> = HashMap::default();
+ let mut lcnames: FxHashMap<PathBuf, Vec<(PathBuf, FileKind)>> = FxHashMap::default();
- let mut doublenames: HashMap<PathBuf, Vec<PathBuf>> = HashMap::default();
+ let mut doublenames: FxHashMap<PathBuf, Vec<PathBuf>> = FxHashMap::default();
- let mut inodes = HashSet::default();
+ let mut inodes = FxHashSet::default();
- let ut = Utils::new(utils::CheckType::PACKAGE);
+ let ut = Utils::new(utils::CheckType::Package);
i0002!(root);
// This hash contains all package file names.
@@ -972,7 +973,7 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> {
// ReadmeKind: is it a certain README, file or symlink?
// A special case of a README file is a file with has a different name but
// was pointed to by a symlink. Example: README --> README.rst
- let mut file_names: FileNamesHashMap = HashMap::default();
+ let mut file_names: FileNamesHashMap = FxHashMap::default();
let mut readme_found = false;
let root_absolute = PathBuf::from(root)
@@ -1010,7 +1011,7 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> {
// we check for weird stuff like socket files aso.
let ft = get_filetype(&dir_entry);
- if found_unwanted_filetype(&dir_entry_str, &ft) {
+ if found_unwanted_filetype(dir_entry_str, &ft) {
continue;
}
@@ -1036,7 +1037,7 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> {
}
let lc_dir_entry_str = dir_entry_str.to_lowercase();
- if let Some(_dir_name) = filename(&dir_entry_str) {
+ if let Some(_dir_name) = filename(dir_entry_str) {
let lcnref = &mut lcnames;
lcnref
.entry(PathBuf::from(lc_dir_entry_str))
@@ -1063,44 +1064,44 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> {
}
}
- let p = get_perms(&dir_entry.path());
+ let p = get_perms(dir_entry.path());
// 2. dealing with directories
if ft == FType::Directory {
let lc_dir_entry_str = dir_entry_str.to_lowercase();
- if let Some(_dir_name) = filename(&dir_entry_str) {
+ if let Some(_dir_name) = filename(dir_entry_str) {
let lcnref = &mut lcnames;
lcnref
.entry(PathBuf::from(lc_dir_entry_str))
.or_insert_with(Vec::new)
- .push((PathBuf::from(&dir_entry_str), FileKind::Directory));
+ .push((PathBuf::from(dir_entry_str), FileKind::Directory));
}
if !owner_has(p, 5) || !others_have(p, 5) {
e0011!(&dir_entry_str, perms_to_string(p));
if ARGS.correct_perms {
i0005!(&dir_entry_str);
- set_perms(&dir_entry_str, 0o775);
+ set_perms(dir_entry_str, 0o775);
}
}
- ut.check_for_empty_directory(&dir_entry_str, &dir_entry_str);
- ut.check_for_hidden_directory(&file_name, &dir_entry_str);
+ ut.check_for_empty_directory(dir_entry_str, dir_entry_str);
+ ut.check_for_hidden_directory(&file_name, dir_entry_str);
ut.is_unwanted_directory(&file_name, dir_entry_str);
continue;
}
// 3. dealing with regular files
- ut.check_for_hidden_file(&file_name, &dir_entry_str);
- ut.check_for_temporary_file(&dir_entry_str);
+ ut.check_for_hidden_file(&file_name, dir_entry_str);
+ ut.check_for_temporary_file(dir_entry_str);
// if is_temporary_file(&dir_entry_str) {
// e0008!(&dir_entry_str);
// }
- if let Some(file_name) = filename(&dir_entry_str) {
+ if let Some(file_name) = filename(dir_entry_str) {
let doubleref = &mut doublenames;
doubleref
@@ -1164,8 +1165,8 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> {
let lc = LinkCheck::new(4, false);
let mut detective = filemagic::Filetype::new();
- let mut sizes: SizesHashMap = HashMap::default();
- let mut generated: GeneratedHashMap = HashMap::default();
+ let mut sizes: SizesHashMap = FxHashMap::default();
+ let mut generated: GeneratedHashMap = FxHashMap::default();
// Processing a single file entry, with the "sizes" hashmap collecting
// same-size files. Entries are either Found::One or Found::Multiple,
@@ -1190,13 +1191,13 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> {
let fsize = meta.len();
ut.check_filesize(fsize, dir_entry_str);
- let p = get_perms(&path);
+ let p = get_perms(path);
if !owner_has(p, 4) {
e0002!(&dir_entry_str, perms_to_string(p));
- fix_perms(&dir_entry_str);
+ fix_perms(dir_entry_str);
continue;
}
- let ftr = detective.analyze(&dir_entry_str);
+ let ftr = detective.analyze(dir_entry_str);
//println!(">>> {:?}", ftr);
// we ignore errors from filetype recognition
if ftr.is_err() {
@@ -1207,87 +1208,87 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> {
// DEBUG !readme_symlinked.contains(&dir_entry_str)
if ReadmeKind::No != *is_readme {
- if !check_readme(&dir_entry_str, is_readme, &ft) {
+ if !check_readme(dir_entry_str, is_readme, &ft) {
continue;
}
if ARGS.urlcheck {
- lc.check_urls(&dir_entry_str);
+ lc.check_urls(dir_entry_str);
}
}
match ft {
filemagic::Mimetype::Text(_) => {
- check_and_correct_perms(&dir_entry_str, p);
- let fext = get_extension_from_filename(&dir_entry_str);
+ check_and_correct_perms(dir_entry_str, p);
+ let fext = get_extension_from_filename(dir_entry_str);
if fext == Some("ins") || fext == Some("dtx") {
- check_generated_files(&dir_entry_str, &mut generated);
+ check_generated_files(dir_entry_str, &mut generated);
}
match fext {
// deal with Windows files
Some("bat") | Some("cmd") | Some("nsh") | Some("reg") => match ft {
- filemagic::Mimetype::Text(LineEnding::CRLF) => (),
- filemagic::Mimetype::Text(LineEnding::CR) => {
+ filemagic::Mimetype::Text(LineEnding::Crlf) => (),
+ filemagic::Mimetype::Text(LineEnding::Cr) => {
e0037!(&dir_entry_str);
if ARGS.correct_le {
- make_crlf(&dir_entry_str);
+ make_crlf(dir_entry_str);
}
}
- filemagic::Mimetype::Text(LineEnding::MIXED(0, 0, 0)) => (),
- filemagic::Mimetype::Text(LineEnding::MIXED(cr, lf, crlf)) => {
+ filemagic::Mimetype::Text(LineEnding::Mixed(0, 0, 0)) => (),
+ filemagic::Mimetype::Text(LineEnding::Mixed(cr, lf, crlf)) => {
e0038!(&dir_entry_str, cr, lf, crlf);
if ARGS.correct_le {
- fix_inconsistent_le(&dir_entry_str);
+ fix_inconsistent_le(dir_entry_str);
}
}
- filemagic::Mimetype::Text(LineEnding::LF) => {
+ filemagic::Mimetype::Text(LineEnding::Lf) => {
w0008!(&dir_entry_str);
}
fmm => println!("Should not occur: {} has {:?}", dir_entry_str, fmm),
},
Some(_) | None => {
match ft {
- filemagic::Mimetype::Text(LineEnding::CRLF) => {
+ filemagic::Mimetype::Text(LineEnding::Crlf) => {
e0012!(&dir_entry_str);
if ARGS.correct_le {
- fix_inconsistent_le(&dir_entry_str);
+ fix_inconsistent_le(dir_entry_str);
}
}
- filemagic::Mimetype::Text(LineEnding::CR) => {
+ filemagic::Mimetype::Text(LineEnding::Cr) => {
e0037!(&dir_entry_str);
if ARGS.correct_le {
- fix_inconsistent_le(&dir_entry_str);
+ fix_inconsistent_le(dir_entry_str);
}
}
- filemagic::Mimetype::Text(LineEnding::MIXED(0, 0, 0)) => (),
- filemagic::Mimetype::Text(LineEnding::MIXED(cr, lf, crlf)) => {
+ filemagic::Mimetype::Text(LineEnding::Mixed(0, 0, 0)) => (),
+ filemagic::Mimetype::Text(LineEnding::Mixed(cr, lf, crlf)) => {
//println!(">>>{}: {:?} {},{},{}", &dir_entry_str, ft, x, y, z);
e0038!(&dir_entry_str, cr, lf, crlf);
if ARGS.correct_le {
- fix_inconsistent_le(&dir_entry_str);
+ fix_inconsistent_le(dir_entry_str);
}
}
- filemagic::Mimetype::Text(LineEnding::LF) => (),
+ filemagic::Mimetype::Text(LineEnding::Lf) => (),
fmm => println!("Should not occur: {} has {:?}", dir_entry_str, fmm),
}
}
}
}
- filemagic::Mimetype::BOM(b) => {
+ filemagic::Mimetype::Bom(b) => {
//println!("{}: {} with BOM detected", dir_entry_str, b.as_ref());
w0004!(&dir_entry_str, b.as_ref());
- check_and_correct_perms(&dir_entry_str, p);
+ check_and_correct_perms(dir_entry_str, p);
}
filemagic::Mimetype::Binary | filemagic::Mimetype::Script(_) => {
if !owner_has(p, 4) || !others_have(p, 4) {
e0002!(&dir_entry_str, perms_to_string(p));
};
- fix_perms(&dir_entry_str);
+ fix_perms(dir_entry_str);
}
filemagic::Mimetype::Pdf => {
- check_and_correct_perms(&dir_entry_str, p);
- let ret = is_pdf_ok(&dir_entry_str);
+ check_and_correct_perms(dir_entry_str, p);
+ let ret = is_pdf_ok(dir_entry_str);
if !ret.status {
e0017!(&dir_entry_str);
if let Some(output) = ret.output {
@@ -1302,11 +1303,11 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> {
} else {
w0001!(&dir_entry_str);
}
- check_and_correct_perms(&dir_entry_str, p);
+ check_and_correct_perms(dir_entry_str, p);
}
- filemagic::Mimetype::Data => check_and_correct_perms(&dir_entry_str, p),
- filemagic::Mimetype::Zerofile => check_and_correct_perms(&dir_entry_str, p),
+ filemagic::Mimetype::Data => check_and_correct_perms(dir_entry_str, p),
+ filemagic::Mimetype::Zerofile => check_and_correct_perms(dir_entry_str, p),
_ => continue,
}
@@ -1314,17 +1315,19 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> {
e0002!(&dir_entry_str, perms_to_string(p));
if ARGS.correct_perms {
i0005!(&dir_entry_str);
- set_perms(&dir_entry_str, 0o664);
+ set_perms(dir_entry_str, 0o664);
}
}
if !(ARGS.ignore_dupes && tds_zip.is_none()) {
- process(fsize, &path);
+ process(fsize, path);
}
}
print_casefolding(&lcnames);
print_generated(&doublenames, &generated);
- print_doublenames(&doublenames);
+ if ! ARGS.ignore_same_named {
+ print_doublenames(&doublenames);
+ }
if ARGS.ignore_dupes && tds_zip.is_none() {
return None;
@@ -1335,7 +1338,7 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> {
// doing mostly IO.
let mut pool = Pool::new(num_cpus::get() as u32 + 1);
- let mut hashes: HashMap<(u64, Vec<u8>), DupPath> = HashMap::default();
+ let mut hashes: FxHashMap<(u64, Vec<u8>), DupPath> = FxHashMap::default();
pool.scoped(|scope| {
let (tx, rx) = channel();
@@ -1368,7 +1371,7 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> {
Some(hashes)
}
-fn print_casefolding(hashes: &HashMap<PathBuf, Vec<(PathBuf, FileKind)>>) {
+fn print_casefolding(hashes: &FxHashMap<PathBuf, Vec<(PathBuf, FileKind)>>) {
for (k, eles) in hashes.iter() {
//println!("pcf: {:?}, {:?}", k, &eles);
if eles.len() == 1 {
@@ -1383,7 +1386,7 @@ fn print_casefolding(hashes: &HashMap<PathBuf, Vec<(PathBuf, FileKind)>>) {
}
}
-fn print_generated(doublenames: &HashMap<PathBuf, Vec<PathBuf>>, generated: &GeneratedHashMap) {
+fn print_generated(doublenames: &FxHashMap<PathBuf, Vec<PathBuf>>, generated: &GeneratedHashMap) {
// `k` is generated by `gen`
for (k, gen) in generated.iter() {
let path = PathBuf::from(k);
@@ -1400,7 +1403,7 @@ fn print_generated(doublenames: &HashMap<PathBuf, Vec<PathBuf>>, generated: &Gen
}
}
-fn print_doublenames(hashes: &HashMap<PathBuf, Vec<PathBuf>>) {
+fn print_doublenames(hashes: &FxHashMap<PathBuf, Vec<PathBuf>>) {
for (k, paths) in hashes.iter() {
if paths.len() == 1 {
continue;
diff --git a/support/pkgcheck/src/messages/errorsd.rs b/support/pkgcheck/src/messages/errorsd.rs
index c034301ad9..6ef9eed62e 100644
--- a/support/pkgcheck/src/messages/errorsd.rs
+++ b/support/pkgcheck/src/messages/errorsd.rs
@@ -237,7 +237,7 @@ package which easily can be generated from other files in the
submission.
Exceptions are the README files of the package, i.e. README, README.md
-or README.txt, or .pdf files.
+or README.txt, .pdf, .html, or .css files.
pkgcheck detects generated files anywhere in the package directory tree.
diff --git a/support/pkgcheck/src/messages/mod.rs b/support/pkgcheck/src/messages/mod.rs
index 65dcbc502a..e3ec350143 100644
--- a/support/pkgcheck/src/messages/mod.rs
+++ b/support/pkgcheck/src/messages/mod.rs
@@ -19,15 +19,15 @@ macro_rules! error_occured {
$crate::ERROR_OCCURED.store(true, Ordering::Relaxed);
};
}
-macro_rules! yellow {
- ($fmt:expr) => {
- if no_colors!() {
- $fmt.clear()
- } else {
- $fmt.bright_yellow()
- }
- };
-}
+// macro_rules! yellow {
+// ($fmt:expr) => {
+// if no_colors!() {
+// $fmt.clear()
+// } else {
+// $fmt.bright_yellow()
+// }
+// };
+// }
// macro_rules! red {
// ($fmt:expr) => {
@@ -44,7 +44,7 @@ macro_rules! msgid {
match (no_colors!(), &$fmt[..1]) {
(true, _) => $fmt.clear(),
(false, "E") => $fmt.bright_red().bold(),
- (false, "I") => $fmt.bright_yellow(),
+ (false, "I") => $fmt.bright_yellow().bold(),
(false, "W") => $fmt.bright_red(),
(false, "F") => $fmt.bright_red().bold(),
(_, _) => $fmt.clear(),
@@ -639,8 +639,8 @@ macro_rules! i0002 {
print!(
"{} {} {}\n",
msgid!("I0002"),
- yellow!("Checking package files in directory"),
- yellow!($fmt)
+ "Checking package files in directory",
+ $fmt
);
};
}
@@ -650,8 +650,8 @@ macro_rules! i0003 {
print!(
"{} {} {}\n",
msgid!("I0003"),
- yellow!("Checking TDS zip archive"),
- yellow!($fmt)
+ "Checking TDS zip archive",
+ $fmt
);
};
}
diff --git a/support/pkgcheck/src/utils.rs b/support/pkgcheck/src/utils.rs
index 2dedc94ecd..a8f2d56c0f 100644
--- a/support/pkgcheck/src/utils.rs
+++ b/support/pkgcheck/src/utils.rs
@@ -17,8 +17,8 @@ use std::fs;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CheckType {
- TDS,
- PACKAGE,
+ Tds,
+ Package,
}
pub struct Utils {
@@ -38,7 +38,7 @@ impl Utils {
return false;
}
if fsize > 40 * 1024 * 1024 {
- if self.kind == CheckType::PACKAGE {
+ if self.kind == CheckType::Package {
w0005!(dir_entry_str, fsize / 1024 / 1024);
} else {
w0006!(dir_entry_str, fsize / 1024 / 1024);
@@ -102,10 +102,10 @@ impl Utils {
pub fn check_for_hidden_file(&self, entry: &str, dir_entry_str: &str) {
if entry.starts_with('.') {
match self.kind {
- CheckType::PACKAGE => {
+ CheckType::Package => {
e0007!(dir_entry_str);
}
- CheckType::TDS => {
+ CheckType::Tds => {
e0007t!(dir_entry_str);
}
}
@@ -123,10 +123,10 @@ impl Utils {
//if fname.map(|s| s.starts_with('.')).unwrap_or(false) {
if fname.starts_with('.') {
match self.kind {
- CheckType::PACKAGE => {
+ CheckType::Package => {
e0006!(dir_entry_str);
}
- CheckType::TDS => {
+ CheckType::Tds => {
e0006t!(dir_entry_str);
}
}
@@ -140,10 +140,10 @@ impl Utils {
if RE.is_match(dir_entry_str) {
match self.kind {
- CheckType::PACKAGE => {
+ CheckType::Package => {
e0008!(dir_entry_str);
}
- CheckType::TDS => {
+ CheckType::Tds => {
e0008t!(dir_entry_str);
}
}
@@ -156,10 +156,10 @@ impl Utils {
Ok(contents) => {
if contents.count() == 0 {
match self.kind {
- CheckType::PACKAGE => {
+ CheckType::Package => {
e0004!(dir_entry_display);
}
- CheckType::TDS => {
+ CheckType::Tds => {
// Note: Karl Berry recommended to issue a warning only
// as an empty directory in a TDS zip archive
// - is automatically deleted before including
@@ -176,10 +176,10 @@ impl Utils {
}
}
-pub fn temp_file_endings() -> Vec<(&'static str, &'static str)> {
+pub fn temp_file_endings() -> Vec<(String, String)> {
// https://github.com/github/gitignore/blob/master/TeX.gitignore
// http://hopf.math.purdue.edu/doc/html/suffixes.html
- vec![
+ let v = vec![
("-blx.aux", "bibliography auxiliary file"),
("-blx.bib", "bibliography auxiliary file"),
(".4ct", "htlatex related"),
@@ -212,6 +212,7 @@ pub fn temp_file_endings() -> Vec<(&'static str, &'static str)> {
(".gls", "glossary related"),
(".glsdefs", "glossaries related"),
(".gtex", "generated by gregoriotex"),
+ (".hd", ""),
(".idv", "htlatex related"),
(".idx", "makeidx related"),
(".ilg", "makeidx related"),
@@ -250,6 +251,7 @@ pub fn temp_file_endings() -> Vec<(&'static str, &'static str)> {
(".synctex(busy)", "synctex related"),
(".synctex.gz", "synctex related"),
(".synctex.gz(busy)", "synctex related"),
+ (".tpt", ""),
(".tdo", "generated by todonotes (list of todos)"),
(".thm", "amsthm related"),
(".tmb", "generated by thumbs package"),
@@ -267,7 +269,9 @@ pub fn temp_file_endings() -> Vec<(&'static str, &'static str)> {
(".xref", "htlatex related"),
("~", "a file name ending with ~ (tilde) is temporary anyway"),
// ( ".lyx~", "LyX related backup file" ),
- ]
+ ];
+
+ v.into_iter().map(|(i, j)| (i.to_string(), j.to_string())).collect()
}
pub fn regex_temporary_file_endings() -> Regex {
@@ -280,7 +284,7 @@ pub fn regex_temporary_file_endings() -> Regex {
} else {
rv.push('|');
}
- let px = str::replace(p, ".", "\\.");
+ let px = str::replace(&p, ".", "\\.");
rv.push_str(&px);
}
@@ -452,12 +456,13 @@ fn test_dirname() {
#[allow(dead_code)]
pub fn dirname(entry: &str) -> Option<&str> {
- if entry.ends_with('/') {
+ if let Some(stripped) = entry.strip_suffix('/') {
if entry.len() == 1 {
return Some(entry);
}
- return Some(&entry[..entry.len() - 1]);
+ return Some(stripped)
}
+
let pos = entry.rfind('/');
match pos {
None => None,