summaryrefslogtreecommitdiff
path: root/support/pkgcheck
diff options
context:
space:
mode:
authorNorbert Preining <norbert@preining.info>2020-01-18 03:01:23 +0000
committerNorbert Preining <norbert@preining.info>2020-01-18 03:01:23 +0000
commit22d3a7ddeb2b2637308cf883f9bd3590e649aa20 (patch)
treee5887f868c532734a8cac1c7833f5492370effb5 /support/pkgcheck
parente71ffee2103e3f137bbc9ee526858bc277e95908 (diff)
CTAN sync 202001180301
Diffstat (limited to 'support/pkgcheck')
-rw-r--r--support/pkgcheck/CHANGES.md53
-rw-r--r--support/pkgcheck/Cargo.toml15
-rwxr-xr-xsupport/pkgcheck/bin/pkgcheckbin7898920 -> 7042864 bytes
-rw-r--r--support/pkgcheck/docs/errorsd.tex67
-rw-r--r--support/pkgcheck/docs/fatald.tex8
-rw-r--r--support/pkgcheck/docs/informationd.tex10
-rw-r--r--support/pkgcheck/docs/pkgcheck.pdfbin66749 -> 70100 bytes
-rw-r--r--support/pkgcheck/docs/title.tex2
-rw-r--r--support/pkgcheck/docs/warningsd.tex27
-rw-r--r--support/pkgcheck/src/filemagic.rs73
-rw-r--r--support/pkgcheck/src/gparser.rs2
-rw-r--r--support/pkgcheck/src/linkcheck.rs21
-rw-r--r--support/pkgcheck/src/main.rs926
-rw-r--r--support/pkgcheck/src/messages/errorsd.rs68
-rw-r--r--support/pkgcheck/src/messages/fatald.rs10
-rw-r--r--support/pkgcheck/src/messages/informationd.rs13
-rw-r--r--support/pkgcheck/src/messages/mod.rs113
-rw-r--r--support/pkgcheck/src/messages/warningsd.rs28
-rw-r--r--support/pkgcheck/src/recode.rs245
-rw-r--r--support/pkgcheck/src/utils.rs227
20 files changed, 1262 insertions, 646 deletions
diff --git a/support/pkgcheck/CHANGES.md b/support/pkgcheck/CHANGES.md
index ac5dc5a1e6..f41ce9b393 100644
--- a/support/pkgcheck/CHANGES.md
+++ b/support/pkgcheck/CHANGES.md
@@ -13,6 +13,7 @@
- quick_intro.txt: improve wording
- build_ctan_zip.p6: add source files to be included into CTAN zip archive
- devnotes.md: add documentation how to build your own binary
+
2018-12-26 (1.1.0)
- enhance check for generated files which now could reside in any
subdirectory in the package tree. This changes the format of the
@@ -21,11 +22,13 @@
- change ordering of the sections in the pkgcheck.pdf document. First come the
informational messages, then warnings and error messages and finally fatal messages.
- add a short info to the PDF documention about how to install pkgcheck
+
2019-01-02 (1.2.0)
- some code simplification
- **New feature**: checks that path names in the TDS zip archive contain the package name
(only exception is a man page path)
- add test cases for messages e0026, e0028
+
2019-03-09 (1.3.0)
- recompile with newest http library
- checking URLs
@@ -35,30 +38,38 @@
- ignoring .tfm files when checking for duplicates
- now checking for UTF BOMs. If a README file contains a BOM we issue an error message,
if other files contain BOMs we issue a warning message
+
2019-03-11 (1.4.0)
- recognizing generated files in a .dtx file when they are included using a
filecontents resp. filecontents* environment.
+
2019-03-14 (1.5.0)
- a README can be a symlink which will now be detected properly
- if a symlink points to a file object outside of the package directory tree error message
e0030 will be issued
+
2019-03-29 (1.6.0)
- new error message e0031 when a filename contains invalid UTF-8 characters
+
2019-06-30 (1.7.0)
- when symlinks occur in e0025 they are reported as files
- improved error message e0025 now displaying the paths of the affected
files/directories/symlinks
- if a found http(s) link ends with "`" then the trailing "`" will be discarded
- compiled with rust edition 2018
+
2019-07-20 (1.8.0)
- add a check to detect temporary files in the TDS zip archive
+
2019-07-21 (1.8.1)
- correct typo in e0008 message text
+
2019-08-07 (1.8.2)
- updating crates, requiring slight code changes in integration.rs
- ignore windows files .nsh and .reg when checking for LF line endings
- new error message e0034 for unwanted files in top level directory of a TDS zip archive
- e0020 now reporting only unwanted directory in top level directory of a TDS zip archive
+
2019-10-01 (1.8.3)
- (Experimental) issue warning if a file in the package tree (W0005) or in the TDS zip archive (W0006) is
larger than 40MiB
@@ -67,8 +78,48 @@
Both features are regarded as experimental because both contain a hard coded check. If required
things will be changed in the future.
+
2019-11-04 (1.9.0)
- (Experimental) for latex-base, latex-tools, latex-graphics and latex-amsmath use the real
- package name 'latex' when checking path names in the TDS zip archive
+ directory name 'latex' when checking path names in the TDS zip archive
- New error message e0035 if a TDS zip archive is found in the unpacked directory tree
- add testcases for w0005, w0006 and e0035
+
+2019-11-27 (1.9.1)
+ - recognize .bat files as script type
+ - add test case for .bat file recognition
+
+2019-11-28 (1.10.0)
+ - remove message E0020 to report unwanted top level directory in a TDS zip archive
+ - more checks for a TDS zip archive
+ - check for temporary files or directories
+
+2020-01-17 (2.0.0)
+ - more error checking in TDS zip archives
+ - detecting if a .dtx or .ins file is not in a subdirectory of source/ or doc/ toplevel directories (E0036)
+ - detecting empty directories (W0007)
+ - detecting hidden directories (E0006)
+ - detecting temporary files (E0008)
+ - as a special case tex/latex/tools/.tex is allowed for package latex-tools. The
+ same applies for latex-tools-dev package
+ - revamped handling of line ending checks
+ - reporting if a file has CR line endings only (E0037)
+ - reporting if a file has inconsistent line endings (E0038)
+ - reporting if a Windows file does not have Windows (CRLF) line endings (W0008)
+ - a file is regarded a Windows file if its name ends with one of .bat, .cmd, .nsh or .reg
+ - files with CR, or CRLF line endings will be corrected to LF
+ - but windows files will be corrected to CRLF line endings
+ - when correcting line endings a line ending at the end of a file will be added if required
+ - renamed command line switch
+ - old: -L, --correct-crlf Correct CRLF line endings
+ - new: -L, --correct-le Correct line endings
+ - other changes
+ - new message F0007 when tmp directory is not writable or the tmp directory is full
+ - some code simplifications
+ - crates update
+ - when detecting an empty file also permissions will be checked and (if requested) corrected
+ - (Experimental) for latex-base-dev, latex-tools-dev, latex-graphics-dev and latex-amsmath-dev
+ check for latex-dev in the path names in the TDS zip archive
+ - if a file in a generated file statement contains a directory part it was correctly
+ recognized but not compared correctly to the contents of the TDS zip archive
+
diff --git a/support/pkgcheck/Cargo.toml b/support/pkgcheck/Cargo.toml
index 7908036125..94ed0aaf5b 100644
--- a/support/pkgcheck/Cargo.toml
+++ b/support/pkgcheck/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "pkgcheck"
-version = "1.9.0"
+version = "2.0.0"
authors = ["Manfred Lotz <manfred@ctan.org>"]
description = "Command-line tool to check packages uploaded to CTAN."
license = "MIT/Apache-2.0"
@@ -8,7 +8,7 @@ edition = "2018"
[dependencies]
structopt = "0.3"
-walkdir = "2.2"
+walkdir = "2.3"
scoped_threadpool = "0.1"
num_cpus = "1"
blake2 = "0.8"
@@ -18,13 +18,15 @@ regex = "1"
glob = "0.3"
colored = "1"
linkify = "0.4"
-reqwest = "0.9"
+
+reqwest = { version = "0.10", features = ["blocking", "gzip"] }
+
lazy_static = "1"
tempdir = "0.3"
-assert_cmd = "0.11"
-predicates = "1.0"
+assert_cmd = "0.12"
+predicates = "1"
openssl-probe = "0.1.2"
-pest = "^2.1"
+pest = "2.1"
pest_derive = "2.1"
escargot = "0.5"
@@ -32,3 +34,4 @@ threadpool = "1.7.1"
unicode-bom = "1"
url = "2.1"
+
diff --git a/support/pkgcheck/bin/pkgcheck b/support/pkgcheck/bin/pkgcheck
index 1ac3e63302..ebf41ce1b3 100755
--- a/support/pkgcheck/bin/pkgcheck
+++ b/support/pkgcheck/bin/pkgcheck
Binary files differ
diff --git a/support/pkgcheck/docs/errorsd.tex b/support/pkgcheck/docs/errorsd.tex
index 25a68c1d79..e3e9cc9b5f 100644
--- a/support/pkgcheck/docs/errorsd.tex
+++ b/support/pkgcheck/docs/errorsd.tex
@@ -33,7 +33,8 @@ file but it isn't.
allowed}\label{e0004----empty-directory-not-allowed}}
Empty directories are considered as rubbish, and are usually not
-accepted as part of a package.
+accepted as part of a package, neither in the package tree nor in the
+TDS zip archive.
For more details refer to:
\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noemptyfiles}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noemptyfiles}
@@ -52,7 +53,8 @@ For more details refer to:
\subsection{E0006 -\/- Hidden directories not
allowed}\label{e0006----hidden-directories-not-allowed}}
-A package should not contain hidden directories.
+A package should not contain hidden directories, neither in the package
+tree nor in the TDS zip archive.
For more details refer to:
\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles}
@@ -61,7 +63,8 @@ For more details refer to:
\subsection{E0007 -\/- Hidden files not
allowed}\label{e0007----hidden-files-not-allowed}}
-A package should not contain hidden files.
+A package should not contain hidden files, neither in the package tree
+nor in the TDS zip archive.
For more details refer to:
\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles}
@@ -188,34 +191,6 @@ files if they are in a different directory in the package.
For more details refer to:
\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#nogeneratedfiles}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#nogeneratedfiles}
-\hypertarget{e0020----unwanted-directory-detected-in-the-top-level-directory-in-tds-zip-archive}{%
-\subsection{E0020 -\/- Unwanted directory detected in the top level
-directory in TDS zip
-archive}\label{e0020----unwanted-directory-detected-in-the-top-level-directory-in-tds-zip-archive}}
-
-A top level directory of a TDS archive should only contain all or some
-of the following directories:
-
-\begin{itemize}
-\tightlist
-\item
- \texttt{tex}
-\item
- \texttt{fonts}
-\item
- \texttt{metafont}
-\item
- \texttt{metapost}
-\item
- \texttt{bibtex}
-\item
- \texttt{scripts}
-\item
- \texttt{doc}
-\item
- \texttt{source}
-\end{itemize}
-
\hypertarget{e0021----error-when-reading-a-file}{%
\subsection{E0021 -\/- Error when reading a
file}\label{e0021----error-when-reading-a-file}}
@@ -337,3 +312,33 @@ directory
tree}\label{e0035----unwanted-tds-archive-detected-in-package-directory-tree}}
A package directory should not contain a TDS zip archive.
+
+\hypertarget{e0036----dtxins-files-found-in-wrong-directory-in-tds-zip-archive}{%
+\subsection{E0036 -\/- .dtx/.ins files found in wrong directory in TDS
+zip
+archive}\label{e0036----dtxins-files-found-in-wrong-directory-in-tds-zip-archive}}
+
+In a TDS zip archive a \texttt{.dtx} resp. \texttt{.ins} file must be in
+a subdirectory of either of \texttt{source/} or \texttt{doc/} top level
+directories.
+
+\hypertarget{e0037----cr-line-endings-detected}{%
+\subsection{E0037 -\/- CR line endings
+detected}\label{e0037----cr-line-endings-detected}}
+
+The file specified in the error message contains CR line endings. Text
+files should have UNIX style line endings.
+
+For more details refer to:
+\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#crlf}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#crlf}
+
+\hypertarget{e0038----file-has-inconsistent-line-endings-cr-x-lf-y-crlf-z}{%
+\subsection{E0038 -\/- File has inconsistent line endings: CR: x, LF: y,
+CRLF:
+z}\label{e0038----file-has-inconsistent-line-endings-cr-x-lf-y-crlf-z}}
+
+The file specified in the error message contains CR line endings. Text
+files should have UNIX style line endings.
+
+For more details refer to:
+\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#crlf}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#crlf}
diff --git a/support/pkgcheck/docs/fatald.tex b/support/pkgcheck/docs/fatald.tex
index b4c175a7c0..ef26ff30cf 100644
--- a/support/pkgcheck/docs/fatald.tex
+++ b/support/pkgcheck/docs/fatald.tex
@@ -37,3 +37,11 @@ Exiting...}\label{f0006----unknown-error-code-specified-with-option--e-resp----e
\texttt{pkgcheck} was called with option \texttt{-e} resp.
\texttt{-\/-explain}, and an unknown error code was specified.
+
+\hypertarget{f0007----could-not-create-temporary-directory-for-unzipping-the-tds-zip-archive}{%
+\subsection{F0007 -\/- Could not create temporary directory for
+unzipping the TDS zip
+archive}\label{f0007----could-not-create-temporary-directory-for-unzipping-the-tds-zip-archive}}
+
+Make sure the temp directory is writable and/or the temp directory has
+enough space.
diff --git a/support/pkgcheck/docs/informationd.tex b/support/pkgcheck/docs/informationd.tex
index 88a2305f48..b1a870ebba 100644
--- a/support/pkgcheck/docs/informationd.tex
+++ b/support/pkgcheck/docs/informationd.tex
@@ -49,3 +49,13 @@ temporary file endings and their meanings.
For more details refer to:
\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles}
+
+\hypertarget{i0007----successfully-corrected-wrong-line-endings-to-lf-resp-crlf}{%
+\subsection{I0007 -\/- Successfully corrected wrong line endings to LF
+resp.
+CRLF}\label{i0007----successfully-corrected-wrong-line-endings-to-lf-resp-crlf}}
+
+\texttt{pkgcheck} successfully converted wrong line endings to LF line
+endings or to CRLF line endings if it the file was a Windows text file.
+
+Wrong line endings could be CR, CRLF or a mixture of line endings.
diff --git a/support/pkgcheck/docs/pkgcheck.pdf b/support/pkgcheck/docs/pkgcheck.pdf
index 87d258b13d..5949a88fed 100644
--- a/support/pkgcheck/docs/pkgcheck.pdf
+++ b/support/pkgcheck/docs/pkgcheck.pdf
Binary files differ
diff --git a/support/pkgcheck/docs/title.tex b/support/pkgcheck/docs/title.tex
index 9961a041b9..1b5df7c036 100644
--- a/support/pkgcheck/docs/title.tex
+++ b/support/pkgcheck/docs/title.tex
@@ -1 +1 @@
-\title{pkgcheck Utility, v1.9.0}
+\title{pkgcheck Utility, v2.0.0}
diff --git a/support/pkgcheck/docs/warningsd.tex b/support/pkgcheck/docs/warningsd.tex
index 7189f196e1..416661ff00 100644
--- a/support/pkgcheck/docs/warningsd.tex
+++ b/support/pkgcheck/docs/warningsd.tex
@@ -53,3 +53,30 @@ archive}{W0006 -\/- Very large file with size \textless size\textgreater{} detec
(Experimental) We issue the message if there is a file larger than 40MiB
in the TDS zip archive.
+
+\hypertarget{w0007----empty-directory-detected-in-the-tds-zip-archive}{%
+\subsection{W0007 -\/- Empty directory detected in the TDS zip
+archive}\label{w0007----empty-directory-detected-in-the-tds-zip-archive}}
+
+Empty directories in a TDS zip archive are discouraged. As they usually
+don't create errors in the distribution we issue a warning only.
+
+\hypertarget{w0008----windows-file-has-unix-line-endings}{%
+\subsection{W0008 -\/- Windows file has Unix line
+endings}\label{w0008----windows-file-has-unix-line-endings}}
+
+A Windows file with Unix line endings was detected.
+
+We regard a file as a Windows file if its name ends with:
+
+\begin{itemize}
+\tightlist
+\item
+ \texttt{.bat}
+\item
+ \texttt{.cmd}
+\item
+ \texttt{.nsh}, or
+\item
+ \texttt{.reg}
+\end{itemize}
diff --git a/support/pkgcheck/src/filemagic.rs b/support/pkgcheck/src/filemagic.rs
index 40edd18351..2ae8bfae6e 100644
--- a/support/pkgcheck/src/filemagic.rs
+++ b/support/pkgcheck/src/filemagic.rs
@@ -19,16 +19,23 @@ use unicode_bom::Bom;
// - https://filesignatures.net/
// - https://github.com/7h3rAm/cigma/blob/master/cigma/magicbytes.json
+
+#[derive(Debug, PartialEq)]
+pub enum LineEnding {
+ LF,
+ CR,
+ CRLF,
+ MIXED(usize, usize, usize),
+}
+
#[derive(Debug, PartialEq)]
pub enum Mimetype {
Binary,
- Script,
- ScriptCRLF,
+ Script(LineEnding),
Pdf,
Archive,
Zip,
- Text,
- TextCRLF,
+ Text(LineEnding),
Data,
Unknown,
BlockDevice,
@@ -42,6 +49,7 @@ pub enum Mimetype {
BOM(Bom),
}
+
pub struct Filetype {
buffer: Vec<u8>,
}
@@ -91,7 +99,7 @@ fn _is_crlf(buffer: &[u8], len: usize) -> bool {
false
}
-fn is_crlf(buffer: &[u8], len: usize) -> bool {
+fn is_crlf(buffer: &[u8], len: usize) -> LineEnding {
let mut seen_cr = false;
let mut n_crlf = 0;
let mut n_lf = 0;
@@ -114,18 +122,18 @@ fn is_crlf(buffer: &[u8], len: usize) -> bool {
seen_cr = *c == CR;
}
+ // println!("LF / CR / CRLF: {} / {} / {}", n_lf, n_cr, n_crlf);
+
// println!("cr: {}, lf: {}, crlf: {}", n_cr, n_lf, n_crlf);
// if (n_crlf == 0 && n_cr == 0 && n_nel == 0 && n_lf == 0)
// --> no line terminators
- if n_crlf == 0 && n_cr == 0 && n_lf == 0 {
- return false;
- }
- if n_crlf > 0 || n_cr > 0 {
- return true;
+ match (n_cr, n_lf, n_crlf) {
+ (0, 0, z) if z > 0 => LineEnding::CRLF,
+ (x, 0, 0) if x > 0 => LineEnding::CR,
+ (0, y, 0) if y > 0 => LineEnding::LF,
+ (x, y, z) => LineEnding::MIXED(x, y, z),
}
-
- false
}
impl Filetype {
@@ -175,10 +183,6 @@ impl Filetype {
return Ok(Mimetype::Pdf);
}
- if bytes_read >= 5 && &self.buffer[0..5] == b"<?php" {
- return Ok(Mimetype::Script);
- }
-
// rtf document
if bytes_read >= 6 && &self.buffer[0..6] == b"\x7B\x5C\x72\x74\x66\x31" {
return Ok(Mimetype::Data);
@@ -224,27 +228,36 @@ impl Filetype {
}
}
+ //println!("Filename: {}", fname);
+
let crlf = is_crlf(&self.buffer, bytes_read);
+ //println!("{:?}", crlf);
if bytes_read >= 5 && (self.buffer.starts_with(b"#!") || self.buffer.starts_with(b"<?php"))
{
- if crlf {
- return Ok(Mimetype::ScriptCRLF);
- } else {
- return Ok(Mimetype::Script);
- }
+ return Ok(Mimetype::Script(crlf));
}
+ // recognize files as script files by file endings
let is_script = Regex::new(r"\.(csh|ksh|py|pl|sh|tcsh)$")
.unwrap()
.is_match(fname.as_bytes());
- match (crlf, is_script) {
- (false, false) => Ok(Mimetype::Text),
- (true, false) => Ok(Mimetype::TextCRLF),
- (false, true) => Ok(Mimetype::Script),
- (true, true) => Ok(Mimetype::ScriptCRLF),
+ //println!("le/is_script: {:?}, {:?}", crlf, is_script);
+ if is_script {
+ Ok(Mimetype::Script(crlf))
+ } else {
+ Ok(Mimetype::Text(crlf))
}
+ // match (crlf, is_script) {
+ // (LineEnding::LF, false) => Ok(Mimetype::Text(LineEnding::LF)),
+ // (LineEnding::CR, false) => Ok(Mimetype::Text(LineEnding::CR)),
+ // (LineEnding::CRLF, false) => Ok(Mimetype::Text(LineEnding::CRLF)),
+ // (LineEnding::LF, true) => Ok(Mimetype::Script(LineEnding::LF)),
+ // (LineEnding::CR, true) => Ok(Mimetype::Script(LineEnding::CR)),
+ // (LineEnding::CRLF, true) => Ok(Mimetype::Script(LineEnding::CRLF)),
+ // (_, _) => Ok(Mimetype::Text(LineEnding::LF)),
+ // }
}
}
@@ -388,8 +401,8 @@ fn test_filetype() {
// analyze() says TextCRLF
//assert!(ft.analyze("tests_filemagic/musterlogo.pdf").ok() == Some(Mimetype::Script));
- assert!(ft.analyze("tests_filemagic/x.pl").ok() == Some(Mimetype::Script));
- assert!(ft.analyze("tests_filemagic/main.php").ok() == Some(Mimetype::Script));
+ assert!(ft.analyze("tests_filemagic/x.pl").ok() == Some(Mimetype::Script(LineEnding::LF)));
+ assert!(ft.analyze("tests_filemagic/main.php").ok() == Some(Mimetype::Script(LineEnding::LF)));
assert!(ft.analyze("tests_filemagic/test.7z").ok() == Some(Mimetype::Archive));
assert!(ft.analyze("tests_filemagic/x.tgz").ok() == Some(Mimetype::Archive));
@@ -404,7 +417,7 @@ fn test_filetype() {
assert!(ft.analyze("tests_filemagic/empty.zip").ok() == Some(Mimetype::Zip));
- assert!(ft.analyze("tests_filemagic/README").ok() == Some(Mimetype::Text));
+ assert!(ft.analyze("tests_filemagic/README").ok() == Some(Mimetype::Text(LineEnding::MIXED(0,0,0))));
// assert!(ft.analyze("tests_filemagic/README1").ok() == Some(Mimetype::Text));
assert!(ft.analyze("tests_filemagic/cp").ok() == Some(Mimetype::Binary));
@@ -418,5 +431,5 @@ fn test_filetype() {
assert!(ft.analyze("tests_filemagic/chap5.rtf").ok() == Some(Mimetype::Data));
assert!(ft.analyze("tests_filemagic/commons-math.jar").ok() == Some(Mimetype::Data));
- assert!(ft.analyze("tests_filemagic/8stbu11h.htm").ok() == Some(Mimetype::TextCRLF));
+ assert!(ft.analyze("tests_filemagic/8stbu11h.htm").ok() == Some(Mimetype::Text(LineEnding::MIXED(0,1,8710))));
}
diff --git a/support/pkgcheck/src/gparser.rs b/support/pkgcheck/src/gparser.rs
index 6f9d7f393a..ac3e1e7289 100644
--- a/support/pkgcheck/src/gparser.rs
+++ b/support/pkgcheck/src/gparser.rs
@@ -1,7 +1,7 @@
use pest::Parser;
#[cfg(debug_assertions)]
-const _GRAMMAR: &'static str = include_str!("generate.pest");
+const _GRAMMAR: &str = include_str!("generate.pest");
#[derive(Parser)]
#[grammar = "generate.pest"]
diff --git a/support/pkgcheck/src/linkcheck.rs b/support/pkgcheck/src/linkcheck.rs
index 60b993e960..f73a05c02b 100644
--- a/support/pkgcheck/src/linkcheck.rs
+++ b/support/pkgcheck/src/linkcheck.rs
@@ -89,7 +89,7 @@ fn check_link(url: &str, fname: &str, urlhash: &Arc<Mutex<UrlHash>>, print_all:
let mut urlhash = urlhash.lock().unwrap();
if !urlhash.contains_key(&url) {
let mut hs = HashSet::default();
- hs.insert(f.clone());
+ hs.insert(f);
let url1 = url.clone();
urlhash.insert(
@@ -178,11 +178,10 @@ fn get_links_inner(s: &str) -> Option<Vec<String>> {
}
// This is a workaround to prevent URLs ending with `
if r.ends_with('`') {
- links.push(String::from(&r[..r.len()-1]));
+ links.push(String::from(&r[..r.len() - 1]));
} else {
links.push(String::from(r));
}
-
}
if !links.is_empty() {
Some(links)
@@ -200,8 +199,8 @@ fn check_link_inner(l: &str, head: bool) -> UrlStatus {
),
);
- let default_policy = reqwest::RedirectPolicy::default();
- let policy = reqwest::RedirectPolicy::custom(move |attempt| {
+ let default_policy = reqwest::redirect::Policy::default();
+ let policy = reqwest::redirect::Policy::custom(move |attempt| {
if attempt.url().host_str() == Some("127.0.0.1") {
attempt.stop()
} else {
@@ -209,18 +208,18 @@ fn check_link_inner(l: &str, head: bool) -> UrlStatus {
}
});
- let cb = reqwest::Client::builder()
+ let cb = reqwest::blocking::Client::builder()
.gzip(true)
.redirect(policy)
.default_headers(headers)
.timeout(Duration::from_secs(7))
.build()
.unwrap();
- // let url: Url =
- // match l.parse() {
- // Ok(url) => url,
- // Err(e) => { println!("Error: {:?}", e); panic!("Scheiss"); }
- // };
+ // let url: Url =
+ // match l.parse() {
+ // Ok(url) => url,
+ // Err(e) => { println!("Error: {:?}", e); panic!("Scheiss"); }
+ // };
let resp = if head {
cb.head(l).send()
} else {
diff --git a/support/pkgcheck/src/main.rs b/support/pkgcheck/src/main.rs
index 6fb4136f91..88b9323692 100644
--- a/support/pkgcheck/src/main.rs
+++ b/support/pkgcheck/src/main.rs
@@ -1,4 +1,3 @@
-extern crate pest;
#[macro_use]
extern crate pest_derive;
@@ -8,18 +7,19 @@ mod gparser;
mod messages;
use messages::{explains, explains_all};
-#[macro_use]
-mod utils;
mod linkcheck;
+mod utils;
mod recode;
-use recode::crlf2lf;
+use recode::{wrong_line_endings2lf, wrong_line_endings2crlf};
mod filemagic;
+use filemagic::{LineEnding};
use linkcheck::LinkCheck;
use std::fmt;
use std::fmt::Display;
+
use std::str;
use utils::*;
@@ -27,7 +27,6 @@ use std::os::unix::fs::MetadataExt;
use scoped_threadpool::Pool;
-
use tempdir::TempDir;
#[macro_use]
extern crate lazy_static;
@@ -63,6 +62,15 @@ fn err(path: &PathBuf, err: &io::Error) {
type HashSender = Sender<(u64, PathBuf, Vec<u8>)>;
//type DupeSender = Sender<(u64, Vec<PathBuf>)>;
+// SizesHashMap contains
+// - file sizes
+// - and a vector of file names having that size
+type SizesHashMap = HashMap<u64, Vec<PathBuf>>;
+
+type GeneratedHashMap = HashMap<String, String>;
+
+type FileNamesHashMap = HashMap<PathBuf, (Metadata, String, ReadmeKind)>;
+
const BLOCKSIZE: usize = 4096;
fn hash_file_inner(path: &PathBuf) -> io::Result<Vec<u8>> {
@@ -87,48 +95,29 @@ fn hash_file(fsize: u64, path: PathBuf, tx: &HashSender) {
}
}
-fn fix_perms(entry: &str) {
- i0005!(entry);
- let rc = run_cmd("/bin/chmod", &["-v", "ug=rwX,o=rX", entry]);
- if rc.status {
- if let Some(op) = rc.output {
- print!("{}", op);
+// returns false if an error occured
+fn fix_inconsistent_le(fname: &str) -> bool {
+ i0004!(fname);
+ match wrong_line_endings2lf(fname) {
+ Ok(_) => {
+ i0007!(fname, "LF");
+ true
+ }
+ Err(e) => {
+ e0027!(fname, e);
+ false
}
}
}
-fn filename_has_bad_chars(entry: &DirEntry) -> Option<(usize, char)> {
- let s = entry.file_name().to_str().unwrap();
-
- for (i, c) in s.char_indices() {
- match c {
- '\x00'..='\x1f' => return Some((i, c)),
- ' ' | '!' | '"' | '#' | '$' | '%' | '&' | '\'' => return Some((i, c)),
- '(' | ')' | '*' => return Some((i, c)),
- '+' | ',' | '-' | '.' => (),
- '0'..='9' => (),
- ':' | ';' | '<' | '=' | '>' | '?' | '@' => return Some((i, c)),
- 'A'..='Z' => (),
- '[' | '\\' | ']' | '^' => return Some((i, c)),
- '_' => (),
- '`' => return Some((i, c)),
- 'a'..='z' => (),
- '{' | '|' | '}' | '~' => return Some((i, c)),
- '/' => return None,
- _ => return Some((i, c)),
- };
- }
- None
-}
-// returns false if CRLF, otherwise true
-fn fix_crlf(fname: &str) -> bool {
+// returns false if an error occured
+fn make_crlf(fname: &str) -> bool {
i0004!(fname);
- //run_cmd("/usr/bin/fromdos", &["-p", "-e", "-v", fname]).status
- match crlf2lf(fname) {
+ match wrong_line_endings2crlf(fname) {
Ok(_) => {
- i0007!(fname);
+ i0007!(fname, "CRLF");
true
}
Err(e) => {
@@ -157,7 +146,7 @@ fn check_readme(dir_entry: &str, is_readme: &ReadmeKind, ft: &filemagic::Mimetyp
e0029!(msg_name, b.as_ref());
return false;
}
- filemagic::Mimetype::Text | filemagic::Mimetype::TextCRLF => match File::open(dir_entry) {
+ filemagic::Mimetype::Text(_le) => match File::open(dir_entry) {
Ok(f) => {
if !check_readme_inner(&msg_name, &f) {
return false;
@@ -216,8 +205,8 @@ struct Args {
ignore_dupes: bool,
#[structopt(short = "v", long = "verbose", help = "Verbose operation?")]
verbose: bool,
- #[structopt(short = "L", long = "correct-crlf", help = "Correct CRLF line endings")]
- correct_crlf: bool,
+ #[structopt(short = "L", long = "correct-le", help = "Correct line endings")]
+ correct_le: bool,
#[structopt(short = "C", long = "correct-perms", help = "Correct permissions")]
correct_perms: bool,
#[structopt(long = "no-colors", help = "Don't display messages in color")]
@@ -309,7 +298,7 @@ impl DupPath {
}
}
-type Hashes = HashMap<(u64, Vec<u8>), DupPath>;
+type DupHashes = HashMap<(u64, Vec<u8>), DupPath>;
fn main() {
match &ARGS.explain {
@@ -336,11 +325,18 @@ fn main() {
process::exit(1);
}
Some(d) => {
- if !exists_dir(&d) {
- f0002!(d);
+ // make sure the given directory ends with a '/' (slash)
+ let ds: String = if d.ends_with('/') {
+ d.to_string()
+ } else {
+ let d_s = d.to_string();
+ d_s + "/"
+ };
+ if !exists_dir(&ds) {
+ f0002!(&ds);
process::exit(1);
};
- d
+ ds
}
};
@@ -365,16 +361,11 @@ fn main() {
}
};
- if let Some(hashes) = check_package(pkg_dir, tds_zip) {
+ if let Some(hashes) = check_package(&pkg_dir, tds_zip) {
if tds_zip.is_some() {
if let Some(pn) = pkg_name {
if let Some(s) = ARGS.tds_zip.as_ref() {
- match check_tds_archive(&pn, s, &hashes) {
- Ok(_) => (),
- Err(e) => {
- e0027!(s, e);
- }
- }
+ check_tds_archive(&pn, s, &hashes);
}
}
}
@@ -387,7 +378,7 @@ fn main() {
}
}
-fn print_duplicates(hashes: &Hashes) {
+fn print_duplicates(hashes: &DupHashes) {
let mut total_dupes = 0;
let mut total_files = 0;
let mut total_size = 0;
@@ -471,7 +462,7 @@ fn get_filetype(entry: &DirEntry) -> FType {
// read file into buffer[u8]
// then convert into string
//
-fn check_generated_files(entry: &str, generated: &mut HashMap<PathBuf, PathBuf>) {
+fn check_generated_files(entry: &str, generated: &mut GeneratedHashMap) {
// unwrap() is ok here as we only call this function for files,
// specifically .ins or .dtx files
let entry_fname = filename(entry).unwrap().to_string();
@@ -508,9 +499,12 @@ fn check_generated_files(entry: &str, generated: &mut HashMap<PathBuf, PathBuf>)
fname.to_string()
};
- let pb = PathBuf::from(fname_x);
-
- generated.entry(pb).or_insert_with(|| PathBuf::from(entry));
+ // If the filename in the generate statement contains a path component
+ // we ignore it so that a generated file will be reported even if it is
+ // in a different place in the package directory
+ if let Some(pb) = utils::filename(&fname_x) {
+ generated.entry(pb.to_string()).or_insert_with(|| entry.to_string());
+ }
}
};
}
@@ -542,7 +536,7 @@ fn check_tds_archive_name(tds_zip: &Option<String>) -> Option<String> {
let mut pname = String::from(utils::basename(&tz));
let plen = pname.len();
pname.truncate(plen - 8);
- Some(pname.to_string())
+ Some(pname)
}
}
}
@@ -564,11 +558,7 @@ fn unzip_tds_archive(tds_zip: &str, tmp_dir: &str) {
}
}
-fn check_tds_archive(
- pkg_name: &str,
- tds_zip: &str,
- hashes: &HashMap<(u64, Vec<u8>), DupPath>,
-) -> Result<(), io::Error> {
+fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) {
i0003!(tds_zip);
let dir_entry = Path::new(tds_zip);
@@ -581,77 +571,48 @@ fn check_tds_archive(
}
};
+ let ut = Utils::new(utils::CheckType::TDS);
+
// We have a discrepancy between
// TDS zip archive name: babel-base.tds.zip
// and package name: babel
// which yields a false test when checking for the package name
- // the path names of the TDS zip archive files.
+ // the path names of the TDS zip archive files.
// Therefore, we correct here.
// latex-tools, latex-graphics, latex-amsmath, latex-base => latex
- //let mut pkg_replacements: HashMap<String,String> = HashMap::default();
- //pkg_replacements.insert("babel-base".to_string(), "babel".to_string());
- let mut pkg_replacements: HashMap<&str,&str> = HashMap::default();
+ // latex-tools-dev, latex-graphics-dev, latex-amsmath-dev, latex-base-dev => latex-dev
+ let mut pkg_replacements: HashMap<&str, &str> = HashMap::default();
pkg_replacements.insert("babel-base", "babel");
pkg_replacements.insert("latex-base", "latex");
pkg_replacements.insert("latex-tools", "latex");
pkg_replacements.insert("latex-graphics", "latex");
pkg_replacements.insert("latex-amsmath", "latex");
- let real_pkg_name = if let Some(real_name) = pkg_replacements.get(pkg_name) {
+ pkg_replacements.insert("latex-base-dev", "latex-dev");
+ pkg_replacements.insert("latex-tools-dev", "latex-dev");
+ pkg_replacements.insert("latex-graphics-dev", "latex-dev");
+ pkg_replacements.insert("latex-amsmath-dev", "latex-dev");
+ let real_pkg_name = if let Some(real_name) = pkg_replacements.get(pkg_name) {
real_name
} else {
pkg_name
};
- let tmp_dir = TempDir::new("pkgcheck")?;
- let tmp_dir_offset = tmp_dir.path().to_str().unwrap().len() + 1;
- unzip_tds_archive(tds_zip, tmp_dir.path().to_str().unwrap());
-
- let tds_toplevel_dirs: HashSet<String> = [
- "tex", "fonts", "metafont", "metapost", "bibtex", "scripts", "doc", "source",
- ]
- .iter()
- .map(|&s| s.to_string())
- .collect();
-
- for entry in fs::read_dir(tmp_dir.path())? {
- let entry = entry?;
- let path = entry.path();
-
- match path.file_stem() {
- None => (),
- Some(file_stem) => {
- let fs = file_stem.to_str().expect("bad file name");
- if !tds_toplevel_dirs.contains(fs) {
- if path.is_dir() {
- e0020!(fs);
- } else {
- e0034!(fs);
- }
- }
- }
+ let tmp_dir = match TempDir::new("pkgcheck") {
+ Ok(tdir) => tdir,
+ Err(e) => {
+ f0007!(e);
+ process::exit(1);
}
+ };
- // if path.is_dir() {
- // match path.file_stem() {
- // None => (),
- // Some(file_stem) => {
- // let fs = file_stem.to_str().expect("bad file name");
- // if !tds_toplevel_dirs.contains(fs) {
- // e0020!(fs);
- // }
- // }
- // }
- // } else {
- // println!("mist");
- // e0020!(path.display());
- // }
- }
+ let tmp_dir_offset = tmp_dir.path().to_str().unwrap().len() + 1;
+ unzip_tds_archive(tds_zip, tmp_dir.path().to_str().unwrap());
// in order to compare the package files with the content of the
// tds zip archive we need to checksum the files in the tds zip
// archive.
- let mut sizes = HashMap::default();
+ let mut sizes: SizesHashMap = HashMap::default();
let mut pool = Pool::new(num_cpus::get() as u32 + 1);
{
// Processing a single file entry, with the "sizes" hashmap collecting
@@ -672,63 +633,93 @@ fn check_tds_archive(
for dir_entry in WalkDir::new(tmp_dir.path().to_str().unwrap()).follow_links(false) {
match dir_entry {
Ok(dir_entry) => {
- //let dir_entry_str = dir_entry.path().to_str().unwrap();
+ let dir_entry_str = match dir_entry.path().to_str() {
+ Some(d) => d,
+ None => {
+ e0031!(dir_entry.path().to_string_lossy());
+ continue;
+ }
+ };
+
+ // this is the file_name without the directory part
+ // unwrap() is ok here as we covered potential UTF-8 related errors
+ // above in the definition of dir_entry_str
+ let file_name = dir_entry.file_name().to_str().unwrap().to_string();
+
+ let meta = match dir_entry.metadata() {
+ Ok(meta) => meta,
+ Err(e) => {
+ e0027!(dir_entry.path().display(), e);
+ continue;
+ }
+ };
let ft = get_filetype(&dir_entry);
+ if let FType::Error(e) = ft {
+ e0023!(e);
+ continue;
+ }
+
+ let dir_entry_display = if dir_entry.depth() == 0 {
+ &dir_entry_str[tmp_dir_offset - 1..]
+ } else {
+ &dir_entry_str[tmp_dir_offset..]
+ };
+
if ft == FType::Directory {
+ ut.check_for_empty_directory(&dir_entry_str, &dir_entry_display);
+ ut.check_for_hidden_directory(&file_name, &dir_entry_display);
+ ut.is_unwanted_directory(&file_name, dir_entry_str);
continue;
}
- if let FType::Error(e) = ft {
- e0023!(e);
+ // We don't want to have other than directories in the top level
+ // directory of a TDS zip archive
+ if dir_entry.depth() == 1 && ft == FType::Regular {
+ e0034!(dir_entry_display);
continue;
}
- match dir_entry.metadata() {
- Ok(meta) => {
- let fsize = meta.len();
- process(fsize, &dir_entry);
+ ut.check_for_temporary_file(&dir_entry_display);
- let dir_entry_str = match dir_entry.path().to_str() {
- Some(d) => d,
- None => {
- e0031!(dir_entry.path().to_string_lossy());
- continue;
- }
- };
- let dir_entry_display = &dir_entry_str[tmp_dir_offset..];
+ // The LaTeX team provides the file `.tex` as a file with an empty name
+ // in order to make `\input\relax` work (explained by David Carlisle)
+ // Therefore, we don't call check_for_hidden_file() in this case
+ match (pkg_name, dir_entry_display) {
+ ("latex-tools", "tex/latex/tools/.tex") => (),
+ ("latex-tools-dev", "tex/latex-dev/tools/.tex") => (),
+ (_, _) => ut.check_for_hidden_file(&file_name, &dir_entry_display)
+ };
- if is_temporary_file(&dir_entry_str) {
- e0008!(dir_entry_display);
- }
- if fsize > 40 * 1024 * 1024 {
- w0006!(dir_entry_display, fsize / 1024 / 1024);
- }
+ let fsize = meta.len();
+ process(fsize, &dir_entry);
+ ut.check_filesize(fsize, &dir_entry_display);
- // if the path doesn't contain a man page...
- if dir_entry_str.find("/man/").is_none() {
- let pkg_name_s = format!("/{}/", real_pkg_name);
- // ...then we want to have the package name in the path
- if dir_entry_str.find(&pkg_name_s).is_none() {
- e0028!(real_pkg_name, dir_entry_display);
- }
- }
-
- if dir_entry_str.ends_with(".map") {
- map_files_found = true;
- let re: Regex = Regex::new(r"fonts[/]map[/]dvips[/]").unwrap();
- if re.is_match(dir_entry_str) {
- map_dvips_found = true;
- }
+ // if we encounter a .dtx or .ins file we check
+ // that it is in a subdirectory of either source/ or doc/
+ if (dir_entry_str.ends_with(".dtx") || dir_entry_str.ends_with(".ins"))
+ && !(dir_entry_display.starts_with("source/")
+ || dir_entry_display.starts_with("doc/"))
+ {
+ e0036!(dir_entry_display);
+ continue;
+ }
- // if ( $fname =~ m{ fonts [/] map [/] dvips [/] }xms ) {
- // $map_dvips_found = 1;
- // }
- }
+ // if the path doesn't contain a man page...
+ if dir_entry_str.find("/man/").is_none() {
+ let pkg_name_s = format!("/{}/", real_pkg_name);
+ // ...then we want to have the package name in the path
+ if dir_entry_str.find(&pkg_name_s).is_none() {
+ e0028!(real_pkg_name, dir_entry_display);
}
- Err(e) => {
- e0027!(dir_entry.path().display(), e);
+ }
+
+ if dir_entry_str.ends_with(".map") {
+ map_files_found = true;
+ let re: Regex = Regex::new(r"fonts[/]map[/]dvips[/]").unwrap();
+ if re.is_match(dir_entry_str) {
+ map_dvips_found = true;
}
}
}
@@ -765,7 +756,6 @@ fn check_tds_archive(
}
});
- tmp_dir.close()?;
// now check if each package file is in the tds archive
for (k, paths) in hashes.iter() {
if !tds_hashes.contains_key(k) {
@@ -773,8 +763,6 @@ fn check_tds_archive(
e0026!(p);
}
}
-
- Ok(())
}
fn get_extension_from_filename(filename: &str) -> Option<&str> {
@@ -807,6 +795,21 @@ fn found_unwanted_filetype(fname: &str, ft: &FType) -> bool {
}
}
+fn fix_perms(entry: &str) {
+ if !ARGS.correct_perms {
+ return;
+ }
+
+ i0005!(entry);
+ let rc = run_cmd("/bin/chmod", &["-v", "ug=rwX,o=rX", entry]);
+
+ if rc.status {
+ if let Some(op) = rc.output {
+ print!("{}", op);
+ }
+ }
+}
+
fn check_and_correct_perms(dir_entry: &str, p: u32) {
if owner_has(p, 5) || !others_have(p, 4) {
e0002!(dir_entry, perms_to_string(p));
@@ -817,6 +820,21 @@ fn check_and_correct_perms(dir_entry: &str, p: u32) {
};
}
+// Sets permissions for a file or directory
+// Sample invocation: set_perms("somfile", 0o644);
+fn set_perms(entry: &str, p: u32) -> bool {
+ let ps = &format!("{:o}", p);
+ let rc = run_cmd("chmod", &["-v", ps, entry]);
+
+ if rc.status {
+ if let Some(op) = rc.output {
+ print!("{}", op);
+ }
+ true
+ } else {
+ false
+ }
+}
#[derive(Debug, Clone, PartialEq)]
enum FileKind {
@@ -825,7 +843,6 @@ enum FileKind {
Symlink(String),
}
-
impl Display for FileKind {
fn fmt(&self, f: &mut ::std::fmt::Formatter) -> Result<(), ::std::fmt::Error> {
match *self {
@@ -836,8 +853,6 @@ impl Display for FileKind {
}
}
-
-
#[derive(Debug, Clone, PartialEq)]
enum ReadmeKind {
No,
@@ -845,15 +860,17 @@ enum ReadmeKind {
Symlink(String),
}
-fn check_package(root: &str, tds_zip: &Option<String>) -> Option<Hashes> {
- //let mut urls = HashMap::default(); //: <String,UrlStatus>
-
+fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> {
let mut lcnames: HashMap<PathBuf, Vec<(PathBuf, FileKind)>> = HashMap::default();
let mut doublenames: HashMap<PathBuf, Vec<PathBuf>> = HashMap::default();
let mut inodes = HashSet::default();
+ let ut = Utils::new(utils::CheckType::PACKAGE);
+
+
+
i0002!(root);
// This hash contains all package file names.
//
@@ -862,185 +879,162 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<Hashes> {
// String: the file name without any directory part
// ReadmeKind: is it a certain README, file or symlink?
// A special case of a README file is a file with has a different name but
- // was poinded to by a symlink. Example: README --> README.rst
- let mut file_names: HashMap<PathBuf, (Metadata, String, ReadmeKind)> = HashMap::default();
- {
- let mut readme_found = false;
-
- let root_absolute = PathBuf::from(root)
- .canonicalize()
- .unwrap()
- .to_string_lossy()
- .to_string();
- for dir_entry in WalkDir::new(root).follow_links(false) {
- match dir_entry {
- Ok(dir_entry) => {
- let dir_entry_str = match dir_entry.path().to_str() {
- Some(d) => d,
- None => {
- e0031!(dir_entry.path().to_string_lossy());
- continue;
- }
- };
- let meta = match dir_entry.metadata() {
- Ok(meta) => meta,
- Err(e) => {
- e0023!(e);
- continue;
- }
- };
-
- if !check_inode(&mut inodes, &meta) {
+ // was pointed to by a symlink. Example: README --> README.rst
+ let mut file_names: FileNamesHashMap = HashMap::default();
+ let mut readme_found = false;
+
+ let root_absolute = PathBuf::from(root)
+ .canonicalize()
+ .unwrap()
+ .to_string_lossy()
+ .to_string();
+
+ for dir_entry in WalkDir::new(root).follow_links(false) {
+ match dir_entry {
+ Ok(dir_entry) => {
+ let dir_entry_str = match dir_entry.path().to_str() {
+ Some(d) => d,
+ None => {
+ e0031!(dir_entry.path().to_string_lossy());
continue;
}
+ };
+ let meta = match dir_entry.metadata() {
+ Ok(meta) => meta,
+ Err(e) => {
+ e0023!(e);
+ continue;
+ }
+ };
- // this is the file_name without the directory part
- // unwrap() is ok here as we covered potential UTF-8 related errors
- // above in the definition of dir_entry_str
- let file_name = dir_entry.file_name().to_str().unwrap().to_string();
+ if !check_inode(&mut inodes, &meta) {
+ continue;
+ }
- let ft = get_filetype(&dir_entry);
+ // this is the file_name without the directory part
+ // unwrap() is ok here as we covered potential UTF-8 related errors
+ // above in the definition of dir_entry_str
+ let file_name = dir_entry.file_name().to_str().unwrap().to_string();
- if found_unwanted_filetype(dir_entry_str, &ft) {
- continue;
- }
+ // we check for weird stuff like socket files aso.
+ let ft = get_filetype(&dir_entry);
+ if found_unwanted_filetype(&dir_entry_str, &ft) {
+ continue;
+ }
- if let Some((offset, ch)) = filename_has_bad_chars(&dir_entry) {
- e0001!(ch, dir_entry_str, offset);
- }
+ ut.filename_has_bad_chars(&dir_entry, dir_entry_str);
- // 1. dealing with symlinks
- if ft == FType::Symlink {
- match get_symlink(&dir_entry) {
- Ok(None) => {
- e0010!(dir_entry_str);
- continue;
- }
- Err(e) => {
- e0027!(dir_entry_str, e);
+ // 1. dealing with symlinks
+ if ft == FType::Symlink {
+ match get_symlink(&dir_entry) {
+ Ok(None) => {
+ e0010!(&dir_entry_str);
+ continue;
+ }
+ Err(e) => {
+ e0027!(&dir_entry_str, e);
+ continue;
+ }
+ Ok(Some(p)) => {
+ let pd: String =
+ p.canonicalize().unwrap().to_string_lossy().to_string();
+ if !pd.starts_with(&root_absolute) {
+ e0030!(&dir_entry_str, p.display());
continue;
}
- Ok(Some(p)) => {
- let pd: String =
- p.canonicalize().unwrap().to_string_lossy().to_string();
- if !pd.starts_with(&root_absolute) {
- e0030!(dir_entry_str, p.display());
- continue;
- }
- let lc_dir_entry_str = dir_entry_str.to_lowercase();
- if let Some(_dir_name) = filename(dir_entry_str) {
- let lcnref = &mut lcnames;
- lcnref
- .entry(PathBuf::from(lc_dir_entry_str))
- .or_insert_with(Vec::new)
- .push((
- PathBuf::from(dir_entry_str),
- //FileKind::Symlink(dir_entry_str.into()),
- FileKind::Symlink(pd.clone()),
- ));
- }
- if is_readme(&dir_entry) {
- readme_found = true;
- file_names.insert(
- p,
- (
- meta,
- file_name,
- ReadmeKind::Symlink(dir_entry_str.to_string()),
- ),
- );
- }
- continue;
+ let lc_dir_entry_str = dir_entry_str.to_lowercase();
+ if let Some(_dir_name) = filename(&dir_entry_str) {
+ let lcnref = &mut lcnames;
+ lcnref
+ .entry(PathBuf::from(lc_dir_entry_str))
+ .or_insert_with(Vec::new)
+ .push((
+ PathBuf::from(&dir_entry_str),
+ //FileKind::Symlink(&dir_entry_str.into()),
+ FileKind::Symlink(pd.clone()),
+ ));
}
+ if is_readme(&dir_entry) {
+ readme_found = true;
+ file_names.insert(
+ p,
+ (
+ meta,
+ file_name,
+ ReadmeKind::Symlink(dir_entry_str.to_string()),
+ ),
+ );
+ }
+ continue;
}
}
+ }
- let p = get_perms(&dir_entry.path());
+ let p = get_perms(&dir_entry.path());
- // 2. dealing with directories
- if ft == FType::Directory {
- let lc_dir_entry_str = dir_entry_str.to_lowercase();
- if let Some(_dir_name) = filename(dir_entry_str) {
- let lcnref = &mut lcnames;
-
- lcnref
- .entry(PathBuf::from(lc_dir_entry_str))
- .or_insert_with(Vec::new)
- .push((PathBuf::from(dir_entry_str), FileKind::Directory));
- }
+ // 2. dealing with directories
+ if ft == FType::Directory {
- if !owner_has(p, 5) || !others_have(p, 5) {
- e0011!(dir_entry_str, perms_to_string(p));
- if ARGS.correct_perms {
- i0005!(&dir_entry_str);
- set_perms(&dir_entry_str, 0o775);
- }
- }
+ let lc_dir_entry_str = dir_entry_str.to_lowercase();
+ if let Some(_dir_name) = filename(&dir_entry_str) {
+ let lcnref = &mut lcnames;
- match is_empty_directory(&dir_entry) {
- Ok(true) => {
- e0004!(dir_entry_str);
- }
- Err(e) => {
- e0027!(dir_entry_str, e);
- }
- Ok(false) => (),
- }
- if is_hidden_directory(&dir_entry) {
- e0006!(dir_entry_str);
- }
+ lcnref
+ .entry(PathBuf::from(lc_dir_entry_str))
+ .or_insert_with(Vec::new)
+ .push((PathBuf::from(&dir_entry_str), FileKind::Directory));
+ }
- if is_unwanted_directory(&dir_entry.file_name().to_str().unwrap()) {
- e0018!(dir_entry_str);
+ if !owner_has(p, 5) || !others_have(p, 5) {
+ e0011!(&dir_entry_str, perms_to_string(p));
+ if ARGS.correct_perms {
+ i0005!(&dir_entry_str);
+ set_perms(&dir_entry_str, 0o775);
}
- continue;
}
- // 3. dealing with regular files
- if is_hidden(&file_name) {
- e0007!(dir_entry_str);
- }
- if is_temporary_file(&dir_entry_str) {
- e0008!(dir_entry_str);
- }
+ ut.check_for_empty_directory(&dir_entry_str, &dir_entry_str);
+ ut.check_for_hidden_directory(&file_name, &dir_entry_str);
+ ut.is_unwanted_directory(&file_name, dir_entry_str);
+ continue;
+ }
- if let Some(file_name) = filename(dir_entry_str) {
- let doubleref = &mut doublenames;
+ // 3. dealing with regular files
+ ut.check_for_hidden_file(&file_name, &dir_entry_str);
+ ut.check_for_temporary_file(&dir_entry_str);
- doubleref
- .entry(PathBuf::from(file_name))
- .or_insert_with(Vec::new)
- .push(PathBuf::from(dir_entry_str));
- }
+ // if is_temporary_file(&dir_entry_str) {
+ // e0008!(&dir_entry_str);
+ // }
- // detecting large files in package directory
-
- if is_readme(&dir_entry) {
- // We want to deal with README files only if they are
- // in the root directory of the package.
- let f = format!(
- "{}{}{}",
- root,
- // we have to pay attention if `root` ends already with '/'
- if root.ends_with('/') { "" } else { "/" },
- &file_name
- );
+ if let Some(file_name) = filename(&dir_entry_str) {
+ let doubleref = &mut doublenames;
- if dir_entry_str == f {
- readme_found = true;
- file_names.insert(
- dir_entry.path().to_path_buf(),
- (meta, file_name.clone(), ReadmeKind::Yes),
- );
- } else {
- file_names.entry(dir_entry.path().to_path_buf()).or_insert((
- meta,
- file_name.clone(),
- ReadmeKind::No,
- ));
- }
+ doubleref
+ .entry(PathBuf::from(file_name))
+ .or_insert_with(Vec::new)
+ .push(PathBuf::from(&dir_entry_str));
+ }
+
+ if is_readme(&dir_entry) {
+ // We want to deal with README files only if they are
+ // in the root directory of the package.
+ let f = format!(
+ "{}{}{}",
+ root,
+ // we have to pay attention if `root` ends already with '/'
+ if root.ends_with('/') { "" } else { "/" },
+ &file_name
+ );
+
+ if dir_entry_str == f {
+ readme_found = true;
+ file_names.insert(
+ dir_entry.path().to_path_buf(),
+ (meta, file_name.clone(), ReadmeKind::Yes),
+ );
} else {
file_names.entry(dir_entry.path().to_path_buf()).or_insert((
meta,
@@ -1048,168 +1042,203 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<Hashes> {
ReadmeKind::No,
));
}
+ } else {
+ file_names.entry(dir_entry.path().to_path_buf()).or_insert((
+ meta,
+ file_name.clone(),
+ ReadmeKind::No,
+ ));
+ }
- let lc_dir_entry_str = dir_entry_str.to_lowercase();
-
- let lcnref = &mut lcnames;
+ let lc_dir_entry_str = dir_entry_str.to_lowercase();
- lcnref
- .entry(PathBuf::from(lc_dir_entry_str))
- .or_insert_with(Vec::new)
- .push((PathBuf::from(dir_entry_str), FileKind::File));
+ let lcnref = &mut lcnames;
- }
+ lcnref
+ .entry(PathBuf::from(lc_dir_entry_str))
+ .or_insert_with(Vec::new)
+ .push((PathBuf::from(&dir_entry_str), FileKind::File));
+ }
- Err(e) => {
- eprintln!("{}", e);
- }
+ Err(e) => {
+ eprintln!("{}", e);
}
}
+ }
- if !readme_found {
- e0009!();
- }
+ if !readme_found {
+ e0009!();
}
let lc = LinkCheck::new(4, false);
let mut detective = filemagic::Filetype::new();
- let mut sizes = HashMap::default();
- let mut generated: HashMap<PathBuf, PathBuf> = HashMap::default();
+ let mut sizes: SizesHashMap = HashMap::default();
+ let mut generated: GeneratedHashMap = HashMap::default();
- {
- // Processing a single file entry, with the "sizes" hashmap collecting
- // same-size files. Entries are either Found::One or Found::Multiple,
- // so that we can submit the first file's path as a hashing job when the
- // first duplicate is found. Hashing each file is submitted as a job to
- // the pool.
- let mut process = |fsize, path: &PathBuf| {
- let sizeref = &mut sizes;
+ // Processing a single file entry, with the "sizes" hashmap collecting
+ // same-size files. Entries are either Found::One or Found::Multiple,
+ // so that we can submit the first file's path as a hashing job when the
+ // first duplicate is found. Hashing each file is submitted as a job to
+ // the pool.
+ let mut process = |fsize, path: &PathBuf| {
+ let sizeref = &mut sizes;
- let path = path.clone();
- sizeref.entry(fsize).or_insert_with(Vec::new).push(path);
+ let path = path.clone();
+ sizeref.entry(fsize).or_insert_with(Vec::new).push(path);
+ };
+ for (path, (meta, _file_name, is_readme)) in file_names.iter() {
+ //println!("DEBUG: path {:?}, file_name {:?}, is_readme: {} ", &path, &file_name, is_readme);
+ let dir_entry_str = match path.to_str() {
+ Some(d) => d,
+ None => {
+ e0031!(&path.to_string_lossy());
+ continue;
+ }
};
- for (path, (meta, _file_name, is_readme)) in file_names.iter() {
- //println!("DEBUG: path {:?}, file_name {:?}, is_readme: {} ", &path, &file_name, is_readme);
- let dir_entry_str = match path.to_str() {
- Some(d) => d,
- None => {
- e0031!(&path.to_string_lossy());
- continue;
- }
- };
+ let fsize = meta.len();
+ ut.check_filesize(fsize, dir_entry_str);
- let fsize = meta.len();
+ let p = get_perms(&path);
+ if !owner_has(p, 4) {
+ e0002!(&dir_entry_str, perms_to_string(p));
+ fix_perms(&dir_entry_str);
+ continue;
+ }
+ let ftr = detective.analyze(&dir_entry_str);
+ //println!(">>> {:?}", ftr);
+ // we ignore errors from filetype recognition
+ if ftr.is_err() {
+ continue;
+ }
- // if it is an empty file we don't need
- // to check further
- if fsize == 0 {
- e0005!(dir_entry_str);
- continue;
- }
- if fsize > 40 * 1024 * 1024 {
- w0005!(dir_entry_str, fsize / 1024 / 1024);
- continue;
- }
+ let ft = ftr.unwrap();
- let p = get_perms(&path);
- if !owner_has(p, 4) {
- e0002!(dir_entry_str, perms_to_string(p));
- if ARGS.correct_perms {
- fix_perms(&dir_entry_str);
- }
+ // DEBUG !readme_symlinked.contains(&dir_entry_str)
+ if ReadmeKind::No != *is_readme {
+ if !check_readme(&dir_entry_str, is_readme, &ft) {
continue;
}
- let ftr = detective.analyze(&dir_entry_str);
- //println!(">>> {:?}", ftr);
- // we ignore errors from filetype recognition
- if ftr.is_err() {
- continue;
+ if ARGS.urlcheck {
+ lc.check_urls(&dir_entry_str);
}
+ }
- let ft = ftr.unwrap();
-
- // DEBUG !readme_symlinked.contains(dir_entry_str)
- if ReadmeKind::No != *is_readme {
- if !check_readme(dir_entry_str, is_readme, &ft) {
- continue;
- }
- if ARGS.urlcheck {
- lc.check_urls(dir_entry_str);
+ match ft {
+ filemagic::Mimetype::Text(_) => {
+ check_and_correct_perms(&dir_entry_str, p);
+ let fext = get_extension_from_filename(&dir_entry_str);
+ if fext == Some("ins") || fext == Some("dtx") {
+ check_generated_files(&dir_entry_str, &mut generated);
}
- }
- match ft {
- filemagic::Mimetype::Text | filemagic::Mimetype::TextCRLF => {
- check_and_correct_perms(dir_entry_str, p);
- let fext = get_extension_from_filename(&dir_entry_str);
- if fext == Some("ins") || fext == Some("dtx") {
- check_generated_files(&dir_entry_str, &mut generated);
- }
- match fext {
- Some("bat") | Some("cmd") |
- Some("nsh") | Some("reg") => (),
- Some(_) | None => {
- if ft == filemagic::Mimetype::TextCRLF {
- e0012!(dir_entry_str);
- if ARGS.correct_crlf {
- fix_crlf(dir_entry_str);
+ match fext {
+ // deal with Windows files
+ Some("bat") | Some("cmd") | Some("nsh") | Some("reg") => {
+ match ft {
+ filemagic::Mimetype::Text(LineEnding::CRLF) => (),
+ filemagic::Mimetype::Text(LineEnding::CR) =>
+ {
+ e0037!(&dir_entry_str);
+ if ARGS.correct_le {
+ make_crlf(&dir_entry_str);
}
- }
+ },
+ filemagic::Mimetype::Text(LineEnding::MIXED(0, 0, 0) ) => (),
+ filemagic::Mimetype::Text(LineEnding::MIXED(cr, lf, crlf) ) =>
+ {
+ e0038!(&dir_entry_str, cr, lf, crlf);
+ if ARGS.correct_le {
+ fix_inconsistent_le(&dir_entry_str);
+ }
+ },
+ filemagic::Mimetype::Text(LineEnding::LF) =>
+ {
+ w0008!(&dir_entry_str);
+ },
+ fmm => println!("Should not occur: {} has {:?}", dir_entry_str, fmm),
}
- };
+ },
+ Some(_) | None => {
+ match ft {
+ filemagic::Mimetype::Text(LineEnding::CRLF) =>
+ {
+ e0012!(&dir_entry_str);
+ if ARGS.correct_le {
+ fix_inconsistent_le(&dir_entry_str);
+ }
+ },
+ filemagic::Mimetype::Text(LineEnding::CR) =>
+ {
+ e0037!(&dir_entry_str);
+ if ARGS.correct_le {
+ fix_inconsistent_le(&dir_entry_str);
+ }
+ },
+ filemagic::Mimetype::Text(LineEnding::MIXED(0, 0, 0) ) => (),
+ filemagic::Mimetype::Text(LineEnding::MIXED(cr, lf, crlf) ) =>
+ {
+ //println!(">>>{}: {:?} {},{},{}", &dir_entry_str, ft, x, y, z);
+ e0038!(&dir_entry_str, cr, lf, crlf);
+ if ARGS.correct_le {
+ fix_inconsistent_le(&dir_entry_str);
+ }
+ },
+ filemagic::Mimetype::Text(LineEnding::LF) => (),
+ fmm => println!("Should not occur: {} has {:?}", dir_entry_str, fmm),
+ }
+ },
}
+ },
- filemagic::Mimetype::BOM(b) => {
- //println!("{}: {} with BOM detected", dir_entry_str, b.as_ref());
- w0004!(dir_entry_str, b.as_ref());
- check_and_correct_perms(dir_entry_str, p);
- }
- filemagic::Mimetype::Binary
- | filemagic::Mimetype::Script
- | filemagic::Mimetype::ScriptCRLF => {
- if !owner_has(p, 4) || !others_have(p, 4) {
- e0002!(dir_entry_str, perms_to_string(p));
+ filemagic::Mimetype::BOM(b) => {
+ //println!("{}: {} with BOM detected", dir_entry_str, b.as_ref());
+ w0004!(&dir_entry_str, b.as_ref());
+ check_and_correct_perms(&dir_entry_str, p);
+ }
+ filemagic::Mimetype::Binary
+ | filemagic::Mimetype::Script(_) => {
+ if !owner_has(p, 4) || !others_have(p, 4) {
+ e0002!(&dir_entry_str, perms_to_string(p));
+ };
+ fix_perms(&dir_entry_str);
+ }
+ filemagic::Mimetype::Pdf => {
+ check_and_correct_perms(&dir_entry_str, p);
+ let ret = is_pdf_ok(&dir_entry_str);
+ if !ret.status {
+ e0017!(&dir_entry_str);
+ if let Some(output) = ret.output {
+ println!("{}", &output);
};
- if ARGS.correct_perms {
- fix_perms(&dir_entry_str);
- }
- }
- filemagic::Mimetype::Pdf => {
- check_and_correct_perms(dir_entry_str, p);
- let ret = is_pdf_ok(&dir_entry_str);
- if !ret.status {
- e0017!(dir_entry_str);
- if let Some(output) = ret.output {
- println!("{}", &output);
- };
- }
- }
-
- filemagic::Mimetype::Archive | filemagic::Mimetype::Zip => {
- if dir_entry_str.ends_with(".tds.zip") {
- e0035!(dir_entry_str);
- } else {
- w0001!(dir_entry_str);
- }
- check_and_correct_perms(dir_entry_str, p);
}
-
- filemagic::Mimetype::Data => check_and_correct_perms(dir_entry_str, p),
- _ => continue,
}
- if others_match(p, 0) {
- e0002!(dir_entry_str, perms_to_string(p));
- if ARGS.correct_perms {
- i0005!(&dir_entry_str);
- set_perms(&dir_entry_str, 0o664);
+
+ filemagic::Mimetype::Archive | filemagic::Mimetype::Zip => {
+ if dir_entry_str.ends_with(".tds.zip") {
+ e0035!(&dir_entry_str);
+ } else {
+ w0001!(&dir_entry_str);
}
+ check_and_correct_perms(&dir_entry_str, p);
}
- if !(ARGS.ignore_dupes && tds_zip.is_none()) {
- process(fsize, &path);
+
+ filemagic::Mimetype::Data => check_and_correct_perms(&dir_entry_str, p),
+ filemagic::Mimetype::Zerofile => check_and_correct_perms(&dir_entry_str, p),
+ _ => continue,
+ }
+
+ if others_match(p, 0) {
+ e0002!(&dir_entry_str, perms_to_string(p));
+ if ARGS.correct_perms {
+ i0005!(&dir_entry_str);
+ set_perms(&dir_entry_str, 0o664);
}
}
+ if !(ARGS.ignore_dupes && tds_zip.is_none()) {
+ process(fsize, &path);
+ }
}
print_casefolding(&lcnames);
@@ -1259,7 +1288,6 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<Hashes> {
}
fn print_casefolding(hashes: &HashMap<PathBuf, Vec<(PathBuf, FileKind)>>) {
-
for (k, eles) in hashes.iter() {
//println!("pcf: {:?}, {:?}", k, &eles);
if eles.len() == 1 {
@@ -1274,12 +1302,14 @@ fn print_casefolding(hashes: &HashMap<PathBuf, Vec<(PathBuf, FileKind)>>) {
}
}
-fn print_generated(hashes: &HashMap<PathBuf, Vec<PathBuf>>, generated: &HashMap<PathBuf, PathBuf>) {
+fn print_generated(hashes: &HashMap<PathBuf, Vec<PathBuf>>, generated: &GeneratedHashMap) {
for (k, gen) in generated.iter() {
- if hashes.contains_key(k) {
- let v = &hashes[k];
+ let mut path = PathBuf::new();
+ path.push(k);
+ if hashes.contains_key(&path) {
+ let v = &hashes[&path];
for fname in v {
- e0019!(fname.to_str().unwrap(), gen.to_str().unwrap());
+ e0019!(fname.to_str().unwrap(), gen.as_str());
}
}
}
@@ -1317,3 +1347,5 @@ fn show_tmp_endings() {
println!("{:23} {}", t, c);
}
}
+
+
diff --git a/support/pkgcheck/src/messages/errorsd.rs b/support/pkgcheck/src/messages/errorsd.rs
index f839f5538b..09c2f4cc53 100644
--- a/support/pkgcheck/src/messages/errorsd.rs
+++ b/support/pkgcheck/src/messages/errorsd.rs
@@ -49,7 +49,8 @@ pub fn e0004d() {
E0004 -- Empty directory not allowed
Empty directories are considered as rubbish, and are usually not
-accepted as part of a package.
+accepted as part of a package, neither in the package tree nor in the
+TDS zip archive.
For more details refer to:
http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#noemptyfiles
@@ -76,7 +77,8 @@ pub fn e0006d() {
r#"
E0006 -- Hidden directories not allowed
-A package should not contain hidden directories.
+A package should not contain hidden directories, neither in the package
+tree nor in the TDS zip archive.
For more details refer to:
http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#noauxfiles
@@ -89,7 +91,8 @@ pub fn e0007d() {
r#"
E0007 -- Hidden files not allowed
-A package should not contain hidden files.
+A package should not contain hidden files, neither in the package tree
+nor in the TDS zip archive.
For more details refer to:
http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#noauxfiles
@@ -264,26 +267,6 @@ http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#nogeneratedfiles
)
}
-pub fn e0020d() {
- println!(
- r#"
-E0020 -- Unwanted directory detected in the top level directory in TDS zip archive
-
-A top level directory of a TDS archive should only contain all or some
-of the following directories:
-
-- tex
-- fonts
-- metafont
-- metapost
-- bibtex
-- scripts
-- doc
-- source
-"#
- )
-}
-
pub fn e0021d() {
println!(
r#"
@@ -453,3 +436,42 @@ A package directory should not contain a TDS zip archive.
)
}
+pub fn e0036d() {
+ println!(
+ r#"
+E0036 -- .dtx/.ins files found in wrong directory in TDS zip archive
+
+In a TDS zip archive a .dtx resp. .ins file must be in a subdirectory of
+either of source/ or doc/ top level directories.
+"#
+ )
+}
+
+pub fn e0037d() {
+ println!(
+ r#"
+E0037 -- CR line endings detected
+
+The file specified in the error message contains CR line endings. Text
+files should have UNIX style line endings.
+
+For more details refer to:
+http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#crlf
+"#
+ )
+}
+
+pub fn e0038d() {
+ println!(
+ r#"
+E0038 -- File has inconsistent line endings: CR: x, LF: y, CRLF: z
+
+The file specified in the error message contains CR line endings. Text
+files should have UNIX style line endings.
+
+For more details refer to:
+http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#crlf
+"#
+ )
+}
+
diff --git a/support/pkgcheck/src/messages/fatald.rs b/support/pkgcheck/src/messages/fatald.rs
index b41cab83b1..ee02e47bde 100644
--- a/support/pkgcheck/src/messages/fatald.rs
+++ b/support/pkgcheck/src/messages/fatald.rs
@@ -64,3 +64,13 @@ code was specified.
)
}
+pub fn f0007d() {
+ println!(
+ r#"
+F0007 -- Could not create temporary directory for unzipping the TDS zip archive
+
+Make sure the temp directory is writable and/or the temp directory has
+enough space.
+"#
+ )
+}
diff --git a/support/pkgcheck/src/messages/informationd.rs b/support/pkgcheck/src/messages/informationd.rs
index d4e3ea43b3..340ade4766 100644
--- a/support/pkgcheck/src/messages/informationd.rs
+++ b/support/pkgcheck/src/messages/informationd.rs
@@ -75,3 +75,16 @@ http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#noauxfiles
)
}
+pub fn i0007d() {
+ println!(
+ r#"
+I0007 -- Successfully corrected wrong line endings to LF resp. CRLF
+
+pkgcheck successfully converted wrong line endings to LF line endings or
+to CRLF line endings if it the file was a Windows text file.
+
+Wrong line endings could be CR, CRLF or a mixture of line endings.
+"#
+ )
+}
+
diff --git a/support/pkgcheck/src/messages/mod.rs b/support/pkgcheck/src/messages/mod.rs
index 2edda3c536..842c693718 100644
--- a/support/pkgcheck/src/messages/mod.rs
+++ b/support/pkgcheck/src/messages/mod.rs
@@ -117,6 +117,17 @@ macro_rules! f0005 {
// };
// }
+macro_rules! f0007 {
+ ($fmt1:expr) => {
+ error_occured!();
+ print!(
+ "{} Unable to create temporary directory for unzipping the TDS zip archive: {}. Exiting...\n",
+ msgid!("F0007"),
+ $fmt1
+ );
+ };
+}
+
macro_rules! e0001 {
($fmt1:expr, $fmt2:expr, $fmt3:expr) => {
error_occured!();
@@ -171,6 +182,17 @@ macro_rules! e0006 {
};
}
+macro_rules! e0006t {
+ ($fmt:expr) => {
+ error_occured!();
+ print!(
+ "{} Hidden directory {} detected in TDS zip archive\n",
+ msgid!("E0006"),
+ $fmt
+ );
+ };
+}
+
macro_rules! e0007 {
($fmt:expr) => {
error_occured!();
@@ -178,6 +200,17 @@ macro_rules! e0007 {
};
}
+macro_rules! e0007t {
+ ($fmt:expr) => {
+ error_occured!();
+ print!(
+ "{} Hidden file {} detected in TDS zip archive\n",
+ msgid!("E0007"),
+ $fmt
+ );
+ };
+}
+
macro_rules! e0008 {
($fmt:expr) => {
error_occured!();
@@ -185,6 +218,16 @@ macro_rules! e0008 {
};
}
+macro_rules! e0008t {
+ ($fmt:expr) => {
+ error_occured!();
+ print!(
+ "{} Temporary file {} detected in TDS zip archive\n",
+ msgid!("E0008"),
+ $fmt
+ );
+ };
+}
macro_rules! e0009 {
() => {
error_occured!();
@@ -217,7 +260,7 @@ macro_rules! e0011 {
macro_rules! e0012 {
($fmt:expr) => {
error_occured!();
- print!("{} CRLF detected: {}\n", msgid!("E0012"), $fmt);
+ print!("{} File {} has CRLF line endings\n", msgid!("E0012"), $fmt);
};
}
@@ -283,17 +326,6 @@ macro_rules! e0019 {
};
}
-macro_rules! e0020 {
- ($fmt:expr) => {
- error_occured!();
- print!(
- "{} Unwanted directory `{}` detected in the top level directory of a TDS archive\n",
- msgid!("E0020"),
- $fmt
- );
- };
-}
-
macro_rules! e0021 {
($fmt1:expr, $fmt2:expr, $fmt3:expr) => {
error_occured!();
@@ -448,8 +480,30 @@ macro_rules! e0035 {
};
}
+macro_rules! e0036 {
+ ($fmt:expr) => {
+ error_occured!();
+ print!(
+ "{} File `{}` not below `source/` or `doc/` top level directory in TDS zip archive\n",
+ msgid!("E0036"),
+ $fmt
+ );
+ };
+}
+macro_rules! e0037 {
+ ($fmt:expr) => {
+ error_occured!();
+ print!("{} File {} has CR line endings\n", msgid!("E0037"), $fmt);
+ };
+}
+macro_rules! e0038 {
+ ($fmt1:expr, $fmt2:expr, $fmt3:expr, $fmt4:expr) => {
+ error_occured!();
+ print!("{} File {} has inconsistent line endings: CR={}, LF={}, CRLF={}\n", msgid!("E0038"), $fmt1, $fmt2, $fmt3, $fmt4);
+ };
+}
macro_rules! w0001 {
($fmt:expr) => {
print!(
@@ -513,6 +567,20 @@ macro_rules! w0006 {
};
}
+macro_rules! w0007 {
+ ($fmt:expr) => {
+ error_occured!();
+ print!("{} Empty directory {} detected in TDS zip archive\n", msgid!("W0007"), $fmt);
+ };
+}
+
+macro_rules! w0008 {
+ ($fmt:expr) => {
+ error_occured!();
+ print!("{} Windows file {} has Unix line endings\n", msgid!("W0008"), $fmt);
+ };
+}
+
macro_rules! i0002 {
($fmt:expr) => {
print!(
@@ -565,15 +633,17 @@ macro_rules! i0006 {
}
macro_rules! i0007 {
- ($fmt:expr) => {
+ ($fmt1:expr, $fmt2:expr) => {
print!(
- "{} {}: Successfully converted from CRLF to LF\n",
+ "{} {}: Successfully corrected wrong line endings to {}\n",
msgid!("I0007"),
- $fmt
+ $fmt1,
+ $fmt2
);
};
}
+
pub fn explains(err: &str) {
let err_upp = err.to_ascii_uppercase();
match err_upp.as_str() {
@@ -583,6 +653,7 @@ pub fn explains(err: &str) {
"F0004" => f0004d(),
"F0005" => f0005d(),
"F0006" => f0006d(),
+ "F0007" => f0007d(),
"E0001" => e0001d(),
"E0002" => e0002d(),
@@ -603,7 +674,6 @@ pub fn explains(err: &str) {
"E0017" => e0017d(),
"E0018" => e0018d(),
"E0019" => e0019d(),
- "E0020" => e0020d(),
"E0021" => e0021d(),
"E0022" => e0022d(),
"E0023" => e0023d(),
@@ -618,6 +688,9 @@ pub fn explains(err: &str) {
"E0033" => e0033d(),
"E0034" => e0034d(),
"E0035" => e0035d(),
+ "E0036" => e0036d(),
+ "E0037" => e0037d(),
+ "E0038" => e0038d(),
// "I0001" => i0001d!(),
"I0001" => i0001d(),
@@ -626,6 +699,7 @@ pub fn explains(err: &str) {
"I0004" => i0004d(),
"I0005" => i0005d(),
"I0006" => i0006d(),
+ "I0007" => i0007d(),
"W0001" => w0001d(),
"W0002" => w0002d(),
@@ -633,6 +707,8 @@ pub fn explains(err: &str) {
"W0004" => w0004d(),
"W0005" => w0005d(),
"W0006" => w0006d(),
+ "W0007" => w0007d(),
+ "W0008" => w0008d(),
e => println!(
"F0006 Unknown error code `{}` specified with option -e resp. --explain. Exiting...",
@@ -668,7 +744,6 @@ pub fn explains_all() {
explains("E0017");
explains("E0018");
explains("E0019");
- explains("E0020");
explains("E0021");
explains("E0022");
explains("E0023");
@@ -683,6 +758,7 @@ pub fn explains_all() {
explains("E0033");
explains("E0034");
explains("E0035");
+ explains("E0036");
explains("I0001");
explains("I0002");
@@ -690,6 +766,7 @@ pub fn explains_all() {
explains("I0004");
explains("I0005");
explains("I0006");
+ explains("I0007");
explains("W0001");
explains("W0002");
@@ -697,4 +774,6 @@ pub fn explains_all() {
explains("W0004");
explains("W0005");
explains("W0006");
+ explains("W0007");
+ explains("W0008");
}
diff --git a/support/pkgcheck/src/messages/warningsd.rs b/support/pkgcheck/src/messages/warningsd.rs
index b8a1aa8ef3..cea814c8ce 100644
--- a/support/pkgcheck/src/messages/warningsd.rs
+++ b/support/pkgcheck/src/messages/warningsd.rs
@@ -78,3 +78,31 @@ in the TDS zip archive.
)
}
+pub fn w0007d() {
+ println!(
+ r#"
+W0007 -- Empty directory detected in the TDS zip archive
+
+Empty directories in a TDS zip archive are discouraged. As they usually
+don't create errors in the distribution we issue a warning only.
+"#
+ )
+}
+
+pub fn w0008d() {
+ println!(
+ r#"
+W0008 -- Windows file has Unix line endings
+
+A Windows file with Unix line endings was detected.
+
+We regard a file as a Windows file if its name ends with:
+
+- .bat
+- .cmd
+- .nsh, or
+- .reg
+"#
+ )
+}
+
diff --git a/support/pkgcheck/src/recode.rs b/support/pkgcheck/src/recode.rs
index 39a85bacd2..9a6b3d426d 100644
--- a/support/pkgcheck/src/recode.rs
+++ b/support/pkgcheck/src/recode.rs
@@ -9,9 +9,52 @@ use std::io::prelude::*;
use std::io::{self, Read};
use std::path::Path;
+struct Bytes {
+ bytes: Vec<u8>,
+ len: usize,
+ index: usize,
+}
+
+impl Bytes {
+ fn new(bytes: Vec<u8>) -> Bytes {
+ let l = bytes.len();
+ Bytes {
+ bytes,
+ len: l,
+ index: 0,
+ }
+ }
+}
+
+#[derive(Debug)]
+enum Item {
+ Pair(u8, u8),
+ Single(u8),
+}
+
+impl Iterator for Bytes {
+ type Item = Item;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ let cur_index = self.index;
+ self.index += 1;
+
+ if cur_index == self.len {
+ return None;
+ }
+
+ if cur_index + 1 == self.len {
+ Some(Item::Single(self.bytes[cur_index]))
+ } else {
+ // if self.index < self.len {
+ Some(Item::Pair(self.bytes[cur_index], self.bytes[cur_index + 1]))
+ }
+ }
+}
+
// The caller has to make sure that crlf2lf is only
// used for text files
-pub fn crlf2lf(fname: &str) -> Result<(), io::Error> {
+pub fn wrong_line_endings2crlf(fname: &str) -> Result<(), io::Error> {
let path = Path::new(fname);
let mut hdl_in;
@@ -28,23 +71,43 @@ pub fn crlf2lf(fname: &str) -> Result<(), io::Error> {
return Err(e);
};
- // convert \r\n to \n
- let mut another_vec: Vec<u8> = Vec::new();
- const CR: u8 = 0x0d; // 13
- const LF: u8 = 0x0a; // 10
-
- for i in 0..buffer.len() {
- if buffer[i] == CR {
- if i < buffer.len() - 1 && buffer[i + 1] == LF {
- continue;
- }
+ let another_vec: Vec<u8> = fix_line_endings2crlf(&buffer);
- if i > 0 && buffer[i - 1] == LF {
- continue;
- }
+ let mut hdl_out;
+ match File::create(path) {
+ Ok(f) => {
+ hdl_out = f;
}
- another_vec.push(buffer[i]);
+ Err(e) => return Err(e),
+ };
+
+ // write back
+ match hdl_out.write(&another_vec) {
+ Ok(_) => Ok(()),
+ Err(e) => Err(e),
}
+}
+
+// The caller has to make sure that crlf2lf is only
+// used for text files
+pub fn wrong_line_endings2lf(fname: &str) -> Result<(), io::Error> {
+ let path = Path::new(fname);
+
+ let mut hdl_in;
+ match File::open(path) {
+ Ok(f) => {
+ hdl_in = f;
+ }
+ Err(e) => return Err(e),
+ };
+ let mut buffer: Vec<u8> = Vec::new();
+
+ // read the whole file
+ if let Err(e) = hdl_in.read_to_end(&mut buffer) {
+ return Err(e);
+ };
+
+ let another_vec: Vec<u8> = fix_line_endings2lf(&buffer);
let mut hdl_out;
match File::create(path) {
@@ -60,3 +123,155 @@ pub fn crlf2lf(fname: &str) -> Result<(), io::Error> {
Err(e) => Err(e),
}
}
+
+pub fn fix_line_endings2lf(buffer: &[u8]) -> Vec<u8> {
+ let mut another_vec: Vec<u8> = Vec::new();
+
+ const CR: u8 = 0x0d; // 13
+ const LF: u8 = 0x0a; // 10
+
+ let b = Bytes::new(buffer.to_vec());
+
+ let mut skip = false;
+ for v in b {
+ if skip {
+ skip = false;
+ continue;
+ }
+ match v {
+ Item::Pair(CR, LF) | Item::Pair(LF, CR) => {
+ another_vec.push(LF);
+ skip = true;
+ }
+ Item::Pair(CR, _) | Item::Pair(LF, _) | Item::Single(CR) | Item::Single(LF) => {
+ another_vec.push(LF);
+ }
+ Item::Pair(c, _) | Item::Single(c) => {
+ another_vec.push(c);
+ }
+ }
+ }
+
+ // if there is no line ending at the end of a fill add it
+ if another_vec[another_vec.len() - 1] != LF {
+ another_vec.push(LF);
+ }
+ another_vec
+}
+
+pub fn fix_line_endings2crlf(buffer: &[u8]) -> Vec<u8> {
+ let mut another_vec: Vec<u8> = Vec::new();
+
+ const CR: u8 = 0x0d; // 13 \r
+ const LF: u8 = 0x0a; // 10 \n
+
+ let b = Bytes::new(buffer.to_vec());
+
+ let mut skip = false;
+ for v in b {
+ if skip {
+ skip = false;
+ continue;
+ }
+ match v {
+ Item::Pair(CR, LF) | Item::Pair(LF, CR) => {
+ another_vec.push(CR);
+ another_vec.push(LF);
+ skip = true;
+ }
+ Item::Pair(CR, _) | Item::Pair(LF, _) | Item::Single(CR) | Item::Single(LF) => {
+ another_vec.push(CR);
+ another_vec.push(LF);
+ }
+ Item::Pair(c, _) | Item::Single(c) => {
+ another_vec.push(c);
+ }
+ }
+ }
+
+ let avlen = another_vec.len();
+
+ if avlen == 1 || (another_vec[avlen - 1] != LF && another_vec[avlen - 2] != CR) {
+ another_vec.push(CR);
+ another_vec.push(LF);
+ }
+
+ another_vec
+}
+
+#[test]
+fn test_fix_line_endings2crlf() {
+ let testfiles = vec![
+ ("a", "a\r\n"),
+ ("no line ending at all", "no line ending at all\r\n"),
+ ("\rbla", "\r\nbla\r\n"),
+ ("\r\r", "\r\n\r\n"),
+ ("\r\nbla", "\r\nbla\r\n"),
+ ("\r\n\r\nbla", "\r\n\r\nbla\r\n"),
+ ("bla\r\nbla", "bla\r\nbla\r\n"),
+ ("bla\r\n\r\nbla", "bla\r\n\r\nbla\r\n"),
+ ("bla\r\nbla\r\nbla", "bla\r\nbla\r\nbla\r\n"),
+ ("\n\r", "\r\n"),
+ ("\n\r\n\r", "\r\n\r\n"),
+ ("\n\rbla\n\r", "\r\nbla\r\n"),
+ ];
+
+ for (s, rs) in testfiles {
+ let bs = s.as_bytes().to_vec();
+ let brs = rs.as_bytes().to_vec();
+ assert_eq!(fix_line_endings2crlf(&bs), brs);
+ }
+}
+
+#[test]
+fn test_fix_line_endings() {
+ let testfiles = vec![
+ ("no line ending at all", "no line ending at all\n"),
+ ("\r", "\n"),
+ ("\n\r", "\n"),
+ ("\r\n", "\n"),
+ ("\r\n\n", "\n\n"),
+ ("CR at end of file\r", "CR at end of file\n"),
+ ("LF at end of file\n", "LF at end of file\n"),
+ ("CRLF at end of file\r\n", "CRLF at end of file\n"),
+ ("Two CR at end of file\r\r", "Two CR at end of file\n\n"),
+ ("Two LF at end of file\n\n", "Two LF at end of file\n\n"),
+ (
+ "Two CRLF at end of file\r\n\r\n",
+ "Two CRLF at end of file\n\n",
+ ),
+ (
+ "Two CR in the middle\r\r of file",
+ "Two CR in the middle\n\n of file\n",
+ ),
+ (
+ "Two LF in the middle\n\n of file",
+ "Two LF in the middle\n\n of file\n",
+ ),
+ (
+ "Two CRLF in the middle\r\n\r\n of file",
+ "Two CRLF in the middle\n\n of file\n",
+ ),
+ (
+ "Mixed CR, CRLF\rline2\r\nline3\r\n",
+ "Mixed CR, CRLF\nline2\nline3\n",
+ ),
+ (
+ "Mixed CR, CRLF\r\r\nline2\r\nline3\r\n",
+ "Mixed CR, CRLF\n\nline2\nline3\n",
+ ),
+ // ("Mixed CR, CRLF\n\r\r\nline2\r\nline3\r\n", "Mixed CR, CRLF\n\n\nline2\nline3\n"),
+ (
+ "Mixed CR, LFCR\rline2\n\rline3\r\n",
+ "Mixed CR, LFCR\nline2\nline3\n",
+ ),
+ //("line1\nebla", "line1\rebla"),
+ //("mor\r\nebla", "mor\rebla"),
+ ];
+
+ for (s, rs) in testfiles {
+ let bs = s.as_bytes().to_vec();
+ let brs = rs.as_bytes().to_vec();
+ assert_eq!(fix_line_endings2lf(&bs), brs);
+ }
+}
diff --git a/support/pkgcheck/src/utils.rs b/support/pkgcheck/src/utils.rs
index 6b2d0a8753..8a34bfe7c1 100644
--- a/support/pkgcheck/src/utils.rs
+++ b/support/pkgcheck/src/utils.rs
@@ -10,9 +10,173 @@ use std::borrow::Cow;
use std::io;
use std::process::Command;
+use colored::*;
+use std::sync::atomic::Ordering;
use std::fs;
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum CheckType {
+ TDS,
+ PACKAGE,
+}
+
+pub struct Utils {
+ kind: CheckType,
+}
+
+impl Utils {
+ pub fn new(kind: CheckType) -> Utils {
+ Utils { kind }
+ }
+
+ pub fn check_filesize(&self, fsize: u64, dir_entry_str: &str) -> bool {
+ // if it is an empty file we don't need
+ // to check further
+ if fsize == 0 {
+ e0005!(dir_entry_str);
+ return false;
+ }
+ if fsize > 40 * 1024 * 1024 {
+ if self.kind == CheckType::PACKAGE {
+ w0005!(dir_entry_str, fsize / 1024 / 1024);
+ } else {
+ w0006!(dir_entry_str, fsize / 1024 / 1024);
+ }
+ return false;
+ }
+
+ true
+ }
+
+ pub fn filename_has_bad_chars(&self, entry: &DirEntry, dir_entry_str: &str) {
+ let s = entry.file_name().to_str().unwrap();
+
+ for (i, c) in s.char_indices() {
+ match c {
+ '\x00'..='\x1f' => {
+ e0001!(c, dir_entry_str, i);
+ }
+ ' ' | '!' | '"' | '#' | '$' | '%' | '&' | '\'' => {
+ e0001!(c, dir_entry_str, i);
+ }
+ '(' | ')' | '*' => {
+ e0001!(c, dir_entry_str, i);
+ }
+ '+' | ',' | '-' | '.' => (),
+ '0'..='9' => (),
+ ':' | ';' | '<' | '=' | '>' | '?' | '@' => {
+ e0001!(c, dir_entry_str, i);
+ }
+ 'A'..='Z' => (),
+ '[' | '\\' | ']' | '^' => {
+ e0001!(c, dir_entry_str, i);
+ }
+ '_' => (),
+ '`' => {
+ e0001!(c, dir_entry_str, i);
+ }
+ 'a'..='z' => (),
+ '{' | '|' | '}' | '~' => {
+ e0001!(c, dir_entry_str, i);
+ }
+ '/' => return,
+ _ => {
+ return {
+ e0001!(c, dir_entry_str, i);
+ }
+ }
+ };
+ }
+ }
+
+ pub fn is_unwanted_directory(&self, entry: &str, dir_entry_str: &str) {
+ let names = ["__MACOSX"];
+ for n in &names {
+ if *n == entry {
+ e0018!(dir_entry_str);
+ };
+ }
+ }
+
+ pub fn check_for_hidden_file(&self, entry: &str, dir_entry_str: &str) {
+ if entry.starts_with('.') {
+ match self.kind {
+ CheckType::PACKAGE => {
+ e0007!(dir_entry_str);
+ }
+ CheckType::TDS => {
+ e0007t!(dir_entry_str);
+ }
+ }
+ }
+ }
+
+ // fname is the base name without the directory part
+ pub fn check_for_hidden_directory(&self, fname: &str, dir_entry_str: &str) {
+
+ // We have to single out '.' and './' as they denote just the current directory
+ // That case may happen if pkgcheck gets called like: `pkgcheck -d .` or `pkgcheck -d ./`
+ if fname == "." || fname == "./" {
+ return;
+ };
+
+ //if fname.map(|s| s.starts_with('.')).unwrap_or(false) {
+ if fname.starts_with('.') {
+ match self.kind {
+ CheckType::PACKAGE => {
+ e0006!(dir_entry_str);
+ }
+ CheckType::TDS => {
+ e0006t!(dir_entry_str);
+ }
+ }
+ }
+ }
+
+ pub fn check_for_temporary_file(&self, dir_entry_str: &str) {
+ lazy_static! {
+ static ref RE: Regex = regex_temporary_file_endings();
+ }
+
+ if RE.is_match(dir_entry_str) {
+ match self.kind {
+ CheckType::PACKAGE => {
+ e0008!(dir_entry_str);
+ }
+ CheckType::TDS => {
+ e0008t!(dir_entry_str);
+ }
+ }
+ }
+ }
+
+ pub fn check_for_empty_directory(&self, dir_entry_str: &str, dir_entry_display: &str) {
+ match read_dir(dir_entry_str) {
+ // try to read the directory specified
+ Ok(contents) => {
+ if contents.count() == 0 {
+ match self.kind {
+ CheckType::PACKAGE => {
+ e0004!(dir_entry_display);
+ }
+ CheckType::TDS => {
+ // Note: Karl Berry recommended to issue a warning only
+ // as an empty directory in a TDS zip archive
+ // - is automatically deleted before including
+ // it in texlive
+ w0007!(dir_entry_display);
+ }
+ }
+ }
+ }
+ Err(e) => {
+ e0027!(dir_entry_display, e);
+ }
+ }
+ }
+}
+
pub fn temp_file_endings() -> Vec<(&'static str, &'static str)> {
// https://github.com/github/gitignore/blob/master/TeX.gitignore
// http://hopf.math.purdue.edu/doc/html/suffixes.html
@@ -125,53 +289,6 @@ pub fn regex_temporary_file_endings() -> Regex {
Regex::new(&rv).unwrap()
}
-pub fn is_temporary_file(entry: &str) -> bool {
- lazy_static! {
- static ref RE: Regex = regex_temporary_file_endings();
- }
-
- RE.is_match(entry)
-}
-
-pub fn is_empty_directory(entry: &DirEntry) -> Result<bool, io::Error> {
- let s = entry.path().to_str().unwrap();
- match read_dir(s) {
- // try to read the directory specified
- Ok(contents) => Ok(contents.count() == 0),
- Err(e) => Err(e),
- }
-}
-
-pub fn is_unwanted_directory(entry: &str) -> bool {
- let names = ["__MACOSX"];
- for n in &names {
- if *n == entry {
- return true;
- };
- }
-
- false
-}
-
-// is_hidden checks for files or directories starting with a dot
-pub fn is_hidden(entry: &str) -> bool {
- entry.starts_with('.')
-}
-
-// is_hidden checks for files or directories starting with a dot
-pub fn is_hidden_directory(entry: &DirEntry) -> bool {
- let fname = entry.file_name();
-
- if fname == "." {
- return false;
- };
- if fname == "./" {
- return false;
- };
-
- fname.to_str().map(|s| s.starts_with('.')).unwrap_or(false)
-}
-
pub fn get_symlink(entry: &DirEntry) -> Result<Option<PathBuf>, io::Error> {
let r = entry.path().to_str().unwrap();
@@ -309,22 +426,6 @@ pub fn run_cmd(cmd: &str, argument: &[&str]) -> CmdReturn {
}
}
-// Sets permissions for a file or directory
-// Sample invocation: set_perms("somfile", 0o644);
-pub fn set_perms(entry: &str, p: u32) -> bool {
- let ps = &format!("{:o}", p);
- let rc = run_cmd("chmod", &["-v", ps, entry]);
-
- if rc.status {
- if let Some(op) = rc.output {
- print!("{}", op);
- }
- true
- } else {
- false
- }
-}
-
// returns true if file is a directory and does exist
// returns false otherwise
pub fn exists_dir(file: &str) -> bool {
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-26 08:06:03 +0000