From 22d3a7ddeb2b2637308cf883f9bd3590e649aa20 Mon Sep 17 00:00:00 2001 From: Norbert Preining Date: Sat, 18 Jan 2020 03:01:23 +0000 Subject: CTAN sync 202001180301 --- support/pkgcheck/CHANGES.md | 53 +- support/pkgcheck/Cargo.toml | 15 +- support/pkgcheck/bin/pkgcheck | Bin 7898920 -> 7042864 bytes support/pkgcheck/docs/errorsd.tex | 67 +- support/pkgcheck/docs/fatald.tex | 8 + support/pkgcheck/docs/informationd.tex | 10 + support/pkgcheck/docs/pkgcheck.pdf | Bin 66749 -> 70100 bytes support/pkgcheck/docs/title.tex | 2 +- support/pkgcheck/docs/warningsd.tex | 27 + support/pkgcheck/src/filemagic.rs | 73 +- support/pkgcheck/src/gparser.rs | 2 +- support/pkgcheck/src/linkcheck.rs | 21 +- support/pkgcheck/src/main.rs | 926 +++++++++++++------------- support/pkgcheck/src/messages/errorsd.rs | 68 +- support/pkgcheck/src/messages/fatald.rs | 10 + support/pkgcheck/src/messages/informationd.rs | 13 + support/pkgcheck/src/messages/mod.rs | 113 +++- support/pkgcheck/src/messages/warningsd.rs | 28 + support/pkgcheck/src/recode.rs | 245 ++++++- support/pkgcheck/src/utils.rs | 227 +++++-- 20 files changed, 1262 insertions(+), 646 deletions(-) (limited to 'support/pkgcheck') diff --git a/support/pkgcheck/CHANGES.md b/support/pkgcheck/CHANGES.md index ac5dc5a1e6..f41ce9b393 100644 --- a/support/pkgcheck/CHANGES.md +++ b/support/pkgcheck/CHANGES.md @@ -13,6 +13,7 @@ - quick_intro.txt: improve wording - build_ctan_zip.p6: add source files to be included into CTAN zip archive - devnotes.md: add documentation how to build your own binary + 2018-12-26 (1.1.0) - enhance check for generated files which now could reside in any subdirectory in the package tree. This changes the format of the @@ -21,11 +22,13 @@ - change ordering of the sections in the pkgcheck.pdf document. First come the informational messages, then warnings and error messages and finally fatal messages. - add a short info to the PDF documention about how to install pkgcheck + 2019-01-02 (1.2.0) - some code simplification - **New feature**: checks that path names in the TDS zip archive contain the package name (only exception is a man page path) - add test cases for messages e0026, e0028 + 2019-03-09 (1.3.0) - recompile with newest http library - checking URLs @@ -35,30 +38,38 @@ - ignoring .tfm files when checking for duplicates - now checking for UTF BOMs. If a README file contains a BOM we issue an error message, if other files contain BOMs we issue a warning message + 2019-03-11 (1.4.0) - recognizing generated files in a .dtx file when they are included using a filecontents resp. filecontents* environment. + 2019-03-14 (1.5.0) - a README can be a symlink which will now be detected properly - if a symlink points to a file object outside of the package directory tree error message e0030 will be issued + 2019-03-29 (1.6.0) - new error message e0031 when a filename contains invalid UTF-8 characters + 2019-06-30 (1.7.0) - when symlinks occur in e0025 they are reported as files - improved error message e0025 now displaying the paths of the affected files/directories/symlinks - if a found http(s) link ends with "`" then the trailing "`" will be discarded - compiled with rust edition 2018 + 2019-07-20 (1.8.0) - add a check to detect temporary files in the TDS zip archive + 2019-07-21 (1.8.1) - correct typo in e0008 message text + 2019-08-07 (1.8.2) - updating crates, requiring slight code changes in integration.rs - ignore windows files .nsh and .reg when checking for LF line endings - new error message e0034 for unwanted files in top level directory of a TDS zip archive - e0020 now reporting only unwanted directory in top level directory of a TDS zip archive + 2019-10-01 (1.8.3) - (Experimental) issue warning if a file in the package tree (W0005) or in the TDS zip archive (W0006) is larger than 40MiB @@ -67,8 +78,48 @@ Both features are regarded as experimental because both contain a hard coded check. If required things will be changed in the future. + 2019-11-04 (1.9.0) - (Experimental) for latex-base, latex-tools, latex-graphics and latex-amsmath use the real - package name 'latex' when checking path names in the TDS zip archive + directory name 'latex' when checking path names in the TDS zip archive - New error message e0035 if a TDS zip archive is found in the unpacked directory tree - add testcases for w0005, w0006 and e0035 + +2019-11-27 (1.9.1) + - recognize .bat files as script type + - add test case for .bat file recognition + +2019-11-28 (1.10.0) + - remove message E0020 to report unwanted top level directory in a TDS zip archive + - more checks for a TDS zip archive + - check for temporary files or directories + +2020-01-17 (2.0.0) + - more error checking in TDS zip archives + - detecting if a .dtx or .ins file is not in a subdirectory of source/ or doc/ toplevel directories (E0036) + - detecting empty directories (W0007) + - detecting hidden directories (E0006) + - detecting temporary files (E0008) + - as a special case tex/latex/tools/.tex is allowed for package latex-tools. The + same applies for latex-tools-dev package + - revamped handling of line ending checks + - reporting if a file has CR line endings only (E0037) + - reporting if a file has inconsistent line endings (E0038) + - reporting if a Windows file does not have Windows (CRLF) line endings (W0008) + - a file is regarded a Windows file if its name ends with one of .bat, .cmd, .nsh or .reg + - files with CR, or CRLF line endings will be corrected to LF + - but windows files will be corrected to CRLF line endings + - when correcting line endings a line ending at the end of a file will be added if required + - renamed command line switch + - old: -L, --correct-crlf Correct CRLF line endings + - new: -L, --correct-le Correct line endings + - other changes + - new message F0007 when tmp directory is not writable or the tmp directory is full + - some code simplifications + - crates update + - when detecting an empty file also permissions will be checked and (if requested) corrected + - (Experimental) for latex-base-dev, latex-tools-dev, latex-graphics-dev and latex-amsmath-dev + check for latex-dev in the path names in the TDS zip archive + - if a file in a generated file statement contains a directory part it was correctly + recognized but not compared correctly to the contents of the TDS zip archive + diff --git a/support/pkgcheck/Cargo.toml b/support/pkgcheck/Cargo.toml index 7908036125..94ed0aaf5b 100644 --- a/support/pkgcheck/Cargo.toml +++ b/support/pkgcheck/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pkgcheck" -version = "1.9.0" +version = "2.0.0" authors = ["Manfred Lotz "] description = "Command-line tool to check packages uploaded to CTAN." license = "MIT/Apache-2.0" @@ -8,7 +8,7 @@ edition = "2018" [dependencies] structopt = "0.3" -walkdir = "2.2" +walkdir = "2.3" scoped_threadpool = "0.1" num_cpus = "1" blake2 = "0.8" @@ -18,13 +18,15 @@ regex = "1" glob = "0.3" colored = "1" linkify = "0.4" -reqwest = "0.9" + +reqwest = { version = "0.10", features = ["blocking", "gzip"] } + lazy_static = "1" tempdir = "0.3" -assert_cmd = "0.11" -predicates = "1.0" +assert_cmd = "0.12" +predicates = "1" openssl-probe = "0.1.2" -pest = "^2.1" +pest = "2.1" pest_derive = "2.1" escargot = "0.5" @@ -32,3 +34,4 @@ threadpool = "1.7.1" unicode-bom = "1" url = "2.1" + diff --git a/support/pkgcheck/bin/pkgcheck b/support/pkgcheck/bin/pkgcheck index 1ac3e63302..ebf41ce1b3 100755 Binary files a/support/pkgcheck/bin/pkgcheck and b/support/pkgcheck/bin/pkgcheck differ diff --git a/support/pkgcheck/docs/errorsd.tex b/support/pkgcheck/docs/errorsd.tex index 25a68c1d79..e3e9cc9b5f 100644 --- a/support/pkgcheck/docs/errorsd.tex +++ b/support/pkgcheck/docs/errorsd.tex @@ -33,7 +33,8 @@ file but it isn't. allowed}\label{e0004----empty-directory-not-allowed}} Empty directories are considered as rubbish, and are usually not -accepted as part of a package. +accepted as part of a package, neither in the package tree nor in the +TDS zip archive. For more details refer to: \href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noemptyfiles}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noemptyfiles} @@ -52,7 +53,8 @@ For more details refer to: \subsection{E0006 -\/- Hidden directories not allowed}\label{e0006----hidden-directories-not-allowed}} -A package should not contain hidden directories. +A package should not contain hidden directories, neither in the package +tree nor in the TDS zip archive. For more details refer to: \href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles} @@ -61,7 +63,8 @@ For more details refer to: \subsection{E0007 -\/- Hidden files not allowed}\label{e0007----hidden-files-not-allowed}} -A package should not contain hidden files. +A package should not contain hidden files, neither in the package tree +nor in the TDS zip archive. For more details refer to: \href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles} @@ -188,34 +191,6 @@ files if they are in a different directory in the package. For more details refer to: \href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#nogeneratedfiles}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#nogeneratedfiles} -\hypertarget{e0020----unwanted-directory-detected-in-the-top-level-directory-in-tds-zip-archive}{% -\subsection{E0020 -\/- Unwanted directory detected in the top level -directory in TDS zip -archive}\label{e0020----unwanted-directory-detected-in-the-top-level-directory-in-tds-zip-archive}} - -A top level directory of a TDS archive should only contain all or some -of the following directories: - -\begin{itemize} -\tightlist -\item - \texttt{tex} -\item - \texttt{fonts} -\item - \texttt{metafont} -\item - \texttt{metapost} -\item - \texttt{bibtex} -\item - \texttt{scripts} -\item - \texttt{doc} -\item - \texttt{source} -\end{itemize} - \hypertarget{e0021----error-when-reading-a-file}{% \subsection{E0021 -\/- Error when reading a file}\label{e0021----error-when-reading-a-file}} @@ -337,3 +312,33 @@ directory tree}\label{e0035----unwanted-tds-archive-detected-in-package-directory-tree}} A package directory should not contain a TDS zip archive. + +\hypertarget{e0036----dtxins-files-found-in-wrong-directory-in-tds-zip-archive}{% +\subsection{E0036 -\/- .dtx/.ins files found in wrong directory in TDS +zip +archive}\label{e0036----dtxins-files-found-in-wrong-directory-in-tds-zip-archive}} + +In a TDS zip archive a \texttt{.dtx} resp. \texttt{.ins} file must be in +a subdirectory of either of \texttt{source/} or \texttt{doc/} top level +directories. + +\hypertarget{e0037----cr-line-endings-detected}{% +\subsection{E0037 -\/- CR line endings +detected}\label{e0037----cr-line-endings-detected}} + +The file specified in the error message contains CR line endings. Text +files should have UNIX style line endings. + +For more details refer to: +\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#crlf}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#crlf} + +\hypertarget{e0038----file-has-inconsistent-line-endings-cr-x-lf-y-crlf-z}{% +\subsection{E0038 -\/- File has inconsistent line endings: CR: x, LF: y, +CRLF: +z}\label{e0038----file-has-inconsistent-line-endings-cr-x-lf-y-crlf-z}} + +The file specified in the error message contains CR line endings. Text +files should have UNIX style line endings. + +For more details refer to: +\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#crlf}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#crlf} diff --git a/support/pkgcheck/docs/fatald.tex b/support/pkgcheck/docs/fatald.tex index b4c175a7c0..ef26ff30cf 100644 --- a/support/pkgcheck/docs/fatald.tex +++ b/support/pkgcheck/docs/fatald.tex @@ -37,3 +37,11 @@ Exiting...}\label{f0006----unknown-error-code-specified-with-option--e-resp----e \texttt{pkgcheck} was called with option \texttt{-e} resp. \texttt{-\/-explain}, and an unknown error code was specified. + +\hypertarget{f0007----could-not-create-temporary-directory-for-unzipping-the-tds-zip-archive}{% +\subsection{F0007 -\/- Could not create temporary directory for +unzipping the TDS zip +archive}\label{f0007----could-not-create-temporary-directory-for-unzipping-the-tds-zip-archive}} + +Make sure the temp directory is writable and/or the temp directory has +enough space. diff --git a/support/pkgcheck/docs/informationd.tex b/support/pkgcheck/docs/informationd.tex index 88a2305f48..b1a870ebba 100644 --- a/support/pkgcheck/docs/informationd.tex +++ b/support/pkgcheck/docs/informationd.tex @@ -49,3 +49,13 @@ temporary file endings and their meanings. For more details refer to: \href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles} + +\hypertarget{i0007----successfully-corrected-wrong-line-endings-to-lf-resp-crlf}{% +\subsection{I0007 -\/- Successfully corrected wrong line endings to LF +resp. +CRLF}\label{i0007----successfully-corrected-wrong-line-endings-to-lf-resp-crlf}} + +\texttt{pkgcheck} successfully converted wrong line endings to LF line +endings or to CRLF line endings if it the file was a Windows text file. + +Wrong line endings could be CR, CRLF or a mixture of line endings. diff --git a/support/pkgcheck/docs/pkgcheck.pdf b/support/pkgcheck/docs/pkgcheck.pdf index 87d258b13d..5949a88fed 100644 Binary files a/support/pkgcheck/docs/pkgcheck.pdf and b/support/pkgcheck/docs/pkgcheck.pdf differ diff --git a/support/pkgcheck/docs/title.tex b/support/pkgcheck/docs/title.tex index 9961a041b9..1b5df7c036 100644 --- a/support/pkgcheck/docs/title.tex +++ b/support/pkgcheck/docs/title.tex @@ -1 +1 @@ -\title{pkgcheck Utility, v1.9.0} +\title{pkgcheck Utility, v2.0.0} diff --git a/support/pkgcheck/docs/warningsd.tex b/support/pkgcheck/docs/warningsd.tex index 7189f196e1..416661ff00 100644 --- a/support/pkgcheck/docs/warningsd.tex +++ b/support/pkgcheck/docs/warningsd.tex @@ -53,3 +53,30 @@ archive}{W0006 -\/- Very large file with size \textless size\textgreater{} detec (Experimental) We issue the message if there is a file larger than 40MiB in the TDS zip archive. + +\hypertarget{w0007----empty-directory-detected-in-the-tds-zip-archive}{% +\subsection{W0007 -\/- Empty directory detected in the TDS zip +archive}\label{w0007----empty-directory-detected-in-the-tds-zip-archive}} + +Empty directories in a TDS zip archive are discouraged. As they usually +don't create errors in the distribution we issue a warning only. + +\hypertarget{w0008----windows-file-has-unix-line-endings}{% +\subsection{W0008 -\/- Windows file has Unix line +endings}\label{w0008----windows-file-has-unix-line-endings}} + +A Windows file with Unix line endings was detected. + +We regard a file as a Windows file if its name ends with: + +\begin{itemize} +\tightlist +\item + \texttt{.bat} +\item + \texttt{.cmd} +\item + \texttt{.nsh}, or +\item + \texttt{.reg} +\end{itemize} diff --git a/support/pkgcheck/src/filemagic.rs b/support/pkgcheck/src/filemagic.rs index 40edd18351..2ae8bfae6e 100644 --- a/support/pkgcheck/src/filemagic.rs +++ b/support/pkgcheck/src/filemagic.rs @@ -19,16 +19,23 @@ use unicode_bom::Bom; // - https://filesignatures.net/ // - https://github.com/7h3rAm/cigma/blob/master/cigma/magicbytes.json + +#[derive(Debug, PartialEq)] +pub enum LineEnding { + LF, + CR, + CRLF, + MIXED(usize, usize, usize), +} + #[derive(Debug, PartialEq)] pub enum Mimetype { Binary, - Script, - ScriptCRLF, + Script(LineEnding), Pdf, Archive, Zip, - Text, - TextCRLF, + Text(LineEnding), Data, Unknown, BlockDevice, @@ -42,6 +49,7 @@ pub enum Mimetype { BOM(Bom), } + pub struct Filetype { buffer: Vec, } @@ -91,7 +99,7 @@ fn _is_crlf(buffer: &[u8], len: usize) -> bool { false } -fn is_crlf(buffer: &[u8], len: usize) -> bool { +fn is_crlf(buffer: &[u8], len: usize) -> LineEnding { let mut seen_cr = false; let mut n_crlf = 0; let mut n_lf = 0; @@ -114,18 +122,18 @@ fn is_crlf(buffer: &[u8], len: usize) -> bool { seen_cr = *c == CR; } + // println!("LF / CR / CRLF: {} / {} / {}", n_lf, n_cr, n_crlf); + // println!("cr: {}, lf: {}, crlf: {}", n_cr, n_lf, n_crlf); // if (n_crlf == 0 && n_cr == 0 && n_nel == 0 && n_lf == 0) // --> no line terminators - if n_crlf == 0 && n_cr == 0 && n_lf == 0 { - return false; - } - if n_crlf > 0 || n_cr > 0 { - return true; + match (n_cr, n_lf, n_crlf) { + (0, 0, z) if z > 0 => LineEnding::CRLF, + (x, 0, 0) if x > 0 => LineEnding::CR, + (0, y, 0) if y > 0 => LineEnding::LF, + (x, y, z) => LineEnding::MIXED(x, y, z), } - - false } impl Filetype { @@ -175,10 +183,6 @@ impl Filetype { return Ok(Mimetype::Pdf); } - if bytes_read >= 5 && &self.buffer[0..5] == b"= 6 && &self.buffer[0..6] == b"\x7B\x5C\x72\x74\x66\x31" { return Ok(Mimetype::Data); @@ -224,27 +228,36 @@ impl Filetype { } } + //println!("Filename: {}", fname); + let crlf = is_crlf(&self.buffer, bytes_read); + //println!("{:?}", crlf); if bytes_read >= 5 && (self.buffer.starts_with(b"#!") || self.buffer.starts_with(b" Ok(Mimetype::Text), - (true, false) => Ok(Mimetype::TextCRLF), - (false, true) => Ok(Mimetype::Script), - (true, true) => Ok(Mimetype::ScriptCRLF), + //println!("le/is_script: {:?}, {:?}", crlf, is_script); + if is_script { + Ok(Mimetype::Script(crlf)) + } else { + Ok(Mimetype::Text(crlf)) } + // match (crlf, is_script) { + // (LineEnding::LF, false) => Ok(Mimetype::Text(LineEnding::LF)), + // (LineEnding::CR, false) => Ok(Mimetype::Text(LineEnding::CR)), + // (LineEnding::CRLF, false) => Ok(Mimetype::Text(LineEnding::CRLF)), + // (LineEnding::LF, true) => Ok(Mimetype::Script(LineEnding::LF)), + // (LineEnding::CR, true) => Ok(Mimetype::Script(LineEnding::CR)), + // (LineEnding::CRLF, true) => Ok(Mimetype::Script(LineEnding::CRLF)), + // (_, _) => Ok(Mimetype::Text(LineEnding::LF)), + // } } } @@ -388,8 +401,8 @@ fn test_filetype() { // analyze() says TextCRLF //assert!(ft.analyze("tests_filemagic/musterlogo.pdf").ok() == Some(Mimetype::Script)); - assert!(ft.analyze("tests_filemagic/x.pl").ok() == Some(Mimetype::Script)); - assert!(ft.analyze("tests_filemagic/main.php").ok() == Some(Mimetype::Script)); + assert!(ft.analyze("tests_filemagic/x.pl").ok() == Some(Mimetype::Script(LineEnding::LF))); + assert!(ft.analyze("tests_filemagic/main.php").ok() == Some(Mimetype::Script(LineEnding::LF))); assert!(ft.analyze("tests_filemagic/test.7z").ok() == Some(Mimetype::Archive)); assert!(ft.analyze("tests_filemagic/x.tgz").ok() == Some(Mimetype::Archive)); @@ -404,7 +417,7 @@ fn test_filetype() { assert!(ft.analyze("tests_filemagic/empty.zip").ok() == Some(Mimetype::Zip)); - assert!(ft.analyze("tests_filemagic/README").ok() == Some(Mimetype::Text)); + assert!(ft.analyze("tests_filemagic/README").ok() == Some(Mimetype::Text(LineEnding::MIXED(0,0,0)))); // assert!(ft.analyze("tests_filemagic/README1").ok() == Some(Mimetype::Text)); assert!(ft.analyze("tests_filemagic/cp").ok() == Some(Mimetype::Binary)); @@ -418,5 +431,5 @@ fn test_filetype() { assert!(ft.analyze("tests_filemagic/chap5.rtf").ok() == Some(Mimetype::Data)); assert!(ft.analyze("tests_filemagic/commons-math.jar").ok() == Some(Mimetype::Data)); - assert!(ft.analyze("tests_filemagic/8stbu11h.htm").ok() == Some(Mimetype::TextCRLF)); + assert!(ft.analyze("tests_filemagic/8stbu11h.htm").ok() == Some(Mimetype::Text(LineEnding::MIXED(0,1,8710)))); } diff --git a/support/pkgcheck/src/gparser.rs b/support/pkgcheck/src/gparser.rs index 6f9d7f393a..ac3e1e7289 100644 --- a/support/pkgcheck/src/gparser.rs +++ b/support/pkgcheck/src/gparser.rs @@ -1,7 +1,7 @@ use pest::Parser; #[cfg(debug_assertions)] -const _GRAMMAR: &'static str = include_str!("generate.pest"); +const _GRAMMAR: &str = include_str!("generate.pest"); #[derive(Parser)] #[grammar = "generate.pest"] diff --git a/support/pkgcheck/src/linkcheck.rs b/support/pkgcheck/src/linkcheck.rs index 60b993e960..f73a05c02b 100644 --- a/support/pkgcheck/src/linkcheck.rs +++ b/support/pkgcheck/src/linkcheck.rs @@ -89,7 +89,7 @@ fn check_link(url: &str, fname: &str, urlhash: &Arc>, print_all: let mut urlhash = urlhash.lock().unwrap(); if !urlhash.contains_key(&url) { let mut hs = HashSet::default(); - hs.insert(f.clone()); + hs.insert(f); let url1 = url.clone(); urlhash.insert( @@ -178,11 +178,10 @@ fn get_links_inner(s: &str) -> Option> { } // This is a workaround to prevent URLs ending with ` if r.ends_with('`') { - links.push(String::from(&r[..r.len()-1])); + links.push(String::from(&r[..r.len() - 1])); } else { links.push(String::from(r)); } - } if !links.is_empty() { Some(links) @@ -200,8 +199,8 @@ fn check_link_inner(l: &str, head: bool) -> UrlStatus { ), ); - let default_policy = reqwest::RedirectPolicy::default(); - let policy = reqwest::RedirectPolicy::custom(move |attempt| { + let default_policy = reqwest::redirect::Policy::default(); + let policy = reqwest::redirect::Policy::custom(move |attempt| { if attempt.url().host_str() == Some("127.0.0.1") { attempt.stop() } else { @@ -209,18 +208,18 @@ fn check_link_inner(l: &str, head: bool) -> UrlStatus { } }); - let cb = reqwest::Client::builder() + let cb = reqwest::blocking::Client::builder() .gzip(true) .redirect(policy) .default_headers(headers) .timeout(Duration::from_secs(7)) .build() .unwrap(); - // let url: Url = - // match l.parse() { - // Ok(url) => url, - // Err(e) => { println!("Error: {:?}", e); panic!("Scheiss"); } - // }; + // let url: Url = + // match l.parse() { + // Ok(url) => url, + // Err(e) => { println!("Error: {:?}", e); panic!("Scheiss"); } + // }; let resp = if head { cb.head(l).send() } else { diff --git a/support/pkgcheck/src/main.rs b/support/pkgcheck/src/main.rs index 6fb4136f91..88b9323692 100644 --- a/support/pkgcheck/src/main.rs +++ b/support/pkgcheck/src/main.rs @@ -1,4 +1,3 @@ -extern crate pest; #[macro_use] extern crate pest_derive; @@ -8,18 +7,19 @@ mod gparser; mod messages; use messages::{explains, explains_all}; -#[macro_use] -mod utils; mod linkcheck; +mod utils; mod recode; -use recode::crlf2lf; +use recode::{wrong_line_endings2lf, wrong_line_endings2crlf}; mod filemagic; +use filemagic::{LineEnding}; use linkcheck::LinkCheck; use std::fmt; use std::fmt::Display; + use std::str; use utils::*; @@ -27,7 +27,6 @@ use std::os::unix::fs::MetadataExt; use scoped_threadpool::Pool; - use tempdir::TempDir; #[macro_use] extern crate lazy_static; @@ -63,6 +62,15 @@ fn err(path: &PathBuf, err: &io::Error) { type HashSender = Sender<(u64, PathBuf, Vec)>; //type DupeSender = Sender<(u64, Vec)>; +// SizesHashMap contains +// - file sizes +// - and a vector of file names having that size +type SizesHashMap = HashMap>; + +type GeneratedHashMap = HashMap; + +type FileNamesHashMap = HashMap; + const BLOCKSIZE: usize = 4096; fn hash_file_inner(path: &PathBuf) -> io::Result> { @@ -87,48 +95,29 @@ fn hash_file(fsize: u64, path: PathBuf, tx: &HashSender) { } } -fn fix_perms(entry: &str) { - i0005!(entry); - let rc = run_cmd("/bin/chmod", &["-v", "ug=rwX,o=rX", entry]); - if rc.status { - if let Some(op) = rc.output { - print!("{}", op); +// returns false if an error occured +fn fix_inconsistent_le(fname: &str) -> bool { + i0004!(fname); + match wrong_line_endings2lf(fname) { + Ok(_) => { + i0007!(fname, "LF"); + true + } + Err(e) => { + e0027!(fname, e); + false } } } -fn filename_has_bad_chars(entry: &DirEntry) -> Option<(usize, char)> { - let s = entry.file_name().to_str().unwrap(); - - for (i, c) in s.char_indices() { - match c { - '\x00'..='\x1f' => return Some((i, c)), - ' ' | '!' | '"' | '#' | '$' | '%' | '&' | '\'' => return Some((i, c)), - '(' | ')' | '*' => return Some((i, c)), - '+' | ',' | '-' | '.' => (), - '0'..='9' => (), - ':' | ';' | '<' | '=' | '>' | '?' | '@' => return Some((i, c)), - 'A'..='Z' => (), - '[' | '\\' | ']' | '^' => return Some((i, c)), - '_' => (), - '`' => return Some((i, c)), - 'a'..='z' => (), - '{' | '|' | '}' | '~' => return Some((i, c)), - '/' => return None, - _ => return Some((i, c)), - }; - } - None -} -// returns false if CRLF, otherwise true -fn fix_crlf(fname: &str) -> bool { +// returns false if an error occured +fn make_crlf(fname: &str) -> bool { i0004!(fname); - //run_cmd("/usr/bin/fromdos", &["-p", "-e", "-v", fname]).status - match crlf2lf(fname) { + match wrong_line_endings2crlf(fname) { Ok(_) => { - i0007!(fname); + i0007!(fname, "CRLF"); true } Err(e) => { @@ -157,7 +146,7 @@ fn check_readme(dir_entry: &str, is_readme: &ReadmeKind, ft: &filemagic::Mimetyp e0029!(msg_name, b.as_ref()); return false; } - filemagic::Mimetype::Text | filemagic::Mimetype::TextCRLF => match File::open(dir_entry) { + filemagic::Mimetype::Text(_le) => match File::open(dir_entry) { Ok(f) => { if !check_readme_inner(&msg_name, &f) { return false; @@ -216,8 +205,8 @@ struct Args { ignore_dupes: bool, #[structopt(short = "v", long = "verbose", help = "Verbose operation?")] verbose: bool, - #[structopt(short = "L", long = "correct-crlf", help = "Correct CRLF line endings")] - correct_crlf: bool, + #[structopt(short = "L", long = "correct-le", help = "Correct line endings")] + correct_le: bool, #[structopt(short = "C", long = "correct-perms", help = "Correct permissions")] correct_perms: bool, #[structopt(long = "no-colors", help = "Don't display messages in color")] @@ -309,7 +298,7 @@ impl DupPath { } } -type Hashes = HashMap<(u64, Vec), DupPath>; +type DupHashes = HashMap<(u64, Vec), DupPath>; fn main() { match &ARGS.explain { @@ -336,11 +325,18 @@ fn main() { process::exit(1); } Some(d) => { - if !exists_dir(&d) { - f0002!(d); + // make sure the given directory ends with a '/' (slash) + let ds: String = if d.ends_with('/') { + d.to_string() + } else { + let d_s = d.to_string(); + d_s + "/" + }; + if !exists_dir(&ds) { + f0002!(&ds); process::exit(1); }; - d + ds } }; @@ -365,16 +361,11 @@ fn main() { } }; - if let Some(hashes) = check_package(pkg_dir, tds_zip) { + if let Some(hashes) = check_package(&pkg_dir, tds_zip) { if tds_zip.is_some() { if let Some(pn) = pkg_name { if let Some(s) = ARGS.tds_zip.as_ref() { - match check_tds_archive(&pn, s, &hashes) { - Ok(_) => (), - Err(e) => { - e0027!(s, e); - } - } + check_tds_archive(&pn, s, &hashes); } } } @@ -387,7 +378,7 @@ fn main() { } } -fn print_duplicates(hashes: &Hashes) { +fn print_duplicates(hashes: &DupHashes) { let mut total_dupes = 0; let mut total_files = 0; let mut total_size = 0; @@ -471,7 +462,7 @@ fn get_filetype(entry: &DirEntry) -> FType { // read file into buffer[u8] // then convert into string // -fn check_generated_files(entry: &str, generated: &mut HashMap) { +fn check_generated_files(entry: &str, generated: &mut GeneratedHashMap) { // unwrap() is ok here as we only call this function for files, // specifically .ins or .dtx files let entry_fname = filename(entry).unwrap().to_string(); @@ -508,9 +499,12 @@ fn check_generated_files(entry: &str, generated: &mut HashMap) fname.to_string() }; - let pb = PathBuf::from(fname_x); - - generated.entry(pb).or_insert_with(|| PathBuf::from(entry)); + // If the filename in the generate statement contains a path component + // we ignore it so that a generated file will be reported even if it is + // in a different place in the package directory + if let Some(pb) = utils::filename(&fname_x) { + generated.entry(pb.to_string()).or_insert_with(|| entry.to_string()); + } } }; } @@ -542,7 +536,7 @@ fn check_tds_archive_name(tds_zip: &Option) -> Option { let mut pname = String::from(utils::basename(&tz)); let plen = pname.len(); pname.truncate(plen - 8); - Some(pname.to_string()) + Some(pname) } } } @@ -564,11 +558,7 @@ fn unzip_tds_archive(tds_zip: &str, tmp_dir: &str) { } } -fn check_tds_archive( - pkg_name: &str, - tds_zip: &str, - hashes: &HashMap<(u64, Vec), DupPath>, -) -> Result<(), io::Error> { +fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { i0003!(tds_zip); let dir_entry = Path::new(tds_zip); @@ -581,77 +571,48 @@ fn check_tds_archive( } }; + let ut = Utils::new(utils::CheckType::TDS); + // We have a discrepancy between // TDS zip archive name: babel-base.tds.zip // and package name: babel // which yields a false test when checking for the package name - // the path names of the TDS zip archive files. + // the path names of the TDS zip archive files. // Therefore, we correct here. // latex-tools, latex-graphics, latex-amsmath, latex-base => latex - //let mut pkg_replacements: HashMap = HashMap::default(); - //pkg_replacements.insert("babel-base".to_string(), "babel".to_string()); - let mut pkg_replacements: HashMap<&str,&str> = HashMap::default(); + // latex-tools-dev, latex-graphics-dev, latex-amsmath-dev, latex-base-dev => latex-dev + let mut pkg_replacements: HashMap<&str, &str> = HashMap::default(); pkg_replacements.insert("babel-base", "babel"); pkg_replacements.insert("latex-base", "latex"); pkg_replacements.insert("latex-tools", "latex"); pkg_replacements.insert("latex-graphics", "latex"); pkg_replacements.insert("latex-amsmath", "latex"); - let real_pkg_name = if let Some(real_name) = pkg_replacements.get(pkg_name) { + pkg_replacements.insert("latex-base-dev", "latex-dev"); + pkg_replacements.insert("latex-tools-dev", "latex-dev"); + pkg_replacements.insert("latex-graphics-dev", "latex-dev"); + pkg_replacements.insert("latex-amsmath-dev", "latex-dev"); + let real_pkg_name = if let Some(real_name) = pkg_replacements.get(pkg_name) { real_name } else { pkg_name }; - let tmp_dir = TempDir::new("pkgcheck")?; - let tmp_dir_offset = tmp_dir.path().to_str().unwrap().len() + 1; - unzip_tds_archive(tds_zip, tmp_dir.path().to_str().unwrap()); - - let tds_toplevel_dirs: HashSet = [ - "tex", "fonts", "metafont", "metapost", "bibtex", "scripts", "doc", "source", - ] - .iter() - .map(|&s| s.to_string()) - .collect(); - - for entry in fs::read_dir(tmp_dir.path())? { - let entry = entry?; - let path = entry.path(); - - match path.file_stem() { - None => (), - Some(file_stem) => { - let fs = file_stem.to_str().expect("bad file name"); - if !tds_toplevel_dirs.contains(fs) { - if path.is_dir() { - e0020!(fs); - } else { - e0034!(fs); - } - } - } + let tmp_dir = match TempDir::new("pkgcheck") { + Ok(tdir) => tdir, + Err(e) => { + f0007!(e); + process::exit(1); } + }; - // if path.is_dir() { - // match path.file_stem() { - // None => (), - // Some(file_stem) => { - // let fs = file_stem.to_str().expect("bad file name"); - // if !tds_toplevel_dirs.contains(fs) { - // e0020!(fs); - // } - // } - // } - // } else { - // println!("mist"); - // e0020!(path.display()); - // } - } + let tmp_dir_offset = tmp_dir.path().to_str().unwrap().len() + 1; + unzip_tds_archive(tds_zip, tmp_dir.path().to_str().unwrap()); // in order to compare the package files with the content of the // tds zip archive we need to checksum the files in the tds zip // archive. - let mut sizes = HashMap::default(); + let mut sizes: SizesHashMap = HashMap::default(); let mut pool = Pool::new(num_cpus::get() as u32 + 1); { // Processing a single file entry, with the "sizes" hashmap collecting @@ -672,63 +633,93 @@ fn check_tds_archive( for dir_entry in WalkDir::new(tmp_dir.path().to_str().unwrap()).follow_links(false) { match dir_entry { Ok(dir_entry) => { - //let dir_entry_str = dir_entry.path().to_str().unwrap(); + let dir_entry_str = match dir_entry.path().to_str() { + Some(d) => d, + None => { + e0031!(dir_entry.path().to_string_lossy()); + continue; + } + }; + + // this is the file_name without the directory part + // unwrap() is ok here as we covered potential UTF-8 related errors + // above in the definition of dir_entry_str + let file_name = dir_entry.file_name().to_str().unwrap().to_string(); + + let meta = match dir_entry.metadata() { + Ok(meta) => meta, + Err(e) => { + e0027!(dir_entry.path().display(), e); + continue; + } + }; let ft = get_filetype(&dir_entry); + if let FType::Error(e) = ft { + e0023!(e); + continue; + } + + let dir_entry_display = if dir_entry.depth() == 0 { + &dir_entry_str[tmp_dir_offset - 1..] + } else { + &dir_entry_str[tmp_dir_offset..] + }; + if ft == FType::Directory { + ut.check_for_empty_directory(&dir_entry_str, &dir_entry_display); + ut.check_for_hidden_directory(&file_name, &dir_entry_display); + ut.is_unwanted_directory(&file_name, dir_entry_str); continue; } - if let FType::Error(e) = ft { - e0023!(e); + // We don't want to have other than directories in the top level + // directory of a TDS zip archive + if dir_entry.depth() == 1 && ft == FType::Regular { + e0034!(dir_entry_display); continue; } - match dir_entry.metadata() { - Ok(meta) => { - let fsize = meta.len(); - process(fsize, &dir_entry); + ut.check_for_temporary_file(&dir_entry_display); - let dir_entry_str = match dir_entry.path().to_str() { - Some(d) => d, - None => { - e0031!(dir_entry.path().to_string_lossy()); - continue; - } - }; - let dir_entry_display = &dir_entry_str[tmp_dir_offset..]; + // The LaTeX team provides the file `.tex` as a file with an empty name + // in order to make `\input\relax` work (explained by David Carlisle) + // Therefore, we don't call check_for_hidden_file() in this case + match (pkg_name, dir_entry_display) { + ("latex-tools", "tex/latex/tools/.tex") => (), + ("latex-tools-dev", "tex/latex-dev/tools/.tex") => (), + (_, _) => ut.check_for_hidden_file(&file_name, &dir_entry_display) + }; - if is_temporary_file(&dir_entry_str) { - e0008!(dir_entry_display); - } - if fsize > 40 * 1024 * 1024 { - w0006!(dir_entry_display, fsize / 1024 / 1024); - } + let fsize = meta.len(); + process(fsize, &dir_entry); + ut.check_filesize(fsize, &dir_entry_display); - // if the path doesn't contain a man page... - if dir_entry_str.find("/man/").is_none() { - let pkg_name_s = format!("/{}/", real_pkg_name); - // ...then we want to have the package name in the path - if dir_entry_str.find(&pkg_name_s).is_none() { - e0028!(real_pkg_name, dir_entry_display); - } - } - - if dir_entry_str.ends_with(".map") { - map_files_found = true; - let re: Regex = Regex::new(r"fonts[/]map[/]dvips[/]").unwrap(); - if re.is_match(dir_entry_str) { - map_dvips_found = true; - } + // if we encounter a .dtx or .ins file we check + // that it is in a subdirectory of either source/ or doc/ + if (dir_entry_str.ends_with(".dtx") || dir_entry_str.ends_with(".ins")) + && !(dir_entry_display.starts_with("source/") + || dir_entry_display.starts_with("doc/")) + { + e0036!(dir_entry_display); + continue; + } - // if ( $fname =~ m{ fonts [/] map [/] dvips [/] }xms ) { - // $map_dvips_found = 1; - // } - } + // if the path doesn't contain a man page... + if dir_entry_str.find("/man/").is_none() { + let pkg_name_s = format!("/{}/", real_pkg_name); + // ...then we want to have the package name in the path + if dir_entry_str.find(&pkg_name_s).is_none() { + e0028!(real_pkg_name, dir_entry_display); } - Err(e) => { - e0027!(dir_entry.path().display(), e); + } + + if dir_entry_str.ends_with(".map") { + map_files_found = true; + let re: Regex = Regex::new(r"fonts[/]map[/]dvips[/]").unwrap(); + if re.is_match(dir_entry_str) { + map_dvips_found = true; } } } @@ -765,7 +756,6 @@ fn check_tds_archive( } }); - tmp_dir.close()?; // now check if each package file is in the tds archive for (k, paths) in hashes.iter() { if !tds_hashes.contains_key(k) { @@ -773,8 +763,6 @@ fn check_tds_archive( e0026!(p); } } - - Ok(()) } fn get_extension_from_filename(filename: &str) -> Option<&str> { @@ -807,6 +795,21 @@ fn found_unwanted_filetype(fname: &str, ft: &FType) -> bool { } } +fn fix_perms(entry: &str) { + if !ARGS.correct_perms { + return; + } + + i0005!(entry); + let rc = run_cmd("/bin/chmod", &["-v", "ug=rwX,o=rX", entry]); + + if rc.status { + if let Some(op) = rc.output { + print!("{}", op); + } + } +} + fn check_and_correct_perms(dir_entry: &str, p: u32) { if owner_has(p, 5) || !others_have(p, 4) { e0002!(dir_entry, perms_to_string(p)); @@ -817,6 +820,21 @@ fn check_and_correct_perms(dir_entry: &str, p: u32) { }; } +// Sets permissions for a file or directory +// Sample invocation: set_perms("somfile", 0o644); +fn set_perms(entry: &str, p: u32) -> bool { + let ps = &format!("{:o}", p); + let rc = run_cmd("chmod", &["-v", ps, entry]); + + if rc.status { + if let Some(op) = rc.output { + print!("{}", op); + } + true + } else { + false + } +} #[derive(Debug, Clone, PartialEq)] enum FileKind { @@ -825,7 +843,6 @@ enum FileKind { Symlink(String), } - impl Display for FileKind { fn fmt(&self, f: &mut ::std::fmt::Formatter) -> Result<(), ::std::fmt::Error> { match *self { @@ -836,8 +853,6 @@ impl Display for FileKind { } } - - #[derive(Debug, Clone, PartialEq)] enum ReadmeKind { No, @@ -845,15 +860,17 @@ enum ReadmeKind { Symlink(String), } -fn check_package(root: &str, tds_zip: &Option) -> Option { - //let mut urls = HashMap::default(); //: - +fn check_package(root: &str, tds_zip: &Option) -> Option { let mut lcnames: HashMap> = HashMap::default(); let mut doublenames: HashMap> = HashMap::default(); let mut inodes = HashSet::default(); + let ut = Utils::new(utils::CheckType::PACKAGE); + + + i0002!(root); // This hash contains all package file names. // @@ -862,185 +879,162 @@ fn check_package(root: &str, tds_zip: &Option) -> Option { // String: the file name without any directory part // ReadmeKind: is it a certain README, file or symlink? // A special case of a README file is a file with has a different name but - // was poinded to by a symlink. Example: README --> README.rst - let mut file_names: HashMap = HashMap::default(); - { - let mut readme_found = false; - - let root_absolute = PathBuf::from(root) - .canonicalize() - .unwrap() - .to_string_lossy() - .to_string(); - for dir_entry in WalkDir::new(root).follow_links(false) { - match dir_entry { - Ok(dir_entry) => { - let dir_entry_str = match dir_entry.path().to_str() { - Some(d) => d, - None => { - e0031!(dir_entry.path().to_string_lossy()); - continue; - } - }; - let meta = match dir_entry.metadata() { - Ok(meta) => meta, - Err(e) => { - e0023!(e); - continue; - } - }; - - if !check_inode(&mut inodes, &meta) { + // was pointed to by a symlink. Example: README --> README.rst + let mut file_names: FileNamesHashMap = HashMap::default(); + let mut readme_found = false; + + let root_absolute = PathBuf::from(root) + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(); + + for dir_entry in WalkDir::new(root).follow_links(false) { + match dir_entry { + Ok(dir_entry) => { + let dir_entry_str = match dir_entry.path().to_str() { + Some(d) => d, + None => { + e0031!(dir_entry.path().to_string_lossy()); continue; } + }; + let meta = match dir_entry.metadata() { + Ok(meta) => meta, + Err(e) => { + e0023!(e); + continue; + } + }; - // this is the file_name without the directory part - // unwrap() is ok here as we covered potential UTF-8 related errors - // above in the definition of dir_entry_str - let file_name = dir_entry.file_name().to_str().unwrap().to_string(); + if !check_inode(&mut inodes, &meta) { + continue; + } - let ft = get_filetype(&dir_entry); + // this is the file_name without the directory part + // unwrap() is ok here as we covered potential UTF-8 related errors + // above in the definition of dir_entry_str + let file_name = dir_entry.file_name().to_str().unwrap().to_string(); - if found_unwanted_filetype(dir_entry_str, &ft) { - continue; - } + // we check for weird stuff like socket files aso. + let ft = get_filetype(&dir_entry); + if found_unwanted_filetype(&dir_entry_str, &ft) { + continue; + } - if let Some((offset, ch)) = filename_has_bad_chars(&dir_entry) { - e0001!(ch, dir_entry_str, offset); - } + ut.filename_has_bad_chars(&dir_entry, dir_entry_str); - // 1. dealing with symlinks - if ft == FType::Symlink { - match get_symlink(&dir_entry) { - Ok(None) => { - e0010!(dir_entry_str); - continue; - } - Err(e) => { - e0027!(dir_entry_str, e); + // 1. dealing with symlinks + if ft == FType::Symlink { + match get_symlink(&dir_entry) { + Ok(None) => { + e0010!(&dir_entry_str); + continue; + } + Err(e) => { + e0027!(&dir_entry_str, e); + continue; + } + Ok(Some(p)) => { + let pd: String = + p.canonicalize().unwrap().to_string_lossy().to_string(); + if !pd.starts_with(&root_absolute) { + e0030!(&dir_entry_str, p.display()); continue; } - Ok(Some(p)) => { - let pd: String = - p.canonicalize().unwrap().to_string_lossy().to_string(); - if !pd.starts_with(&root_absolute) { - e0030!(dir_entry_str, p.display()); - continue; - } - let lc_dir_entry_str = dir_entry_str.to_lowercase(); - if let Some(_dir_name) = filename(dir_entry_str) { - let lcnref = &mut lcnames; - lcnref - .entry(PathBuf::from(lc_dir_entry_str)) - .or_insert_with(Vec::new) - .push(( - PathBuf::from(dir_entry_str), - //FileKind::Symlink(dir_entry_str.into()), - FileKind::Symlink(pd.clone()), - )); - } - if is_readme(&dir_entry) { - readme_found = true; - file_names.insert( - p, - ( - meta, - file_name, - ReadmeKind::Symlink(dir_entry_str.to_string()), - ), - ); - } - continue; + let lc_dir_entry_str = dir_entry_str.to_lowercase(); + if let Some(_dir_name) = filename(&dir_entry_str) { + let lcnref = &mut lcnames; + lcnref + .entry(PathBuf::from(lc_dir_entry_str)) + .or_insert_with(Vec::new) + .push(( + PathBuf::from(&dir_entry_str), + //FileKind::Symlink(&dir_entry_str.into()), + FileKind::Symlink(pd.clone()), + )); } + if is_readme(&dir_entry) { + readme_found = true; + file_names.insert( + p, + ( + meta, + file_name, + ReadmeKind::Symlink(dir_entry_str.to_string()), + ), + ); + } + continue; } } + } - let p = get_perms(&dir_entry.path()); + let p = get_perms(&dir_entry.path()); - // 2. dealing with directories - if ft == FType::Directory { - let lc_dir_entry_str = dir_entry_str.to_lowercase(); - if let Some(_dir_name) = filename(dir_entry_str) { - let lcnref = &mut lcnames; - - lcnref - .entry(PathBuf::from(lc_dir_entry_str)) - .or_insert_with(Vec::new) - .push((PathBuf::from(dir_entry_str), FileKind::Directory)); - } + // 2. dealing with directories + if ft == FType::Directory { - if !owner_has(p, 5) || !others_have(p, 5) { - e0011!(dir_entry_str, perms_to_string(p)); - if ARGS.correct_perms { - i0005!(&dir_entry_str); - set_perms(&dir_entry_str, 0o775); - } - } + let lc_dir_entry_str = dir_entry_str.to_lowercase(); + if let Some(_dir_name) = filename(&dir_entry_str) { + let lcnref = &mut lcnames; - match is_empty_directory(&dir_entry) { - Ok(true) => { - e0004!(dir_entry_str); - } - Err(e) => { - e0027!(dir_entry_str, e); - } - Ok(false) => (), - } - if is_hidden_directory(&dir_entry) { - e0006!(dir_entry_str); - } + lcnref + .entry(PathBuf::from(lc_dir_entry_str)) + .or_insert_with(Vec::new) + .push((PathBuf::from(&dir_entry_str), FileKind::Directory)); + } - if is_unwanted_directory(&dir_entry.file_name().to_str().unwrap()) { - e0018!(dir_entry_str); + if !owner_has(p, 5) || !others_have(p, 5) { + e0011!(&dir_entry_str, perms_to_string(p)); + if ARGS.correct_perms { + i0005!(&dir_entry_str); + set_perms(&dir_entry_str, 0o775); } - continue; } - // 3. dealing with regular files - if is_hidden(&file_name) { - e0007!(dir_entry_str); - } - if is_temporary_file(&dir_entry_str) { - e0008!(dir_entry_str); - } + ut.check_for_empty_directory(&dir_entry_str, &dir_entry_str); + ut.check_for_hidden_directory(&file_name, &dir_entry_str); + ut.is_unwanted_directory(&file_name, dir_entry_str); + continue; + } - if let Some(file_name) = filename(dir_entry_str) { - let doubleref = &mut doublenames; + // 3. dealing with regular files + ut.check_for_hidden_file(&file_name, &dir_entry_str); + ut.check_for_temporary_file(&dir_entry_str); - doubleref - .entry(PathBuf::from(file_name)) - .or_insert_with(Vec::new) - .push(PathBuf::from(dir_entry_str)); - } + // if is_temporary_file(&dir_entry_str) { + // e0008!(&dir_entry_str); + // } - // detecting large files in package directory - - if is_readme(&dir_entry) { - // We want to deal with README files only if they are - // in the root directory of the package. - let f = format!( - "{}{}{}", - root, - // we have to pay attention if `root` ends already with '/' - if root.ends_with('/') { "" } else { "/" }, - &file_name - ); + if let Some(file_name) = filename(&dir_entry_str) { + let doubleref = &mut doublenames; - if dir_entry_str == f { - readme_found = true; - file_names.insert( - dir_entry.path().to_path_buf(), - (meta, file_name.clone(), ReadmeKind::Yes), - ); - } else { - file_names.entry(dir_entry.path().to_path_buf()).or_insert(( - meta, - file_name.clone(), - ReadmeKind::No, - )); - } + doubleref + .entry(PathBuf::from(file_name)) + .or_insert_with(Vec::new) + .push(PathBuf::from(&dir_entry_str)); + } + + if is_readme(&dir_entry) { + // We want to deal with README files only if they are + // in the root directory of the package. + let f = format!( + "{}{}{}", + root, + // we have to pay attention if `root` ends already with '/' + if root.ends_with('/') { "" } else { "/" }, + &file_name + ); + + if dir_entry_str == f { + readme_found = true; + file_names.insert( + dir_entry.path().to_path_buf(), + (meta, file_name.clone(), ReadmeKind::Yes), + ); } else { file_names.entry(dir_entry.path().to_path_buf()).or_insert(( meta, @@ -1048,168 +1042,203 @@ fn check_package(root: &str, tds_zip: &Option) -> Option { ReadmeKind::No, )); } + } else { + file_names.entry(dir_entry.path().to_path_buf()).or_insert(( + meta, + file_name.clone(), + ReadmeKind::No, + )); + } - let lc_dir_entry_str = dir_entry_str.to_lowercase(); - - let lcnref = &mut lcnames; + let lc_dir_entry_str = dir_entry_str.to_lowercase(); - lcnref - .entry(PathBuf::from(lc_dir_entry_str)) - .or_insert_with(Vec::new) - .push((PathBuf::from(dir_entry_str), FileKind::File)); + let lcnref = &mut lcnames; - } + lcnref + .entry(PathBuf::from(lc_dir_entry_str)) + .or_insert_with(Vec::new) + .push((PathBuf::from(&dir_entry_str), FileKind::File)); + } - Err(e) => { - eprintln!("{}", e); - } + Err(e) => { + eprintln!("{}", e); } } + } - if !readme_found { - e0009!(); - } + if !readme_found { + e0009!(); } let lc = LinkCheck::new(4, false); let mut detective = filemagic::Filetype::new(); - let mut sizes = HashMap::default(); - let mut generated: HashMap = HashMap::default(); + let mut sizes: SizesHashMap = HashMap::default(); + let mut generated: GeneratedHashMap = HashMap::default(); - { - // Processing a single file entry, with the "sizes" hashmap collecting - // same-size files. Entries are either Found::One or Found::Multiple, - // so that we can submit the first file's path as a hashing job when the - // first duplicate is found. Hashing each file is submitted as a job to - // the pool. - let mut process = |fsize, path: &PathBuf| { - let sizeref = &mut sizes; + // Processing a single file entry, with the "sizes" hashmap collecting + // same-size files. Entries are either Found::One or Found::Multiple, + // so that we can submit the first file's path as a hashing job when the + // first duplicate is found. Hashing each file is submitted as a job to + // the pool. + let mut process = |fsize, path: &PathBuf| { + let sizeref = &mut sizes; - let path = path.clone(); - sizeref.entry(fsize).or_insert_with(Vec::new).push(path); + let path = path.clone(); + sizeref.entry(fsize).or_insert_with(Vec::new).push(path); + }; + for (path, (meta, _file_name, is_readme)) in file_names.iter() { + //println!("DEBUG: path {:?}, file_name {:?}, is_readme: {} ", &path, &file_name, is_readme); + let dir_entry_str = match path.to_str() { + Some(d) => d, + None => { + e0031!(&path.to_string_lossy()); + continue; + } }; - for (path, (meta, _file_name, is_readme)) in file_names.iter() { - //println!("DEBUG: path {:?}, file_name {:?}, is_readme: {} ", &path, &file_name, is_readme); - let dir_entry_str = match path.to_str() { - Some(d) => d, - None => { - e0031!(&path.to_string_lossy()); - continue; - } - }; + let fsize = meta.len(); + ut.check_filesize(fsize, dir_entry_str); - let fsize = meta.len(); + let p = get_perms(&path); + if !owner_has(p, 4) { + e0002!(&dir_entry_str, perms_to_string(p)); + fix_perms(&dir_entry_str); + continue; + } + let ftr = detective.analyze(&dir_entry_str); + //println!(">>> {:?}", ftr); + // we ignore errors from filetype recognition + if ftr.is_err() { + continue; + } - // if it is an empty file we don't need - // to check further - if fsize == 0 { - e0005!(dir_entry_str); - continue; - } - if fsize > 40 * 1024 * 1024 { - w0005!(dir_entry_str, fsize / 1024 / 1024); - continue; - } + let ft = ftr.unwrap(); - let p = get_perms(&path); - if !owner_has(p, 4) { - e0002!(dir_entry_str, perms_to_string(p)); - if ARGS.correct_perms { - fix_perms(&dir_entry_str); - } + // DEBUG !readme_symlinked.contains(&dir_entry_str) + if ReadmeKind::No != *is_readme { + if !check_readme(&dir_entry_str, is_readme, &ft) { continue; } - let ftr = detective.analyze(&dir_entry_str); - //println!(">>> {:?}", ftr); - // we ignore errors from filetype recognition - if ftr.is_err() { - continue; + if ARGS.urlcheck { + lc.check_urls(&dir_entry_str); } + } - let ft = ftr.unwrap(); - - // DEBUG !readme_symlinked.contains(dir_entry_str) - if ReadmeKind::No != *is_readme { - if !check_readme(dir_entry_str, is_readme, &ft) { - continue; - } - if ARGS.urlcheck { - lc.check_urls(dir_entry_str); + match ft { + filemagic::Mimetype::Text(_) => { + check_and_correct_perms(&dir_entry_str, p); + let fext = get_extension_from_filename(&dir_entry_str); + if fext == Some("ins") || fext == Some("dtx") { + check_generated_files(&dir_entry_str, &mut generated); } - } - match ft { - filemagic::Mimetype::Text | filemagic::Mimetype::TextCRLF => { - check_and_correct_perms(dir_entry_str, p); - let fext = get_extension_from_filename(&dir_entry_str); - if fext == Some("ins") || fext == Some("dtx") { - check_generated_files(&dir_entry_str, &mut generated); - } - match fext { - Some("bat") | Some("cmd") | - Some("nsh") | Some("reg") => (), - Some(_) | None => { - if ft == filemagic::Mimetype::TextCRLF { - e0012!(dir_entry_str); - if ARGS.correct_crlf { - fix_crlf(dir_entry_str); + match fext { + // deal with Windows files + Some("bat") | Some("cmd") | Some("nsh") | Some("reg") => { + match ft { + filemagic::Mimetype::Text(LineEnding::CRLF) => (), + filemagic::Mimetype::Text(LineEnding::CR) => + { + e0037!(&dir_entry_str); + if ARGS.correct_le { + make_crlf(&dir_entry_str); } - } + }, + filemagic::Mimetype::Text(LineEnding::MIXED(0, 0, 0) ) => (), + filemagic::Mimetype::Text(LineEnding::MIXED(cr, lf, crlf) ) => + { + e0038!(&dir_entry_str, cr, lf, crlf); + if ARGS.correct_le { + fix_inconsistent_le(&dir_entry_str); + } + }, + filemagic::Mimetype::Text(LineEnding::LF) => + { + w0008!(&dir_entry_str); + }, + fmm => println!("Should not occur: {} has {:?}", dir_entry_str, fmm), } - }; + }, + Some(_) | None => { + match ft { + filemagic::Mimetype::Text(LineEnding::CRLF) => + { + e0012!(&dir_entry_str); + if ARGS.correct_le { + fix_inconsistent_le(&dir_entry_str); + } + }, + filemagic::Mimetype::Text(LineEnding::CR) => + { + e0037!(&dir_entry_str); + if ARGS.correct_le { + fix_inconsistent_le(&dir_entry_str); + } + }, + filemagic::Mimetype::Text(LineEnding::MIXED(0, 0, 0) ) => (), + filemagic::Mimetype::Text(LineEnding::MIXED(cr, lf, crlf) ) => + { + //println!(">>>{}: {:?} {},{},{}", &dir_entry_str, ft, x, y, z); + e0038!(&dir_entry_str, cr, lf, crlf); + if ARGS.correct_le { + fix_inconsistent_le(&dir_entry_str); + } + }, + filemagic::Mimetype::Text(LineEnding::LF) => (), + fmm => println!("Should not occur: {} has {:?}", dir_entry_str, fmm), + } + }, } + }, - filemagic::Mimetype::BOM(b) => { - //println!("{}: {} with BOM detected", dir_entry_str, b.as_ref()); - w0004!(dir_entry_str, b.as_ref()); - check_and_correct_perms(dir_entry_str, p); - } - filemagic::Mimetype::Binary - | filemagic::Mimetype::Script - | filemagic::Mimetype::ScriptCRLF => { - if !owner_has(p, 4) || !others_have(p, 4) { - e0002!(dir_entry_str, perms_to_string(p)); + filemagic::Mimetype::BOM(b) => { + //println!("{}: {} with BOM detected", dir_entry_str, b.as_ref()); + w0004!(&dir_entry_str, b.as_ref()); + check_and_correct_perms(&dir_entry_str, p); + } + filemagic::Mimetype::Binary + | filemagic::Mimetype::Script(_) => { + if !owner_has(p, 4) || !others_have(p, 4) { + e0002!(&dir_entry_str, perms_to_string(p)); + }; + fix_perms(&dir_entry_str); + } + filemagic::Mimetype::Pdf => { + check_and_correct_perms(&dir_entry_str, p); + let ret = is_pdf_ok(&dir_entry_str); + if !ret.status { + e0017!(&dir_entry_str); + if let Some(output) = ret.output { + println!("{}", &output); }; - if ARGS.correct_perms { - fix_perms(&dir_entry_str); - } - } - filemagic::Mimetype::Pdf => { - check_and_correct_perms(dir_entry_str, p); - let ret = is_pdf_ok(&dir_entry_str); - if !ret.status { - e0017!(dir_entry_str); - if let Some(output) = ret.output { - println!("{}", &output); - }; - } - } - - filemagic::Mimetype::Archive | filemagic::Mimetype::Zip => { - if dir_entry_str.ends_with(".tds.zip") { - e0035!(dir_entry_str); - } else { - w0001!(dir_entry_str); - } - check_and_correct_perms(dir_entry_str, p); } - - filemagic::Mimetype::Data => check_and_correct_perms(dir_entry_str, p), - _ => continue, } - if others_match(p, 0) { - e0002!(dir_entry_str, perms_to_string(p)); - if ARGS.correct_perms { - i0005!(&dir_entry_str); - set_perms(&dir_entry_str, 0o664); + + filemagic::Mimetype::Archive | filemagic::Mimetype::Zip => { + if dir_entry_str.ends_with(".tds.zip") { + e0035!(&dir_entry_str); + } else { + w0001!(&dir_entry_str); } + check_and_correct_perms(&dir_entry_str, p); } - if !(ARGS.ignore_dupes && tds_zip.is_none()) { - process(fsize, &path); + + filemagic::Mimetype::Data => check_and_correct_perms(&dir_entry_str, p), + filemagic::Mimetype::Zerofile => check_and_correct_perms(&dir_entry_str, p), + _ => continue, + } + + if others_match(p, 0) { + e0002!(&dir_entry_str, perms_to_string(p)); + if ARGS.correct_perms { + i0005!(&dir_entry_str); + set_perms(&dir_entry_str, 0o664); } } + if !(ARGS.ignore_dupes && tds_zip.is_none()) { + process(fsize, &path); + } } print_casefolding(&lcnames); @@ -1259,7 +1288,6 @@ fn check_package(root: &str, tds_zip: &Option) -> Option { } fn print_casefolding(hashes: &HashMap>) { - for (k, eles) in hashes.iter() { //println!("pcf: {:?}, {:?}", k, &eles); if eles.len() == 1 { @@ -1274,12 +1302,14 @@ fn print_casefolding(hashes: &HashMap>) { } } -fn print_generated(hashes: &HashMap>, generated: &HashMap) { +fn print_generated(hashes: &HashMap>, generated: &GeneratedHashMap) { for (k, gen) in generated.iter() { - if hashes.contains_key(k) { - let v = &hashes[k]; + let mut path = PathBuf::new(); + path.push(k); + if hashes.contains_key(&path) { + let v = &hashes[&path]; for fname in v { - e0019!(fname.to_str().unwrap(), gen.to_str().unwrap()); + e0019!(fname.to_str().unwrap(), gen.as_str()); } } } @@ -1317,3 +1347,5 @@ fn show_tmp_endings() { println!("{:23} {}", t, c); } } + + diff --git a/support/pkgcheck/src/messages/errorsd.rs b/support/pkgcheck/src/messages/errorsd.rs index f839f5538b..09c2f4cc53 100644 --- a/support/pkgcheck/src/messages/errorsd.rs +++ b/support/pkgcheck/src/messages/errorsd.rs @@ -49,7 +49,8 @@ pub fn e0004d() { E0004 -- Empty directory not allowed Empty directories are considered as rubbish, and are usually not -accepted as part of a package. +accepted as part of a package, neither in the package tree nor in the +TDS zip archive. For more details refer to: http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#noemptyfiles @@ -76,7 +77,8 @@ pub fn e0006d() { r#" E0006 -- Hidden directories not allowed -A package should not contain hidden directories. +A package should not contain hidden directories, neither in the package +tree nor in the TDS zip archive. For more details refer to: http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#noauxfiles @@ -89,7 +91,8 @@ pub fn e0007d() { r#" E0007 -- Hidden files not allowed -A package should not contain hidden files. +A package should not contain hidden files, neither in the package tree +nor in the TDS zip archive. For more details refer to: http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#noauxfiles @@ -264,26 +267,6 @@ http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#nogeneratedfiles ) } -pub fn e0020d() { - println!( - r#" -E0020 -- Unwanted directory detected in the top level directory in TDS zip archive - -A top level directory of a TDS archive should only contain all or some -of the following directories: - -- tex -- fonts -- metafont -- metapost -- bibtex -- scripts -- doc -- source -"# - ) -} - pub fn e0021d() { println!( r#" @@ -453,3 +436,42 @@ A package directory should not contain a TDS zip archive. ) } +pub fn e0036d() { + println!( + r#" +E0036 -- .dtx/.ins files found in wrong directory in TDS zip archive + +In a TDS zip archive a .dtx resp. .ins file must be in a subdirectory of +either of source/ or doc/ top level directories. +"# + ) +} + +pub fn e0037d() { + println!( + r#" +E0037 -- CR line endings detected + +The file specified in the error message contains CR line endings. Text +files should have UNIX style line endings. + +For more details refer to: +http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#crlf +"# + ) +} + +pub fn e0038d() { + println!( + r#" +E0038 -- File has inconsistent line endings: CR: x, LF: y, CRLF: z + +The file specified in the error message contains CR line endings. Text +files should have UNIX style line endings. + +For more details refer to: +http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#crlf +"# + ) +} + diff --git a/support/pkgcheck/src/messages/fatald.rs b/support/pkgcheck/src/messages/fatald.rs index b41cab83b1..ee02e47bde 100644 --- a/support/pkgcheck/src/messages/fatald.rs +++ b/support/pkgcheck/src/messages/fatald.rs @@ -64,3 +64,13 @@ code was specified. ) } +pub fn f0007d() { + println!( + r#" +F0007 -- Could not create temporary directory for unzipping the TDS zip archive + +Make sure the temp directory is writable and/or the temp directory has +enough space. +"# + ) +} diff --git a/support/pkgcheck/src/messages/informationd.rs b/support/pkgcheck/src/messages/informationd.rs index d4e3ea43b3..340ade4766 100644 --- a/support/pkgcheck/src/messages/informationd.rs +++ b/support/pkgcheck/src/messages/informationd.rs @@ -75,3 +75,16 @@ http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#noauxfiles ) } +pub fn i0007d() { + println!( + r#" +I0007 -- Successfully corrected wrong line endings to LF resp. CRLF + +pkgcheck successfully converted wrong line endings to LF line endings or +to CRLF line endings if it the file was a Windows text file. + +Wrong line endings could be CR, CRLF or a mixture of line endings. +"# + ) +} + diff --git a/support/pkgcheck/src/messages/mod.rs b/support/pkgcheck/src/messages/mod.rs index 2edda3c536..842c693718 100644 --- a/support/pkgcheck/src/messages/mod.rs +++ b/support/pkgcheck/src/messages/mod.rs @@ -117,6 +117,17 @@ macro_rules! f0005 { // }; // } +macro_rules! f0007 { + ($fmt1:expr) => { + error_occured!(); + print!( + "{} Unable to create temporary directory for unzipping the TDS zip archive: {}. Exiting...\n", + msgid!("F0007"), + $fmt1 + ); + }; +} + macro_rules! e0001 { ($fmt1:expr, $fmt2:expr, $fmt3:expr) => { error_occured!(); @@ -171,6 +182,17 @@ macro_rules! e0006 { }; } +macro_rules! e0006t { + ($fmt:expr) => { + error_occured!(); + print!( + "{} Hidden directory {} detected in TDS zip archive\n", + msgid!("E0006"), + $fmt + ); + }; +} + macro_rules! e0007 { ($fmt:expr) => { error_occured!(); @@ -178,6 +200,17 @@ macro_rules! e0007 { }; } +macro_rules! e0007t { + ($fmt:expr) => { + error_occured!(); + print!( + "{} Hidden file {} detected in TDS zip archive\n", + msgid!("E0007"), + $fmt + ); + }; +} + macro_rules! e0008 { ($fmt:expr) => { error_occured!(); @@ -185,6 +218,16 @@ macro_rules! e0008 { }; } +macro_rules! e0008t { + ($fmt:expr) => { + error_occured!(); + print!( + "{} Temporary file {} detected in TDS zip archive\n", + msgid!("E0008"), + $fmt + ); + }; +} macro_rules! e0009 { () => { error_occured!(); @@ -217,7 +260,7 @@ macro_rules! e0011 { macro_rules! e0012 { ($fmt:expr) => { error_occured!(); - print!("{} CRLF detected: {}\n", msgid!("E0012"), $fmt); + print!("{} File {} has CRLF line endings\n", msgid!("E0012"), $fmt); }; } @@ -283,17 +326,6 @@ macro_rules! e0019 { }; } -macro_rules! e0020 { - ($fmt:expr) => { - error_occured!(); - print!( - "{} Unwanted directory `{}` detected in the top level directory of a TDS archive\n", - msgid!("E0020"), - $fmt - ); - }; -} - macro_rules! e0021 { ($fmt1:expr, $fmt2:expr, $fmt3:expr) => { error_occured!(); @@ -448,8 +480,30 @@ macro_rules! e0035 { }; } +macro_rules! e0036 { + ($fmt:expr) => { + error_occured!(); + print!( + "{} File `{}` not below `source/` or `doc/` top level directory in TDS zip archive\n", + msgid!("E0036"), + $fmt + ); + }; +} +macro_rules! e0037 { + ($fmt:expr) => { + error_occured!(); + print!("{} File {} has CR line endings\n", msgid!("E0037"), $fmt); + }; +} +macro_rules! e0038 { + ($fmt1:expr, $fmt2:expr, $fmt3:expr, $fmt4:expr) => { + error_occured!(); + print!("{} File {} has inconsistent line endings: CR={}, LF={}, CRLF={}\n", msgid!("E0038"), $fmt1, $fmt2, $fmt3, $fmt4); + }; +} macro_rules! w0001 { ($fmt:expr) => { print!( @@ -513,6 +567,20 @@ macro_rules! w0006 { }; } +macro_rules! w0007 { + ($fmt:expr) => { + error_occured!(); + print!("{} Empty directory {} detected in TDS zip archive\n", msgid!("W0007"), $fmt); + }; +} + +macro_rules! w0008 { + ($fmt:expr) => { + error_occured!(); + print!("{} Windows file {} has Unix line endings\n", msgid!("W0008"), $fmt); + }; +} + macro_rules! i0002 { ($fmt:expr) => { print!( @@ -565,15 +633,17 @@ macro_rules! i0006 { } macro_rules! i0007 { - ($fmt:expr) => { + ($fmt1:expr, $fmt2:expr) => { print!( - "{} {}: Successfully converted from CRLF to LF\n", + "{} {}: Successfully corrected wrong line endings to {}\n", msgid!("I0007"), - $fmt + $fmt1, + $fmt2 ); }; } + pub fn explains(err: &str) { let err_upp = err.to_ascii_uppercase(); match err_upp.as_str() { @@ -583,6 +653,7 @@ pub fn explains(err: &str) { "F0004" => f0004d(), "F0005" => f0005d(), "F0006" => f0006d(), + "F0007" => f0007d(), "E0001" => e0001d(), "E0002" => e0002d(), @@ -603,7 +674,6 @@ pub fn explains(err: &str) { "E0017" => e0017d(), "E0018" => e0018d(), "E0019" => e0019d(), - "E0020" => e0020d(), "E0021" => e0021d(), "E0022" => e0022d(), "E0023" => e0023d(), @@ -618,6 +688,9 @@ pub fn explains(err: &str) { "E0033" => e0033d(), "E0034" => e0034d(), "E0035" => e0035d(), + "E0036" => e0036d(), + "E0037" => e0037d(), + "E0038" => e0038d(), // "I0001" => i0001d!(), "I0001" => i0001d(), @@ -626,6 +699,7 @@ pub fn explains(err: &str) { "I0004" => i0004d(), "I0005" => i0005d(), "I0006" => i0006d(), + "I0007" => i0007d(), "W0001" => w0001d(), "W0002" => w0002d(), @@ -633,6 +707,8 @@ pub fn explains(err: &str) { "W0004" => w0004d(), "W0005" => w0005d(), "W0006" => w0006d(), + "W0007" => w0007d(), + "W0008" => w0008d(), e => println!( "F0006 Unknown error code `{}` specified with option -e resp. --explain. Exiting...", @@ -668,7 +744,6 @@ pub fn explains_all() { explains("E0017"); explains("E0018"); explains("E0019"); - explains("E0020"); explains("E0021"); explains("E0022"); explains("E0023"); @@ -683,6 +758,7 @@ pub fn explains_all() { explains("E0033"); explains("E0034"); explains("E0035"); + explains("E0036"); explains("I0001"); explains("I0002"); @@ -690,6 +766,7 @@ pub fn explains_all() { explains("I0004"); explains("I0005"); explains("I0006"); + explains("I0007"); explains("W0001"); explains("W0002"); @@ -697,4 +774,6 @@ pub fn explains_all() { explains("W0004"); explains("W0005"); explains("W0006"); + explains("W0007"); + explains("W0008"); } diff --git a/support/pkgcheck/src/messages/warningsd.rs b/support/pkgcheck/src/messages/warningsd.rs index b8a1aa8ef3..cea814c8ce 100644 --- a/support/pkgcheck/src/messages/warningsd.rs +++ b/support/pkgcheck/src/messages/warningsd.rs @@ -78,3 +78,31 @@ in the TDS zip archive. ) } +pub fn w0007d() { + println!( + r#" +W0007 -- Empty directory detected in the TDS zip archive + +Empty directories in a TDS zip archive are discouraged. As they usually +don't create errors in the distribution we issue a warning only. +"# + ) +} + +pub fn w0008d() { + println!( + r#" +W0008 -- Windows file has Unix line endings + +A Windows file with Unix line endings was detected. + +We regard a file as a Windows file if its name ends with: + +- .bat +- .cmd +- .nsh, or +- .reg +"# + ) +} + diff --git a/support/pkgcheck/src/recode.rs b/support/pkgcheck/src/recode.rs index 39a85bacd2..9a6b3d426d 100644 --- a/support/pkgcheck/src/recode.rs +++ b/support/pkgcheck/src/recode.rs @@ -9,9 +9,52 @@ use std::io::prelude::*; use std::io::{self, Read}; use std::path::Path; +struct Bytes { + bytes: Vec, + len: usize, + index: usize, +} + +impl Bytes { + fn new(bytes: Vec) -> Bytes { + let l = bytes.len(); + Bytes { + bytes, + len: l, + index: 0, + } + } +} + +#[derive(Debug)] +enum Item { + Pair(u8, u8), + Single(u8), +} + +impl Iterator for Bytes { + type Item = Item; + + fn next(&mut self) -> Option { + let cur_index = self.index; + self.index += 1; + + if cur_index == self.len { + return None; + } + + if cur_index + 1 == self.len { + Some(Item::Single(self.bytes[cur_index])) + } else { + // if self.index < self.len { + Some(Item::Pair(self.bytes[cur_index], self.bytes[cur_index + 1])) + } + } +} + // The caller has to make sure that crlf2lf is only // used for text files -pub fn crlf2lf(fname: &str) -> Result<(), io::Error> { +pub fn wrong_line_endings2crlf(fname: &str) -> Result<(), io::Error> { let path = Path::new(fname); let mut hdl_in; @@ -28,23 +71,43 @@ pub fn crlf2lf(fname: &str) -> Result<(), io::Error> { return Err(e); }; - // convert \r\n to \n - let mut another_vec: Vec = Vec::new(); - const CR: u8 = 0x0d; // 13 - const LF: u8 = 0x0a; // 10 - - for i in 0..buffer.len() { - if buffer[i] == CR { - if i < buffer.len() - 1 && buffer[i + 1] == LF { - continue; - } + let another_vec: Vec = fix_line_endings2crlf(&buffer); - if i > 0 && buffer[i - 1] == LF { - continue; - } + let mut hdl_out; + match File::create(path) { + Ok(f) => { + hdl_out = f; } - another_vec.push(buffer[i]); + Err(e) => return Err(e), + }; + + // write back + match hdl_out.write(&another_vec) { + Ok(_) => Ok(()), + Err(e) => Err(e), } +} + +// The caller has to make sure that crlf2lf is only +// used for text files +pub fn wrong_line_endings2lf(fname: &str) -> Result<(), io::Error> { + let path = Path::new(fname); + + let mut hdl_in; + match File::open(path) { + Ok(f) => { + hdl_in = f; + } + Err(e) => return Err(e), + }; + let mut buffer: Vec = Vec::new(); + + // read the whole file + if let Err(e) = hdl_in.read_to_end(&mut buffer) { + return Err(e); + }; + + let another_vec: Vec = fix_line_endings2lf(&buffer); let mut hdl_out; match File::create(path) { @@ -60,3 +123,155 @@ pub fn crlf2lf(fname: &str) -> Result<(), io::Error> { Err(e) => Err(e), } } + +pub fn fix_line_endings2lf(buffer: &[u8]) -> Vec { + let mut another_vec: Vec = Vec::new(); + + const CR: u8 = 0x0d; // 13 + const LF: u8 = 0x0a; // 10 + + let b = Bytes::new(buffer.to_vec()); + + let mut skip = false; + for v in b { + if skip { + skip = false; + continue; + } + match v { + Item::Pair(CR, LF) | Item::Pair(LF, CR) => { + another_vec.push(LF); + skip = true; + } + Item::Pair(CR, _) | Item::Pair(LF, _) | Item::Single(CR) | Item::Single(LF) => { + another_vec.push(LF); + } + Item::Pair(c, _) | Item::Single(c) => { + another_vec.push(c); + } + } + } + + // if there is no line ending at the end of a fill add it + if another_vec[another_vec.len() - 1] != LF { + another_vec.push(LF); + } + another_vec +} + +pub fn fix_line_endings2crlf(buffer: &[u8]) -> Vec { + let mut another_vec: Vec = Vec::new(); + + const CR: u8 = 0x0d; // 13 \r + const LF: u8 = 0x0a; // 10 \n + + let b = Bytes::new(buffer.to_vec()); + + let mut skip = false; + for v in b { + if skip { + skip = false; + continue; + } + match v { + Item::Pair(CR, LF) | Item::Pair(LF, CR) => { + another_vec.push(CR); + another_vec.push(LF); + skip = true; + } + Item::Pair(CR, _) | Item::Pair(LF, _) | Item::Single(CR) | Item::Single(LF) => { + another_vec.push(CR); + another_vec.push(LF); + } + Item::Pair(c, _) | Item::Single(c) => { + another_vec.push(c); + } + } + } + + let avlen = another_vec.len(); + + if avlen == 1 || (another_vec[avlen - 1] != LF && another_vec[avlen - 2] != CR) { + another_vec.push(CR); + another_vec.push(LF); + } + + another_vec +} + +#[test] +fn test_fix_line_endings2crlf() { + let testfiles = vec![ + ("a", "a\r\n"), + ("no line ending at all", "no line ending at all\r\n"), + ("\rbla", "\r\nbla\r\n"), + ("\r\r", "\r\n\r\n"), + ("\r\nbla", "\r\nbla\r\n"), + ("\r\n\r\nbla", "\r\n\r\nbla\r\n"), + ("bla\r\nbla", "bla\r\nbla\r\n"), + ("bla\r\n\r\nbla", "bla\r\n\r\nbla\r\n"), + ("bla\r\nbla\r\nbla", "bla\r\nbla\r\nbla\r\n"), + ("\n\r", "\r\n"), + ("\n\r\n\r", "\r\n\r\n"), + ("\n\rbla\n\r", "\r\nbla\r\n"), + ]; + + for (s, rs) in testfiles { + let bs = s.as_bytes().to_vec(); + let brs = rs.as_bytes().to_vec(); + assert_eq!(fix_line_endings2crlf(&bs), brs); + } +} + +#[test] +fn test_fix_line_endings() { + let testfiles = vec![ + ("no line ending at all", "no line ending at all\n"), + ("\r", "\n"), + ("\n\r", "\n"), + ("\r\n", "\n"), + ("\r\n\n", "\n\n"), + ("CR at end of file\r", "CR at end of file\n"), + ("LF at end of file\n", "LF at end of file\n"), + ("CRLF at end of file\r\n", "CRLF at end of file\n"), + ("Two CR at end of file\r\r", "Two CR at end of file\n\n"), + ("Two LF at end of file\n\n", "Two LF at end of file\n\n"), + ( + "Two CRLF at end of file\r\n\r\n", + "Two CRLF at end of file\n\n", + ), + ( + "Two CR in the middle\r\r of file", + "Two CR in the middle\n\n of file\n", + ), + ( + "Two LF in the middle\n\n of file", + "Two LF in the middle\n\n of file\n", + ), + ( + "Two CRLF in the middle\r\n\r\n of file", + "Two CRLF in the middle\n\n of file\n", + ), + ( + "Mixed CR, CRLF\rline2\r\nline3\r\n", + "Mixed CR, CRLF\nline2\nline3\n", + ), + ( + "Mixed CR, CRLF\r\r\nline2\r\nline3\r\n", + "Mixed CR, CRLF\n\nline2\nline3\n", + ), + // ("Mixed CR, CRLF\n\r\r\nline2\r\nline3\r\n", "Mixed CR, CRLF\n\n\nline2\nline3\n"), + ( + "Mixed CR, LFCR\rline2\n\rline3\r\n", + "Mixed CR, LFCR\nline2\nline3\n", + ), + //("line1\nebla", "line1\rebla"), + //("mor\r\nebla", "mor\rebla"), + ]; + + for (s, rs) in testfiles { + let bs = s.as_bytes().to_vec(); + let brs = rs.as_bytes().to_vec(); + assert_eq!(fix_line_endings2lf(&bs), brs); + } +} diff --git a/support/pkgcheck/src/utils.rs b/support/pkgcheck/src/utils.rs index 6b2d0a8753..8a34bfe7c1 100644 --- a/support/pkgcheck/src/utils.rs +++ b/support/pkgcheck/src/utils.rs @@ -10,9 +10,173 @@ use std::borrow::Cow; use std::io; use std::process::Command; +use colored::*; +use std::sync::atomic::Ordering; use std::fs; +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CheckType { + TDS, + PACKAGE, +} + +pub struct Utils { + kind: CheckType, +} + +impl Utils { + pub fn new(kind: CheckType) -> Utils { + Utils { kind } + } + + pub fn check_filesize(&self, fsize: u64, dir_entry_str: &str) -> bool { + // if it is an empty file we don't need + // to check further + if fsize == 0 { + e0005!(dir_entry_str); + return false; + } + if fsize > 40 * 1024 * 1024 { + if self.kind == CheckType::PACKAGE { + w0005!(dir_entry_str, fsize / 1024 / 1024); + } else { + w0006!(dir_entry_str, fsize / 1024 / 1024); + } + return false; + } + + true + } + + pub fn filename_has_bad_chars(&self, entry: &DirEntry, dir_entry_str: &str) { + let s = entry.file_name().to_str().unwrap(); + + for (i, c) in s.char_indices() { + match c { + '\x00'..='\x1f' => { + e0001!(c, dir_entry_str, i); + } + ' ' | '!' | '"' | '#' | '$' | '%' | '&' | '\'' => { + e0001!(c, dir_entry_str, i); + } + '(' | ')' | '*' => { + e0001!(c, dir_entry_str, i); + } + '+' | ',' | '-' | '.' => (), + '0'..='9' => (), + ':' | ';' | '<' | '=' | '>' | '?' | '@' => { + e0001!(c, dir_entry_str, i); + } + 'A'..='Z' => (), + '[' | '\\' | ']' | '^' => { + e0001!(c, dir_entry_str, i); + } + '_' => (), + '`' => { + e0001!(c, dir_entry_str, i); + } + 'a'..='z' => (), + '{' | '|' | '}' | '~' => { + e0001!(c, dir_entry_str, i); + } + '/' => return, + _ => { + return { + e0001!(c, dir_entry_str, i); + } + } + }; + } + } + + pub fn is_unwanted_directory(&self, entry: &str, dir_entry_str: &str) { + let names = ["__MACOSX"]; + for n in &names { + if *n == entry { + e0018!(dir_entry_str); + }; + } + } + + pub fn check_for_hidden_file(&self, entry: &str, dir_entry_str: &str) { + if entry.starts_with('.') { + match self.kind { + CheckType::PACKAGE => { + e0007!(dir_entry_str); + } + CheckType::TDS => { + e0007t!(dir_entry_str); + } + } + } + } + + // fname is the base name without the directory part + pub fn check_for_hidden_directory(&self, fname: &str, dir_entry_str: &str) { + + // We have to single out '.' and './' as they denote just the current directory + // That case may happen if pkgcheck gets called like: `pkgcheck -d .` or `pkgcheck -d ./` + if fname == "." || fname == "./" { + return; + }; + + //if fname.map(|s| s.starts_with('.')).unwrap_or(false) { + if fname.starts_with('.') { + match self.kind { + CheckType::PACKAGE => { + e0006!(dir_entry_str); + } + CheckType::TDS => { + e0006t!(dir_entry_str); + } + } + } + } + + pub fn check_for_temporary_file(&self, dir_entry_str: &str) { + lazy_static! { + static ref RE: Regex = regex_temporary_file_endings(); + } + + if RE.is_match(dir_entry_str) { + match self.kind { + CheckType::PACKAGE => { + e0008!(dir_entry_str); + } + CheckType::TDS => { + e0008t!(dir_entry_str); + } + } + } + } + + pub fn check_for_empty_directory(&self, dir_entry_str: &str, dir_entry_display: &str) { + match read_dir(dir_entry_str) { + // try to read the directory specified + Ok(contents) => { + if contents.count() == 0 { + match self.kind { + CheckType::PACKAGE => { + e0004!(dir_entry_display); + } + CheckType::TDS => { + // Note: Karl Berry recommended to issue a warning only + // as an empty directory in a TDS zip archive + // - is automatically deleted before including + // it in texlive + w0007!(dir_entry_display); + } + } + } + } + Err(e) => { + e0027!(dir_entry_display, e); + } + } + } +} + pub fn temp_file_endings() -> Vec<(&'static str, &'static str)> { // https://github.com/github/gitignore/blob/master/TeX.gitignore // http://hopf.math.purdue.edu/doc/html/suffixes.html @@ -125,53 +289,6 @@ pub fn regex_temporary_file_endings() -> Regex { Regex::new(&rv).unwrap() } -pub fn is_temporary_file(entry: &str) -> bool { - lazy_static! { - static ref RE: Regex = regex_temporary_file_endings(); - } - - RE.is_match(entry) -} - -pub fn is_empty_directory(entry: &DirEntry) -> Result { - let s = entry.path().to_str().unwrap(); - match read_dir(s) { - // try to read the directory specified - Ok(contents) => Ok(contents.count() == 0), - Err(e) => Err(e), - } -} - -pub fn is_unwanted_directory(entry: &str) -> bool { - let names = ["__MACOSX"]; - for n in &names { - if *n == entry { - return true; - }; - } - - false -} - -// is_hidden checks for files or directories starting with a dot -pub fn is_hidden(entry: &str) -> bool { - entry.starts_with('.') -} - -// is_hidden checks for files or directories starting with a dot -pub fn is_hidden_directory(entry: &DirEntry) -> bool { - let fname = entry.file_name(); - - if fname == "." { - return false; - }; - if fname == "./" { - return false; - }; - - fname.to_str().map(|s| s.starts_with('.')).unwrap_or(false) -} - pub fn get_symlink(entry: &DirEntry) -> Result, io::Error> { let r = entry.path().to_str().unwrap(); @@ -309,22 +426,6 @@ pub fn run_cmd(cmd: &str, argument: &[&str]) -> CmdReturn { } } -// Sets permissions for a file or directory -// Sample invocation: set_perms("somfile", 0o644); -pub fn set_perms(entry: &str, p: u32) -> bool { - let ps = &format!("{:o}", p); - let rc = run_cmd("chmod", &["-v", ps, entry]); - - if rc.status { - if let Some(op) = rc.output { - print!("{}", op); - } - true - } else { - false - } -} - // returns true if file is a directory and does exist // returns false otherwise pub fn exists_dir(file: &str) -> bool { -- cgit v1.2.3