summaryrefslogtreecommitdiff
path: root/support/pkgcheck/src/main.rs
diff options
context:
space:
mode:
authorNorbert Preining <norbert@preining.info>2020-11-30 03:00:37 +0000
committerNorbert Preining <norbert@preining.info>2020-11-30 03:00:37 +0000
commitcb68d38823870fa7adb5f0c2b68cf59205dd4bec (patch)
tree8d6e29ee5d082dcbf1ad936b7025db071e9cf573 /support/pkgcheck/src/main.rs
parent0cc87411bcebcb1b0d2a3488d7a7093024b58650 (diff)
CTAN sync 202011300300
Diffstat (limited to 'support/pkgcheck/src/main.rs')
-rw-r--r--support/pkgcheck/src/main.rs207
1 files changed, 142 insertions, 65 deletions
diff --git a/support/pkgcheck/src/main.rs b/support/pkgcheck/src/main.rs
index cbb678729f..d4ef7d6db2 100644
--- a/support/pkgcheck/src/main.rs
+++ b/support/pkgcheck/src/main.rs
@@ -11,10 +11,10 @@ mod linkcheck;
mod utils;
mod recode;
-use recode::{wrong_line_endings2lf, wrong_line_endings2crlf};
+use recode::{wrong_line_endings2crlf, wrong_line_endings2lf};
mod filemagic;
-use filemagic::{LineEnding};
+use filemagic::LineEnding;
use linkcheck::LinkCheck;
use std::fmt;
@@ -27,7 +27,8 @@ use std::os::unix::fs::MetadataExt;
use scoped_threadpool::Pool;
-use tempdir::TempDir;
+use tempfile::Builder;
+
#[macro_use]
extern crate lazy_static;
use colored::*;
@@ -51,6 +52,8 @@ use std::sync::atomic::{AtomicBool, Ordering};
use blake2::{Blake2b, Digest};
use fnv::{FnvHashMap as HashMap, FnvHashSet as HashSet};
use std::sync::mpsc::{channel, Sender};
+
+use structopt::clap::Shell;
use structopt::StructOpt;
#[cfg(unix)]
use walkdir::{DirEntry, WalkDir};
@@ -77,15 +80,13 @@ fn hash_file_inner(path: &PathBuf) -> io::Result<Vec<u8>> {
let mut buf = [0u8; BLOCKSIZE];
let mut fp = File::open(&path)?;
let mut digest = Blake2b::default();
- // When we compare byte-by-byte, we don't need to hash the whole file.
- // Instead, hash a block of 4kB, skipping 100kB.
loop {
match fp.read(&mut buf)? {
0 => break,
- n => digest.input(&buf[..n]),
+ n => digest.update(&buf[..n]),
}
}
- Ok(digest.result().to_vec())
+ Ok(digest.finalize().to_vec())
}
fn hash_file(fsize: u64, path: PathBuf, tx: &HashSender) {
@@ -95,7 +96,6 @@ fn hash_file(fsize: u64, path: PathBuf, tx: &HashSender) {
}
}
-
// returns false if an error occured
fn fix_inconsistent_le(fname: &str) -> bool {
i0004!(fname);
@@ -111,7 +111,6 @@ fn fix_inconsistent_le(fname: &str) -> bool {
}
}
-
// returns false if an error occured
fn make_crlf(fname: &str) -> bool {
i0004!(fname);
@@ -209,6 +208,8 @@ struct Args {
correct_perms: bool,
#[structopt(long = "no-colors", help = "Don't display messages in color")]
no_colors: bool,
+ // #[structopt(long = "install-completion", help = "Install completion for the current shell")]
+ // install_completion: Option<String>,
#[structopt(long = "urlcheck", help = "Check URLs found in README files")]
urlcheck: bool,
#[structopt(short = "T", long = "tds-zip", help = "tds zip archive", group = "tds")]
@@ -296,6 +297,18 @@ impl DupPath {
}
}
+// fn install_completion_bash() {
+// Args::clap().gen_completions(env!("CARGO_PKG_NAME"), Shell::Bash, "bla");
+// }
+
+// fn install_completion_zsh() {
+// Args::clap().gen_completions(env!("CARGO_PKG_NAME"), Shell::Zsh, "/home/manfred/.zfunc");
+// }
+
+// fn install_completion_fish() {
+// Args::clap().gen_completions(env!("CARGO_PKG_NAME"), Shell::Fish, "target");
+// }
+
type DupHashes = HashMap<(u64, Vec<u8>), DupPath>;
fn main() {
@@ -307,6 +320,24 @@ fn main() {
}
}
+ //
+ // !!!!!! this should be activated when structop uses
+ // !!!!!! the new clap 3.0
+ //
+ // match &ARGS.install_completion {
+ // None => { }
+ // Some(d) => {
+ // println!("{}", &d);
+ // match d.as_str() {
+ // "bash" => install_completion_bash(),
+ // "zsh" => install_completion_zsh(),
+ // "fish" => install_completion_fish(),
+ // _ => println!("Invalid shell name {} given", d)
+ // }
+ // process::exit(0);
+ // }
+ // }
+
if ARGS.explain_all {
explains_all();
process::exit(0);
@@ -506,7 +537,9 @@ fn check_generated_files(entry: &str, generated: &mut GeneratedHashMap) {
continue;
}
- generated.entry(filename.to_string()).or_insert_with(|| entry.to_string());
+ generated
+ .entry(filename.to_string())
+ .or_insert_with(|| entry.to_string());
}
};
}
@@ -544,7 +577,7 @@ fn check_tds_archive_name(tds_zip: &Option<String>) -> Option<String> {
}
fn unzip_tds_archive(tds_zip: &str, tmp_dir: &str) {
- match run_cmd("unzip", &["-a", "-q", "-d", tmp_dir, &tds_zip]) {
+ match run_cmd("unzip", &["-q", "-d", tmp_dir, &tds_zip]) {
CmdReturn { status: true, .. } => (),
CmdReturn {
status: false,
@@ -593,13 +626,14 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) {
pkg_replacements.insert("latex-tools-dev", "latex-dev");
pkg_replacements.insert("latex-graphics-dev", "latex-dev");
pkg_replacements.insert("latex-amsmath-dev", "latex-dev");
+ pkg_replacements.insert("latex-firstaid", "latex/firstaid");
let real_pkg_name = if let Some(real_name) = pkg_replacements.get(pkg_name) {
real_name
} else {
pkg_name
};
- let tmp_dir = match TempDir::new("pkgcheck") {
+ let tmp_dir = match Builder::new().prefix("pkgcheck").tempdir() {
Ok(tdir) => tdir,
Err(e) => {
f0007!(e);
@@ -632,6 +666,45 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) {
let mut map_files_found = false;
let mut map_dvips_found = false;
+ // those top level directories are the directories found in the
+ // texmf-dist/ directory of a texlive installation
+ let tds_toplevel_dirs: HashSet<String> = [
+ "asymptote",
+ "bibtex",
+ "chktex",
+ "context",
+ "doc",
+ "dvipdfmx",
+ "dvips",
+ "fonts",
+ "hbf2gf",
+ "makeindex",
+ "metafont",
+ "metapost",
+ "mft",
+ "omega",
+ "pbibtex",
+ "psutils",
+ "scripts",
+ "source",
+ "tex",
+ "tex4ht",
+ "texconfig",
+ "texdoc",
+ "texdoctk",
+ "ttf2pk",
+ "web2c",
+ "xdvi",
+ "xindy",
+ ]
+ .iter()
+ .map(|&s| s.to_string())
+ .collect();
+
+ // set to True if the TDS zip archive contains a top level directory doc/
+ let mut doc_found = false;
+ // we track the number of toplevel directories which must at least be 2
+ let mut number_of_toplevel_dirs = 0;
for dir_entry in WalkDir::new(tmp_dir.path().to_str().unwrap()).follow_links(false) {
match dir_entry {
Ok(dir_entry) => {
@@ -669,29 +742,42 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) {
&dir_entry_str[tmp_dir_offset..]
};
+ ut.check_for_temporary_file(&dir_entry_display);
+
+ // In the top level directory of a TDS zip archive
+ // ... no files are allowed
+ // ... only specific directories are allowed
+ if dir_entry.depth() == 1 {
+ if ft == FType::Regular {
+ e0034!(dir_entry_display);
+ continue;
+ }
+ if !tds_toplevel_dirs.contains(&file_name) {
+ e0020!(&file_name);
+ } else {
+ number_of_toplevel_dirs += 1;
+ if &file_name == "doc" {
+ doc_found = true;
+ }
+ }
+ continue;
+ }
+
if ft == FType::Directory {
ut.check_for_empty_directory(&dir_entry_str, &dir_entry_display);
ut.check_for_hidden_directory(&file_name, &dir_entry_display);
ut.is_unwanted_directory(&file_name, dir_entry_str);
- continue;
- }
- // We don't want to have other than directories in the top level
- // directory of a TDS zip archive
- if dir_entry.depth() == 1 && ft == FType::Regular {
- e0034!(dir_entry_display);
continue;
}
- ut.check_for_temporary_file(&dir_entry_display);
-
// The LaTeX team provides the file `.tex` as a file with an empty name
// in order to make `\input\relax` work (explained by David Carlisle)
// Therefore, we don't call check_for_hidden_file() in this case
match (pkg_name, dir_entry_display) {
("latex-tools", "tex/latex/tools/.tex") => (),
("latex-tools-dev", "tex/latex-dev/tools/.tex") => (),
- (_, _) => ut.check_for_hidden_file(&file_name, &dir_entry_display)
+ (_, _) => ut.check_for_hidden_file(&file_name, &dir_entry_display),
};
let fsize = meta.len();
@@ -731,8 +817,14 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) {
}
}
+ if !doc_found {
+ e0039!();
+ }
+ if number_of_toplevel_dirs < 2 {
+ e0040!();
+ }
if map_files_found && !map_dvips_found {
- println!( "Map files found for package {} but none of them is in a path starting with fonts/map/dvips", pkg_name);
+ e0041!();
}
};
@@ -871,8 +963,6 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> {
let ut = Utils::new(utils::CheckType::PACKAGE);
-
-
i0002!(root);
// This hash contains all package file names.
//
@@ -977,7 +1067,6 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> {
// 2. dealing with directories
if ft == FType::Directory {
-
let lc_dir_entry_str = dir_entry_str.to_lowercase();
if let Some(_dir_name) = filename(&dir_entry_str) {
let lcnref = &mut lcnames;
@@ -1090,7 +1179,6 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> {
sizeref.entry(fsize).or_insert_with(Vec::new).push(path);
};
for (path, (meta, _file_name, is_readme)) in file_names.iter() {
- //println!("DEBUG: path {:?}, file_name {:?}, is_readme: {} ", &path, &file_name, is_readme);
let dir_entry_str = match path.to_str() {
Some(d) => d,
None => {
@@ -1137,70 +1225,61 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> {
match fext {
// deal with Windows files
- Some("bat") | Some("cmd") | Some("nsh") | Some("reg") => {
- match ft {
- filemagic::Mimetype::Text(LineEnding::CRLF) => (),
- filemagic::Mimetype::Text(LineEnding::CR) =>
- {
- e0037!(&dir_entry_str);
- if ARGS.correct_le {
- make_crlf(&dir_entry_str);
- }
- },
- filemagic::Mimetype::Text(LineEnding::MIXED(0, 0, 0) ) => (),
- filemagic::Mimetype::Text(LineEnding::MIXED(cr, lf, crlf) ) =>
- {
- e0038!(&dir_entry_str, cr, lf, crlf);
- if ARGS.correct_le {
- fix_inconsistent_le(&dir_entry_str);
- }
- },
- filemagic::Mimetype::Text(LineEnding::LF) =>
- {
- w0008!(&dir_entry_str);
- },
- fmm => println!("Should not occur: {} has {:?}", dir_entry_str, fmm),
+ Some("bat") | Some("cmd") | Some("nsh") | Some("reg") => match ft {
+ filemagic::Mimetype::Text(LineEnding::CRLF) => (),
+ filemagic::Mimetype::Text(LineEnding::CR) => {
+ e0037!(&dir_entry_str);
+ if ARGS.correct_le {
+ make_crlf(&dir_entry_str);
+ }
+ }
+ filemagic::Mimetype::Text(LineEnding::MIXED(0, 0, 0)) => (),
+ filemagic::Mimetype::Text(LineEnding::MIXED(cr, lf, crlf)) => {
+ e0038!(&dir_entry_str, cr, lf, crlf);
+ if ARGS.correct_le {
+ fix_inconsistent_le(&dir_entry_str);
+ }
}
+ filemagic::Mimetype::Text(LineEnding::LF) => {
+ w0008!(&dir_entry_str);
+ }
+ fmm => println!("Should not occur: {} has {:?}", dir_entry_str, fmm),
},
Some(_) | None => {
match ft {
- filemagic::Mimetype::Text(LineEnding::CRLF) =>
- {
+ filemagic::Mimetype::Text(LineEnding::CRLF) => {
e0012!(&dir_entry_str);
if ARGS.correct_le {
fix_inconsistent_le(&dir_entry_str);
}
- },
- filemagic::Mimetype::Text(LineEnding::CR) =>
- {
+ }
+ filemagic::Mimetype::Text(LineEnding::CR) => {
e0037!(&dir_entry_str);
if ARGS.correct_le {
fix_inconsistent_le(&dir_entry_str);
}
- },
- filemagic::Mimetype::Text(LineEnding::MIXED(0, 0, 0) ) => (),
- filemagic::Mimetype::Text(LineEnding::MIXED(cr, lf, crlf) ) =>
- {
+ }
+ filemagic::Mimetype::Text(LineEnding::MIXED(0, 0, 0)) => (),
+ filemagic::Mimetype::Text(LineEnding::MIXED(cr, lf, crlf)) => {
//println!(">>>{}: {:?} {},{},{}", &dir_entry_str, ft, x, y, z);
e0038!(&dir_entry_str, cr, lf, crlf);
if ARGS.correct_le {
fix_inconsistent_le(&dir_entry_str);
}
- },
+ }
filemagic::Mimetype::Text(LineEnding::LF) => (),
fmm => println!("Should not occur: {} has {:?}", dir_entry_str, fmm),
}
- },
+ }
}
- },
+ }
filemagic::Mimetype::BOM(b) => {
//println!("{}: {} with BOM detected", dir_entry_str, b.as_ref());
w0004!(&dir_entry_str, b.as_ref());
check_and_correct_perms(&dir_entry_str, p);
}
- filemagic::Mimetype::Binary
- | filemagic::Mimetype::Script(_) => {
+ filemagic::Mimetype::Binary | filemagic::Mimetype::Script(_) => {
if !owner_has(p, 4) || !others_have(p, 4) {
e0002!(&dir_entry_str, perms_to_string(p));
};
@@ -1353,5 +1432,3 @@ fn show_tmp_endings() {
println!("{:23} {}", t, c);
}
}
-
-