From 4a258bbb349d86735a9d1be59c3a5eae8756012c Mon Sep 17 00:00:00 2001 From: Norbert Preining Date: Mon, 4 Oct 2021 03:02:00 +0000 Subject: CTAN sync 202110040301 --- support/pkgcheck/src/main.rs | 191 ++++++++++++++++++++++--------------------- 1 file changed, 97 insertions(+), 94 deletions(-) (limited to 'support/pkgcheck/src/main.rs') diff --git a/support/pkgcheck/src/main.rs b/support/pkgcheck/src/main.rs index d4ef7d6db2..bb8e90b76e 100644 --- a/support/pkgcheck/src/main.rs +++ b/support/pkgcheck/src/main.rs @@ -50,7 +50,8 @@ use std::path::PathBuf; use std::sync::atomic::{AtomicBool, Ordering}; use blake2::{Blake2b, Digest}; -use fnv::{FnvHashMap as HashMap, FnvHashSet as HashSet}; +use rustc_hash::{FxHashMap, FxHashSet}; + use std::sync::mpsc::{channel, Sender}; use structopt::clap::Shell; @@ -58,7 +59,7 @@ use structopt::StructOpt; #[cfg(unix)] use walkdir::{DirEntry, WalkDir}; -fn err(path: &PathBuf, err: &io::Error) { +fn err(path: &Path, err: &io::Error) { e0027!(path.display(), err); } @@ -68,15 +69,15 @@ type HashSender = Sender<(u64, PathBuf, Vec)>; // SizesHashMap contains // - file sizes // - and a vector of file names having that size -type SizesHashMap = HashMap>; +type SizesHashMap = FxHashMap>; -type GeneratedHashMap = HashMap; +type GeneratedHashMap = FxHashMap; -type FileNamesHashMap = HashMap; +type FileNamesHashMap = FxHashMap; const BLOCKSIZE: usize = 4096; -fn hash_file_inner(path: &PathBuf) -> io::Result> { +fn hash_file_inner(path: &Path) -> io::Result> { let mut buf = [0u8; BLOCKSIZE]; let mut fp = File::open(&path)?; let mut digest = Blake2b::default(); @@ -141,7 +142,7 @@ fn check_readme(dir_entry: &str, is_readme: &ReadmeKind, ft: &filemagic::Mimetyp e0003!(msg_name); return false; } - filemagic::Mimetype::BOM(b) => { + filemagic::Mimetype::Bom(b) => { e0029!(msg_name, b.as_ref()); return false; } @@ -177,10 +178,7 @@ fn check_readme_inner(fname: &str, f: &std::fs::File) -> bool { } fn is_readme(entry: &str) -> bool { - match entry { - "README" | "README.txt" | "README.md" => true, - _ => false, - } + matches!(entry, "README" | "README.txt" | "README.md") } fn get_devno(meta: &Metadata) -> u64 { @@ -200,6 +198,8 @@ fn _get_devno(entry: &DirEntry) -> u64 { struct Args { #[structopt(short = "I", long = "ignore-dupes", help = "Ignore dupes")] ignore_dupes: bool, + #[structopt(long = "ignore-same-named", help = "Ignore same-named files")] + ignore_same_named: bool, #[structopt(short = "v", long = "verbose", help = "Verbose operation?")] verbose: bool, #[structopt(short = "L", long = "correct-le", help = "Correct line endings")] @@ -240,12 +240,12 @@ struct Args { // We take care to avoid visiting a single inode twice, // which takes care of (false positive) hardlinks. #[cfg(unix)] -fn check_inode(set: &mut HashSet<(u64, u64)>, meta: &Metadata) -> bool { +fn check_inode(set: &mut FxHashSet<(u64, u64)>, meta: &Metadata) -> bool { set.insert((get_devno(meta), meta.ino())) } #[cfg(not(unix))] -fn check_inode(_: &mut HashSet, _: &Metadata) -> bool { +fn check_inode(_: &mut FxHashSet, _: &Metadata) -> bool { true } @@ -309,13 +309,13 @@ impl DupPath { // Args::clap().gen_completions(env!("CARGO_PKG_NAME"), Shell::Fish, "target"); // } -type DupHashes = HashMap<(u64, Vec), DupPath>; +type DupHashes = FxHashMap<(u64, Vec), DupPath>; fn main() { match &ARGS.explain { None => (), Some(e) => { - explains(&e); + explains(e); process::exit(0); } } @@ -374,12 +374,12 @@ fn main() { None => None, Some(tz) => { let pn = check_tds_archive_name(tds_zip); - if !exists_file(&tz) { + if !exists_file(tz) { f0003!(&tz); process::exit(1); } let mut fmagic = filemagic::Filetype::new(); - match fmagic.analyze(&tz) { + match fmagic.analyze(tz) { Ok(filemagic::Mimetype::Zip) => (), _ => { f0004!(&tz); @@ -528,12 +528,12 @@ fn check_generated_files(entry: &str, generated: &mut GeneratedHashMap) { // a package we ignore if a README was generated by an // .ins or .dtx file // CAVEAT: If this happens in a subdirectory it could be an error!!!! - if is_readme(&filename) { + if is_readme(filename) { continue; } - // Ignore generated pdf files - if fname.ends_with(".pdf") { + // Ignore generated pdf, html, and css files + if fname.ends_with(".pdf") || fname.ends_with(".html") || fname.ends_with(".css") { continue; } @@ -568,7 +568,7 @@ fn check_tds_archive_name(tds_zip: &Option) -> Option { f0005!(tz); process::exit(1); } - let mut pname = String::from(utils::basename(&tz)); + let mut pname = String::from(utils::basename(tz)); let plen = pname.len(); pname.truncate(plen - 8); Some(pname) @@ -577,7 +577,7 @@ fn check_tds_archive_name(tds_zip: &Option) -> Option { } fn unzip_tds_archive(tds_zip: &str, tmp_dir: &str) { - match run_cmd("unzip", &["-q", "-d", tmp_dir, &tds_zip]) { + match run_cmd("unzip", &["-q", "-d", tmp_dir, tds_zip]) { CmdReturn { status: true, .. } => (), CmdReturn { status: false, @@ -597,16 +597,16 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { i0003!(tds_zip); let dir_entry = Path::new(tds_zip); - let p = get_perms(&dir_entry); + let p = get_perms(dir_entry); if !owner_has(p, 4) || !others_have(p, 4) || x_bit_set(p) { e0024!(tds_zip, perms_to_string(p)); if ARGS.correct_perms { i0005!(&tds_zip); - set_perms(&tds_zip, 0o664); + set_perms(tds_zip, 0o664); } }; - let ut = Utils::new(utils::CheckType::TDS); + let ut = Utils::new(utils::CheckType::Tds); // We have a discrepancy between // TDS zip archive name: babel-base.tds.zip @@ -616,17 +616,18 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { // Therefore, we correct here. // latex-tools, latex-graphics, latex-amsmath, latex-base => latex // latex-tools-dev, latex-graphics-dev, latex-amsmath-dev, latex-base-dev => latex-dev - let mut pkg_replacements: HashMap<&str, &str> = HashMap::default(); + let mut pkg_replacements: FxHashMap<&str, &str> = FxHashMap::default(); pkg_replacements.insert("babel-base", "babel"); pkg_replacements.insert("latex-base", "latex"); pkg_replacements.insert("latex-tools", "latex"); pkg_replacements.insert("latex-graphics", "latex"); pkg_replacements.insert("latex-amsmath", "latex"); + pkg_replacements.insert("latex-firstaid", "latex/firstaid"); pkg_replacements.insert("latex-base-dev", "latex-dev"); pkg_replacements.insert("latex-tools-dev", "latex-dev"); pkg_replacements.insert("latex-graphics-dev", "latex-dev"); pkg_replacements.insert("latex-amsmath-dev", "latex-dev"); - pkg_replacements.insert("latex-firstaid", "latex/firstaid"); + pkg_replacements.insert("latex-firstaid-dev", "latex-dev/firstaid"); let real_pkg_name = if let Some(real_name) = pkg_replacements.get(pkg_name) { real_name } else { @@ -648,7 +649,7 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { // tds zip archive we need to checksum the files in the tds zip // archive. - let mut sizes: SizesHashMap = HashMap::default(); + let mut sizes: SizesHashMap = FxHashMap::default(); let mut pool = Pool::new(num_cpus::get() as u32 + 1); { // Processing a single file entry, with the "sizes" hashmap collecting @@ -668,7 +669,7 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { // those top level directories are the directories found in the // texmf-dist/ directory of a texlive installation - let tds_toplevel_dirs: HashSet = [ + let tds_toplevel_dirs: FxHashSet = [ "asymptote", "bibtex", "chktex", @@ -742,7 +743,7 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { &dir_entry_str[tmp_dir_offset..] }; - ut.check_for_temporary_file(&dir_entry_display); + ut.check_for_temporary_file(dir_entry_display); // In the top level directory of a TDS zip archive // ... no files are allowed @@ -764,8 +765,8 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { } if ft == FType::Directory { - ut.check_for_empty_directory(&dir_entry_str, &dir_entry_display); - ut.check_for_hidden_directory(&file_name, &dir_entry_display); + ut.check_for_empty_directory(dir_entry_str, dir_entry_display); + ut.check_for_hidden_directory(&file_name, dir_entry_display); ut.is_unwanted_directory(&file_name, dir_entry_str); continue; @@ -777,12 +778,12 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { match (pkg_name, dir_entry_display) { ("latex-tools", "tex/latex/tools/.tex") => (), ("latex-tools-dev", "tex/latex-dev/tools/.tex") => (), - (_, _) => ut.check_for_hidden_file(&file_name, &dir_entry_display), + (_, _) => ut.check_for_hidden_file(&file_name, dir_entry_display), }; let fsize = meta.len(); process(fsize, &dir_entry); - ut.check_filesize(fsize, &dir_entry_display); + ut.check_filesize(fsize, dir_entry_display); // if we encounter a .dtx or .ins file we check // that it is in a subdirectory of either source/ or doc/ @@ -795,10 +796,10 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { } // if the path doesn't contain a man page... - if dir_entry_str.find("/man/").is_none() { + if !dir_entry_str.contains("/man/") { let pkg_name_s = format!("/{}/", real_pkg_name); // ...then we want to have the package name in the path - if dir_entry_str.find(&pkg_name_s).is_none() { + if !dir_entry_str.contains(&pkg_name_s) { e0028!(real_pkg_name, dir_entry_display); } } @@ -828,7 +829,7 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { } }; - let mut tds_hashes: HashMap<(u64, Vec), Vec> = HashMap::default(); + let mut tds_hashes: FxHashMap<(u64, Vec), Vec> = FxHashMap::default(); pool.scoped(|scope| { let (tx, rx) = channel(); @@ -909,7 +910,7 @@ fn check_and_correct_perms(dir_entry: &str, p: u32) { e0002!(dir_entry, perms_to_string(p)); if ARGS.correct_perms { i0005!(&dir_entry); - set_perms(&dir_entry, 0o664); + set_perms(dir_entry, 0o664); } }; } @@ -955,13 +956,13 @@ enum ReadmeKind { } fn check_package(root: &str, tds_zip: &Option) -> Option { - let mut lcnames: HashMap> = HashMap::default(); + let mut lcnames: FxHashMap> = FxHashMap::default(); - let mut doublenames: HashMap> = HashMap::default(); + let mut doublenames: FxHashMap> = FxHashMap::default(); - let mut inodes = HashSet::default(); + let mut inodes = FxHashSet::default(); - let ut = Utils::new(utils::CheckType::PACKAGE); + let ut = Utils::new(utils::CheckType::Package); i0002!(root); // This hash contains all package file names. @@ -972,7 +973,7 @@ fn check_package(root: &str, tds_zip: &Option) -> Option { // ReadmeKind: is it a certain README, file or symlink? // A special case of a README file is a file with has a different name but // was pointed to by a symlink. Example: README --> README.rst - let mut file_names: FileNamesHashMap = HashMap::default(); + let mut file_names: FileNamesHashMap = FxHashMap::default(); let mut readme_found = false; let root_absolute = PathBuf::from(root) @@ -1010,7 +1011,7 @@ fn check_package(root: &str, tds_zip: &Option) -> Option { // we check for weird stuff like socket files aso. let ft = get_filetype(&dir_entry); - if found_unwanted_filetype(&dir_entry_str, &ft) { + if found_unwanted_filetype(dir_entry_str, &ft) { continue; } @@ -1036,7 +1037,7 @@ fn check_package(root: &str, tds_zip: &Option) -> Option { } let lc_dir_entry_str = dir_entry_str.to_lowercase(); - if let Some(_dir_name) = filename(&dir_entry_str) { + if let Some(_dir_name) = filename(dir_entry_str) { let lcnref = &mut lcnames; lcnref .entry(PathBuf::from(lc_dir_entry_str)) @@ -1063,44 +1064,44 @@ fn check_package(root: &str, tds_zip: &Option) -> Option { } } - let p = get_perms(&dir_entry.path()); + let p = get_perms(dir_entry.path()); // 2. dealing with directories if ft == FType::Directory { let lc_dir_entry_str = dir_entry_str.to_lowercase(); - if let Some(_dir_name) = filename(&dir_entry_str) { + if let Some(_dir_name) = filename(dir_entry_str) { let lcnref = &mut lcnames; lcnref .entry(PathBuf::from(lc_dir_entry_str)) .or_insert_with(Vec::new) - .push((PathBuf::from(&dir_entry_str), FileKind::Directory)); + .push((PathBuf::from(dir_entry_str), FileKind::Directory)); } if !owner_has(p, 5) || !others_have(p, 5) { e0011!(&dir_entry_str, perms_to_string(p)); if ARGS.correct_perms { i0005!(&dir_entry_str); - set_perms(&dir_entry_str, 0o775); + set_perms(dir_entry_str, 0o775); } } - ut.check_for_empty_directory(&dir_entry_str, &dir_entry_str); - ut.check_for_hidden_directory(&file_name, &dir_entry_str); + ut.check_for_empty_directory(dir_entry_str, dir_entry_str); + ut.check_for_hidden_directory(&file_name, dir_entry_str); ut.is_unwanted_directory(&file_name, dir_entry_str); continue; } // 3. dealing with regular files - ut.check_for_hidden_file(&file_name, &dir_entry_str); - ut.check_for_temporary_file(&dir_entry_str); + ut.check_for_hidden_file(&file_name, dir_entry_str); + ut.check_for_temporary_file(dir_entry_str); // if is_temporary_file(&dir_entry_str) { // e0008!(&dir_entry_str); // } - if let Some(file_name) = filename(&dir_entry_str) { + if let Some(file_name) = filename(dir_entry_str) { let doubleref = &mut doublenames; doubleref @@ -1164,8 +1165,8 @@ fn check_package(root: &str, tds_zip: &Option) -> Option { let lc = LinkCheck::new(4, false); let mut detective = filemagic::Filetype::new(); - let mut sizes: SizesHashMap = HashMap::default(); - let mut generated: GeneratedHashMap = HashMap::default(); + let mut sizes: SizesHashMap = FxHashMap::default(); + let mut generated: GeneratedHashMap = FxHashMap::default(); // Processing a single file entry, with the "sizes" hashmap collecting // same-size files. Entries are either Found::One or Found::Multiple, @@ -1190,13 +1191,13 @@ fn check_package(root: &str, tds_zip: &Option) -> Option { let fsize = meta.len(); ut.check_filesize(fsize, dir_entry_str); - let p = get_perms(&path); + let p = get_perms(path); if !owner_has(p, 4) { e0002!(&dir_entry_str, perms_to_string(p)); - fix_perms(&dir_entry_str); + fix_perms(dir_entry_str); continue; } - let ftr = detective.analyze(&dir_entry_str); + let ftr = detective.analyze(dir_entry_str); //println!(">>> {:?}", ftr); // we ignore errors from filetype recognition if ftr.is_err() { @@ -1207,87 +1208,87 @@ fn check_package(root: &str, tds_zip: &Option) -> Option { // DEBUG !readme_symlinked.contains(&dir_entry_str) if ReadmeKind::No != *is_readme { - if !check_readme(&dir_entry_str, is_readme, &ft) { + if !check_readme(dir_entry_str, is_readme, &ft) { continue; } if ARGS.urlcheck { - lc.check_urls(&dir_entry_str); + lc.check_urls(dir_entry_str); } } match ft { filemagic::Mimetype::Text(_) => { - check_and_correct_perms(&dir_entry_str, p); - let fext = get_extension_from_filename(&dir_entry_str); + check_and_correct_perms(dir_entry_str, p); + let fext = get_extension_from_filename(dir_entry_str); if fext == Some("ins") || fext == Some("dtx") { - check_generated_files(&dir_entry_str, &mut generated); + check_generated_files(dir_entry_str, &mut generated); } match fext { // deal with Windows files Some("bat") | Some("cmd") | Some("nsh") | Some("reg") => match ft { - filemagic::Mimetype::Text(LineEnding::CRLF) => (), - filemagic::Mimetype::Text(LineEnding::CR) => { + filemagic::Mimetype::Text(LineEnding::Crlf) => (), + filemagic::Mimetype::Text(LineEnding::Cr) => { e0037!(&dir_entry_str); if ARGS.correct_le { - make_crlf(&dir_entry_str); + make_crlf(dir_entry_str); } } - filemagic::Mimetype::Text(LineEnding::MIXED(0, 0, 0)) => (), - filemagic::Mimetype::Text(LineEnding::MIXED(cr, lf, crlf)) => { + filemagic::Mimetype::Text(LineEnding::Mixed(0, 0, 0)) => (), + filemagic::Mimetype::Text(LineEnding::Mixed(cr, lf, crlf)) => { e0038!(&dir_entry_str, cr, lf, crlf); if ARGS.correct_le { - fix_inconsistent_le(&dir_entry_str); + fix_inconsistent_le(dir_entry_str); } } - filemagic::Mimetype::Text(LineEnding::LF) => { + filemagic::Mimetype::Text(LineEnding::Lf) => { w0008!(&dir_entry_str); } fmm => println!("Should not occur: {} has {:?}", dir_entry_str, fmm), }, Some(_) | None => { match ft { - filemagic::Mimetype::Text(LineEnding::CRLF) => { + filemagic::Mimetype::Text(LineEnding::Crlf) => { e0012!(&dir_entry_str); if ARGS.correct_le { - fix_inconsistent_le(&dir_entry_str); + fix_inconsistent_le(dir_entry_str); } } - filemagic::Mimetype::Text(LineEnding::CR) => { + filemagic::Mimetype::Text(LineEnding::Cr) => { e0037!(&dir_entry_str); if ARGS.correct_le { - fix_inconsistent_le(&dir_entry_str); + fix_inconsistent_le(dir_entry_str); } } - filemagic::Mimetype::Text(LineEnding::MIXED(0, 0, 0)) => (), - filemagic::Mimetype::Text(LineEnding::MIXED(cr, lf, crlf)) => { + filemagic::Mimetype::Text(LineEnding::Mixed(0, 0, 0)) => (), + filemagic::Mimetype::Text(LineEnding::Mixed(cr, lf, crlf)) => { //println!(">>>{}: {:?} {},{},{}", &dir_entry_str, ft, x, y, z); e0038!(&dir_entry_str, cr, lf, crlf); if ARGS.correct_le { - fix_inconsistent_le(&dir_entry_str); + fix_inconsistent_le(dir_entry_str); } } - filemagic::Mimetype::Text(LineEnding::LF) => (), + filemagic::Mimetype::Text(LineEnding::Lf) => (), fmm => println!("Should not occur: {} has {:?}", dir_entry_str, fmm), } } } } - filemagic::Mimetype::BOM(b) => { + filemagic::Mimetype::Bom(b) => { //println!("{}: {} with BOM detected", dir_entry_str, b.as_ref()); w0004!(&dir_entry_str, b.as_ref()); - check_and_correct_perms(&dir_entry_str, p); + check_and_correct_perms(dir_entry_str, p); } filemagic::Mimetype::Binary | filemagic::Mimetype::Script(_) => { if !owner_has(p, 4) || !others_have(p, 4) { e0002!(&dir_entry_str, perms_to_string(p)); }; - fix_perms(&dir_entry_str); + fix_perms(dir_entry_str); } filemagic::Mimetype::Pdf => { - check_and_correct_perms(&dir_entry_str, p); - let ret = is_pdf_ok(&dir_entry_str); + check_and_correct_perms(dir_entry_str, p); + let ret = is_pdf_ok(dir_entry_str); if !ret.status { e0017!(&dir_entry_str); if let Some(output) = ret.output { @@ -1302,11 +1303,11 @@ fn check_package(root: &str, tds_zip: &Option) -> Option { } else { w0001!(&dir_entry_str); } - check_and_correct_perms(&dir_entry_str, p); + check_and_correct_perms(dir_entry_str, p); } - filemagic::Mimetype::Data => check_and_correct_perms(&dir_entry_str, p), - filemagic::Mimetype::Zerofile => check_and_correct_perms(&dir_entry_str, p), + filemagic::Mimetype::Data => check_and_correct_perms(dir_entry_str, p), + filemagic::Mimetype::Zerofile => check_and_correct_perms(dir_entry_str, p), _ => continue, } @@ -1314,17 +1315,19 @@ fn check_package(root: &str, tds_zip: &Option) -> Option { e0002!(&dir_entry_str, perms_to_string(p)); if ARGS.correct_perms { i0005!(&dir_entry_str); - set_perms(&dir_entry_str, 0o664); + set_perms(dir_entry_str, 0o664); } } if !(ARGS.ignore_dupes && tds_zip.is_none()) { - process(fsize, &path); + process(fsize, path); } } print_casefolding(&lcnames); print_generated(&doublenames, &generated); - print_doublenames(&doublenames); + if ! ARGS.ignore_same_named { + print_doublenames(&doublenames); + } if ARGS.ignore_dupes && tds_zip.is_none() { return None; @@ -1335,7 +1338,7 @@ fn check_package(root: &str, tds_zip: &Option) -> Option { // doing mostly IO. let mut pool = Pool::new(num_cpus::get() as u32 + 1); - let mut hashes: HashMap<(u64, Vec), DupPath> = HashMap::default(); + let mut hashes: FxHashMap<(u64, Vec), DupPath> = FxHashMap::default(); pool.scoped(|scope| { let (tx, rx) = channel(); @@ -1368,7 +1371,7 @@ fn check_package(root: &str, tds_zip: &Option) -> Option { Some(hashes) } -fn print_casefolding(hashes: &HashMap>) { +fn print_casefolding(hashes: &FxHashMap>) { for (k, eles) in hashes.iter() { //println!("pcf: {:?}, {:?}", k, &eles); if eles.len() == 1 { @@ -1383,7 +1386,7 @@ fn print_casefolding(hashes: &HashMap>) { } } -fn print_generated(doublenames: &HashMap>, generated: &GeneratedHashMap) { +fn print_generated(doublenames: &FxHashMap>, generated: &GeneratedHashMap) { // `k` is generated by `gen` for (k, gen) in generated.iter() { let path = PathBuf::from(k); @@ -1400,7 +1403,7 @@ fn print_generated(doublenames: &HashMap>, generated: &Gen } } -fn print_doublenames(hashes: &HashMap>) { +fn print_doublenames(hashes: &FxHashMap>) { for (k, paths) in hashes.iter() { if paths.len() == 1 { continue; -- cgit v1.2.3