diff options
author | Norbert Preining <norbert@preining.info> | 2021-10-04 03:02:00 +0000 |
---|---|---|
committer | Norbert Preining <norbert@preining.info> | 2021-10-04 03:02:00 +0000 |
commit | 4a258bbb349d86735a9d1be59c3a5eae8756012c (patch) | |
tree | fbf34e5440862041423deb0b41262aad30b05425 /support/pkgcheck/src | |
parent | c7bf3f5a728fcb426d1aa38fbec211d4344a3725 (diff) |
CTAN sync 202110040301
Diffstat (limited to 'support/pkgcheck/src')
-rw-r--r-- | support/pkgcheck/src/filemagic.rs | 53 | ||||
-rw-r--r-- | support/pkgcheck/src/linkcheck.rs | 24 | ||||
-rw-r--r-- | support/pkgcheck/src/main.rs | 191 | ||||
-rw-r--r-- | support/pkgcheck/src/messages/errorsd.rs | 2 | ||||
-rw-r--r-- | support/pkgcheck/src/messages/mod.rs | 28 | ||||
-rw-r--r-- | support/pkgcheck/src/utils.rs | 39 |
6 files changed, 174 insertions, 163 deletions
diff --git a/support/pkgcheck/src/filemagic.rs b/support/pkgcheck/src/filemagic.rs index aa9c50feda..f3a4bcc391 100644 --- a/support/pkgcheck/src/filemagic.rs +++ b/support/pkgcheck/src/filemagic.rs @@ -21,10 +21,10 @@ use unicode_bom::Bom; #[derive(Debug, PartialEq)] pub enum LineEnding { - LF, - CR, - CRLF, - MIXED(usize, usize, usize), + Lf, + Cr, + Crlf, + Mixed(usize, usize, usize), } #[derive(Debug, PartialEq)] @@ -45,7 +45,7 @@ pub enum Mimetype { Socket, Zerofile, VeryShort, - BOM(Bom), + Bom(Bom), } pub struct Filetype { @@ -83,7 +83,7 @@ fn _is_crlf(buffer: &[u8], len: usize) -> bool { } //println!("cr: {}, lf: {}", cr, lf); - // Heuristics: we accept if only a few lines are not CRLF + // Heuristics: we accept if only a few lines are not Crlf match (cr, lf) { (0, _lf) => return false, (_cr, 0) => return true, @@ -120,17 +120,17 @@ fn is_crlf(buffer: &[u8], len: usize) -> LineEnding { seen_cr = *c == CR; } - // println!("LF / CR / CRLF: {} / {} / {}", n_lf, n_cr, n_crlf); + // println!("Lf / Cr / Crlf: {} / {} / {}", n_lf, n_cr, n_crlf); // println!("cr: {}, lf: {}, crlf: {}", n_cr, n_lf, n_crlf); // if (n_crlf == 0 && n_cr == 0 && n_nel == 0 && n_lf == 0) // --> no line terminators match (n_cr, n_lf, n_crlf) { - (0, 0, z) if z > 0 => LineEnding::CRLF, - (x, 0, 0) if x > 0 => LineEnding::CR, - (0, y, 0) if y > 0 => LineEnding::LF, - (x, y, z) => LineEnding::MIXED(x, y, z), + (0, 0, z) if z > 0 => LineEnding::Crlf, + (x, 0, 0) if x > 0 => LineEnding::Cr, + (0, y, 0) if y > 0 => LineEnding::Lf, + (x, y, z) => LineEnding::Mixed(x, y, z), } } @@ -194,7 +194,7 @@ impl Filetype { let bom: Bom = Bom::from(&self.buffer[0..]); if bom != Bom::Null { - return Ok(Mimetype::BOM(bom)); + return Ok(Mimetype::Bom(bom)); } if is_binary_data(&self.buffer, bytes_read) { @@ -231,13 +231,16 @@ impl Filetype { let crlf = is_crlf(&self.buffer, bytes_read); //println!("{:?}", crlf); + // checks for + // - shebang + // - php indicator if bytes_read >= 5 && (self.buffer.starts_with(b"#!") || self.buffer.starts_with(b"<?php")) { return Ok(Mimetype::Script(crlf)); } // recognize files as script files by file endings - let is_script = Regex::new(r"\.(csh|ksh|py|pl|sh|tcsh)$") + let is_script = Regex::new(r"\.(csh|ksh|py|pl|sh|tcsh|lua)$") .unwrap() .is_match(fname.as_bytes()); @@ -248,13 +251,13 @@ impl Filetype { Ok(Mimetype::Text(crlf)) } // match (crlf, is_script) { - // (LineEnding::LF, false) => Ok(Mimetype::Text(LineEnding::LF)), - // (LineEnding::CR, false) => Ok(Mimetype::Text(LineEnding::CR)), - // (LineEnding::CRLF, false) => Ok(Mimetype::Text(LineEnding::CRLF)), - // (LineEnding::LF, true) => Ok(Mimetype::Script(LineEnding::LF)), - // (LineEnding::CR, true) => Ok(Mimetype::Script(LineEnding::CR)), - // (LineEnding::CRLF, true) => Ok(Mimetype::Script(LineEnding::CRLF)), - // (_, _) => Ok(Mimetype::Text(LineEnding::LF)), + // (LineEnding::Lf, false) => Ok(Mimetype::Text(LineEnding::Lf)), + // (LineEnding::Cr, false) => Ok(Mimetype::Text(LineEnding::Cr)), + // (LineEnding::Crlf, false) => Ok(Mimetype::Text(LineEnding::Crlf)), + // (LineEnding::Lf, true) => Ok(Mimetype::Script(LineEnding::Lf)), + // (LineEnding::Cr, true) => Ok(Mimetype::Script(LineEnding::Cr)), + // (LineEnding::Crlf, true) => Ok(Mimetype::Script(LineEnding::Crlf)), + // (_, _) => Ok(Mimetype::Text(LineEnding::Lf)), // } } } @@ -396,11 +399,11 @@ fn test_filetype() { // This file is a pdf but has lines starting with % before the pdf signature shows up // The unix `file` command) says: data - // analyze() says TextCRLF + // analyze() says TextCrlf //assert!(ft.analyze("tests_filemagic/musterlogo.pdf").ok() == Some(Mimetype::Script)); - assert!(ft.analyze("tests_filemagic/x.pl").ok() == Some(Mimetype::Script(LineEnding::LF))); - assert!(ft.analyze("tests_filemagic/main.php").ok() == Some(Mimetype::Script(LineEnding::LF))); + assert!(ft.analyze("tests_filemagic/x.pl").ok() == Some(Mimetype::Script(LineEnding::Lf))); + assert!(ft.analyze("tests_filemagic/main.php").ok() == Some(Mimetype::Script(LineEnding::Lf))); assert!(ft.analyze("tests_filemagic/test.7z").ok() == Some(Mimetype::Archive)); assert!(ft.analyze("tests_filemagic/x.tgz").ok() == Some(Mimetype::Archive)); @@ -417,7 +420,7 @@ fn test_filetype() { assert!( ft.analyze("tests_filemagic/README").ok() - == Some(Mimetype::Text(LineEnding::MIXED(0, 0, 0))) + == Some(Mimetype::Text(LineEnding::Mixed(0, 0, 0))) ); // assert!(ft.analyze("tests_filemagic/README1").ok() == Some(Mimetype::Text)); @@ -434,6 +437,6 @@ fn test_filetype() { assert!( ft.analyze("tests_filemagic/8stbu11h.htm").ok() - == Some(Mimetype::Text(LineEnding::MIXED(0, 1, 8710))) + == Some(Mimetype::Text(LineEnding::Mixed(0, 1, 8710))) ); } diff --git a/support/pkgcheck/src/linkcheck.rs b/support/pkgcheck/src/linkcheck.rs index d7d1671dcf..cc7aa36e31 100644 --- a/support/pkgcheck/src/linkcheck.rs +++ b/support/pkgcheck/src/linkcheck.rs @@ -15,8 +15,7 @@ use std::sync::Mutex; use std::sync::atomic::Ordering; -use std::collections::HashMap; -use std::collections::HashSet; +use rustc_hash::{FxHashMap, FxHashSet}; enum UrlStatus { Unknown, @@ -25,11 +24,11 @@ enum UrlStatus { } struct HashVal { - paths: HashSet<String>, + paths: FxHashSet<String>, status: UrlStatus, } -type UrlHash = HashMap<String, HashVal>; +type UrlHash = FxHashMap<String, HashVal>; pub struct LinkCheck { pool: Mutex<ThreadPool>, @@ -43,7 +42,7 @@ impl LinkCheck { let pool = Mutex::new(ThreadPool::new(num_threads)); LinkCheck { pool, - urlhash: Arc::new(Mutex::new(HashMap::default())), + urlhash: Arc::new(Mutex::new(FxHashMap::default())), print_all, } } @@ -86,7 +85,7 @@ fn check_link(url: &str, fname: &str, urlhash: &Arc<Mutex<UrlHash>>, print_all: let mut urlhash = urlhash.lock().unwrap(); if !urlhash.contains_key(&url) { - let mut hs = HashSet::default(); + let mut hs = FxHashSet::default(); hs.insert(f); let url1 = url.clone(); @@ -166,6 +165,8 @@ fn get_links(fname: &str) -> Option<Vec<String>> { fn get_links_inner(s: &str) -> Option<Vec<String>> { let mut finder = LinkFinder::new(); finder.kinds(&[LinkKind::Url]); + + // finder.links() does the actual search for URLs let links: Vec<_> = finder.links(s).collect(); let result: Vec<&str> = links.iter().map(|e| e.as_str()).collect(); @@ -174,12 +175,11 @@ fn get_links_inner(s: &str) -> Option<Vec<String>> { if !r.starts_with("http://") && !r.starts_with("https://") && !r.starts_with("ftp://") { continue; } - // This is a workaround to prevent URLs ending with ` - if r.ends_with('`') { - links.push(String::from(&r[..r.len() - 1])); - } else { - links.push(String::from(r)); - } + + // This is a workaround to prevent URLs ending with certain characters + let url = r.trim_end_matches(|c| c == '。' || c == '`'); + + links.push(String::from(url)); } if !links.is_empty() { Some(links) diff --git a/support/pkgcheck/src/main.rs b/support/pkgcheck/src/main.rs index d4ef7d6db2..bb8e90b76e 100644 --- a/support/pkgcheck/src/main.rs +++ b/support/pkgcheck/src/main.rs @@ -50,7 +50,8 @@ use std::path::PathBuf; use std::sync::atomic::{AtomicBool, Ordering}; use blake2::{Blake2b, Digest}; -use fnv::{FnvHashMap as HashMap, FnvHashSet as HashSet}; +use rustc_hash::{FxHashMap, FxHashSet}; + use std::sync::mpsc::{channel, Sender}; use structopt::clap::Shell; @@ -58,7 +59,7 @@ use structopt::StructOpt; #[cfg(unix)] use walkdir::{DirEntry, WalkDir}; -fn err(path: &PathBuf, err: &io::Error) { +fn err(path: &Path, err: &io::Error) { e0027!(path.display(), err); } @@ -68,15 +69,15 @@ type HashSender = Sender<(u64, PathBuf, Vec<u8>)>; // SizesHashMap contains // - file sizes // - and a vector of file names having that size -type SizesHashMap = HashMap<u64, Vec<PathBuf>>; +type SizesHashMap = FxHashMap<u64, Vec<PathBuf>>; -type GeneratedHashMap = HashMap<String, String>; +type GeneratedHashMap = FxHashMap<String, String>; -type FileNamesHashMap = HashMap<PathBuf, (Metadata, String, ReadmeKind)>; +type FileNamesHashMap = FxHashMap<PathBuf, (Metadata, String, ReadmeKind)>; const BLOCKSIZE: usize = 4096; -fn hash_file_inner(path: &PathBuf) -> io::Result<Vec<u8>> { +fn hash_file_inner(path: &Path) -> io::Result<Vec<u8>> { let mut buf = [0u8; BLOCKSIZE]; let mut fp = File::open(&path)?; let mut digest = Blake2b::default(); @@ -141,7 +142,7 @@ fn check_readme(dir_entry: &str, is_readme: &ReadmeKind, ft: &filemagic::Mimetyp e0003!(msg_name); return false; } - filemagic::Mimetype::BOM(b) => { + filemagic::Mimetype::Bom(b) => { e0029!(msg_name, b.as_ref()); return false; } @@ -177,10 +178,7 @@ fn check_readme_inner(fname: &str, f: &std::fs::File) -> bool { } fn is_readme(entry: &str) -> bool { - match entry { - "README" | "README.txt" | "README.md" => true, - _ => false, - } + matches!(entry, "README" | "README.txt" | "README.md") } fn get_devno(meta: &Metadata) -> u64 { @@ -200,6 +198,8 @@ fn _get_devno(entry: &DirEntry) -> u64 { struct Args { #[structopt(short = "I", long = "ignore-dupes", help = "Ignore dupes")] ignore_dupes: bool, + #[structopt(long = "ignore-same-named", help = "Ignore same-named files")] + ignore_same_named: bool, #[structopt(short = "v", long = "verbose", help = "Verbose operation?")] verbose: bool, #[structopt(short = "L", long = "correct-le", help = "Correct line endings")] @@ -240,12 +240,12 @@ struct Args { // We take care to avoid visiting a single inode twice, // which takes care of (false positive) hardlinks. #[cfg(unix)] -fn check_inode(set: &mut HashSet<(u64, u64)>, meta: &Metadata) -> bool { +fn check_inode(set: &mut FxHashSet<(u64, u64)>, meta: &Metadata) -> bool { set.insert((get_devno(meta), meta.ino())) } #[cfg(not(unix))] -fn check_inode(_: &mut HashSet<u64>, _: &Metadata) -> bool { +fn check_inode(_: &mut FxHashSet<u64>, _: &Metadata) -> bool { true } @@ -309,13 +309,13 @@ impl DupPath { // Args::clap().gen_completions(env!("CARGO_PKG_NAME"), Shell::Fish, "target"); // } -type DupHashes = HashMap<(u64, Vec<u8>), DupPath>; +type DupHashes = FxHashMap<(u64, Vec<u8>), DupPath>; fn main() { match &ARGS.explain { None => (), Some(e) => { - explains(&e); + explains(e); process::exit(0); } } @@ -374,12 +374,12 @@ fn main() { None => None, Some(tz) => { let pn = check_tds_archive_name(tds_zip); - if !exists_file(&tz) { + if !exists_file(tz) { f0003!(&tz); process::exit(1); } let mut fmagic = filemagic::Filetype::new(); - match fmagic.analyze(&tz) { + match fmagic.analyze(tz) { Ok(filemagic::Mimetype::Zip) => (), _ => { f0004!(&tz); @@ -528,12 +528,12 @@ fn check_generated_files(entry: &str, generated: &mut GeneratedHashMap) { // a package we ignore if a README was generated by an // .ins or .dtx file // CAVEAT: If this happens in a subdirectory it could be an error!!!! - if is_readme(&filename) { + if is_readme(filename) { continue; } - // Ignore generated pdf files - if fname.ends_with(".pdf") { + // Ignore generated pdf, html, and css files + if fname.ends_with(".pdf") || fname.ends_with(".html") || fname.ends_with(".css") { continue; } @@ -568,7 +568,7 @@ fn check_tds_archive_name(tds_zip: &Option<String>) -> Option<String> { f0005!(tz); process::exit(1); } - let mut pname = String::from(utils::basename(&tz)); + let mut pname = String::from(utils::basename(tz)); let plen = pname.len(); pname.truncate(plen - 8); Some(pname) @@ -577,7 +577,7 @@ fn check_tds_archive_name(tds_zip: &Option<String>) -> Option<String> { } fn unzip_tds_archive(tds_zip: &str, tmp_dir: &str) { - match run_cmd("unzip", &["-q", "-d", tmp_dir, &tds_zip]) { + match run_cmd("unzip", &["-q", "-d", tmp_dir, tds_zip]) { CmdReturn { status: true, .. } => (), CmdReturn { status: false, @@ -597,16 +597,16 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { i0003!(tds_zip); let dir_entry = Path::new(tds_zip); - let p = get_perms(&dir_entry); + let p = get_perms(dir_entry); if !owner_has(p, 4) || !others_have(p, 4) || x_bit_set(p) { e0024!(tds_zip, perms_to_string(p)); if ARGS.correct_perms { i0005!(&tds_zip); - set_perms(&tds_zip, 0o664); + set_perms(tds_zip, 0o664); } }; - let ut = Utils::new(utils::CheckType::TDS); + let ut = Utils::new(utils::CheckType::Tds); // We have a discrepancy between // TDS zip archive name: babel-base.tds.zip @@ -616,17 +616,18 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { // Therefore, we correct here. // latex-tools, latex-graphics, latex-amsmath, latex-base => latex // latex-tools-dev, latex-graphics-dev, latex-amsmath-dev, latex-base-dev => latex-dev - let mut pkg_replacements: HashMap<&str, &str> = HashMap::default(); + let mut pkg_replacements: FxHashMap<&str, &str> = FxHashMap::default(); pkg_replacements.insert("babel-base", "babel"); pkg_replacements.insert("latex-base", "latex"); pkg_replacements.insert("latex-tools", "latex"); pkg_replacements.insert("latex-graphics", "latex"); pkg_replacements.insert("latex-amsmath", "latex"); + pkg_replacements.insert("latex-firstaid", "latex/firstaid"); pkg_replacements.insert("latex-base-dev", "latex-dev"); pkg_replacements.insert("latex-tools-dev", "latex-dev"); pkg_replacements.insert("latex-graphics-dev", "latex-dev"); pkg_replacements.insert("latex-amsmath-dev", "latex-dev"); - pkg_replacements.insert("latex-firstaid", "latex/firstaid"); + pkg_replacements.insert("latex-firstaid-dev", "latex-dev/firstaid"); let real_pkg_name = if let Some(real_name) = pkg_replacements.get(pkg_name) { real_name } else { @@ -648,7 +649,7 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { // tds zip archive we need to checksum the files in the tds zip // archive. - let mut sizes: SizesHashMap = HashMap::default(); + let mut sizes: SizesHashMap = FxHashMap::default(); let mut pool = Pool::new(num_cpus::get() as u32 + 1); { // Processing a single file entry, with the "sizes" hashmap collecting @@ -668,7 +669,7 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { // those top level directories are the directories found in the // texmf-dist/ directory of a texlive installation - let tds_toplevel_dirs: HashSet<String> = [ + let tds_toplevel_dirs: FxHashSet<String> = [ "asymptote", "bibtex", "chktex", @@ -742,7 +743,7 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { &dir_entry_str[tmp_dir_offset..] }; - ut.check_for_temporary_file(&dir_entry_display); + ut.check_for_temporary_file(dir_entry_display); // In the top level directory of a TDS zip archive // ... no files are allowed @@ -764,8 +765,8 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { } if ft == FType::Directory { - ut.check_for_empty_directory(&dir_entry_str, &dir_entry_display); - ut.check_for_hidden_directory(&file_name, &dir_entry_display); + ut.check_for_empty_directory(dir_entry_str, dir_entry_display); + ut.check_for_hidden_directory(&file_name, dir_entry_display); ut.is_unwanted_directory(&file_name, dir_entry_str); continue; @@ -777,12 +778,12 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { match (pkg_name, dir_entry_display) { ("latex-tools", "tex/latex/tools/.tex") => (), ("latex-tools-dev", "tex/latex-dev/tools/.tex") => (), - (_, _) => ut.check_for_hidden_file(&file_name, &dir_entry_display), + (_, _) => ut.check_for_hidden_file(&file_name, dir_entry_display), }; let fsize = meta.len(); process(fsize, &dir_entry); - ut.check_filesize(fsize, &dir_entry_display); + ut.check_filesize(fsize, dir_entry_display); // if we encounter a .dtx or .ins file we check // that it is in a subdirectory of either source/ or doc/ @@ -795,10 +796,10 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { } // if the path doesn't contain a man page... - if dir_entry_str.find("/man/").is_none() { + if !dir_entry_str.contains("/man/") { let pkg_name_s = format!("/{}/", real_pkg_name); // ...then we want to have the package name in the path - if dir_entry_str.find(&pkg_name_s).is_none() { + if !dir_entry_str.contains(&pkg_name_s) { e0028!(real_pkg_name, dir_entry_display); } } @@ -828,7 +829,7 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { } }; - let mut tds_hashes: HashMap<(u64, Vec<u8>), Vec<PathBuf>> = HashMap::default(); + let mut tds_hashes: FxHashMap<(u64, Vec<u8>), Vec<PathBuf>> = FxHashMap::default(); pool.scoped(|scope| { let (tx, rx) = channel(); @@ -909,7 +910,7 @@ fn check_and_correct_perms(dir_entry: &str, p: u32) { e0002!(dir_entry, perms_to_string(p)); if ARGS.correct_perms { i0005!(&dir_entry); - set_perms(&dir_entry, 0o664); + set_perms(dir_entry, 0o664); } }; } @@ -955,13 +956,13 @@ enum ReadmeKind { } fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> { - let mut lcnames: HashMap<PathBuf, Vec<(PathBuf, FileKind)>> = HashMap::default(); + let mut lcnames: FxHashMap<PathBuf, Vec<(PathBuf, FileKind)>> = FxHashMap::default(); - let mut doublenames: HashMap<PathBuf, Vec<PathBuf>> = HashMap::default(); + let mut doublenames: FxHashMap<PathBuf, Vec<PathBuf>> = FxHashMap::default(); - let mut inodes = HashSet::default(); + let mut inodes = FxHashSet::default(); - let ut = Utils::new(utils::CheckType::PACKAGE); + let ut = Utils::new(utils::CheckType::Package); i0002!(root); // This hash contains all package file names. @@ -972,7 +973,7 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> { // ReadmeKind: is it a certain README, file or symlink? // A special case of a README file is a file with has a different name but // was pointed to by a symlink. Example: README --> README.rst - let mut file_names: FileNamesHashMap = HashMap::default(); + let mut file_names: FileNamesHashMap = FxHashMap::default(); let mut readme_found = false; let root_absolute = PathBuf::from(root) @@ -1010,7 +1011,7 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> { // we check for weird stuff like socket files aso. let ft = get_filetype(&dir_entry); - if found_unwanted_filetype(&dir_entry_str, &ft) { + if found_unwanted_filetype(dir_entry_str, &ft) { continue; } @@ -1036,7 +1037,7 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> { } let lc_dir_entry_str = dir_entry_str.to_lowercase(); - if let Some(_dir_name) = filename(&dir_entry_str) { + if let Some(_dir_name) = filename(dir_entry_str) { let lcnref = &mut lcnames; lcnref .entry(PathBuf::from(lc_dir_entry_str)) @@ -1063,44 +1064,44 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> { } } - let p = get_perms(&dir_entry.path()); + let p = get_perms(dir_entry.path()); // 2. dealing with directories if ft == FType::Directory { let lc_dir_entry_str = dir_entry_str.to_lowercase(); - if let Some(_dir_name) = filename(&dir_entry_str) { + if let Some(_dir_name) = filename(dir_entry_str) { let lcnref = &mut lcnames; lcnref .entry(PathBuf::from(lc_dir_entry_str)) .or_insert_with(Vec::new) - .push((PathBuf::from(&dir_entry_str), FileKind::Directory)); + .push((PathBuf::from(dir_entry_str), FileKind::Directory)); } if !owner_has(p, 5) || !others_have(p, 5) { e0011!(&dir_entry_str, perms_to_string(p)); if ARGS.correct_perms { i0005!(&dir_entry_str); - set_perms(&dir_entry_str, 0o775); + set_perms(dir_entry_str, 0o775); } } - ut.check_for_empty_directory(&dir_entry_str, &dir_entry_str); - ut.check_for_hidden_directory(&file_name, &dir_entry_str); + ut.check_for_empty_directory(dir_entry_str, dir_entry_str); + ut.check_for_hidden_directory(&file_name, dir_entry_str); ut.is_unwanted_directory(&file_name, dir_entry_str); continue; } // 3. dealing with regular files - ut.check_for_hidden_file(&file_name, &dir_entry_str); - ut.check_for_temporary_file(&dir_entry_str); + ut.check_for_hidden_file(&file_name, dir_entry_str); + ut.check_for_temporary_file(dir_entry_str); // if is_temporary_file(&dir_entry_str) { // e0008!(&dir_entry_str); // } - if let Some(file_name) = filename(&dir_entry_str) { + if let Some(file_name) = filename(dir_entry_str) { let doubleref = &mut doublenames; doubleref @@ -1164,8 +1165,8 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> { let lc = LinkCheck::new(4, false); let mut detective = filemagic::Filetype::new(); - let mut sizes: SizesHashMap = HashMap::default(); - let mut generated: GeneratedHashMap = HashMap::default(); + let mut sizes: SizesHashMap = FxHashMap::default(); + let mut generated: GeneratedHashMap = FxHashMap::default(); // Processing a single file entry, with the "sizes" hashmap collecting // same-size files. Entries are either Found::One or Found::Multiple, @@ -1190,13 +1191,13 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> { let fsize = meta.len(); ut.check_filesize(fsize, dir_entry_str); - let p = get_perms(&path); + let p = get_perms(path); if !owner_has(p, 4) { e0002!(&dir_entry_str, perms_to_string(p)); - fix_perms(&dir_entry_str); + fix_perms(dir_entry_str); continue; } - let ftr = detective.analyze(&dir_entry_str); + let ftr = detective.analyze(dir_entry_str); //println!(">>> {:?}", ftr); // we ignore errors from filetype recognition if ftr.is_err() { @@ -1207,87 +1208,87 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> { // DEBUG !readme_symlinked.contains(&dir_entry_str) if ReadmeKind::No != *is_readme { - if !check_readme(&dir_entry_str, is_readme, &ft) { + if !check_readme(dir_entry_str, is_readme, &ft) { continue; } if ARGS.urlcheck { - lc.check_urls(&dir_entry_str); + lc.check_urls(dir_entry_str); } } match ft { filemagic::Mimetype::Text(_) => { - check_and_correct_perms(&dir_entry_str, p); - let fext = get_extension_from_filename(&dir_entry_str); + check_and_correct_perms(dir_entry_str, p); + let fext = get_extension_from_filename(dir_entry_str); if fext == Some("ins") || fext == Some("dtx") { - check_generated_files(&dir_entry_str, &mut generated); + check_generated_files(dir_entry_str, &mut generated); } match fext { // deal with Windows files Some("bat") | Some("cmd") | Some("nsh") | Some("reg") => match ft { - filemagic::Mimetype::Text(LineEnding::CRLF) => (), - filemagic::Mimetype::Text(LineEnding::CR) => { + filemagic::Mimetype::Text(LineEnding::Crlf) => (), + filemagic::Mimetype::Text(LineEnding::Cr) => { e0037!(&dir_entry_str); if ARGS.correct_le { - make_crlf(&dir_entry_str); + make_crlf(dir_entry_str); } } - filemagic::Mimetype::Text(LineEnding::MIXED(0, 0, 0)) => (), - filemagic::Mimetype::Text(LineEnding::MIXED(cr, lf, crlf)) => { + filemagic::Mimetype::Text(LineEnding::Mixed(0, 0, 0)) => (), + filemagic::Mimetype::Text(LineEnding::Mixed(cr, lf, crlf)) => { e0038!(&dir_entry_str, cr, lf, crlf); if ARGS.correct_le { - fix_inconsistent_le(&dir_entry_str); + fix_inconsistent_le(dir_entry_str); } } - filemagic::Mimetype::Text(LineEnding::LF) => { + filemagic::Mimetype::Text(LineEnding::Lf) => { w0008!(&dir_entry_str); } fmm => println!("Should not occur: {} has {:?}", dir_entry_str, fmm), }, Some(_) | None => { match ft { - filemagic::Mimetype::Text(LineEnding::CRLF) => { + filemagic::Mimetype::Text(LineEnding::Crlf) => { e0012!(&dir_entry_str); if ARGS.correct_le { - fix_inconsistent_le(&dir_entry_str); + fix_inconsistent_le(dir_entry_str); } } - filemagic::Mimetype::Text(LineEnding::CR) => { + filemagic::Mimetype::Text(LineEnding::Cr) => { e0037!(&dir_entry_str); if ARGS.correct_le { - fix_inconsistent_le(&dir_entry_str); + fix_inconsistent_le(dir_entry_str); } } - filemagic::Mimetype::Text(LineEnding::MIXED(0, 0, 0)) => (), - filemagic::Mimetype::Text(LineEnding::MIXED(cr, lf, crlf)) => { + filemagic::Mimetype::Text(LineEnding::Mixed(0, 0, 0)) => (), + filemagic::Mimetype::Text(LineEnding::Mixed(cr, lf, crlf)) => { //println!(">>>{}: {:?} {},{},{}", &dir_entry_str, ft, x, y, z); e0038!(&dir_entry_str, cr, lf, crlf); if ARGS.correct_le { - fix_inconsistent_le(&dir_entry_str); + fix_inconsistent_le(dir_entry_str); } } - filemagic::Mimetype::Text(LineEnding::LF) => (), + filemagic::Mimetype::Text(LineEnding::Lf) => (), fmm => println!("Should not occur: {} has {:?}", dir_entry_str, fmm), } } } } - filemagic::Mimetype::BOM(b) => { + filemagic::Mimetype::Bom(b) => { //println!("{}: {} with BOM detected", dir_entry_str, b.as_ref()); w0004!(&dir_entry_str, b.as_ref()); - check_and_correct_perms(&dir_entry_str, p); + check_and_correct_perms(dir_entry_str, p); } filemagic::Mimetype::Binary | filemagic::Mimetype::Script(_) => { if !owner_has(p, 4) || !others_have(p, 4) { e0002!(&dir_entry_str, perms_to_string(p)); }; - fix_perms(&dir_entry_str); + fix_perms(dir_entry_str); } filemagic::Mimetype::Pdf => { - check_and_correct_perms(&dir_entry_str, p); - let ret = is_pdf_ok(&dir_entry_str); + check_and_correct_perms(dir_entry_str, p); + let ret = is_pdf_ok(dir_entry_str); if !ret.status { e0017!(&dir_entry_str); if let Some(output) = ret.output { @@ -1302,11 +1303,11 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> { } else { w0001!(&dir_entry_str); } - check_and_correct_perms(&dir_entry_str, p); + check_and_correct_perms(dir_entry_str, p); } - filemagic::Mimetype::Data => check_and_correct_perms(&dir_entry_str, p), - filemagic::Mimetype::Zerofile => check_and_correct_perms(&dir_entry_str, p), + filemagic::Mimetype::Data => check_and_correct_perms(dir_entry_str, p), + filemagic::Mimetype::Zerofile => check_and_correct_perms(dir_entry_str, p), _ => continue, } @@ -1314,17 +1315,19 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> { e0002!(&dir_entry_str, perms_to_string(p)); if ARGS.correct_perms { i0005!(&dir_entry_str); - set_perms(&dir_entry_str, 0o664); + set_perms(dir_entry_str, 0o664); } } if !(ARGS.ignore_dupes && tds_zip.is_none()) { - process(fsize, &path); + process(fsize, path); } } print_casefolding(&lcnames); print_generated(&doublenames, &generated); - print_doublenames(&doublenames); + if ! ARGS.ignore_same_named { + print_doublenames(&doublenames); + } if ARGS.ignore_dupes && tds_zip.is_none() { return None; @@ -1335,7 +1338,7 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> { // doing mostly IO. let mut pool = Pool::new(num_cpus::get() as u32 + 1); - let mut hashes: HashMap<(u64, Vec<u8>), DupPath> = HashMap::default(); + let mut hashes: FxHashMap<(u64, Vec<u8>), DupPath> = FxHashMap::default(); pool.scoped(|scope| { let (tx, rx) = channel(); @@ -1368,7 +1371,7 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> { Some(hashes) } -fn print_casefolding(hashes: &HashMap<PathBuf, Vec<(PathBuf, FileKind)>>) { +fn print_casefolding(hashes: &FxHashMap<PathBuf, Vec<(PathBuf, FileKind)>>) { for (k, eles) in hashes.iter() { //println!("pcf: {:?}, {:?}", k, &eles); if eles.len() == 1 { @@ -1383,7 +1386,7 @@ fn print_casefolding(hashes: &HashMap<PathBuf, Vec<(PathBuf, FileKind)>>) { } } -fn print_generated(doublenames: &HashMap<PathBuf, Vec<PathBuf>>, generated: &GeneratedHashMap) { +fn print_generated(doublenames: &FxHashMap<PathBuf, Vec<PathBuf>>, generated: &GeneratedHashMap) { // `k` is generated by `gen` for (k, gen) in generated.iter() { let path = PathBuf::from(k); @@ -1400,7 +1403,7 @@ fn print_generated(doublenames: &HashMap<PathBuf, Vec<PathBuf>>, generated: &Gen } } -fn print_doublenames(hashes: &HashMap<PathBuf, Vec<PathBuf>>) { +fn print_doublenames(hashes: &FxHashMap<PathBuf, Vec<PathBuf>>) { for (k, paths) in hashes.iter() { if paths.len() == 1 { continue; diff --git a/support/pkgcheck/src/messages/errorsd.rs b/support/pkgcheck/src/messages/errorsd.rs index c034301ad9..6ef9eed62e 100644 --- a/support/pkgcheck/src/messages/errorsd.rs +++ b/support/pkgcheck/src/messages/errorsd.rs @@ -237,7 +237,7 @@ package which easily can be generated from other files in the submission. Exceptions are the README files of the package, i.e. README, README.md -or README.txt, or .pdf files. +or README.txt, .pdf, .html, or .css files. pkgcheck detects generated files anywhere in the package directory tree. diff --git a/support/pkgcheck/src/messages/mod.rs b/support/pkgcheck/src/messages/mod.rs index 65dcbc502a..e3ec350143 100644 --- a/support/pkgcheck/src/messages/mod.rs +++ b/support/pkgcheck/src/messages/mod.rs @@ -19,15 +19,15 @@ macro_rules! error_occured { $crate::ERROR_OCCURED.store(true, Ordering::Relaxed); }; } -macro_rules! yellow { - ($fmt:expr) => { - if no_colors!() { - $fmt.clear() - } else { - $fmt.bright_yellow() - } - }; -} +// macro_rules! yellow { +// ($fmt:expr) => { +// if no_colors!() { +// $fmt.clear() +// } else { +// $fmt.bright_yellow() +// } +// }; +// } // macro_rules! red { // ($fmt:expr) => { @@ -44,7 +44,7 @@ macro_rules! msgid { match (no_colors!(), &$fmt[..1]) { (true, _) => $fmt.clear(), (false, "E") => $fmt.bright_red().bold(), - (false, "I") => $fmt.bright_yellow(), + (false, "I") => $fmt.bright_yellow().bold(), (false, "W") => $fmt.bright_red(), (false, "F") => $fmt.bright_red().bold(), (_, _) => $fmt.clear(), @@ -639,8 +639,8 @@ macro_rules! i0002 { print!( "{} {} {}\n", msgid!("I0002"), - yellow!("Checking package files in directory"), - yellow!($fmt) + "Checking package files in directory", + $fmt ); }; } @@ -650,8 +650,8 @@ macro_rules! i0003 { print!( "{} {} {}\n", msgid!("I0003"), - yellow!("Checking TDS zip archive"), - yellow!($fmt) + "Checking TDS zip archive", + $fmt ); }; } diff --git a/support/pkgcheck/src/utils.rs b/support/pkgcheck/src/utils.rs index 2dedc94ecd..a8f2d56c0f 100644 --- a/support/pkgcheck/src/utils.rs +++ b/support/pkgcheck/src/utils.rs @@ -17,8 +17,8 @@ use std::fs; #[derive(Debug, Clone, PartialEq, Eq)] pub enum CheckType { - TDS, - PACKAGE, + Tds, + Package, } pub struct Utils { @@ -38,7 +38,7 @@ impl Utils { return false; } if fsize > 40 * 1024 * 1024 { - if self.kind == CheckType::PACKAGE { + if self.kind == CheckType::Package { w0005!(dir_entry_str, fsize / 1024 / 1024); } else { w0006!(dir_entry_str, fsize / 1024 / 1024); @@ -102,10 +102,10 @@ impl Utils { pub fn check_for_hidden_file(&self, entry: &str, dir_entry_str: &str) { if entry.starts_with('.') { match self.kind { - CheckType::PACKAGE => { + CheckType::Package => { e0007!(dir_entry_str); } - CheckType::TDS => { + CheckType::Tds => { e0007t!(dir_entry_str); } } @@ -123,10 +123,10 @@ impl Utils { //if fname.map(|s| s.starts_with('.')).unwrap_or(false) { if fname.starts_with('.') { match self.kind { - CheckType::PACKAGE => { + CheckType::Package => { e0006!(dir_entry_str); } - CheckType::TDS => { + CheckType::Tds => { e0006t!(dir_entry_str); } } @@ -140,10 +140,10 @@ impl Utils { if RE.is_match(dir_entry_str) { match self.kind { - CheckType::PACKAGE => { + CheckType::Package => { e0008!(dir_entry_str); } - CheckType::TDS => { + CheckType::Tds => { e0008t!(dir_entry_str); } } @@ -156,10 +156,10 @@ impl Utils { Ok(contents) => { if contents.count() == 0 { match self.kind { - CheckType::PACKAGE => { + CheckType::Package => { e0004!(dir_entry_display); } - CheckType::TDS => { + CheckType::Tds => { // Note: Karl Berry recommended to issue a warning only // as an empty directory in a TDS zip archive // - is automatically deleted before including @@ -176,10 +176,10 @@ impl Utils { } } -pub fn temp_file_endings() -> Vec<(&'static str, &'static str)> { +pub fn temp_file_endings() -> Vec<(String, String)> { // https://github.com/github/gitignore/blob/master/TeX.gitignore // http://hopf.math.purdue.edu/doc/html/suffixes.html - vec![ + let v = vec![ ("-blx.aux", "bibliography auxiliary file"), ("-blx.bib", "bibliography auxiliary file"), (".4ct", "htlatex related"), @@ -212,6 +212,7 @@ pub fn temp_file_endings() -> Vec<(&'static str, &'static str)> { (".gls", "glossary related"), (".glsdefs", "glossaries related"), (".gtex", "generated by gregoriotex"), + (".hd", ""), (".idv", "htlatex related"), (".idx", "makeidx related"), (".ilg", "makeidx related"), @@ -250,6 +251,7 @@ pub fn temp_file_endings() -> Vec<(&'static str, &'static str)> { (".synctex(busy)", "synctex related"), (".synctex.gz", "synctex related"), (".synctex.gz(busy)", "synctex related"), + (".tpt", ""), (".tdo", "generated by todonotes (list of todos)"), (".thm", "amsthm related"), (".tmb", "generated by thumbs package"), @@ -267,7 +269,9 @@ pub fn temp_file_endings() -> Vec<(&'static str, &'static str)> { (".xref", "htlatex related"), ("~", "a file name ending with ~ (tilde) is temporary anyway"), // ( ".lyx~", "LyX related backup file" ), - ] + ]; + + v.into_iter().map(|(i, j)| (i.to_string(), j.to_string())).collect() } pub fn regex_temporary_file_endings() -> Regex { @@ -280,7 +284,7 @@ pub fn regex_temporary_file_endings() -> Regex { } else { rv.push('|'); } - let px = str::replace(p, ".", "\\."); + let px = str::replace(&p, ".", "\\."); rv.push_str(&px); } @@ -452,12 +456,13 @@ fn test_dirname() { #[allow(dead_code)] pub fn dirname(entry: &str) -> Option<&str> { - if entry.ends_with('/') { + if let Some(stripped) = entry.strip_suffix('/') { if entry.len() == 1 { return Some(entry); } - return Some(&entry[..entry.len() - 1]); + return Some(stripped) } + let pos = entry.rfind('/'); match pos { None => None, |