diff options
author | Norbert Preining <norbert@preining.info> | 2020-11-30 03:00:37 +0000 |
---|---|---|
committer | Norbert Preining <norbert@preining.info> | 2020-11-30 03:00:37 +0000 |
commit | cb68d38823870fa7adb5f0c2b68cf59205dd4bec (patch) | |
tree | 8d6e29ee5d082dcbf1ad936b7025db071e9cf573 /support/pkgcheck/src/main.rs | |
parent | 0cc87411bcebcb1b0d2a3488d7a7093024b58650 (diff) |
CTAN sync 202011300300
Diffstat (limited to 'support/pkgcheck/src/main.rs')
-rw-r--r-- | support/pkgcheck/src/main.rs | 207 |
1 files changed, 142 insertions, 65 deletions
diff --git a/support/pkgcheck/src/main.rs b/support/pkgcheck/src/main.rs index cbb678729f..d4ef7d6db2 100644 --- a/support/pkgcheck/src/main.rs +++ b/support/pkgcheck/src/main.rs @@ -11,10 +11,10 @@ mod linkcheck; mod utils; mod recode; -use recode::{wrong_line_endings2lf, wrong_line_endings2crlf}; +use recode::{wrong_line_endings2crlf, wrong_line_endings2lf}; mod filemagic; -use filemagic::{LineEnding}; +use filemagic::LineEnding; use linkcheck::LinkCheck; use std::fmt; @@ -27,7 +27,8 @@ use std::os::unix::fs::MetadataExt; use scoped_threadpool::Pool; -use tempdir::TempDir; +use tempfile::Builder; + #[macro_use] extern crate lazy_static; use colored::*; @@ -51,6 +52,8 @@ use std::sync::atomic::{AtomicBool, Ordering}; use blake2::{Blake2b, Digest}; use fnv::{FnvHashMap as HashMap, FnvHashSet as HashSet}; use std::sync::mpsc::{channel, Sender}; + +use structopt::clap::Shell; use structopt::StructOpt; #[cfg(unix)] use walkdir::{DirEntry, WalkDir}; @@ -77,15 +80,13 @@ fn hash_file_inner(path: &PathBuf) -> io::Result<Vec<u8>> { let mut buf = [0u8; BLOCKSIZE]; let mut fp = File::open(&path)?; let mut digest = Blake2b::default(); - // When we compare byte-by-byte, we don't need to hash the whole file. - // Instead, hash a block of 4kB, skipping 100kB. loop { match fp.read(&mut buf)? { 0 => break, - n => digest.input(&buf[..n]), + n => digest.update(&buf[..n]), } } - Ok(digest.result().to_vec()) + Ok(digest.finalize().to_vec()) } fn hash_file(fsize: u64, path: PathBuf, tx: &HashSender) { @@ -95,7 +96,6 @@ fn hash_file(fsize: u64, path: PathBuf, tx: &HashSender) { } } - // returns false if an error occured fn fix_inconsistent_le(fname: &str) -> bool { i0004!(fname); @@ -111,7 +111,6 @@ fn fix_inconsistent_le(fname: &str) -> bool { } } - // returns false if an error occured fn make_crlf(fname: &str) -> bool { i0004!(fname); @@ -209,6 +208,8 @@ struct Args { correct_perms: bool, #[structopt(long = "no-colors", help = "Don't display messages in color")] no_colors: bool, + // #[structopt(long = "install-completion", help = "Install completion for the current shell")] + // install_completion: Option<String>, #[structopt(long = "urlcheck", help = "Check URLs found in README files")] urlcheck: bool, #[structopt(short = "T", long = "tds-zip", help = "tds zip archive", group = "tds")] @@ -296,6 +297,18 @@ impl DupPath { } } +// fn install_completion_bash() { +// Args::clap().gen_completions(env!("CARGO_PKG_NAME"), Shell::Bash, "bla"); +// } + +// fn install_completion_zsh() { +// Args::clap().gen_completions(env!("CARGO_PKG_NAME"), Shell::Zsh, "/home/manfred/.zfunc"); +// } + +// fn install_completion_fish() { +// Args::clap().gen_completions(env!("CARGO_PKG_NAME"), Shell::Fish, "target"); +// } + type DupHashes = HashMap<(u64, Vec<u8>), DupPath>; fn main() { @@ -307,6 +320,24 @@ fn main() { } } + // + // !!!!!! this should be activated when structop uses + // !!!!!! the new clap 3.0 + // + // match &ARGS.install_completion { + // None => { } + // Some(d) => { + // println!("{}", &d); + // match d.as_str() { + // "bash" => install_completion_bash(), + // "zsh" => install_completion_zsh(), + // "fish" => install_completion_fish(), + // _ => println!("Invalid shell name {} given", d) + // } + // process::exit(0); + // } + // } + if ARGS.explain_all { explains_all(); process::exit(0); @@ -506,7 +537,9 @@ fn check_generated_files(entry: &str, generated: &mut GeneratedHashMap) { continue; } - generated.entry(filename.to_string()).or_insert_with(|| entry.to_string()); + generated + .entry(filename.to_string()) + .or_insert_with(|| entry.to_string()); } }; } @@ -544,7 +577,7 @@ fn check_tds_archive_name(tds_zip: &Option<String>) -> Option<String> { } fn unzip_tds_archive(tds_zip: &str, tmp_dir: &str) { - match run_cmd("unzip", &["-a", "-q", "-d", tmp_dir, &tds_zip]) { + match run_cmd("unzip", &["-q", "-d", tmp_dir, &tds_zip]) { CmdReturn { status: true, .. } => (), CmdReturn { status: false, @@ -593,13 +626,14 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { pkg_replacements.insert("latex-tools-dev", "latex-dev"); pkg_replacements.insert("latex-graphics-dev", "latex-dev"); pkg_replacements.insert("latex-amsmath-dev", "latex-dev"); + pkg_replacements.insert("latex-firstaid", "latex/firstaid"); let real_pkg_name = if let Some(real_name) = pkg_replacements.get(pkg_name) { real_name } else { pkg_name }; - let tmp_dir = match TempDir::new("pkgcheck") { + let tmp_dir = match Builder::new().prefix("pkgcheck").tempdir() { Ok(tdir) => tdir, Err(e) => { f0007!(e); @@ -632,6 +666,45 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { let mut map_files_found = false; let mut map_dvips_found = false; + // those top level directories are the directories found in the + // texmf-dist/ directory of a texlive installation + let tds_toplevel_dirs: HashSet<String> = [ + "asymptote", + "bibtex", + "chktex", + "context", + "doc", + "dvipdfmx", + "dvips", + "fonts", + "hbf2gf", + "makeindex", + "metafont", + "metapost", + "mft", + "omega", + "pbibtex", + "psutils", + "scripts", + "source", + "tex", + "tex4ht", + "texconfig", + "texdoc", + "texdoctk", + "ttf2pk", + "web2c", + "xdvi", + "xindy", + ] + .iter() + .map(|&s| s.to_string()) + .collect(); + + // set to True if the TDS zip archive contains a top level directory doc/ + let mut doc_found = false; + // we track the number of toplevel directories which must at least be 2 + let mut number_of_toplevel_dirs = 0; for dir_entry in WalkDir::new(tmp_dir.path().to_str().unwrap()).follow_links(false) { match dir_entry { Ok(dir_entry) => { @@ -669,29 +742,42 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { &dir_entry_str[tmp_dir_offset..] }; + ut.check_for_temporary_file(&dir_entry_display); + + // In the top level directory of a TDS zip archive + // ... no files are allowed + // ... only specific directories are allowed + if dir_entry.depth() == 1 { + if ft == FType::Regular { + e0034!(dir_entry_display); + continue; + } + if !tds_toplevel_dirs.contains(&file_name) { + e0020!(&file_name); + } else { + number_of_toplevel_dirs += 1; + if &file_name == "doc" { + doc_found = true; + } + } + continue; + } + if ft == FType::Directory { ut.check_for_empty_directory(&dir_entry_str, &dir_entry_display); ut.check_for_hidden_directory(&file_name, &dir_entry_display); ut.is_unwanted_directory(&file_name, dir_entry_str); - continue; - } - // We don't want to have other than directories in the top level - // directory of a TDS zip archive - if dir_entry.depth() == 1 && ft == FType::Regular { - e0034!(dir_entry_display); continue; } - ut.check_for_temporary_file(&dir_entry_display); - // The LaTeX team provides the file `.tex` as a file with an empty name // in order to make `\input\relax` work (explained by David Carlisle) // Therefore, we don't call check_for_hidden_file() in this case match (pkg_name, dir_entry_display) { ("latex-tools", "tex/latex/tools/.tex") => (), ("latex-tools-dev", "tex/latex-dev/tools/.tex") => (), - (_, _) => ut.check_for_hidden_file(&file_name, &dir_entry_display) + (_, _) => ut.check_for_hidden_file(&file_name, &dir_entry_display), }; let fsize = meta.len(); @@ -731,8 +817,14 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { } } + if !doc_found { + e0039!(); + } + if number_of_toplevel_dirs < 2 { + e0040!(); + } if map_files_found && !map_dvips_found { - println!( "Map files found for package {} but none of them is in a path starting with fonts/map/dvips", pkg_name); + e0041!(); } }; @@ -871,8 +963,6 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> { let ut = Utils::new(utils::CheckType::PACKAGE); - - i0002!(root); // This hash contains all package file names. // @@ -977,7 +1067,6 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> { // 2. dealing with directories if ft == FType::Directory { - let lc_dir_entry_str = dir_entry_str.to_lowercase(); if let Some(_dir_name) = filename(&dir_entry_str) { let lcnref = &mut lcnames; @@ -1090,7 +1179,6 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> { sizeref.entry(fsize).or_insert_with(Vec::new).push(path); }; for (path, (meta, _file_name, is_readme)) in file_names.iter() { - //println!("DEBUG: path {:?}, file_name {:?}, is_readme: {} ", &path, &file_name, is_readme); let dir_entry_str = match path.to_str() { Some(d) => d, None => { @@ -1137,70 +1225,61 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> { match fext { // deal with Windows files - Some("bat") | Some("cmd") | Some("nsh") | Some("reg") => { - match ft { - filemagic::Mimetype::Text(LineEnding::CRLF) => (), - filemagic::Mimetype::Text(LineEnding::CR) => - { - e0037!(&dir_entry_str); - if ARGS.correct_le { - make_crlf(&dir_entry_str); - } - }, - filemagic::Mimetype::Text(LineEnding::MIXED(0, 0, 0) ) => (), - filemagic::Mimetype::Text(LineEnding::MIXED(cr, lf, crlf) ) => - { - e0038!(&dir_entry_str, cr, lf, crlf); - if ARGS.correct_le { - fix_inconsistent_le(&dir_entry_str); - } - }, - filemagic::Mimetype::Text(LineEnding::LF) => - { - w0008!(&dir_entry_str); - }, - fmm => println!("Should not occur: {} has {:?}", dir_entry_str, fmm), + Some("bat") | Some("cmd") | Some("nsh") | Some("reg") => match ft { + filemagic::Mimetype::Text(LineEnding::CRLF) => (), + filemagic::Mimetype::Text(LineEnding::CR) => { + e0037!(&dir_entry_str); + if ARGS.correct_le { + make_crlf(&dir_entry_str); + } + } + filemagic::Mimetype::Text(LineEnding::MIXED(0, 0, 0)) => (), + filemagic::Mimetype::Text(LineEnding::MIXED(cr, lf, crlf)) => { + e0038!(&dir_entry_str, cr, lf, crlf); + if ARGS.correct_le { + fix_inconsistent_le(&dir_entry_str); + } } + filemagic::Mimetype::Text(LineEnding::LF) => { + w0008!(&dir_entry_str); + } + fmm => println!("Should not occur: {} has {:?}", dir_entry_str, fmm), }, Some(_) | None => { match ft { - filemagic::Mimetype::Text(LineEnding::CRLF) => - { + filemagic::Mimetype::Text(LineEnding::CRLF) => { e0012!(&dir_entry_str); if ARGS.correct_le { fix_inconsistent_le(&dir_entry_str); } - }, - filemagic::Mimetype::Text(LineEnding::CR) => - { + } + filemagic::Mimetype::Text(LineEnding::CR) => { e0037!(&dir_entry_str); if ARGS.correct_le { fix_inconsistent_le(&dir_entry_str); } - }, - filemagic::Mimetype::Text(LineEnding::MIXED(0, 0, 0) ) => (), - filemagic::Mimetype::Text(LineEnding::MIXED(cr, lf, crlf) ) => - { + } + filemagic::Mimetype::Text(LineEnding::MIXED(0, 0, 0)) => (), + filemagic::Mimetype::Text(LineEnding::MIXED(cr, lf, crlf)) => { //println!(">>>{}: {:?} {},{},{}", &dir_entry_str, ft, x, y, z); e0038!(&dir_entry_str, cr, lf, crlf); if ARGS.correct_le { fix_inconsistent_le(&dir_entry_str); } - }, + } filemagic::Mimetype::Text(LineEnding::LF) => (), fmm => println!("Should not occur: {} has {:?}", dir_entry_str, fmm), } - }, + } } - }, + } filemagic::Mimetype::BOM(b) => { //println!("{}: {} with BOM detected", dir_entry_str, b.as_ref()); w0004!(&dir_entry_str, b.as_ref()); check_and_correct_perms(&dir_entry_str, p); } - filemagic::Mimetype::Binary - | filemagic::Mimetype::Script(_) => { + filemagic::Mimetype::Binary | filemagic::Mimetype::Script(_) => { if !owner_has(p, 4) || !others_have(p, 4) { e0002!(&dir_entry_str, perms_to_string(p)); }; @@ -1353,5 +1432,3 @@ fn show_tmp_endings() { println!("{:23} {}", t, c); } } - - |