diff options
author | Norbert Preining <norbert@preining.info> | 2022-05-27 03:01:31 +0000 |
---|---|---|
committer | Norbert Preining <norbert@preining.info> | 2022-05-27 03:01:31 +0000 |
commit | 84a5593d3fb9d03aac5677678bcc4c92e8c9a9c4 (patch) | |
tree | 0212ba38147bcfe80e5c1a853071a6e6a45b87d7 /support/pkgcheck/src/main.rs | |
parent | 02d941fa9c9895bb08a84ac9afe3559abd1ba8ad (diff) |
CTAN sync 202205270301
Diffstat (limited to 'support/pkgcheck/src/main.rs')
-rw-r--r-- | support/pkgcheck/src/main.rs | 317 |
1 files changed, 211 insertions, 106 deletions
diff --git a/support/pkgcheck/src/main.rs b/support/pkgcheck/src/main.rs index b18fd8793b..f0766cdeba 100644 --- a/support/pkgcheck/src/main.rs +++ b/support/pkgcheck/src/main.rs @@ -1,6 +1,8 @@ #[macro_use] extern crate pest_derive; +use log::*; + mod gparser; #[macro_use] @@ -14,6 +16,7 @@ mod recode; use recode::{wrong_line_endings2crlf, wrong_line_endings2lf}; mod filemagic; + use filemagic::LineEnding; use linkcheck::LinkCheck; @@ -23,14 +26,15 @@ use std::fmt::Display; use std::str; use utils::*; +use serde::{Deserialize, Serialize}; use std::os::unix::fs::MetadataExt; - +use std::borrow::Cow; use scoped_threadpool::Pool; use tempfile::Builder; -#[macro_use] -extern crate lazy_static; +use once_cell::sync::Lazy; // 1.3.1 + use colored::*; use regex::Regex; @@ -51,13 +55,140 @@ use std::sync::atomic::{AtomicBool, Ordering}; use rustc_hash::{FxHashMap, FxHashSet}; +use std::fmt::Arguments; use std::sync::mpsc::{channel, Sender}; -use structopt::clap::Shell; -use structopt::StructOpt; +use clap::{Command, CommandFactory, Parser, ValueHint}; +use clap_complete::{generate, Generator, Shell}; + #[cfg(unix)] use walkdir::{DirEntry, WalkDir}; +fn format_message(message: &Arguments, no_color: bool) -> Cow<'static, str> { + let msg_str = format!("{}", message); + if msg_str.starts_with(' ') { + return msg_str.into(); + } + let (left, right) = msg_str.split_once(' ').unwrap(); + if no_color { + msg_str.into() + } else { + let colored_msg = match &left.chars().next().unwrap() { + 'E' | 'F' => format!("{} {}", left.bright_red().bold(), right), + 'W' => format!("{} {}", left.bright_red(), right), + 'I' => format!("{} {}", left.bright_yellow().bold(), right), + _ => msg_str, + }; + colored_msg.into() + } +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TdsException { + pub pkg: String, + pub tpkg: String, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PathExceptions { + pub tds_path_exceptions: Vec<TdsException>, +} + +fn get_config_file_name() -> Option<String> { + if let Some(config_file) = &ARGS.config_file { + if Path::new(&config_file).exists() { + return Some(config_file.to_string()) + } else { + f0008!(config_file); + std::process::exit(1); + } + } + let home_dir = match home::home_dir() { + Some(path) => path.display().to_string(), + None => panic!("Impossible to get your home dir!"), + }; + let config_files = [".ctan/pkgcheck.yml", ".config/ctan/pkgcheck.yml"]; + for f in config_files { + let config_file_abs_path = format!("{}/{}", home_dir, f); + if Path::new(&config_file_abs_path).exists() { + return Some(config_file_abs_path); + } + } + None +} + +fn read_yaml_config() -> FxHashMap<String, String> { + let mut pkg_replacements: FxHashMap<String, String> = FxHashMap::default(); + for (p, q) in [ + ("babel-base", "babel"), + ("latex-base", "latex"), + ("latex-lab", "latex"), + ("latex-tools", "latex"), + ("latex-graphics", "latex"), + ("latex-amsmath", "latex"), + ("latex-firstaid", "latex/firstaid"), + ("latex-base-dev", "latex-dev"), + ("latex-lab-dev", "latex-dev"), + ("latex-tools-dev", "latex-dev"), + ("latex-graphics-dev", "latex-dev"), + ("latex-amsmath-dev", "latex-dev"), + ("latex-firstaid-dev", "latex-dev/firstaid"), + ("vntex-nonfree", "vntex"), + ] { + pkg_replacements.insert(p.to_string(), q.to_string()); + } + + match get_config_file_name() { + Some(config_filename) => { + i0008!(config_filename); + + let data = match fs::read_to_string(&config_filename) { + Ok(str) => str, + Err(e) => { f0009!(&config_filename, e); std::process::exit(1); } + }; + + let path_exceptions = serde_yaml::from_str::<PathExceptions>(&data); + + match path_exceptions { + Ok(pb) => { + for play in &pb.tds_path_exceptions { + // check if package name is already in pkg_replacements hash + let old_val = pkg_replacements.get(&play.pkg); + if let Some(ov) = old_val { + if ARGS.verbose { + if ov == &play.tpkg { + w0009!(play.pkg, play.tpkg); + } else { + i0009!(play.pkg, ov, play.tpkg); + } + } + } + pkg_replacements.insert(play.pkg.clone(), play.tpkg.clone()); + } + pb + } + Err(e) => { f0010!(e); std::process::exit(1);}, + }; + pkg_replacements + } + None => pkg_replacements, + } +} + +fn setup_logger(no_color: bool) -> Result<(), fern::InitError> { + fern::Dispatch::new() + .format(move |out, message, _record| { + let msg_txt = format_message(message, no_color); + out.finish(format_args!("{}", msg_txt)) + }) + .level(log::LevelFilter::Info) + .chain(std::io::stdout()) + .apply()?; + Ok(()) +} + fn err(path: &Path, err: &io::Error) { e0027!(path.display(), err); } @@ -83,7 +214,9 @@ fn hash_file_inner(path: &Path) -> io::Result<Vec<u8>> { loop { match fp.read(&mut buf)? { 0 => break, - n => { digest.update(&buf[..n]); }, + n => { + digest.update(&buf[..n]); + } } } Ok(digest.finalize().as_bytes().to_vec()) @@ -96,7 +229,7 @@ fn hash_file(fsize: u64, path: PathBuf, tx: &HashSender) { } } -// returns false if an error occured +// returns false if an error occurred fn fix_inconsistent_le(fname: &str) -> bool { i0004!(fname); match wrong_line_endings2lf(fname) { @@ -111,7 +244,7 @@ fn fix_inconsistent_le(fname: &str) -> bool { } } -// returns false if an error occured +// returns false if an error occurred fn make_crlf(fname: &str) -> bool { i0004!(fname); match wrong_line_endings2crlf(fname) { @@ -192,48 +325,50 @@ fn _get_devno(entry: &DirEntry) -> u64 { } } -#[derive(StructOpt, Debug)] -#[structopt(about = "A checker for uploaded packages to CTAN.")] +#[derive(Parser, Debug, PartialEq)] +#[clap(author, version, about, long_about = None)] struct Args { - #[structopt(short = "I", long = "ignore-dupes", help = "Ignore dupes")] + #[clap(short = 'I', long = "ignore-dupes", help = "Ignore dupes")] ignore_dupes: bool, - #[structopt(long = "ignore-same-named", help = "Ignore same-named files")] + #[clap(long = "ignore-same-named", help = "Ignore same-named files")] ignore_same_named: bool, - #[structopt(short = "v", long = "verbose", help = "Verbose operation?")] + #[clap(short = 'v', long = "verbose", help = "Verbose operation?")] verbose: bool, - #[structopt(short = "L", long = "correct-le", help = "Correct line endings")] + #[clap(short = 'L', long = "correct-le", help = "Correct line endings")] correct_le: bool, - #[structopt(short = "C", long = "correct-perms", help = "Correct permissions")] + #[clap(short = 'C', long = "correct-perms", help = "Correct permissions")] correct_perms: bool, - #[structopt(long = "no-colors", help = "Don't display messages in color")] + #[clap(long = "no-colors", help = "Don't display messages in color")] no_colors: bool, - // #[structopt(long = "install-completion", help = "Install completion for the current shell")] - // install_completion: Option<String>, - #[structopt(long = "urlcheck", help = "Check URLs found in README files")] + #[clap(long = "urlcheck", help = "Check URLs found in README files")] urlcheck: bool, - #[structopt(short = "T", long = "tds-zip", help = "tds zip archive", group = "tds")] + #[clap(short = 'T', long = "tds-zip", help = "tds zip archive", group = "tds", value_hint = ValueHint::FilePath)] tds_zip: Option<String>, - #[structopt( - short = "e", + #[clap( + short = 'e', long = "explain", help = "Explain error or warning message", group = "only_one" )] explain: Option<String>, - #[structopt( + #[clap( long = "explain-all", help = "Explains all error or warning messages", group = "only_one" )] explain_all: bool, - #[structopt( + #[clap(long = "generate-completion", group = "only_one", arg_enum)] + generator: Option<Shell>, + #[clap( long = "show-temp-endings", help = "Show file endings for temporary files", group = "only_one" )] show_tmp_endings: bool, - #[structopt(short = "d", long = "package-dir", help = "Package directory")] + #[clap(short = 'd', long = "package-dir", help = "Package directory", value_hint = ValueHint::DirPath)] pkg_dir: Option<String>, + #[clap(long = "config-file", help = "Specify config file to use", value_hint = ValueHint::FilePath)] + config_file: Option<String>, } // We take care to avoid visiting a single inode twice, @@ -248,10 +383,8 @@ fn check_inode(_: &mut FxHashSet<u64>, _: &Metadata) -> bool { true } -lazy_static! { - static ref ARGS: Args = Args::from_args(); - static ref ERROR_OCCURED: AtomicBool = AtomicBool::new(false); -} +static ARGS: Lazy<Args> = Lazy::new(Args::parse); +static ERROR_OCCURRED: AtomicBool = AtomicBool::new(false); #[derive(Debug, Clone, PartialEq, Eq)] pub enum DPath { @@ -296,21 +429,18 @@ impl DupPath { } } -// fn install_completion_bash() { -// Args::clap().gen_completions(env!("CARGO_PKG_NAME"), Shell::Bash, "bla"); -// } +type DupHashes = FxHashMap<(u64, Vec<u8>), DupPath>; -// fn install_completion_zsh() { -// Args::clap().gen_completions(env!("CARGO_PKG_NAME"), Shell::Zsh, "/home/manfred/.zfunc"); -// } +fn print_completions<G: Generator>(gen: G, cmd: &mut Command) { + generate(gen, cmd, cmd.get_name().to_string(), &mut io::stdout()); +} -// fn install_completion_fish() { -// Args::clap().gen_completions(env!("CARGO_PKG_NAME"), Shell::Fish, "target"); -// } +fn main() { + let _ = setup_logger(ARGS.no_colors); -type DupHashes = FxHashMap<(u64, Vec<u8>), DupPath>; + // read yaml config file if one is given explicitly or implicitly + let pkg_replace : FxHashMap<String, String> = read_yaml_config(); -fn main() { match &ARGS.explain { None => (), Some(e) => { @@ -319,23 +449,12 @@ fn main() { } } - // - // !!!!!! this should be activated when structop uses - // !!!!!! the new clap 3.0 - // - // match &ARGS.install_completion { - // None => { } - // Some(d) => { - // println!("{}", &d); - // match d.as_str() { - // "bash" => install_completion_bash(), - // "zsh" => install_completion_zsh(), - // "fish" => install_completion_fish(), - // _ => println!("Invalid shell name {} given", d) - // } - // process::exit(0); - // } - // } + if let Some(generator) = ARGS.generator { + let mut cmd = Args::command(); + eprintln!("Generating completion file for {:?}...", generator); + print_completions(generator, &mut cmd); + process::exit(0) + } if ARGS.explain_all { explains_all(); @@ -393,13 +512,13 @@ fn main() { if tds_zip.is_some() { if let Some(pn) = pkg_name { if let Some(s) = ARGS.tds_zip.as_ref() { - check_tds_archive(&pn, s, &hashes); + check_tds_archive(&pn, s, &hashes, &pkg_replace); } } } } - if ERROR_OCCURED.load(Ordering::Relaxed) { + if ERROR_OCCURRED.load(Ordering::Relaxed) { process::exit(1); } else { process::exit(0); @@ -428,20 +547,20 @@ fn print_duplicates(hashes: &DupHashes) { total_size += sz * (paths.plen - 1) as u64; total_dupes += (paths.plen - 1) as u64; - println!("Size: {}", sz); + info!("Size: {}", sz); for p in &paths.dupes { if let DPath::Both(p) = p { let ps = p.as_path().to_str().unwrap(); - println!(" >>> {}", ps); + info!(" >>> {}", ps); } } - println!(); + //eprintln!(); } if ARGS.verbose && total_dupes > 0 { - println!("Duplicate statistics"); - println!(" Found {} duplicate files", total_files); - println!(" Size of duplicate files: {}", total_size); + info!("Duplicate statistics"); + info!(" Found {} duplicate files", total_files); + info!(" Size of duplicate files: {}", total_size); } } @@ -532,7 +651,10 @@ fn check_generated_files(entry: &str, generated: &mut GeneratedHashMap) { } // Ignore generated pdf, html, and css files - if fname.ends_with(".pdf") || fname.ends_with(".html") || fname.ends_with(".css") { + if fname.ends_with(".pdf") + || fname.ends_with(".html") + || fname.ends_with(".css") + { continue; } @@ -542,10 +664,10 @@ fn check_generated_files(entry: &str, generated: &mut GeneratedHashMap) { } }; } - Err(e) => println!("Error reading file {}: {:?}", entry, e), + Err(e) => error!("Error reading file {}: {:?}", entry, e), } } - Err(e) => println!("Error opening file {}: {:?}", entry, e), + Err(e) => error!("Error opening file {}: {:?}", entry, e), } } @@ -592,7 +714,7 @@ fn check_tds_archive_name(tds_zip: &Option<String>) -> Option<String> { // } // } -fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { +fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes, pkg_replace: &FxHashMap<String, String>) { i0003!(tds_zip); let dir_entry = Path::new(tds_zip); @@ -607,29 +729,7 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { let ut = Utils::new(utils::CheckType::Tds); - // We have a discrepancy between - // TDS zip archive name: babel-base.tds.zip - // and package name: babel - // which yields a false test when checking for the package name - // the path names of the TDS zip archive files. - // Therefore, we correct here. - // latex-tools, latex-graphics, latex-amsmath, latex-base => latex - // latex-tools-dev, latex-graphics-dev, latex-amsmath-dev, latex-base-dev => latex-dev - let mut pkg_replacements: FxHashMap<&str, &str> = FxHashMap::default(); - pkg_replacements.insert("babel-base", "babel"); - pkg_replacements.insert("latex-base", "latex"); - pkg_replacements.insert("latex-lab", "latex"); - pkg_replacements.insert("latex-tools", "latex"); - pkg_replacements.insert("latex-graphics", "latex"); - pkg_replacements.insert("latex-amsmath", "latex"); - pkg_replacements.insert("latex-firstaid", "latex/firstaid"); - pkg_replacements.insert("latex-base-dev", "latex-dev"); - pkg_replacements.insert("latex-lab-dev", "latex-dev"); - pkg_replacements.insert("latex-tools-dev", "latex-dev"); - pkg_replacements.insert("latex-graphics-dev", "latex-dev"); - pkg_replacements.insert("latex-amsmath-dev", "latex-dev"); - pkg_replacements.insert("latex-firstaid-dev", "latex-dev/firstaid"); - let real_pkg_name = if let Some(real_name) = pkg_replacements.get(pkg_name) { + let real_pkg_name = if let Some(real_name) = pkg_replace.get(pkg_name) { real_name } else { pkg_name @@ -647,10 +747,13 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { let tmp_dir_str = tmp_dir.path().to_str().unwrap(); // unzip the TDS zip archive into a temporary directory match ut.unzip(tds_zip, tmp_dir_str) { - Ok(_) => {}, + Ok(_) => {} Err(e) => { - println!("Could not unpack `{}` into directory `{}`", tds_zip, tmp_dir_str); - println!("Error from unzip: {}", e); + error!( + "Could not unpack `{}` into directory `{}`", + tds_zip, tmp_dir_str + ); + error!("Error from unzip: {}", e); e0033!(&tds_zip, e); process::exit(1) } @@ -824,7 +927,7 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { } } Err(e) => { - eprintln!("{}", e); + error!("{}", e); } } } @@ -911,7 +1014,7 @@ fn fix_perms(entry: &str) { if rc.status { if let Some(op) = rc.output { - print!("{}", op); + info!("{}", op); } } } @@ -934,7 +1037,9 @@ fn set_perms(entry: &str, p: u32) -> bool { if rc.status { if let Some(op) = rc.output { - print!("{}", op); + // we need to remove the `\n` in the `chmod` output + // as `info!` also adds a `\n` + info!("{}", op.trim_end()); } true } else { @@ -1164,7 +1269,7 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> { } Err(e) => { - eprintln!("{}", e); + error!("{}", e); } } } @@ -1255,7 +1360,7 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> { filemagic::Mimetype::Text(LineEnding::Lf) => { w0008!(&dir_entry_str); } - fmm => println!("Should not occur: {} has {:?}", dir_entry_str, fmm), + fmm => error!("Should not occur: {} has {:?}", dir_entry_str, fmm), }, Some(_) | None => { match ft { @@ -1280,7 +1385,7 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> { } } filemagic::Mimetype::Text(LineEnding::Lf) => (), - fmm => println!("Should not occur: {} has {:?}", dir_entry_str, fmm), + fmm => error!("Should not occur: {} has {:?}", dir_entry_str, fmm), } } } @@ -1303,7 +1408,7 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> { if !ret.status { e0017!(&dir_entry_str); if let Some(output) = ret.output { - println!("{}", &output); + info!("{}", &output); }; } } @@ -1336,7 +1441,7 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> { print_casefolding(&lcnames); print_generated(&doublenames, &generated); - if ! ARGS.ignore_same_named { + if !ARGS.ignore_same_named { print_doublenames(&doublenames); } @@ -1392,7 +1497,7 @@ fn print_casefolding(hashes: &FxHashMap<PathBuf, Vec<(PathBuf, FileKind)>>) { e0025!(k.display()); for (p, ty) in eles { - println!(" >>> {} ({})", p.display(), ty); + info!(" >>> {} ({})", p.display(), ty); } } } @@ -1435,7 +1540,7 @@ fn print_doublenames(hashes: &FxHashMap<PathBuf, Vec<PathBuf>>) { // println!(":: {}", k.display()); for p in paths { - println!(" >>> {}", p.display()); + info!(" >>> {}", p.display()); } } } @@ -1443,6 +1548,6 @@ fn print_doublenames(hashes: &FxHashMap<PathBuf, Vec<PathBuf>>) { fn show_tmp_endings() { i0006!(); for (t, c) in temp_file_endings() { - println!("{:23} {}", t, c); + info!("{:23} {}", t, c); } } |