summaryrefslogtreecommitdiff
path: root/support/pkgcheck/src/main.rs
diff options
context:
space:
mode:
authorNorbert Preining <norbert@preining.info>2022-05-27 03:01:31 +0000
committerNorbert Preining <norbert@preining.info>2022-05-27 03:01:31 +0000
commit84a5593d3fb9d03aac5677678bcc4c92e8c9a9c4 (patch)
tree0212ba38147bcfe80e5c1a853071a6e6a45b87d7 /support/pkgcheck/src/main.rs
parent02d941fa9c9895bb08a84ac9afe3559abd1ba8ad (diff)
CTAN sync 202205270301
Diffstat (limited to 'support/pkgcheck/src/main.rs')
-rw-r--r--support/pkgcheck/src/main.rs317
1 files changed, 211 insertions, 106 deletions
diff --git a/support/pkgcheck/src/main.rs b/support/pkgcheck/src/main.rs
index b18fd8793b..f0766cdeba 100644
--- a/support/pkgcheck/src/main.rs
+++ b/support/pkgcheck/src/main.rs
@@ -1,6 +1,8 @@
#[macro_use]
extern crate pest_derive;
+use log::*;
+
mod gparser;
#[macro_use]
@@ -14,6 +16,7 @@ mod recode;
use recode::{wrong_line_endings2crlf, wrong_line_endings2lf};
mod filemagic;
+
use filemagic::LineEnding;
use linkcheck::LinkCheck;
@@ -23,14 +26,15 @@ use std::fmt::Display;
use std::str;
use utils::*;
+use serde::{Deserialize, Serialize};
use std::os::unix::fs::MetadataExt;
-
+use std::borrow::Cow;
use scoped_threadpool::Pool;
use tempfile::Builder;
-#[macro_use]
-extern crate lazy_static;
+use once_cell::sync::Lazy; // 1.3.1
+
use colored::*;
use regex::Regex;
@@ -51,13 +55,140 @@ use std::sync::atomic::{AtomicBool, Ordering};
use rustc_hash::{FxHashMap, FxHashSet};
+use std::fmt::Arguments;
use std::sync::mpsc::{channel, Sender};
-use structopt::clap::Shell;
-use structopt::StructOpt;
+use clap::{Command, CommandFactory, Parser, ValueHint};
+use clap_complete::{generate, Generator, Shell};
+
#[cfg(unix)]
use walkdir::{DirEntry, WalkDir};
+fn format_message(message: &Arguments, no_color: bool) -> Cow<'static, str> {
+ let msg_str = format!("{}", message);
+ if msg_str.starts_with(' ') {
+ return msg_str.into();
+ }
+ let (left, right) = msg_str.split_once(' ').unwrap();
+ if no_color {
+ msg_str.into()
+ } else {
+ let colored_msg = match &left.chars().next().unwrap() {
+ 'E' | 'F' => format!("{} {}", left.bright_red().bold(), right),
+ 'W' => format!("{} {}", left.bright_red(), right),
+ 'I' => format!("{} {}", left.bright_yellow().bold(), right),
+ _ => msg_str,
+ };
+ colored_msg.into()
+ }
+}
+
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct TdsException {
+ pub pkg: String,
+ pub tpkg: String,
+}
+
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct PathExceptions {
+ pub tds_path_exceptions: Vec<TdsException>,
+}
+
+fn get_config_file_name() -> Option<String> {
+ if let Some(config_file) = &ARGS.config_file {
+ if Path::new(&config_file).exists() {
+ return Some(config_file.to_string())
+ } else {
+ f0008!(config_file);
+ std::process::exit(1);
+ }
+ }
+ let home_dir = match home::home_dir() {
+ Some(path) => path.display().to_string(),
+ None => panic!("Impossible to get your home dir!"),
+ };
+ let config_files = [".ctan/pkgcheck.yml", ".config/ctan/pkgcheck.yml"];
+ for f in config_files {
+ let config_file_abs_path = format!("{}/{}", home_dir, f);
+ if Path::new(&config_file_abs_path).exists() {
+ return Some(config_file_abs_path);
+ }
+ }
+ None
+}
+
+fn read_yaml_config() -> FxHashMap<String, String> {
+ let mut pkg_replacements: FxHashMap<String, String> = FxHashMap::default();
+ for (p, q) in [
+ ("babel-base", "babel"),
+ ("latex-base", "latex"),
+ ("latex-lab", "latex"),
+ ("latex-tools", "latex"),
+ ("latex-graphics", "latex"),
+ ("latex-amsmath", "latex"),
+ ("latex-firstaid", "latex/firstaid"),
+ ("latex-base-dev", "latex-dev"),
+ ("latex-lab-dev", "latex-dev"),
+ ("latex-tools-dev", "latex-dev"),
+ ("latex-graphics-dev", "latex-dev"),
+ ("latex-amsmath-dev", "latex-dev"),
+ ("latex-firstaid-dev", "latex-dev/firstaid"),
+ ("vntex-nonfree", "vntex"),
+ ] {
+ pkg_replacements.insert(p.to_string(), q.to_string());
+ }
+
+ match get_config_file_name() {
+ Some(config_filename) => {
+ i0008!(config_filename);
+
+ let data = match fs::read_to_string(&config_filename) {
+ Ok(str) => str,
+ Err(e) => { f0009!(&config_filename, e); std::process::exit(1); }
+ };
+
+ let path_exceptions = serde_yaml::from_str::<PathExceptions>(&data);
+
+ match path_exceptions {
+ Ok(pb) => {
+ for play in &pb.tds_path_exceptions {
+ // check if package name is already in pkg_replacements hash
+ let old_val = pkg_replacements.get(&play.pkg);
+ if let Some(ov) = old_val {
+ if ARGS.verbose {
+ if ov == &play.tpkg {
+ w0009!(play.pkg, play.tpkg);
+ } else {
+ i0009!(play.pkg, ov, play.tpkg);
+ }
+ }
+ }
+ pkg_replacements.insert(play.pkg.clone(), play.tpkg.clone());
+ }
+ pb
+ }
+ Err(e) => { f0010!(e); std::process::exit(1);},
+ };
+ pkg_replacements
+ }
+ None => pkg_replacements,
+ }
+}
+
+fn setup_logger(no_color: bool) -> Result<(), fern::InitError> {
+ fern::Dispatch::new()
+ .format(move |out, message, _record| {
+ let msg_txt = format_message(message, no_color);
+ out.finish(format_args!("{}", msg_txt))
+ })
+ .level(log::LevelFilter::Info)
+ .chain(std::io::stdout())
+ .apply()?;
+ Ok(())
+}
+
fn err(path: &Path, err: &io::Error) {
e0027!(path.display(), err);
}
@@ -83,7 +214,9 @@ fn hash_file_inner(path: &Path) -> io::Result<Vec<u8>> {
loop {
match fp.read(&mut buf)? {
0 => break,
- n => { digest.update(&buf[..n]); },
+ n => {
+ digest.update(&buf[..n]);
+ }
}
}
Ok(digest.finalize().as_bytes().to_vec())
@@ -96,7 +229,7 @@ fn hash_file(fsize: u64, path: PathBuf, tx: &HashSender) {
}
}
-// returns false if an error occured
+// returns false if an error occurred
fn fix_inconsistent_le(fname: &str) -> bool {
i0004!(fname);
match wrong_line_endings2lf(fname) {
@@ -111,7 +244,7 @@ fn fix_inconsistent_le(fname: &str) -> bool {
}
}
-// returns false if an error occured
+// returns false if an error occurred
fn make_crlf(fname: &str) -> bool {
i0004!(fname);
match wrong_line_endings2crlf(fname) {
@@ -192,48 +325,50 @@ fn _get_devno(entry: &DirEntry) -> u64 {
}
}
-#[derive(StructOpt, Debug)]
-#[structopt(about = "A checker for uploaded packages to CTAN.")]
+#[derive(Parser, Debug, PartialEq)]
+#[clap(author, version, about, long_about = None)]
struct Args {
- #[structopt(short = "I", long = "ignore-dupes", help = "Ignore dupes")]
+ #[clap(short = 'I', long = "ignore-dupes", help = "Ignore dupes")]
ignore_dupes: bool,
- #[structopt(long = "ignore-same-named", help = "Ignore same-named files")]
+ #[clap(long = "ignore-same-named", help = "Ignore same-named files")]
ignore_same_named: bool,
- #[structopt(short = "v", long = "verbose", help = "Verbose operation?")]
+ #[clap(short = 'v', long = "verbose", help = "Verbose operation?")]
verbose: bool,
- #[structopt(short = "L", long = "correct-le", help = "Correct line endings")]
+ #[clap(short = 'L', long = "correct-le", help = "Correct line endings")]
correct_le: bool,
- #[structopt(short = "C", long = "correct-perms", help = "Correct permissions")]
+ #[clap(short = 'C', long = "correct-perms", help = "Correct permissions")]
correct_perms: bool,
- #[structopt(long = "no-colors", help = "Don't display messages in color")]
+ #[clap(long = "no-colors", help = "Don't display messages in color")]
no_colors: bool,
- // #[structopt(long = "install-completion", help = "Install completion for the current shell")]
- // install_completion: Option<String>,
- #[structopt(long = "urlcheck", help = "Check URLs found in README files")]
+ #[clap(long = "urlcheck", help = "Check URLs found in README files")]
urlcheck: bool,
- #[structopt(short = "T", long = "tds-zip", help = "tds zip archive", group = "tds")]
+ #[clap(short = 'T', long = "tds-zip", help = "tds zip archive", group = "tds", value_hint = ValueHint::FilePath)]
tds_zip: Option<String>,
- #[structopt(
- short = "e",
+ #[clap(
+ short = 'e',
long = "explain",
help = "Explain error or warning message",
group = "only_one"
)]
explain: Option<String>,
- #[structopt(
+ #[clap(
long = "explain-all",
help = "Explains all error or warning messages",
group = "only_one"
)]
explain_all: bool,
- #[structopt(
+ #[clap(long = "generate-completion", group = "only_one", arg_enum)]
+ generator: Option<Shell>,
+ #[clap(
long = "show-temp-endings",
help = "Show file endings for temporary files",
group = "only_one"
)]
show_tmp_endings: bool,
- #[structopt(short = "d", long = "package-dir", help = "Package directory")]
+ #[clap(short = 'd', long = "package-dir", help = "Package directory", value_hint = ValueHint::DirPath)]
pkg_dir: Option<String>,
+ #[clap(long = "config-file", help = "Specify config file to use", value_hint = ValueHint::FilePath)]
+ config_file: Option<String>,
}
// We take care to avoid visiting a single inode twice,
@@ -248,10 +383,8 @@ fn check_inode(_: &mut FxHashSet<u64>, _: &Metadata) -> bool {
true
}
-lazy_static! {
- static ref ARGS: Args = Args::from_args();
- static ref ERROR_OCCURED: AtomicBool = AtomicBool::new(false);
-}
+static ARGS: Lazy<Args> = Lazy::new(Args::parse);
+static ERROR_OCCURRED: AtomicBool = AtomicBool::new(false);
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum DPath {
@@ -296,21 +429,18 @@ impl DupPath {
}
}
-// fn install_completion_bash() {
-// Args::clap().gen_completions(env!("CARGO_PKG_NAME"), Shell::Bash, "bla");
-// }
+type DupHashes = FxHashMap<(u64, Vec<u8>), DupPath>;
-// fn install_completion_zsh() {
-// Args::clap().gen_completions(env!("CARGO_PKG_NAME"), Shell::Zsh, "/home/manfred/.zfunc");
-// }
+fn print_completions<G: Generator>(gen: G, cmd: &mut Command) {
+ generate(gen, cmd, cmd.get_name().to_string(), &mut io::stdout());
+}
-// fn install_completion_fish() {
-// Args::clap().gen_completions(env!("CARGO_PKG_NAME"), Shell::Fish, "target");
-// }
+fn main() {
+ let _ = setup_logger(ARGS.no_colors);
-type DupHashes = FxHashMap<(u64, Vec<u8>), DupPath>;
+ // read yaml config file if one is given explicitly or implicitly
+ let pkg_replace : FxHashMap<String, String> = read_yaml_config();
-fn main() {
match &ARGS.explain {
None => (),
Some(e) => {
@@ -319,23 +449,12 @@ fn main() {
}
}
- //
- // !!!!!! this should be activated when structop uses
- // !!!!!! the new clap 3.0
- //
- // match &ARGS.install_completion {
- // None => { }
- // Some(d) => {
- // println!("{}", &d);
- // match d.as_str() {
- // "bash" => install_completion_bash(),
- // "zsh" => install_completion_zsh(),
- // "fish" => install_completion_fish(),
- // _ => println!("Invalid shell name {} given", d)
- // }
- // process::exit(0);
- // }
- // }
+ if let Some(generator) = ARGS.generator {
+ let mut cmd = Args::command();
+ eprintln!("Generating completion file for {:?}...", generator);
+ print_completions(generator, &mut cmd);
+ process::exit(0)
+ }
if ARGS.explain_all {
explains_all();
@@ -393,13 +512,13 @@ fn main() {
if tds_zip.is_some() {
if let Some(pn) = pkg_name {
if let Some(s) = ARGS.tds_zip.as_ref() {
- check_tds_archive(&pn, s, &hashes);
+ check_tds_archive(&pn, s, &hashes, &pkg_replace);
}
}
}
}
- if ERROR_OCCURED.load(Ordering::Relaxed) {
+ if ERROR_OCCURRED.load(Ordering::Relaxed) {
process::exit(1);
} else {
process::exit(0);
@@ -428,20 +547,20 @@ fn print_duplicates(hashes: &DupHashes) {
total_size += sz * (paths.plen - 1) as u64;
total_dupes += (paths.plen - 1) as u64;
- println!("Size: {}", sz);
+ info!("Size: {}", sz);
for p in &paths.dupes {
if let DPath::Both(p) = p {
let ps = p.as_path().to_str().unwrap();
- println!(" >>> {}", ps);
+ info!(" >>> {}", ps);
}
}
- println!();
+ //eprintln!();
}
if ARGS.verbose && total_dupes > 0 {
- println!("Duplicate statistics");
- println!(" Found {} duplicate files", total_files);
- println!(" Size of duplicate files: {}", total_size);
+ info!("Duplicate statistics");
+ info!(" Found {} duplicate files", total_files);
+ info!(" Size of duplicate files: {}", total_size);
}
}
@@ -532,7 +651,10 @@ fn check_generated_files(entry: &str, generated: &mut GeneratedHashMap) {
}
// Ignore generated pdf, html, and css files
- if fname.ends_with(".pdf") || fname.ends_with(".html") || fname.ends_with(".css") {
+ if fname.ends_with(".pdf")
+ || fname.ends_with(".html")
+ || fname.ends_with(".css")
+ {
continue;
}
@@ -542,10 +664,10 @@ fn check_generated_files(entry: &str, generated: &mut GeneratedHashMap) {
}
};
}
- Err(e) => println!("Error reading file {}: {:?}", entry, e),
+ Err(e) => error!("Error reading file {}: {:?}", entry, e),
}
}
- Err(e) => println!("Error opening file {}: {:?}", entry, e),
+ Err(e) => error!("Error opening file {}: {:?}", entry, e),
}
}
@@ -592,7 +714,7 @@ fn check_tds_archive_name(tds_zip: &Option<String>) -> Option<String> {
// }
// }
-fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) {
+fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes, pkg_replace: &FxHashMap<String, String>) {
i0003!(tds_zip);
let dir_entry = Path::new(tds_zip);
@@ -607,29 +729,7 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) {
let ut = Utils::new(utils::CheckType::Tds);
- // We have a discrepancy between
- // TDS zip archive name: babel-base.tds.zip
- // and package name: babel
- // which yields a false test when checking for the package name
- // the path names of the TDS zip archive files.
- // Therefore, we correct here.
- // latex-tools, latex-graphics, latex-amsmath, latex-base => latex
- // latex-tools-dev, latex-graphics-dev, latex-amsmath-dev, latex-base-dev => latex-dev
- let mut pkg_replacements: FxHashMap<&str, &str> = FxHashMap::default();
- pkg_replacements.insert("babel-base", "babel");
- pkg_replacements.insert("latex-base", "latex");
- pkg_replacements.insert("latex-lab", "latex");
- pkg_replacements.insert("latex-tools", "latex");
- pkg_replacements.insert("latex-graphics", "latex");
- pkg_replacements.insert("latex-amsmath", "latex");
- pkg_replacements.insert("latex-firstaid", "latex/firstaid");
- pkg_replacements.insert("latex-base-dev", "latex-dev");
- pkg_replacements.insert("latex-lab-dev", "latex-dev");
- pkg_replacements.insert("latex-tools-dev", "latex-dev");
- pkg_replacements.insert("latex-graphics-dev", "latex-dev");
- pkg_replacements.insert("latex-amsmath-dev", "latex-dev");
- pkg_replacements.insert("latex-firstaid-dev", "latex-dev/firstaid");
- let real_pkg_name = if let Some(real_name) = pkg_replacements.get(pkg_name) {
+ let real_pkg_name = if let Some(real_name) = pkg_replace.get(pkg_name) {
real_name
} else {
pkg_name
@@ -647,10 +747,13 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) {
let tmp_dir_str = tmp_dir.path().to_str().unwrap();
// unzip the TDS zip archive into a temporary directory
match ut.unzip(tds_zip, tmp_dir_str) {
- Ok(_) => {},
+ Ok(_) => {}
Err(e) => {
- println!("Could not unpack `{}` into directory `{}`", tds_zip, tmp_dir_str);
- println!("Error from unzip: {}", e);
+ error!(
+ "Could not unpack `{}` into directory `{}`",
+ tds_zip, tmp_dir_str
+ );
+ error!("Error from unzip: {}", e);
e0033!(&tds_zip, e);
process::exit(1)
}
@@ -824,7 +927,7 @@ fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) {
}
}
Err(e) => {
- eprintln!("{}", e);
+ error!("{}", e);
}
}
}
@@ -911,7 +1014,7 @@ fn fix_perms(entry: &str) {
if rc.status {
if let Some(op) = rc.output {
- print!("{}", op);
+ info!("{}", op);
}
}
}
@@ -934,7 +1037,9 @@ fn set_perms(entry: &str, p: u32) -> bool {
if rc.status {
if let Some(op) = rc.output {
- print!("{}", op);
+ // we need to remove the `\n` in the `chmod` output
+ // as `info!` also adds a `\n`
+ info!("{}", op.trim_end());
}
true
} else {
@@ -1164,7 +1269,7 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> {
}
Err(e) => {
- eprintln!("{}", e);
+ error!("{}", e);
}
}
}
@@ -1255,7 +1360,7 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> {
filemagic::Mimetype::Text(LineEnding::Lf) => {
w0008!(&dir_entry_str);
}
- fmm => println!("Should not occur: {} has {:?}", dir_entry_str, fmm),
+ fmm => error!("Should not occur: {} has {:?}", dir_entry_str, fmm),
},
Some(_) | None => {
match ft {
@@ -1280,7 +1385,7 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> {
}
}
filemagic::Mimetype::Text(LineEnding::Lf) => (),
- fmm => println!("Should not occur: {} has {:?}", dir_entry_str, fmm),
+ fmm => error!("Should not occur: {} has {:?}", dir_entry_str, fmm),
}
}
}
@@ -1303,7 +1408,7 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> {
if !ret.status {
e0017!(&dir_entry_str);
if let Some(output) = ret.output {
- println!("{}", &output);
+ info!("{}", &output);
};
}
}
@@ -1336,7 +1441,7 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> {
print_casefolding(&lcnames);
print_generated(&doublenames, &generated);
- if ! ARGS.ignore_same_named {
+ if !ARGS.ignore_same_named {
print_doublenames(&doublenames);
}
@@ -1392,7 +1497,7 @@ fn print_casefolding(hashes: &FxHashMap<PathBuf, Vec<(PathBuf, FileKind)>>) {
e0025!(k.display());
for (p, ty) in eles {
- println!(" >>> {} ({})", p.display(), ty);
+ info!(" >>> {} ({})", p.display(), ty);
}
}
}
@@ -1435,7 +1540,7 @@ fn print_doublenames(hashes: &FxHashMap<PathBuf, Vec<PathBuf>>) {
// println!(":: {}", k.display());
for p in paths {
- println!(" >>> {}", p.display());
+ info!(" >>> {}", p.display());
}
}
}
@@ -1443,6 +1548,6 @@ fn print_doublenames(hashes: &FxHashMap<PathBuf, Vec<PathBuf>>) {
fn show_tmp_endings() {
i0006!();
for (t, c) in temp_file_endings() {
- println!("{:23} {}", t, c);
+ info!("{:23} {}", t, c);
}
}