summaryrefslogtreecommitdiff
path: root/support/pkgcheck/src/main.rs
diff options
context:
space:
mode:
authorNorbert Preining <norbert@preining.info>2019-09-02 13:46:59 +0900
committerNorbert Preining <norbert@preining.info>2019-09-02 13:46:59 +0900
commite0c6872cf40896c7be36b11dcc744620f10adf1d (patch)
tree60335e10d2f4354b0674ec22d7b53f0f8abee672 /support/pkgcheck/src/main.rs
Initial commit
Diffstat (limited to 'support/pkgcheck/src/main.rs')
-rw-r--r--support/pkgcheck/src/main.rs1287
1 files changed, 1287 insertions, 0 deletions
diff --git a/support/pkgcheck/src/main.rs b/support/pkgcheck/src/main.rs
new file mode 100644
index 0000000000..a7be277330
--- /dev/null
+++ b/support/pkgcheck/src/main.rs
@@ -0,0 +1,1287 @@
+extern crate pest;
+#[macro_use]
+extern crate pest_derive;
+
+mod gparser;
+
+#[macro_use]
+mod messages;
+use messages::{explains, explains_all};
+
+#[macro_use]
+mod utils;
+mod linkcheck;
+
+mod recode;
+use recode::crlf2lf;
+
+mod filemagic;
+
+use linkcheck::LinkCheck;
+use std::fmt;
+use std::fmt::Display;
+use std::str;
+use utils::*;
+
+use std::os::unix::fs::MetadataExt;
+
+use scoped_threadpool::Pool;
+
+
+use tempdir::TempDir;
+#[macro_use]
+extern crate lazy_static;
+use colored::*;
+
+use regex::Regex;
+
+use std::fs::File;
+use std::fs::Metadata;
+use std::io::prelude::*;
+use std::io::BufReader;
+use std::os::unix::fs::FileTypeExt;
+use std::{fs, process};
+// use std::os::unix::fs::PermissionsExt;
+
+use std::ffi::OsStr;
+use std::io::{self, Read};
+use std::path::Path;
+use std::path::PathBuf;
+use std::sync::atomic::{AtomicBool, Ordering};
+
+use blake2::{Blake2b, Digest};
+use fnv::{FnvHashMap as HashMap, FnvHashSet as HashSet};
+use std::sync::mpsc::{channel, Sender};
+use structopt::StructOpt;
+#[cfg(unix)]
+use walkdir::{DirEntry, WalkDir};
+
+fn err(path: &PathBuf, err: &io::Error) {
+ e0027!(path.display(), err);
+}
+
+type HashSender = Sender<(u64, PathBuf, Vec<u8>)>;
+//type DupeSender = Sender<(u64, Vec<PathBuf>)>;
+
+const BLOCKSIZE: usize = 4096;
+
+fn hash_file_inner(path: &PathBuf) -> io::Result<Vec<u8>> {
+ let mut buf = [0u8; BLOCKSIZE];
+ let mut fp = File::open(&path)?;
+ let mut digest = Blake2b::default();
+ // When we compare byte-by-byte, we don't need to hash the whole file.
+ // Instead, hash a block of 4kB, skipping 100kB.
+ loop {
+ match fp.read(&mut buf)? {
+ 0 => break,
+ n => digest.input(&buf[..n]),
+ }
+ }
+ Ok(digest.result().to_vec())
+}
+
+fn hash_file(fsize: u64, path: PathBuf, tx: &HashSender) {
+ match hash_file_inner(&path) {
+ Ok(hash) => tx.send((fsize, path, hash)).unwrap(),
+ Err(e) => err(&path, &e),
+ }
+}
+
+fn fix_perms(entry: &str) {
+ i0005!(entry);
+ let rc = run_cmd("/bin/chmod", &["-v", "ug=rwX,o=rX", entry]);
+
+ if rc.status {
+ if let Some(op) = rc.output {
+ print!("{}", op);
+ }
+ }
+}
+
+fn filename_has_bad_chars(entry: &DirEntry) -> Option<(usize, char)> {
+ let s = entry.file_name().to_str().unwrap();
+
+ for (i, c) in s.char_indices() {
+ match c {
+ '\x00'...'\x1f' => return Some((i, c)),
+ ' ' | '!' | '"' | '#' | '$' | '%' | '&' | '\'' => return Some((i, c)),
+ '(' | ')' | '*' => return Some((i, c)),
+ '+' | ',' | '-' | '.' => (),
+ '0'...'9' => (),
+ ':' | ';' | '<' | '=' | '>' | '?' | '@' => return Some((i, c)),
+ 'A'...'Z' => (),
+ '[' | '\\' | ']' | '^' => return Some((i, c)),
+ '_' => (),
+ '`' => return Some((i, c)),
+ 'a'...'z' => (),
+ '{' | '|' | '}' | '~' => return Some((i, c)),
+ '/' => return None,
+ _ => return Some((i, c)),
+ };
+ }
+ None
+}
+
+// returns false if CRLF, otherwise true
+fn fix_crlf(fname: &str) -> bool {
+ i0004!(fname);
+ //run_cmd("/usr/bin/fromdos", &["-p", "-e", "-v", fname]).status
+ match crlf2lf(fname) {
+ Ok(_) => {
+ i0007!(fname);
+ true
+ }
+ Err(e) => {
+ e0027!(fname, e);
+ false
+ }
+ }
+}
+
+fn check_readme(dir_entry: &str, is_readme: &ReadmeKind, ft: &filemagic::Mimetype) -> bool {
+ let msg_name = if let ReadmeKind::Symlink(s) = is_readme {
+ format!("{} (symlinked from {})", dir_entry, &s)
+ } else {
+ dir_entry.to_string()
+ };
+
+ match ft {
+ filemagic::Mimetype::Pdf
+ | filemagic::Mimetype::Binary
+ | filemagic::Mimetype::Archive
+ | filemagic::Mimetype::Zip => {
+ e0003!(msg_name);
+ return false;
+ }
+ filemagic::Mimetype::BOM(b) => {
+ e0029!(msg_name, b.as_ref());
+ return false;
+ }
+ filemagic::Mimetype::Text | filemagic::Mimetype::TextCRLF => match File::open(dir_entry) {
+ Ok(f) => {
+ if !check_readme_inner(&msg_name, &f) {
+ return false;
+ }
+ }
+ Err(e) => {
+ e0027!(msg_name, e);
+ return false;
+ }
+ },
+ _ => (),
+ }
+ true
+}
+
+fn check_readme_inner(fname: &str, f: &std::fs::File) -> bool {
+ let reader = BufReader::new(f);
+
+ let lines = reader.split(b'\n').map(|l| l.unwrap());
+ let mut result = true;
+
+ for (lineno, line) in lines.enumerate() {
+ if let Err(e) = String::from_utf8(line.clone()) {
+ e0021!(fname, lineno + 1, e);
+ result = false;
+ }
+ }
+ result
+}
+
+fn is_readme(entry: &DirEntry) -> bool {
+ let r = entry.file_name().to_str().unwrap();
+
+ match r {
+ "README" | "README.txt" | "README.md" => true,
+ _ => false,
+ }
+}
+
+fn get_devno(meta: &Metadata) -> u64 {
+ meta.dev()
+}
+
+fn _get_devno(entry: &DirEntry) -> u64 {
+ let meta = fs::metadata(entry.path().to_str().unwrap());
+ match meta {
+ Ok(m) => m.dev(),
+ _ => 0,
+ }
+}
+
+#[derive(StructOpt, Debug)]
+#[structopt(about = "A checker for uploaded packages to CTAN.")]
+struct Args {
+ #[structopt(short = "I", long = "ignore-dupes", help = "Ignore dupes")]
+ ignore_dupes: bool,
+ #[structopt(short = "v", long = "verbose", help = "Verbose operation?")]
+ verbose: bool,
+ #[structopt(short = "L", long = "correct-crlf", help = "Correct CRLF line endings")]
+ correct_crlf: bool,
+ #[structopt(short = "C", long = "correct-perms", help = "Correct permissions")]
+ correct_perms: bool,
+ #[structopt(long = "no-colors", help = "Don't display messages in color")]
+ no_colors: bool,
+ #[structopt(long = "urlcheck", help = "Check URLs found in README files")]
+ urlcheck: bool,
+ #[structopt(short = "T", long = "tds-zip", help = "tds zip archive", group = "tds")]
+ tds_zip: Option<String>,
+ #[structopt(
+ short = "e",
+ long = "explain",
+ help = "Explain error or warning message",
+ group = "only_one"
+ )]
+ explain: Option<String>,
+ #[structopt(
+ long = "explain-all",
+ help = "Explains all error or warning messages",
+ group = "only_one"
+ )]
+ explain_all: bool,
+ #[structopt(
+ long = "show-temp-endings",
+ help = "Show file endings for temporary files",
+ group = "only_one"
+ )]
+ show_tmp_endings: bool,
+ #[structopt(short = "d", long = "package-dir", help = "Package directory")]
+ pkg_dir: Option<String>,
+}
+
+// We take care to avoid visiting a single inode twice,
+// which takes care of (false positive) hardlinks.
+#[cfg(unix)]
+fn check_inode(set: &mut HashSet<(u64, u64)>, meta: &Metadata) -> bool {
+ set.insert((get_devno(meta), meta.ino()))
+}
+
+#[cfg(not(unix))]
+fn check_inode(_: &mut HashSet<u64>, _: &Metadata) -> bool {
+ true
+}
+
+lazy_static! {
+ static ref ARGS: Args = Args::from_args();
+ static ref ERROR_OCCURED: AtomicBool = AtomicBool::new(false);
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum DPath {
+ Both(PathBuf),
+ Tds(PathBuf),
+}
+
+impl fmt::Display for DPath {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ match self {
+ DPath::Both(p) => write!(f, "{}", p.display()),
+ DPath::Tds(p) => write!(f, "{}", p.display()),
+ }
+ }
+}
+
+#[derive(Default)]
+pub struct DupPath {
+ len: usize,
+ plen: usize,
+ dupes: Vec<DPath>,
+}
+
+impl DupPath {
+ pub fn new() -> DupPath {
+ DupPath {
+ len: 0,
+ plen: 0,
+ dupes: Vec::new(),
+ }
+ }
+
+ pub fn push(&mut self, pb: PathBuf) {
+ let pname = pb.to_string_lossy();
+ self.len += 1;
+ if pname.ends_with(".tfm") {
+ self.dupes.push(DPath::Tds(pb.clone()));
+ } else {
+ self.plen += 1;
+ self.dupes.push(DPath::Both(pb.clone()));
+ }
+ }
+}
+
+type Hashes = HashMap<(u64, Vec<u8>), DupPath>;
+
+fn main() {
+ match &ARGS.explain {
+ None => (),
+ Some(e) => {
+ explains(&e);
+ process::exit(0);
+ }
+ }
+
+ if ARGS.explain_all {
+ explains_all();
+ process::exit(0);
+ }
+
+ if ARGS.show_tmp_endings {
+ show_tmp_endings();
+ process::exit(0);
+ }
+
+ let pkg_dir = match &ARGS.pkg_dir {
+ None => {
+ f0001!();
+ process::exit(1);
+ }
+ Some(d) => {
+ if !exists_dir(&d) {
+ f0002!(d);
+ process::exit(1);
+ };
+ d
+ }
+ };
+
+ let tds_zip = &ARGS.tds_zip;
+ let pkg_name = match tds_zip {
+ None => None,
+ Some(tz) => {
+ let pn = check_tds_archive_name(tds_zip);
+ if !exists_file(&tz) {
+ f0003!(&tz);
+ process::exit(1);
+ }
+ let mut fmagic = filemagic::Filetype::new();
+ match fmagic.analyze(&tz) {
+ Ok(filemagic::Mimetype::Zip) => (),
+ _ => {
+ f0004!(&tz);
+ process::exit(1)
+ }
+ };
+ pn
+ }
+ };
+
+ if let Some(hashes) = check_package(pkg_dir, tds_zip) {
+ if tds_zip.is_some() {
+ if let Some(pn) = pkg_name {
+ if let Some(s) = ARGS.tds_zip.as_ref() {
+ match check_tds_archive(&pn, s, &hashes) {
+ Ok(_) => (),
+ Err(e) => {
+ e0027!(s, e);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ if ERROR_OCCURED.load(Ordering::Relaxed) {
+ process::exit(1);
+ } else {
+ process::exit(0);
+ }
+}
+
+fn print_duplicates(hashes: &Hashes) {
+ let mut total_dupes = 0;
+ let mut total_files = 0;
+ let mut total_size = 0;
+
+ let mut header_printed = false;
+ for (k, paths) in hashes.iter() {
+ let (sz, _hash) = k;
+
+ if paths.plen <= 1 {
+ total_files += 1;
+ total_size += sz;
+ continue;
+ } else if !header_printed {
+ w0002!();
+ header_printed = true;
+ }
+
+ total_files += paths.plen;
+ total_size += sz * (paths.plen - 1) as u64;
+ total_dupes += (paths.plen - 1) as u64;
+
+ println!("Size: {}", sz);
+ for p in &paths.dupes {
+ if let DPath::Both(p) = p {
+ let ps = p.as_path().to_str().unwrap();
+ println!(" >>> {}", ps);
+ }
+ }
+ println!();
+ }
+
+ if ARGS.verbose && total_dupes > 0 {
+ println!("Duplicate statistics");
+ println!(" Found {} duplicate files", total_files);
+ println!(" Size of duplicate files: {}", total_size);
+ }
+}
+
+//#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, PartialEq, Eq)]
+pub enum FType {
+ Regular,
+ Directory,
+ Symlink,
+ BlockDevice,
+ CharDevice,
+ Fifo,
+ Socket,
+ Error(String),
+}
+
+fn get_filetype(entry: &DirEntry) -> FType {
+ match entry.metadata() {
+ Ok(mt) => {
+ let ft = mt.file_type();
+ if ft.is_symlink() {
+ return FType::Symlink;
+ }
+ if ft.is_dir() {
+ return FType::Directory;
+ }
+ if ft.is_block_device() {
+ return FType::BlockDevice;
+ }
+ if ft.is_char_device() {
+ return FType::CharDevice;
+ }
+ if ft.is_fifo() {
+ return FType::Fifo;
+ }
+ if ft.is_socket() {
+ return FType::Socket;
+ }
+ FType::Regular
+ }
+ Err(e) => FType::Error(format!("{}", e)),
+ }
+}
+
+//
+// read file into buffer[u8]
+// then convert into string
+//
+fn check_generated_files(entry: &str, generated: &mut HashMap<PathBuf, PathBuf>) {
+ // unwrap() is ok here as we only call this function for files,
+ // specifically .ins or .dtx files
+ let entry_fname = filename(entry).unwrap().to_string();
+ let entry_base = &entry_fname[0..entry_fname.len() - 4];
+
+ let fhdl = File::open(&entry);
+ match fhdl {
+ Ok(mut f) => {
+ let mut buf = Vec::new();
+
+ match f.read_to_end(&mut buf) {
+ Ok(_bytes_read) => {
+ if let Some(found) =
+ gparser::parse_generate(&String::from_utf8_lossy(&buf.clone()))
+ {
+ for fname in found {
+ // As we request a README in the top level directory of
+ // a package we ignore if a README was generated by an
+ // .ins or .dtx file
+ // CAVEAT: If this happens in a subdirectory it could be an error!!!!
+ if fname == "README" || fname == "README.txt" || fname == "README.md" {
+ continue;
+ }
+
+ // If the filename in the 'file{} statement is jobname.<ext>
+ // we replace jobname with the filename before we investigate further
+ let fname_x = if fname.starts_with("\\jobname") {
+ let ext = match get_extension_from_filename(&fname) {
+ Some(e) => e,
+ None => "",
+ };
+ format!("{}.{}", entry_base, ext)
+ } else {
+ fname.to_string()
+ };
+
+ let pb = PathBuf::from(fname_x);
+
+ generated.entry(pb).or_insert_with(|| PathBuf::from(entry));
+ }
+ };
+ }
+ Err(e) => println!("Error reading file {}: {:?}", entry, e),
+ }
+ }
+ Err(e) => println!("Error opening file {}: {:?}", entry, e),
+ }
+ return;
+}
+
+fn x_bit_set(p: u32) -> bool {
+ let p1 = p & 0o7777;
+ p1 & 0o111 != 0
+}
+
+// checks that archive name is in the format x.tds.zip and returns:
+// None if not
+// Option<x> if x.tds.zip
+// Caveat: we currently don't know if x is really the package name.
+fn check_tds_archive_name(tds_zip: &Option<String>) -> Option<String> {
+ match tds_zip {
+ None => None,
+ Some(tz) => {
+ // 8 for ".tds.zip", 1 means a package name is at least a character
+ if tz.len() < 8 + 1 || !tz.ends_with(".tds.zip") {
+ f0005!(tz);
+ process::exit(1);
+ }
+ let mut pname = String::from(utils::basename(&tz));
+ let plen = pname.len();
+ pname.truncate(plen - 8);
+ Some(pname.to_string())
+ }
+ }
+}
+
+fn unzip_tds_archive(tds_zip: &str, tmp_dir: &str) {
+ match run_cmd("/usr/bin/unzip", &["-q", "-d", tmp_dir, &tds_zip]) {
+ CmdReturn { status: true, .. } => (),
+ CmdReturn {
+ status: false,
+ output: out,
+ } => {
+ if let Some(o) = out {
+ e0033!(&tds_zip, o);
+ } else {
+ e0033!(&tds_zip, "<no error given");
+ }
+ process::exit(1);
+ }
+ }
+}
+
+fn check_tds_archive(
+ pkg_name: &str,
+ tds_zip: &str,
+ hashes: &HashMap<(u64, Vec<u8>), DupPath>,
+) -> Result<(), io::Error> {
+ i0003!(tds_zip);
+
+ let dir_entry = Path::new(tds_zip);
+ let p = get_perms(&dir_entry);
+ if !owner_has(p, 4) || !others_have(p, 4) || x_bit_set(p) {
+ e0024!(tds_zip, perms_to_string(p));
+ if ARGS.correct_perms {
+ i0005!(&tds_zip);
+ set_perms(&tds_zip, 0o664);
+ }
+ };
+
+ let tmp_dir = TempDir::new("pkgcheck")?;
+ let tmp_dir_offset = tmp_dir.path().to_str().unwrap().len() + 1;
+ unzip_tds_archive(tds_zip, tmp_dir.path().to_str().unwrap());
+
+ let tds_toplevel_dirs: HashSet<String> = [
+ "tex", "fonts", "metafont", "metapost", "bibtex", "scripts", "doc", "source",
+ ]
+ .iter()
+ .map(|&s| s.to_string())
+ .collect();
+
+ for entry in fs::read_dir(tmp_dir.path())? {
+ let entry = entry?;
+ let path = entry.path();
+
+ match path.file_stem() {
+ None => (),
+ Some(file_stem) => {
+ let fs = file_stem.to_str().expect("bad file name");
+ if !tds_toplevel_dirs.contains(fs) {
+ if path.is_dir() {
+ e0020!(fs);
+ } else {
+ e0034!(fs);
+ }
+ }
+ }
+ }
+
+ // if path.is_dir() {
+ // match path.file_stem() {
+ // None => (),
+ // Some(file_stem) => {
+ // let fs = file_stem.to_str().expect("bad file name");
+ // if !tds_toplevel_dirs.contains(fs) {
+ // e0020!(fs);
+ // }
+ // }
+ // }
+ // } else {
+ // println!("mist");
+ // e0020!(path.display());
+ // }
+ }
+
+ // in order to compare the package files with the content of the
+ // tds zip archive we need to checksum the files in the tds zip
+ // archive.
+
+ let mut sizes = HashMap::default();
+ let mut pool = Pool::new(num_cpus::get() as u32 + 1);
+ {
+ // Processing a single file entry, with the "sizes" hashmap collecting
+ // same-size files. Entries are either Found::One or Found::Multiple,
+ // so that we can submit the first file's path as a hashing job when the
+ // first duplicate is found. Hashing each file is submitted as a job to
+ // the pool.
+ let mut process = |fsize, dir_entry: &DirEntry| {
+ let path = dir_entry.path().to_path_buf();
+ let sizeref = &mut sizes;
+
+ sizeref.entry(fsize).or_insert_with(Vec::new).push(path);
+ };
+
+ let mut map_files_found = false;
+ let mut map_dvips_found = false;
+
+ for dir_entry in WalkDir::new(tmp_dir.path().to_str().unwrap()).follow_links(false) {
+ match dir_entry {
+ Ok(dir_entry) => {
+ //let dir_entry_str = dir_entry.path().to_str().unwrap();
+
+ let ft = get_filetype(&dir_entry);
+
+ if ft == FType::Directory {
+ continue;
+ }
+
+ if let FType::Error(e) = ft {
+ e0023!(e);
+ continue;
+ }
+
+ match dir_entry.metadata() {
+ Ok(meta) => {
+ let fsize = meta.len();
+ process(fsize, &dir_entry);
+
+ let dir_entry_str = match dir_entry.path().to_str() {
+ Some(d) => d,
+ None => {
+ e0031!(dir_entry.path().to_string_lossy());
+ continue;
+ }
+ };
+ let dir_entry_display = &dir_entry_str[tmp_dir_offset..];
+
+ if is_temporary_file(&dir_entry_str) {
+ e0008!(dir_entry_display);
+ }
+
+ // if the path doesn't contain a man page...
+ if dir_entry_str.find("/man/").is_none() {
+ let pkg_name_s = format!("/{}/", pkg_name);
+ // ...then we want to have the package name in the path
+ if dir_entry_str.find(&pkg_name_s).is_none() {
+ e0028!(pkg_name, dir_entry_display);
+ }
+ }
+
+ if dir_entry_str.ends_with(".map") {
+ map_files_found = true;
+ let re: Regex = Regex::new(r"fonts[/]map[/]dvips[/]").unwrap();
+ if re.is_match(dir_entry_str) {
+ map_dvips_found = true;
+ }
+
+ // if ( $fname =~ m{ fonts [/] map [/] dvips [/] }xms ) {
+ // $map_dvips_found = 1;
+ // }
+ }
+ }
+ Err(e) => {
+ e0027!(dir_entry.path().display(), e);
+ }
+ }
+ }
+ Err(e) => {
+ eprintln!("{}", e);
+ }
+ }
+ }
+
+ if map_files_found && !map_dvips_found {
+ println!( "Map files found for package {} but none of them is in a path starting with fonts/map/dvips", pkg_name);
+ }
+ };
+
+ let mut tds_hashes: HashMap<(u64, Vec<u8>), Vec<PathBuf>> = HashMap::default();
+ pool.scoped(|scope| {
+ let (tx, rx) = channel();
+
+ let hashref = &mut tds_hashes;
+ scope.execute(move || {
+ for (size, path, hash) in rx.iter() {
+ hashref
+ .entry((size, hash))
+ .or_insert_with(Vec::new)
+ .push(path);
+ }
+ });
+
+ for size in sizes.keys() {
+ for p in &sizes[size] {
+ let txc = tx.clone();
+ scope.execute(move || hash_file(*size, p.to_path_buf(), &txc));
+ }
+ }
+ });
+
+ tmp_dir.close()?;
+ // now check if each package file is in the tds archive
+ for (k, paths) in hashes.iter() {
+ if !tds_hashes.contains_key(k) {
+ let p = &paths.dupes[0];
+ e0026!(p);
+ }
+ }
+
+ Ok(())
+}
+
+fn get_extension_from_filename(filename: &str) -> Option<&str> {
+ Path::new(filename).extension().and_then(OsStr::to_str)
+}
+
+fn found_unwanted_filetype(fname: &str, ft: &FType) -> bool {
+ match ft {
+ FType::Socket => {
+ e0013!(fname);
+ true
+ }
+ FType::Fifo => {
+ e0014!(fname);
+ true
+ }
+ FType::BlockDevice => {
+ e0015!(fname);
+ true
+ }
+ FType::CharDevice => {
+ e0016!(fname);
+ true
+ }
+ FType::Error(e) => {
+ e0023!(e);
+ true
+ }
+ _ => false,
+ }
+}
+
+fn check_and_correct_perms(dir_entry: &str, p: u32) {
+ if owner_has(p, 5) || !others_have(p, 4) {
+ e0002!(dir_entry, perms_to_string(p));
+ if ARGS.correct_perms {
+ i0005!(&dir_entry);
+ set_perms(&dir_entry, 0o664);
+ }
+ };
+}
+
+
+#[derive(Debug, Clone, PartialEq)]
+enum FileKind {
+ File,
+ Directory,
+ Symlink(String),
+}
+
+
+impl Display for FileKind {
+ fn fmt(&self, f: &mut ::std::fmt::Formatter) -> Result<(), ::std::fmt::Error> {
+ match *self {
+ FileKind::File => f.write_str("file"),
+ FileKind::Directory => f.write_str("directory"),
+ FileKind::Symlink(_) => f.write_str("symlink"),
+ }
+ }
+}
+
+
+
+#[derive(Debug, Clone, PartialEq)]
+enum ReadmeKind {
+ No,
+ Yes,
+ Symlink(String),
+}
+
+fn check_package(root: &str, tds_zip: &Option<String>) -> Option<Hashes> {
+ //let mut urls = HashMap::default(); //: <String,UrlStatus>
+
+ let mut lcnames: HashMap<PathBuf, Vec<(PathBuf, FileKind)>> = HashMap::default();
+
+ let mut doublenames: HashMap<PathBuf, Vec<PathBuf>> = HashMap::default();
+
+ let mut inodes = HashSet::default();
+
+ i0002!(root);
+ // This hash contains all package file names.
+ //
+ // PathBuf: the full path starting at the directory specified at the command line
+ // Metadata: the meta data of the file
+ // String: the file name without any directory part
+ // ReadmeKind: is it a certain README, file or symlink?
+ // A special case of a README file is a file with has a different name but
+ // was poinded to by a symlink. Example: README --> README.rst
+ let mut file_names: HashMap<PathBuf, (Metadata, String, ReadmeKind)> = HashMap::default();
+ {
+ let mut readme_found = false;
+
+ let root_absolute = PathBuf::from(root)
+ .canonicalize()
+ .unwrap()
+ .to_string_lossy()
+ .to_string();
+ for dir_entry in WalkDir::new(root).follow_links(false) {
+ match dir_entry {
+ Ok(dir_entry) => {
+ let dir_entry_str = match dir_entry.path().to_str() {
+ Some(d) => d,
+ None => {
+ e0031!(dir_entry.path().to_string_lossy());
+ continue;
+ }
+ };
+ let meta = match dir_entry.metadata() {
+ Ok(meta) => meta,
+ Err(e) => {
+ e0023!(e);
+ continue;
+ }
+ };
+
+ if !check_inode(&mut inodes, &meta) {
+ continue;
+ }
+
+ // this is the file_name without the directory part
+ // unwrap() is ok here as we covered potential UTF-8 related errors
+ // above in the definition of dir_entry_str
+ let file_name = dir_entry.file_name().to_str().unwrap().to_string();
+
+ let ft = get_filetype(&dir_entry);
+
+ if found_unwanted_filetype(dir_entry_str, &ft) {
+ continue;
+ }
+
+ if let Some((offset, ch)) = filename_has_bad_chars(&dir_entry) {
+ e0001!(ch, dir_entry_str, offset);
+ }
+
+ // 1. dealing with symlinks
+ if ft == FType::Symlink {
+ match get_symlink(&dir_entry) {
+ Ok(None) => {
+ e0010!(dir_entry_str);
+ continue;
+ }
+ Err(e) => {
+ e0027!(dir_entry_str, e);
+ continue;
+ }
+ Ok(Some(p)) => {
+ let pd: String =
+ p.canonicalize().unwrap().to_string_lossy().to_string();
+ if !pd.starts_with(&root_absolute) {
+ e0030!(dir_entry_str, p.display());
+ continue;
+ }
+
+ let lc_dir_entry_str = dir_entry_str.to_lowercase();
+ if let Some(_dir_name) = filename(dir_entry_str) {
+ let lcnref = &mut lcnames;
+ lcnref
+ .entry(PathBuf::from(lc_dir_entry_str))
+ .or_insert_with(Vec::new)
+ .push((
+ PathBuf::from(dir_entry_str),
+ //FileKind::Symlink(dir_entry_str.into()),
+ FileKind::Symlink(pd.clone()),
+ ));
+ }
+ if is_readme(&dir_entry) {
+ readme_found = true;
+ file_names.insert(
+ p,
+ (
+ meta,
+ file_name,
+ ReadmeKind::Symlink(dir_entry_str.to_string()),
+ ),
+ );
+ }
+ continue;
+ }
+ }
+ }
+
+ let p = get_perms(&dir_entry.path());
+
+ // 2. dealing with directories
+ if ft == FType::Directory {
+ let lc_dir_entry_str = dir_entry_str.to_lowercase();
+ if let Some(_dir_name) = filename(dir_entry_str) {
+ let lcnref = &mut lcnames;
+
+ lcnref
+ .entry(PathBuf::from(lc_dir_entry_str))
+ .or_insert_with(Vec::new)
+ .push((PathBuf::from(dir_entry_str), FileKind::Directory));
+ }
+
+ if !owner_has(p, 5) || !others_have(p, 5) {
+ e0011!(dir_entry_str, perms_to_string(p));
+ if ARGS.correct_perms {
+ i0005!(&dir_entry_str);
+ set_perms(&dir_entry_str, 0o775);
+ }
+ }
+
+ match is_empty_directory(&dir_entry) {
+ Ok(true) => {
+ e0004!(dir_entry_str);
+ }
+ Err(e) => {
+ e0027!(dir_entry_str, e);
+ }
+ Ok(false) => (),
+ }
+ if is_hidden_directory(&dir_entry) {
+ e0006!(dir_entry_str);
+ }
+
+ if is_unwanted_directory(&dir_entry.file_name().to_str().unwrap()) {
+ e0018!(dir_entry_str);
+ }
+ continue;
+ }
+
+ // 3. dealing with regular files
+ if is_hidden(&file_name) {
+ e0007!(dir_entry_str);
+ }
+ if is_temporary_file(&dir_entry_str) {
+ e0008!(dir_entry_str);
+ }
+
+
+ if let Some(file_name) = filename(dir_entry_str) {
+ let doubleref = &mut doublenames;
+
+ doubleref
+ .entry(PathBuf::from(file_name))
+ .or_insert_with(Vec::new)
+ .push(PathBuf::from(dir_entry_str));
+ }
+
+
+ if is_readme(&dir_entry) {
+ // We want to deal with README files only if they are
+ // in the root directory of the package.
+ let f = format!(
+ "{}{}{}",
+ root,
+ // we have to pay attention if `root` ends already with '/'
+ if root.ends_with('/') { "" } else { "/" },
+ &file_name
+ );
+
+ if dir_entry_str == f {
+ readme_found = true;
+ file_names.insert(
+ dir_entry.path().to_path_buf(),
+ (meta, file_name.clone(), ReadmeKind::Yes),
+ );
+ } else {
+ file_names.entry(dir_entry.path().to_path_buf()).or_insert((
+ meta,
+ file_name.clone(),
+ ReadmeKind::No,
+ ));
+ }
+ } else {
+ file_names.entry(dir_entry.path().to_path_buf()).or_insert((
+ meta,
+ file_name.clone(),
+ ReadmeKind::No,
+ ));
+ }
+
+ let lc_dir_entry_str = dir_entry_str.to_lowercase();
+
+ let lcnref = &mut lcnames;
+
+ lcnref
+ .entry(PathBuf::from(lc_dir_entry_str))
+ .or_insert_with(Vec::new)
+ .push((PathBuf::from(dir_entry_str), FileKind::File));
+
+ }
+
+ Err(e) => {
+ eprintln!("{}", e);
+ }
+ }
+ }
+
+ if !readme_found {
+ e0009!();
+ }
+ }
+
+ let lc = LinkCheck::new(4, false);
+ let mut detective = filemagic::Filetype::new();
+
+ let mut sizes = HashMap::default();
+ let mut generated: HashMap<PathBuf, PathBuf> = HashMap::default();
+
+ {
+ // Processing a single file entry, with the "sizes" hashmap collecting
+ // same-size files. Entries are either Found::One or Found::Multiple,
+ // so that we can submit the first file's path as a hashing job when the
+ // first duplicate is found. Hashing each file is submitted as a job to
+ // the pool.
+ let mut process = |fsize, path: &PathBuf| {
+ let sizeref = &mut sizes;
+
+ let path = path.clone();
+ sizeref.entry(fsize).or_insert_with(Vec::new).push(path);
+ };
+ for (path, (meta, _file_name, is_readme)) in file_names.iter() {
+
+ //println!("DEBUG: path {:?}, file_name {:?}, is_readme: {} ", &path, &file_name, is_readme);
+ let dir_entry_str = match path.to_str() {
+ Some(d) => d,
+ None => {
+ e0031!(&path.to_string_lossy());
+ continue;
+ }
+ };
+
+ let fsize = meta.len();
+
+ // if it is an empty file we don't need
+ // to check further
+ if fsize == 0 {
+ e0005!(dir_entry_str);
+ continue;
+ }
+
+ let p = get_perms(&path);
+ if !owner_has(p, 4) {
+ e0002!(dir_entry_str, perms_to_string(p));
+ if ARGS.correct_perms {
+ fix_perms(&dir_entry_str);
+ }
+ continue;
+ }
+ let ftr = detective.analyze(&dir_entry_str);
+ //println!(">>> {:?}", ftr);
+ // we ignore errors from filetype recognition
+ if ftr.is_err() {
+ continue;
+ }
+
+ let ft = ftr.unwrap();
+
+ // DEBUG !readme_symlinked.contains(dir_entry_str)
+ if ReadmeKind::No != *is_readme {
+ if !check_readme(dir_entry_str, is_readme, &ft) {
+ continue;
+ }
+ if ARGS.urlcheck {
+ lc.check_urls(dir_entry_str);
+ }
+ }
+
+ match ft {
+ filemagic::Mimetype::Text | filemagic::Mimetype::TextCRLF => {
+ check_and_correct_perms(dir_entry_str, p);
+ let fext = get_extension_from_filename(&dir_entry_str);
+ if fext == Some("ins") || fext == Some("dtx") {
+ check_generated_files(&dir_entry_str, &mut generated);
+ }
+ match fext {
+ Some("bat") | Some("cmd") |
+ Some("nsh") | Some("reg") => (),
+ Some(_) | None => {
+ if ft == filemagic::Mimetype::TextCRLF {
+ e0012!(dir_entry_str);
+ if ARGS.correct_crlf {
+ fix_crlf(dir_entry_str);
+ }
+ }
+ }
+ };
+ }
+
+ filemagic::Mimetype::BOM(b) => {
+ //println!("{}: {} with BOM detected", dir_entry_str, b.as_ref());
+ w0004!(dir_entry_str, b.as_ref());
+ check_and_correct_perms(dir_entry_str, p);
+ }
+ filemagic::Mimetype::Binary
+ | filemagic::Mimetype::Script
+ | filemagic::Mimetype::ScriptCRLF => {
+ if !owner_has(p, 4) || !others_have(p, 4) {
+ e0002!(dir_entry_str, perms_to_string(p));
+ };
+ if ARGS.correct_perms {
+ fix_perms(&dir_entry_str);
+ }
+ }
+ filemagic::Mimetype::Pdf => {
+ check_and_correct_perms(dir_entry_str, p);
+ let ret = is_pdf_ok(&dir_entry_str);
+ if !ret.status {
+ e0017!(dir_entry_str);
+ if let Some(output) = ret.output {
+ println!("{}", &output);
+ };
+ }
+ }
+
+ filemagic::Mimetype::Archive | filemagic::Mimetype::Zip => {
+ w0001!(dir_entry_str);
+ check_and_correct_perms(dir_entry_str, p);
+ }
+
+ filemagic::Mimetype::Data => check_and_correct_perms(dir_entry_str, p),
+ _ => continue,
+ }
+ if others_match(p, 0) {
+ e0002!(dir_entry_str, perms_to_string(p));
+ if ARGS.correct_perms {
+ i0005!(&dir_entry_str);
+ set_perms(&dir_entry_str, 0o664);
+ }
+ }
+ if !(ARGS.ignore_dupes && tds_zip.is_none()) {
+ process(fsize, &path);
+ }
+ }
+ }
+
+ print_casefolding(&lcnames);
+ print_generated(&doublenames, &generated);
+ print_doublenames(&doublenames);
+
+ if ARGS.ignore_dupes && tds_zip.is_none() {
+ return None;
+ }
+
+ // Set up thread pool for the task to hash a file. Number of CPUs + 1 has been
+ // found to be a good pool size, likely since the walker thread should be
+ // doing mostly IO.
+ let mut pool = Pool::new(num_cpus::get() as u32 + 1);
+
+ let mut hashes: HashMap<(u64, Vec<u8>), DupPath> = HashMap::default();
+ pool.scoped(|scope| {
+ let (tx, rx) = channel();
+
+ let hashref = &mut hashes;
+ scope.execute(move || {
+ for (size, path, hash) in rx.iter() {
+ hashref
+ .entry((size, hash))
+ .or_insert_with(DupPath::new)
+ .push(path);
+ }
+ });
+
+ for size in sizes.keys() {
+ let paths = &sizes[size];
+ if paths.len() == 1 && tds_zip.is_none() {
+ continue;
+ };
+
+ for p in &sizes[size] {
+ let txc = tx.clone();
+ scope.execute(move || hash_file(*size, p.to_path_buf(), &txc));
+ }
+ }
+ });
+
+ if !ARGS.ignore_dupes {
+ print_duplicates(&hashes);
+ }
+ Some(hashes)
+}
+
+fn print_casefolding(hashes: &HashMap<PathBuf, Vec<(PathBuf, FileKind)>>) {
+
+ for (k, eles) in hashes.iter() {
+ //println!("pcf: {:?}, {:?}", k, &eles);
+ if eles.len() == 1 {
+ continue;
+ }
+
+ e0025!(k.display());
+
+ for (p, ty) in eles {
+ println!(" >>> {} ({})", p.display(), ty);
+ }
+ }
+}
+
+fn print_generated(hashes: &HashMap<PathBuf, Vec<PathBuf>>, generated: &HashMap<PathBuf, PathBuf>) {
+ for (k, gen) in generated.iter() {
+ if hashes.contains_key(k) {
+ let v = &hashes[k];
+ for fname in v {
+ e0019!(fname.to_str().unwrap(), gen.to_str().unwrap());
+ }
+ }
+ }
+}
+
+fn print_doublenames(hashes: &HashMap<PathBuf, Vec<PathBuf>>) {
+ for (k, paths) in hashes.iter() {
+ if paths.len() == 1 {
+ continue;
+ }
+ let ks = k.to_str().unwrap();
+ if ks == "README"
+ || ks == "README.txt"
+ || ks == "README.md"
+ || ks == "Makefile"
+ || ks == "Makefile.am"
+ || ks == "Makefile.in"
+ || ks == "makefile"
+ {
+ continue;
+ }
+
+ w0003!(k.to_str().unwrap());
+ // println!(":: {}", k.display());
+
+ for p in paths {
+ println!(" >>> {}", p.display());
+ }
+ }
+}
+
+fn show_tmp_endings() {
+ i0006!();
+ for (t, c) in temp_file_endings() {
+ println!("{:23} {}", t, c);
+ }
+}