path: root/support/pkgcheck/src/main.rs

extern crate pest;
#[macro_use]
extern crate pest_derive;

mod gparser;

#[macro_use]
mod messages;
use messages::{explains, explains_all};

#[macro_use]
mod utils;
mod linkcheck;

mod recode;
use recode::crlf2lf;

mod filemagic;

use linkcheck::LinkCheck;
use std::fmt;
use std::fmt::Display;
use std::str;
use utils::*;

use std::os::unix::fs::MetadataExt;

use scoped_threadpool::Pool;


use tempdir::TempDir;
#[macro_use]
extern crate lazy_static;
use colored::*;

use regex::Regex;

use std::fs::File;
use std::fs::Metadata;
use std::io::prelude::*;
use std::io::BufReader;
use std::os::unix::fs::FileTypeExt;
use std::{fs, process};
// use std::os::unix::fs::PermissionsExt;

use std::ffi::OsStr;
use std::io::{self, Read};
use std::path::Path;
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering};

use blake2::{Blake2b, Digest};
use fnv::{FnvHashMap as HashMap, FnvHashSet as HashSet};
use std::sync::mpsc::{channel, Sender};
use structopt::StructOpt;
#[cfg(unix)]
use walkdir::{DirEntry, WalkDir};

fn err(path: &PathBuf, err: &io::Error) {
    e0027!(path.display(), err);
}

type HashSender = Sender<(u64, PathBuf, Vec<u8>)>;
//type DupeSender = Sender<(u64, Vec<PathBuf>)>;

const BLOCKSIZE: usize = 4096;

fn hash_file_inner(path: &PathBuf) -> io::Result<Vec<u8>> {
    let mut buf = [0u8; BLOCKSIZE];
    let mut fp = File::open(&path)?;
    let mut digest = Blake2b::default();
    // When we compare byte-by-byte, we don't need to hash the whole file.
    // Instead, hash a block of 4kB, skipping 100kB.
    loop {
        match fp.read(&mut buf)? {
            0 => break,
            n => digest.input(&buf[..n]),
        }
    }
    Ok(digest.result().to_vec())
}

fn hash_file(fsize: u64, path: PathBuf, tx: &HashSender) {
    match hash_file_inner(&path) {
        Ok(hash) => tx.send((fsize, path, hash)).unwrap(),
        Err(e) => err(&path, &e),
    }
}

fn fix_perms(entry: &str) {
    i0005!(entry);
    let rc = run_cmd("/bin/chmod", &["-v", "ug=rwX,o=rX", entry]);

    if rc.status {
        if let Some(op) = rc.output {
            print!("{}", op);
        }
    }
}

fn filename_has_bad_chars(entry: &DirEntry) -> Option<(usize, char)> {
    let s = entry.file_name().to_str().unwrap();

    for (i, c) in s.char_indices() {
        match c {
            '\x00'..='\x1f' => return Some((i, c)),
            ' ' | '!' | '"' | '#' | '$' | '%' | '&' | '\'' => return Some((i, c)),
            '(' | ')' | '*' => return Some((i, c)),
            '+' | ',' | '-' | '.' => (),
            '0'..='9' => (),
            ':' | ';' | '<' | '=' | '>' | '?' | '@' => return Some((i, c)),
            'A'..='Z' => (),
            '[' | '\\' | ']' | '^' => return Some((i, c)),
            '_' => (),
            '`' => return Some((i, c)),
            'a'..='z' => (),
            '{' | '|' | '}' | '~' => return Some((i, c)),
            '/' => return None,
            _ => return Some((i, c)),
        };
    }
    None
}

// returns false if CRLF, otherwise true
fn fix_crlf(fname: &str) -> bool {
    i0004!(fname);
    //run_cmd("/usr/bin/fromdos", &["-p", "-e", "-v", fname]).status
    match crlf2lf(fname) {
        Ok(_) => {
            i0007!(fname);
            true
        }
        Err(e) => {
            e0027!(fname, e);
            false
        }
    }
}

fn check_readme(dir_entry: &str, is_readme: &ReadmeKind, ft: &filemagic::Mimetype) -> bool {
    let msg_name = if let ReadmeKind::Symlink(s) = is_readme {
        format!("{} (symlinked from {})", dir_entry, &s)
    } else {
        dir_entry.to_string()
    };

    match ft {
        filemagic::Mimetype::Pdf
        | filemagic::Mimetype::Binary
        | filemagic::Mimetype::Archive
        | filemagic::Mimetype::Zip => {
            e0003!(msg_name);
            return false;
        }
        filemagic::Mimetype::BOM(b) => {
            e0029!(msg_name, b.as_ref());
            return false;
        }
        filemagic::Mimetype::Text | filemagic::Mimetype::TextCRLF => match File::open(dir_entry) {
            Ok(f) => {
                if !check_readme_inner(&msg_name, &f) {
                    return false;
                }
            }
            Err(e) => {
                e0027!(msg_name, e);
                return false;
            }
        },
        _ => (),
    }
    true
}

fn check_readme_inner(fname: &str, f: &std::fs::File) -> bool {
    let reader = BufReader::new(f);

    let lines = reader.split(b'\n').map(|l| l.unwrap());
    let mut result = true;

    for (lineno, line) in lines.enumerate() {
        if let Err(e) = String::from_utf8(line.clone()) {
            e0021!(fname, lineno + 1, e);
            result = false;
        }
    }
    result
}

fn is_readme(entry: &DirEntry) -> bool {
    let r = entry.file_name().to_str().unwrap();

    match r {
        "README" | "README.txt" | "README.md" => true,
        _ => false,
    }
}

fn get_devno(meta: &Metadata) -> u64 {
    meta.dev()
}

fn _get_devno(entry: &DirEntry) -> u64 {
    let meta = fs::metadata(entry.path().to_str().unwrap());
    match meta {
        Ok(m) => m.dev(),
        _ => 0,
    }
}

#[derive(StructOpt, Debug)]
#[structopt(about = "A checker for uploaded packages to CTAN.")]
struct Args {
    #[structopt(short = "I", long = "ignore-dupes", help = "Ignore dupes")]
    ignore_dupes: bool,
    #[structopt(short = "v", long = "verbose", help = "Verbose operation?")]
    verbose: bool,
    #[structopt(short = "L", long = "correct-crlf", help = "Correct CRLF line endings")]
    correct_crlf: bool,
    #[structopt(short = "C", long = "correct-perms", help = "Correct permissions")]
    correct_perms: bool,
    #[structopt(long = "no-colors", help = "Don't display messages in color")]
    no_colors: bool,
    #[structopt(long = "urlcheck", help = "Check URLs found in README files")]
    urlcheck: bool,
    #[structopt(short = "T", long = "tds-zip", help = "tds zip archive", group = "tds")]
    tds_zip: Option<String>,
    #[structopt(
        short = "e",
        long = "explain",
        help = "Explain error or warning message",
        group = "only_one"
    )]
    explain: Option<String>,
    #[structopt(
        long = "explain-all",
        help = "Explains all error or warning messages",
        group = "only_one"
    )]
    explain_all: bool,
    #[structopt(
        long = "show-temp-endings",
        help = "Show file endings for temporary files",
        group = "only_one"
    )]
    show_tmp_endings: bool,
    #[structopt(short = "d", long = "package-dir", help = "Package directory")]
    pkg_dir: Option<String>,
}

// We take care to avoid visiting a single inode twice,
// which takes care of (false positive) hardlinks.
#[cfg(unix)]
fn check_inode(set: &mut HashSet<(u64, u64)>, meta: &Metadata) -> bool {
    set.insert((get_devno(meta), meta.ino()))
}

#[cfg(not(unix))]
fn check_inode(_: &mut HashSet<u64>, _: &Metadata) -> bool {
    true
}

lazy_static! {
    static ref ARGS: Args = Args::from_args();
    static ref ERROR_OCCURED: AtomicBool = AtomicBool::new(false);
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum DPath {
    Both(PathBuf),
    Tds(PathBuf),
}

impl fmt::Display for DPath {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self {
            DPath::Both(p) => write!(f, "{}", p.display()),
            DPath::Tds(p) => write!(f, "{}", p.display()),
        }
    }
}

#[derive(Default)]
pub struct DupPath {
    len: usize,
    plen: usize,
    dupes: Vec<DPath>,
}

impl DupPath {
    pub fn new() -> DupPath {
        DupPath {
            len: 0,
            plen: 0,
            dupes: Vec::new(),
        }
    }

    pub fn push(&mut self, pb: PathBuf) {
        let pname = pb.to_string_lossy();
        self.len += 1;
        if pname.ends_with(".tfm") {
            self.dupes.push(DPath::Tds(pb.clone()));
        } else {
            self.plen += 1;
            self.dupes.push(DPath::Both(pb.clone()));
        }
    }
}

type Hashes = HashMap<(u64, Vec<u8>), DupPath>;

fn main() {
    match &ARGS.explain {
        None => (),
        Some(e) => {
            explains(&e);
            process::exit(0);
        }
    }

    if ARGS.explain_all {
        explains_all();
        process::exit(0);
    }

    if ARGS.show_tmp_endings {
        show_tmp_endings();
        process::exit(0);
    }

    let pkg_dir = match &ARGS.pkg_dir {
        None => {
            f0001!();
            process::exit(1);
        }
        Some(d) => {
            if !exists_dir(&d) {
                f0002!(d);
                process::exit(1);
            };
            d
        }
    };

    let tds_zip = &ARGS.tds_zip;
    let pkg_name = match tds_zip {
        None => None,
        Some(tz) => {
            let pn = check_tds_archive_name(tds_zip);
            if !exists_file(&tz) {
                f0003!(&tz);
                process::exit(1);
            }
            let mut fmagic = filemagic::Filetype::new();
            match fmagic.analyze(&tz) {
                Ok(filemagic::Mimetype::Zip) => (),
                _ => {
                    f0004!(&tz);
                    process::exit(1)
                }
            };
            pn
        }
    };

    if let Some(hashes) = check_package(pkg_dir, tds_zip) {
        if tds_zip.is_some() {
            if let Some(pn) = pkg_name {
                if let Some(s) = ARGS.tds_zip.as_ref() {
                    match check_tds_archive(&pn, s, &hashes) {
                        Ok(_) => (),
                        Err(e) => {
                            e0027!(s, e);
                        }
                    }
                }
            }
        }
    }

    if ERROR_OCCURED.load(Ordering::Relaxed) {
        process::exit(1);
    } else {
        process::exit(0);
    }
}

fn print_duplicates(hashes: &Hashes) {
    let mut total_dupes = 0;
    let mut total_files = 0;
    let mut total_size = 0;

    let mut header_printed = false;
    for (k, paths) in hashes.iter() {
        let (sz, _hash) = k;

        if paths.plen <= 1 {
            total_files += 1;
            total_size += sz;
            continue;
        } else if !header_printed {
            w0002!();
            header_printed = true;
        }

        total_files += paths.plen;
        total_size += sz * (paths.plen - 1) as u64;
        total_dupes += (paths.plen - 1) as u64;

        println!("Size: {}", sz);
        for p in &paths.dupes {
            if let DPath::Both(p) = p {
                let ps = p.as_path().to_str().unwrap();
                println!("   >>> {}", ps);
            }
        }
        println!();
    }

    if ARGS.verbose && total_dupes > 0 {
        println!("Duplicate statistics");
        println!("  Found {} duplicate files", total_files);
        println!("  Size of duplicate files: {}", total_size);
    }
}

//#[derive(Debug, PartialEq, Eq)]
#[derive(Debug, PartialEq, Eq)]
pub enum FType {
    Regular,
    Directory,
    Symlink,
    BlockDevice,
    CharDevice,
    Fifo,
    Socket,
    Error(String),
}

fn get_filetype(entry: &DirEntry) -> FType {
    match entry.metadata() {
        Ok(mt) => {
            let ft = mt.file_type();
            if ft.is_symlink() {
                return FType::Symlink;
            }
            if ft.is_dir() {
                return FType::Directory;
            }
            if ft.is_block_device() {
                return FType::BlockDevice;
            }
            if ft.is_char_device() {
                return FType::CharDevice;
            }
            if ft.is_fifo() {
                return FType::Fifo;
            }
            if ft.is_socket() {
                return FType::Socket;
            }
            FType::Regular
        }
        Err(e) => FType::Error(format!("{}", e)),
    }
}

//
// read file into buffer[u8]
// then convert into string
//
fn check_generated_files(entry: &str, generated: &mut HashMap<PathBuf, PathBuf>) {
    // unwrap() is ok here as we only call this function for files,
    // specifically .ins or .dtx files
    let entry_fname = filename(entry).unwrap().to_string();
    let entry_base = &entry_fname[0..entry_fname.len() - 4];

    let fhdl = File::open(&entry);
    match fhdl {
        Ok(mut f) => {
            let mut buf = Vec::new();

            match f.read_to_end(&mut buf) {
                Ok(_bytes_read) => {
                    if let Some(found) =
                        gparser::parse_generate(&String::from_utf8_lossy(&buf.clone()))
                    {
                        for fname in found {
                            // As we request a README in the top level directory of
                            // a package we ignore if a README was generated by an
                            // .ins or .dtx file
                            // CAVEAT: If this happens in a subdirectory it could be an error!!!!
                            if fname == "README" || fname == "README.txt" || fname == "README.md" {
                                continue;
                            }

                            // If the filename in the 'file{} statement is jobname.<ext>
                            // we replace jobname with the filename before we investigate further
                            let fname_x = if fname.starts_with("\\jobname") {
                                let ext = match get_extension_from_filename(&fname) {
                                    Some(e) => e,
                                    None => "",
                                };
                                format!("{}.{}", entry_base, ext)
                            } else {
                                fname.to_string()
                            };

                            let pb = PathBuf::from(fname_x);

                            generated.entry(pb).or_insert_with(|| PathBuf::from(entry));
                        }
                    };
                }
                Err(e) => println!("Error reading file {}: {:?}", entry, e),
            }
        }
        Err(e) => println!("Error opening file {}: {:?}", entry, e),
    }
}

fn x_bit_set(p: u32) -> bool {
    let p1 = p & 0o7777;
    p1 & 0o111 != 0
}

// checks that archive name is in the format x.tds.zip and returns:
//  None if not
//  Option<x> if x.tds.zip
// Caveat: we currently don't know if x is really the package name.
fn check_tds_archive_name(tds_zip: &Option<String>) -> Option<String> {
    match tds_zip {
        None => None,
        Some(tz) => {
            // 8 for ".tds.zip", 1 means a package name is at least a character
            if tz.len() < 8 + 1 || !tz.ends_with(".tds.zip") {
                f0005!(tz);
                process::exit(1);
            }
            let mut pname = String::from(utils::basename(&tz));
            let plen = pname.len();
            pname.truncate(plen - 8);
            Some(pname.to_string())
        }
    }
}

fn unzip_tds_archive(tds_zip: &str, tmp_dir: &str) {
    match run_cmd("/usr/bin/unzip", &["-q", "-d", tmp_dir, &tds_zip]) {
        CmdReturn { status: true, .. } => (),
        CmdReturn {
            status: false,
            output: out,
        } => {
            if let Some(o) = out {
                e0033!(&tds_zip, o);
            } else {
                e0033!(&tds_zip, "<no error given");
            }
            process::exit(1);
        }
    }
}

fn check_tds_archive(
    pkg_name: &str,
    tds_zip: &str,
    hashes: &HashMap<(u64, Vec<u8>), DupPath>,
) -> Result<(), io::Error> {
    i0003!(tds_zip);

    let dir_entry = Path::new(tds_zip);
    let p = get_perms(&dir_entry);
    if !owner_has(p, 4) || !others_have(p, 4) || x_bit_set(p) {
        e0024!(tds_zip, perms_to_string(p));
        if ARGS.correct_perms {
            i0005!(&tds_zip);
            set_perms(&tds_zip, 0o664);
        }
    };

    // We have a discrepancy between
    // TDS zip archive name: babel-base.tds.zip
    // and package name: babel
    // which yields a false test when checking for the package name
    // the path names of the TDS zip archive files. 
    //    Therefore, we correct here.
    let real_pkg_name = if pkg_name == "babel-base" {
        "babel"
    } else {
        pkg_name
    };


    let tmp_dir = TempDir::new("pkgcheck")?;
    let tmp_dir_offset = tmp_dir.path().to_str().unwrap().len() + 1;
    unzip_tds_archive(tds_zip, tmp_dir.path().to_str().unwrap());

    let tds_toplevel_dirs: HashSet<String> = [
        "tex", "fonts", "metafont", "metapost", "bibtex", "scripts", "doc", "source",
    ]
    .iter()
    .map(|&s| s.to_string())
    .collect();

    for entry in fs::read_dir(tmp_dir.path())? {
        let entry = entry?;
        let path = entry.path();

        match path.file_stem() {
            None => (),
            Some(file_stem) => {
                let fs = file_stem.to_str().expect("bad file name");
                if !tds_toplevel_dirs.contains(fs) {
                    if path.is_dir() {
                        e0020!(fs);
                    } else {
                        e0034!(fs);
                    }
                }
            }
        }

        // if path.is_dir() {
        //     match path.file_stem() {
        //         None => (),
        //         Some(file_stem) => {
        //             let fs = file_stem.to_str().expect("bad file name");
        //             if !tds_toplevel_dirs.contains(fs) {
        //                 e0020!(fs);
        //             }
        //         }
        //     }
        // } else {
        //     println!("mist");
        //     e0020!(path.display());
        // }
    }

    // in order to compare the package files with the content of the
    // tds zip archive we need to checksum the files in the tds zip
    // archive.

    let mut sizes = HashMap::default();
    let mut pool = Pool::new(num_cpus::get() as u32 + 1);
    {
        // Processing a single file entry, with the "sizes" hashmap collecting
        // same-size files.  Entries are either Found::One or Found::Multiple,
        // so that we can submit the first file's path as a hashing job when the
        // first duplicate is found.  Hashing each file is submitted as a job to
        // the pool.
        let mut process = |fsize, dir_entry: &DirEntry| {
            let path = dir_entry.path().to_path_buf();
            let sizeref = &mut sizes;

            sizeref.entry(fsize).or_insert_with(Vec::new).push(path);
        };

        let mut map_files_found = false;
        let mut map_dvips_found = false;

        for dir_entry in WalkDir::new(tmp_dir.path().to_str().unwrap()).follow_links(false) {
            match dir_entry {
                Ok(dir_entry) => {
                    //let dir_entry_str = dir_entry.path().to_str().unwrap();

                    let ft = get_filetype(&dir_entry);

                    if ft == FType::Directory {
                        continue;
                    }

                    if let FType::Error(e) = ft {
                        e0023!(e);
                        continue;
                    }

                    match dir_entry.metadata() {
                        Ok(meta) => {
                            let fsize = meta.len();
                            process(fsize, &dir_entry);

                            let dir_entry_str = match dir_entry.path().to_str() {
                                Some(d) => d,
                                None => {
                                    e0031!(dir_entry.path().to_string_lossy());
                                    continue;
                                }
                            };
                            let dir_entry_display = &dir_entry_str[tmp_dir_offset..];

                            if is_temporary_file(&dir_entry_str) {
                                e0008!(dir_entry_display);
                            }
                            if fsize > 40 * 1024 * 1024 {
                                w0006!(dir_entry_display, fsize / 1024 / 1024);
                            }

                            // if the path doesn't contain a man page...
                            if dir_entry_str.find("/man/").is_none() {
                                let pkg_name_s = format!("/{}/", real_pkg_name);
                                // ...then we want to have the package name in the path
                                if dir_entry_str.find(&pkg_name_s).is_none() {
                                    e0028!(real_pkg_name, dir_entry_display);
                                }
                            }

                            if dir_entry_str.ends_with(".map") {
                                map_files_found = true;
                                let re: Regex = Regex::new(r"fonts[/]map[/]dvips[/]").unwrap();
                                if re.is_match(dir_entry_str) {
                                    map_dvips_found = true;
                                }

                                // if ( $fname =~ m{ fonts [/] map [/] dvips [/] }xms ) {
                                //     $map_dvips_found = 1;
                                // }
                            }
                        }
                        Err(e) => {
                            e0027!(dir_entry.path().display(), e);
                        }
                    }
                }
                Err(e) => {
                    eprintln!("{}", e);
                }
            }
        }

        if map_files_found && !map_dvips_found {
            println!( "Map files found for package {} but none of them is in a path starting with fonts/map/dvips", pkg_name);
        }
    };

    let mut tds_hashes: HashMap<(u64, Vec<u8>), Vec<PathBuf>> = HashMap::default();
    pool.scoped(|scope| {
        let (tx, rx) = channel();

        let hashref = &mut tds_hashes;
        scope.execute(move || {
            for (size, path, hash) in rx.iter() {
                hashref
                    .entry((size, hash))
                    .or_insert_with(Vec::new)
                    .push(path);
            }
        });

        for size in sizes.keys() {
            for p in &sizes[size] {
                let txc = tx.clone();
                scope.execute(move || hash_file(*size, p.to_path_buf(), &txc));
            }
        }
    });

    tmp_dir.close()?;
    // now check if each package file is in the tds archive
    for (k, paths) in hashes.iter() {
        if !tds_hashes.contains_key(k) {
            let p = &paths.dupes[0];
            e0026!(p);
        }
    }

    Ok(())
}

fn get_extension_from_filename(filename: &str) -> Option<&str> {
    Path::new(filename).extension().and_then(OsStr::to_str)
}

fn found_unwanted_filetype(fname: &str, ft: &FType) -> bool {
    match ft {
        FType::Socket => {
            e0013!(fname);
            true
        }
        FType::Fifo => {
            e0014!(fname);
            true
        }
        FType::BlockDevice => {
            e0015!(fname);
            true
        }
        FType::CharDevice => {
            e0016!(fname);
            true
        }
        FType::Error(e) => {
            e0023!(e);
            true
        }
        _ => false,
    }
}

fn check_and_correct_perms(dir_entry: &str, p: u32) {
    if owner_has(p, 5) || !others_have(p, 4) {
        e0002!(dir_entry, perms_to_string(p));
        if ARGS.correct_perms {
            i0005!(&dir_entry);
            set_perms(&dir_entry, 0o664);
        }
    };
}


#[derive(Debug, Clone, PartialEq)]
enum FileKind {
    File,
    Directory,
    Symlink(String),
}


impl Display for FileKind {
    fn fmt(&self, f: &mut ::std::fmt::Formatter) -> Result<(), ::std::fmt::Error> {
        match *self {
            FileKind::File => f.write_str("file"),
            FileKind::Directory => f.write_str("directory"),
            FileKind::Symlink(_) => f.write_str("symlink"),
        }
    }
}



#[derive(Debug, Clone, PartialEq)]
enum ReadmeKind {
    No,
    Yes,
    Symlink(String),
}

fn check_package(root: &str, tds_zip: &Option<String>) -> Option<Hashes> {
    //let mut urls = HashMap::default(); //: <String,UrlStatus>

    let mut lcnames: HashMap<PathBuf, Vec<(PathBuf, FileKind)>> = HashMap::default();

    let mut doublenames: HashMap<PathBuf, Vec<PathBuf>> = HashMap::default();

    let mut inodes = HashSet::default();

    i0002!(root);
    // This hash contains all package file names.
    //
    //  PathBuf: the full path starting at the directory specified at the command line
    //  Metadata: the meta data of the file
    //  String: the file name without any directory part
    //  ReadmeKind: is it a certain README, file or symlink?
    //      A special case of a README file is a file with has a different name but
    //      was poinded to by a symlink. Example: README --> README.rst
    let mut file_names: HashMap<PathBuf, (Metadata, String, ReadmeKind)> = HashMap::default();
    {
        let mut readme_found = false;

        let root_absolute = PathBuf::from(root)
            .canonicalize()
            .unwrap()
            .to_string_lossy()
            .to_string();
        for dir_entry in WalkDir::new(root).follow_links(false) {
            match dir_entry {
                Ok(dir_entry) => {
                    let dir_entry_str = match dir_entry.path().to_str() {
                        Some(d) => d,
                        None => {
                            e0031!(dir_entry.path().to_string_lossy());
                            continue;
                        }
                    };
                    let meta = match dir_entry.metadata() {
                        Ok(meta) => meta,
                        Err(e) => {
                            e0023!(e);
                            continue;
                        }
                    };

                    if !check_inode(&mut inodes, &meta) {
                        continue;
                    }

                    // this is the file_name without the directory part
                    // unwrap() is ok here as we covered potential UTF-8 related errors
                    // above in the definition of dir_entry_str
                    let file_name = dir_entry.file_name().to_str().unwrap().to_string();

                    let ft = get_filetype(&dir_entry);

                    if found_unwanted_filetype(dir_entry_str, &ft) {
                        continue;
                    }

                    if let Some((offset, ch)) = filename_has_bad_chars(&dir_entry) {
                        e0001!(ch, dir_entry_str, offset);
                    }

                    // 1. dealing with symlinks
                    if ft == FType::Symlink {
                        match get_symlink(&dir_entry) {
                            Ok(None) => {
                                e0010!(dir_entry_str);
                                continue;
                            }
                            Err(e) => {
                                e0027!(dir_entry_str, e);
                                continue;
                            }
                            Ok(Some(p)) => {
                                let pd: String =
                                    p.canonicalize().unwrap().to_string_lossy().to_string();
                                if !pd.starts_with(&root_absolute) {
                                    e0030!(dir_entry_str, p.display());
                                    continue;
                                }

                                let lc_dir_entry_str = dir_entry_str.to_lowercase();
                                if let Some(_dir_name) = filename(dir_entry_str) {
                                    let lcnref = &mut lcnames;
                                    lcnref
                                        .entry(PathBuf::from(lc_dir_entry_str))
                                        .or_insert_with(Vec::new)
                                        .push((
                                            PathBuf::from(dir_entry_str),
                                            //FileKind::Symlink(dir_entry_str.into()),
                                            FileKind::Symlink(pd.clone()),
                                        ));
                                }
                                if is_readme(&dir_entry) {
                                    readme_found = true;
                                    file_names.insert(
                                        p,
                                        (
                                            meta,
                                            file_name,
                                            ReadmeKind::Symlink(dir_entry_str.to_string()),
                                        ),
                                    );
                                }
                                    continue;
                            }
                        }
                    }

                    let p = get_perms(&dir_entry.path());

                    // 2. dealing with directories
                    if ft == FType::Directory {
                        let lc_dir_entry_str = dir_entry_str.to_lowercase();
                        if let Some(_dir_name) = filename(dir_entry_str) {
                            let lcnref = &mut lcnames;

                            lcnref
                                .entry(PathBuf::from(lc_dir_entry_str))
                                .or_insert_with(Vec::new)
                                .push((PathBuf::from(dir_entry_str), FileKind::Directory));
                        }

                        if !owner_has(p, 5) || !others_have(p, 5) {
                            e0011!(dir_entry_str, perms_to_string(p));
                            if ARGS.correct_perms {
                                i0005!(&dir_entry_str);
                                set_perms(&dir_entry_str, 0o775);
                            }
                        }

                        match is_empty_directory(&dir_entry) {
                            Ok(true) => {
                                e0004!(dir_entry_str);
                            }
                            Err(e) => {
                                e0027!(dir_entry_str, e);
                            }
                            Ok(false) => (),
                        }
                        if is_hidden_directory(&dir_entry) {
                            e0006!(dir_entry_str);
                        }

                        if is_unwanted_directory(&dir_entry.file_name().to_str().unwrap()) {
                            e0018!(dir_entry_str);
                        }
                        continue;
                    }

                    // 3. dealing with regular files
                    if is_hidden(&file_name) {
                        e0007!(dir_entry_str);
                    }
                    if is_temporary_file(&dir_entry_str) {
                        e0008!(dir_entry_str);
                    }


                    if let Some(file_name) = filename(dir_entry_str) {
                        let doubleref = &mut doublenames;

                        doubleref
                            .entry(PathBuf::from(file_name))
                            .or_insert_with(Vec::new)
                            .push(PathBuf::from(dir_entry_str));
                    }

                    // detecting large files in package directory

                    if is_readme(&dir_entry) {
                        // We want to deal with README files only if they are
                        // in the root directory of the package.
                        let f = format!(
                            "{}{}{}",
                            root,
                            // we have to pay attention if `root` ends already with '/'
                            if root.ends_with('/') { "" } else { "/" },
                            &file_name
                        );

                        if dir_entry_str == f {
                            readme_found = true;
                            file_names.insert(
                                dir_entry.path().to_path_buf(),
                                (meta, file_name.clone(), ReadmeKind::Yes),
                            );
                        } else {
                            file_names.entry(dir_entry.path().to_path_buf()).or_insert((
                                meta,
                                file_name.clone(),
                                ReadmeKind::No,
                            ));
                        }
                    } else {
                        file_names.entry(dir_entry.path().to_path_buf()).or_insert((
                            meta,
                            file_name.clone(),
                            ReadmeKind::No,
                        ));
                    }

                    let lc_dir_entry_str = dir_entry_str.to_lowercase();

                    let lcnref = &mut lcnames;

                    lcnref
                        .entry(PathBuf::from(lc_dir_entry_str))
                        .or_insert_with(Vec::new)
                        .push((PathBuf::from(dir_entry_str), FileKind::File));

                }

                Err(e) => {
                    eprintln!("{}", e);
                }
            }
        }

        if !readme_found {
            e0009!();
        }
    }

    let lc = LinkCheck::new(4, false);
    let mut detective = filemagic::Filetype::new();

    let mut sizes = HashMap::default();
    let mut generated: HashMap<PathBuf, PathBuf> = HashMap::default();

    {
        // Processing a single file entry, with the "sizes" hashmap collecting
        // same-size files.  Entries are either Found::One or Found::Multiple,
        // so that we can submit the first file's path as a hashing job when the
        // first duplicate is found.  Hashing each file is submitted as a job to
        // the pool.
        let mut process = |fsize, path: &PathBuf| {
            let sizeref = &mut sizes;

            let path = path.clone();
            sizeref.entry(fsize).or_insert_with(Vec::new).push(path);
        };
        for (path, (meta, _file_name, is_readme)) in file_names.iter() {

            //println!("DEBUG: path {:?}, file_name {:?}, is_readme: {} ", &path, &file_name, is_readme);
            let dir_entry_str = match path.to_str() {
                Some(d) => d,
                None => {
                    e0031!(&path.to_string_lossy());
                    continue;
                }
            };

            let fsize = meta.len();

            // if it is an empty file we don't need
            // to check further
            if fsize == 0 {
                e0005!(dir_entry_str);
                continue;
            }
            if fsize > 40 * 1024 * 1024 {
                w0005!(dir_entry_str, fsize / 1024 / 1024);
                continue;
            }

            let p = get_perms(&path);
            if !owner_has(p, 4) {
                e0002!(dir_entry_str, perms_to_string(p));
                if ARGS.correct_perms {
                    fix_perms(&dir_entry_str);
                }
                continue;
            }
            let ftr = detective.analyze(&dir_entry_str);
            //println!(">>> {:?}", ftr);
            // we ignore errors from filetype recognition
            if ftr.is_err() {
                continue;
            }

            let ft = ftr.unwrap();

            // DEBUG !readme_symlinked.contains(dir_entry_str)
            if ReadmeKind::No != *is_readme {
                if !check_readme(dir_entry_str, is_readme, &ft) {
                    continue;
                }
                if ARGS.urlcheck {
                    lc.check_urls(dir_entry_str);
                }
            }

            match ft {
                filemagic::Mimetype::Text | filemagic::Mimetype::TextCRLF => {
                    check_and_correct_perms(dir_entry_str, p);
                    let fext = get_extension_from_filename(&dir_entry_str);
                    if fext == Some("ins") || fext == Some("dtx") {
                        check_generated_files(&dir_entry_str, &mut generated);
                    }
                    match fext {
                        Some("bat") | Some("cmd") |
                        Some("nsh") | Some("reg") => (),
                        Some(_) | None => {
                            if ft == filemagic::Mimetype::TextCRLF {
                                e0012!(dir_entry_str);
                                if ARGS.correct_crlf {
                                    fix_crlf(dir_entry_str);
                                }
                            }
                        }
                    };
                }

                filemagic::Mimetype::BOM(b) => {
                    //println!("{}: {} with BOM detected", dir_entry_str, b.as_ref());
                    w0004!(dir_entry_str, b.as_ref());
                    check_and_correct_perms(dir_entry_str, p);
                }
                filemagic::Mimetype::Binary
                | filemagic::Mimetype::Script
                | filemagic::Mimetype::ScriptCRLF => {
                    if !owner_has(p, 4) || !others_have(p, 4) {
                        e0002!(dir_entry_str, perms_to_string(p));
                    };
                    if ARGS.correct_perms {
                        fix_perms(&dir_entry_str);
                    }
                }
                filemagic::Mimetype::Pdf => {
                    check_and_correct_perms(dir_entry_str, p);
                    let ret = is_pdf_ok(&dir_entry_str);
                    if !ret.status {
                        e0017!(dir_entry_str);
                        if let Some(output) = ret.output {
                            println!("{}", &output);
                        };
                    }
                }

                filemagic::Mimetype::Archive | filemagic::Mimetype::Zip => {
                    w0001!(dir_entry_str);
                    check_and_correct_perms(dir_entry_str, p);
                }

                filemagic::Mimetype::Data => check_and_correct_perms(dir_entry_str, p),
                _ => continue,
            }
            if others_match(p, 0) {
                e0002!(dir_entry_str, perms_to_string(p));
                if ARGS.correct_perms {
                    i0005!(&dir_entry_str);
                    set_perms(&dir_entry_str, 0o664);
                }
            }
            if !(ARGS.ignore_dupes && tds_zip.is_none()) {
                process(fsize, &path);
            }
        }
    }

    print_casefolding(&lcnames);
    print_generated(&doublenames, &generated);
    print_doublenames(&doublenames);

    if ARGS.ignore_dupes && tds_zip.is_none() {
        return None;
    }

    // Set up thread pool for the task to hash a file.  Number of CPUs + 1 has been
    // found to be a good pool size, likely since the walker thread should be
    // doing mostly IO.
    let mut pool = Pool::new(num_cpus::get() as u32 + 1);

    let mut hashes: HashMap<(u64, Vec<u8>), DupPath> = HashMap::default();
    pool.scoped(|scope| {
        let (tx, rx) = channel();

        let hashref = &mut hashes;
        scope.execute(move || {
            for (size, path, hash) in rx.iter() {
                hashref
                    .entry((size, hash))
                    .or_insert_with(DupPath::new)
                    .push(path);
            }
        });

        for size in sizes.keys() {
            let paths = &sizes[size];
            if paths.len() == 1 && tds_zip.is_none() {
                continue;
            };

            for p in &sizes[size] {
                let txc = tx.clone();
                scope.execute(move || hash_file(*size, p.to_path_buf(), &txc));
            }
        }
    });

    if !ARGS.ignore_dupes {
        print_duplicates(&hashes);
    }
    Some(hashes)
}

fn print_casefolding(hashes: &HashMap<PathBuf, Vec<(PathBuf, FileKind)>>) {

    for (k, eles) in hashes.iter() {
        //println!("pcf: {:?}, {:?}", k, &eles);
        if eles.len() == 1 {
            continue;
        }

        e0025!(k.display());

        for (p, ty) in eles {
            println!("   >>> {} ({})", p.display(), ty);
        }
    }
}

fn print_generated(hashes: &HashMap<PathBuf, Vec<PathBuf>>, generated: &HashMap<PathBuf, PathBuf>) {
    for (k, gen) in generated.iter() {
        if hashes.contains_key(k) {
            let v = &hashes[k];
            for fname in v {
                e0019!(fname.to_str().unwrap(), gen.to_str().unwrap());
            }
        }
    }
}

fn print_doublenames(hashes: &HashMap<PathBuf, Vec<PathBuf>>) {
    for (k, paths) in hashes.iter() {
        if paths.len() == 1 {
            continue;
        }
        let ks = k.to_str().unwrap();
        if ks == "README"
            || ks == "README.txt"
            || ks == "README.md"
            || ks == "Makefile"
            || ks == "Makefile.am"
            || ks == "Makefile.in"
            || ks == "makefile"
        {
            continue;
        }

        w0003!(k.to_str().unwrap());
        //        println!(":: {}", k.display());

        for p in paths {
            println!("   >>> {}", p.display());
        }
    }
}

fn show_tmp_endings() {
    i0006!();
    for (t, c) in temp_file_endings() {
        println!("{:23} {}", t, c);
    }
}