diff options
Diffstat (limited to 'support/pkgcheck/src/main.rs')
-rw-r--r-- | support/pkgcheck/src/main.rs | 926 |
1 files changed, 479 insertions, 447 deletions
diff --git a/support/pkgcheck/src/main.rs b/support/pkgcheck/src/main.rs index 6fb4136f91..88b9323692 100644 --- a/support/pkgcheck/src/main.rs +++ b/support/pkgcheck/src/main.rs @@ -1,4 +1,3 @@ -extern crate pest; #[macro_use] extern crate pest_derive; @@ -8,18 +7,19 @@ mod gparser; mod messages; use messages::{explains, explains_all}; -#[macro_use] -mod utils; mod linkcheck; +mod utils; mod recode; -use recode::crlf2lf; +use recode::{wrong_line_endings2lf, wrong_line_endings2crlf}; mod filemagic; +use filemagic::{LineEnding}; use linkcheck::LinkCheck; use std::fmt; use std::fmt::Display; + use std::str; use utils::*; @@ -27,7 +27,6 @@ use std::os::unix::fs::MetadataExt; use scoped_threadpool::Pool; - use tempdir::TempDir; #[macro_use] extern crate lazy_static; @@ -63,6 +62,15 @@ fn err(path: &PathBuf, err: &io::Error) { type HashSender = Sender<(u64, PathBuf, Vec<u8>)>; //type DupeSender = Sender<(u64, Vec<PathBuf>)>; +// SizesHashMap contains +// - file sizes +// - and a vector of file names having that size +type SizesHashMap = HashMap<u64, Vec<PathBuf>>; + +type GeneratedHashMap = HashMap<String, String>; + +type FileNamesHashMap = HashMap<PathBuf, (Metadata, String, ReadmeKind)>; + const BLOCKSIZE: usize = 4096; fn hash_file_inner(path: &PathBuf) -> io::Result<Vec<u8>> { @@ -87,48 +95,29 @@ fn hash_file(fsize: u64, path: PathBuf, tx: &HashSender) { } } -fn fix_perms(entry: &str) { - i0005!(entry); - let rc = run_cmd("/bin/chmod", &["-v", "ug=rwX,o=rX", entry]); - if rc.status { - if let Some(op) = rc.output { - print!("{}", op); +// returns false if an error occured +fn fix_inconsistent_le(fname: &str) -> bool { + i0004!(fname); + match wrong_line_endings2lf(fname) { + Ok(_) => { + i0007!(fname, "LF"); + true + } + Err(e) => { + e0027!(fname, e); + false } } } -fn filename_has_bad_chars(entry: &DirEntry) -> Option<(usize, char)> { - let s = entry.file_name().to_str().unwrap(); - - for (i, c) in s.char_indices() { - match c { - '\x00'..='\x1f' => return Some((i, c)), - ' ' | '!' | '"' | '#' | '$' | '%' | '&' | '\'' => return Some((i, c)), - '(' | ')' | '*' => return Some((i, c)), - '+' | ',' | '-' | '.' => (), - '0'..='9' => (), - ':' | ';' | '<' | '=' | '>' | '?' | '@' => return Some((i, c)), - 'A'..='Z' => (), - '[' | '\\' | ']' | '^' => return Some((i, c)), - '_' => (), - '`' => return Some((i, c)), - 'a'..='z' => (), - '{' | '|' | '}' | '~' => return Some((i, c)), - '/' => return None, - _ => return Some((i, c)), - }; - } - None -} -// returns false if CRLF, otherwise true -fn fix_crlf(fname: &str) -> bool { +// returns false if an error occured +fn make_crlf(fname: &str) -> bool { i0004!(fname); - //run_cmd("/usr/bin/fromdos", &["-p", "-e", "-v", fname]).status - match crlf2lf(fname) { + match wrong_line_endings2crlf(fname) { Ok(_) => { - i0007!(fname); + i0007!(fname, "CRLF"); true } Err(e) => { @@ -157,7 +146,7 @@ fn check_readme(dir_entry: &str, is_readme: &ReadmeKind, ft: &filemagic::Mimetyp e0029!(msg_name, b.as_ref()); return false; } - filemagic::Mimetype::Text | filemagic::Mimetype::TextCRLF => match File::open(dir_entry) { + filemagic::Mimetype::Text(_le) => match File::open(dir_entry) { Ok(f) => { if !check_readme_inner(&msg_name, &f) { return false; @@ -216,8 +205,8 @@ struct Args { ignore_dupes: bool, #[structopt(short = "v", long = "verbose", help = "Verbose operation?")] verbose: bool, - #[structopt(short = "L", long = "correct-crlf", help = "Correct CRLF line endings")] - correct_crlf: bool, + #[structopt(short = "L", long = "correct-le", help = "Correct line endings")] + correct_le: bool, #[structopt(short = "C", long = "correct-perms", help = "Correct permissions")] correct_perms: bool, #[structopt(long = "no-colors", help = "Don't display messages in color")] @@ -309,7 +298,7 @@ impl DupPath { } } -type Hashes = HashMap<(u64, Vec<u8>), DupPath>; +type DupHashes = HashMap<(u64, Vec<u8>), DupPath>; fn main() { match &ARGS.explain { @@ -336,11 +325,18 @@ fn main() { process::exit(1); } Some(d) => { - if !exists_dir(&d) { - f0002!(d); + // make sure the given directory ends with a '/' (slash) + let ds: String = if d.ends_with('/') { + d.to_string() + } else { + let d_s = d.to_string(); + d_s + "/" + }; + if !exists_dir(&ds) { + f0002!(&ds); process::exit(1); }; - d + ds } }; @@ -365,16 +361,11 @@ fn main() { } }; - if let Some(hashes) = check_package(pkg_dir, tds_zip) { + if let Some(hashes) = check_package(&pkg_dir, tds_zip) { if tds_zip.is_some() { if let Some(pn) = pkg_name { if let Some(s) = ARGS.tds_zip.as_ref() { - match check_tds_archive(&pn, s, &hashes) { - Ok(_) => (), - Err(e) => { - e0027!(s, e); - } - } + check_tds_archive(&pn, s, &hashes); } } } @@ -387,7 +378,7 @@ fn main() { } } -fn print_duplicates(hashes: &Hashes) { +fn print_duplicates(hashes: &DupHashes) { let mut total_dupes = 0; let mut total_files = 0; let mut total_size = 0; @@ -471,7 +462,7 @@ fn get_filetype(entry: &DirEntry) -> FType { // read file into buffer[u8] // then convert into string // -fn check_generated_files(entry: &str, generated: &mut HashMap<PathBuf, PathBuf>) { +fn check_generated_files(entry: &str, generated: &mut GeneratedHashMap) { // unwrap() is ok here as we only call this function for files, // specifically .ins or .dtx files let entry_fname = filename(entry).unwrap().to_string(); @@ -508,9 +499,12 @@ fn check_generated_files(entry: &str, generated: &mut HashMap<PathBuf, PathBuf>) fname.to_string() }; - let pb = PathBuf::from(fname_x); - - generated.entry(pb).or_insert_with(|| PathBuf::from(entry)); + // If the filename in the generate statement contains a path component + // we ignore it so that a generated file will be reported even if it is + // in a different place in the package directory + if let Some(pb) = utils::filename(&fname_x) { + generated.entry(pb.to_string()).or_insert_with(|| entry.to_string()); + } } }; } @@ -542,7 +536,7 @@ fn check_tds_archive_name(tds_zip: &Option<String>) -> Option<String> { let mut pname = String::from(utils::basename(&tz)); let plen = pname.len(); pname.truncate(plen - 8); - Some(pname.to_string()) + Some(pname) } } } @@ -564,11 +558,7 @@ fn unzip_tds_archive(tds_zip: &str, tmp_dir: &str) { } } -fn check_tds_archive( - pkg_name: &str, - tds_zip: &str, - hashes: &HashMap<(u64, Vec<u8>), DupPath>, -) -> Result<(), io::Error> { +fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) { i0003!(tds_zip); let dir_entry = Path::new(tds_zip); @@ -581,77 +571,48 @@ fn check_tds_archive( } }; + let ut = Utils::new(utils::CheckType::TDS); + // We have a discrepancy between // TDS zip archive name: babel-base.tds.zip // and package name: babel // which yields a false test when checking for the package name - // the path names of the TDS zip archive files. + // the path names of the TDS zip archive files. // Therefore, we correct here. // latex-tools, latex-graphics, latex-amsmath, latex-base => latex - //let mut pkg_replacements: HashMap<String,String> = HashMap::default(); - //pkg_replacements.insert("babel-base".to_string(), "babel".to_string()); - let mut pkg_replacements: HashMap<&str,&str> = HashMap::default(); + // latex-tools-dev, latex-graphics-dev, latex-amsmath-dev, latex-base-dev => latex-dev + let mut pkg_replacements: HashMap<&str, &str> = HashMap::default(); pkg_replacements.insert("babel-base", "babel"); pkg_replacements.insert("latex-base", "latex"); pkg_replacements.insert("latex-tools", "latex"); pkg_replacements.insert("latex-graphics", "latex"); pkg_replacements.insert("latex-amsmath", "latex"); - let real_pkg_name = if let Some(real_name) = pkg_replacements.get(pkg_name) { + pkg_replacements.insert("latex-base-dev", "latex-dev"); + pkg_replacements.insert("latex-tools-dev", "latex-dev"); + pkg_replacements.insert("latex-graphics-dev", "latex-dev"); + pkg_replacements.insert("latex-amsmath-dev", "latex-dev"); + let real_pkg_name = if let Some(real_name) = pkg_replacements.get(pkg_name) { real_name } else { pkg_name }; - let tmp_dir = TempDir::new("pkgcheck")?; - let tmp_dir_offset = tmp_dir.path().to_str().unwrap().len() + 1; - unzip_tds_archive(tds_zip, tmp_dir.path().to_str().unwrap()); - - let tds_toplevel_dirs: HashSet<String> = [ - "tex", "fonts", "metafont", "metapost", "bibtex", "scripts", "doc", "source", - ] - .iter() - .map(|&s| s.to_string()) - .collect(); - - for entry in fs::read_dir(tmp_dir.path())? { - let entry = entry?; - let path = entry.path(); - - match path.file_stem() { - None => (), - Some(file_stem) => { - let fs = file_stem.to_str().expect("bad file name"); - if !tds_toplevel_dirs.contains(fs) { - if path.is_dir() { - e0020!(fs); - } else { - e0034!(fs); - } - } - } + let tmp_dir = match TempDir::new("pkgcheck") { + Ok(tdir) => tdir, + Err(e) => { + f0007!(e); + process::exit(1); } + }; - // if path.is_dir() { - // match path.file_stem() { - // None => (), - // Some(file_stem) => { - // let fs = file_stem.to_str().expect("bad file name"); - // if !tds_toplevel_dirs.contains(fs) { - // e0020!(fs); - // } - // } - // } - // } else { - // println!("mist"); - // e0020!(path.display()); - // } - } + let tmp_dir_offset = tmp_dir.path().to_str().unwrap().len() + 1; + unzip_tds_archive(tds_zip, tmp_dir.path().to_str().unwrap()); // in order to compare the package files with the content of the // tds zip archive we need to checksum the files in the tds zip // archive. - let mut sizes = HashMap::default(); + let mut sizes: SizesHashMap = HashMap::default(); let mut pool = Pool::new(num_cpus::get() as u32 + 1); { // Processing a single file entry, with the "sizes" hashmap collecting @@ -672,63 +633,93 @@ fn check_tds_archive( for dir_entry in WalkDir::new(tmp_dir.path().to_str().unwrap()).follow_links(false) { match dir_entry { Ok(dir_entry) => { - //let dir_entry_str = dir_entry.path().to_str().unwrap(); + let dir_entry_str = match dir_entry.path().to_str() { + Some(d) => d, + None => { + e0031!(dir_entry.path().to_string_lossy()); + continue; + } + }; + + // this is the file_name without the directory part + // unwrap() is ok here as we covered potential UTF-8 related errors + // above in the definition of dir_entry_str + let file_name = dir_entry.file_name().to_str().unwrap().to_string(); + + let meta = match dir_entry.metadata() { + Ok(meta) => meta, + Err(e) => { + e0027!(dir_entry.path().display(), e); + continue; + } + }; let ft = get_filetype(&dir_entry); + if let FType::Error(e) = ft { + e0023!(e); + continue; + } + + let dir_entry_display = if dir_entry.depth() == 0 { + &dir_entry_str[tmp_dir_offset - 1..] + } else { + &dir_entry_str[tmp_dir_offset..] + }; + if ft == FType::Directory { + ut.check_for_empty_directory(&dir_entry_str, &dir_entry_display); + ut.check_for_hidden_directory(&file_name, &dir_entry_display); + ut.is_unwanted_directory(&file_name, dir_entry_str); continue; } - if let FType::Error(e) = ft { - e0023!(e); + // We don't want to have other than directories in the top level + // directory of a TDS zip archive + if dir_entry.depth() == 1 && ft == FType::Regular { + e0034!(dir_entry_display); continue; } - match dir_entry.metadata() { - Ok(meta) => { - let fsize = meta.len(); - process(fsize, &dir_entry); + ut.check_for_temporary_file(&dir_entry_display); - let dir_entry_str = match dir_entry.path().to_str() { - Some(d) => d, - None => { - e0031!(dir_entry.path().to_string_lossy()); - continue; - } - }; - let dir_entry_display = &dir_entry_str[tmp_dir_offset..]; + // The LaTeX team provides the file `.tex` as a file with an empty name + // in order to make `\input\relax` work (explained by David Carlisle) + // Therefore, we don't call check_for_hidden_file() in this case + match (pkg_name, dir_entry_display) { + ("latex-tools", "tex/latex/tools/.tex") => (), + ("latex-tools-dev", "tex/latex-dev/tools/.tex") => (), + (_, _) => ut.check_for_hidden_file(&file_name, &dir_entry_display) + }; - if is_temporary_file(&dir_entry_str) { - e0008!(dir_entry_display); - } - if fsize > 40 * 1024 * 1024 { - w0006!(dir_entry_display, fsize / 1024 / 1024); - } + let fsize = meta.len(); + process(fsize, &dir_entry); + ut.check_filesize(fsize, &dir_entry_display); - // if the path doesn't contain a man page... - if dir_entry_str.find("/man/").is_none() { - let pkg_name_s = format!("/{}/", real_pkg_name); - // ...then we want to have the package name in the path - if dir_entry_str.find(&pkg_name_s).is_none() { - e0028!(real_pkg_name, dir_entry_display); - } - } - - if dir_entry_str.ends_with(".map") { - map_files_found = true; - let re: Regex = Regex::new(r"fonts[/]map[/]dvips[/]").unwrap(); - if re.is_match(dir_entry_str) { - map_dvips_found = true; - } + // if we encounter a .dtx or .ins file we check + // that it is in a subdirectory of either source/ or doc/ + if (dir_entry_str.ends_with(".dtx") || dir_entry_str.ends_with(".ins")) + && !(dir_entry_display.starts_with("source/") + || dir_entry_display.starts_with("doc/")) + { + e0036!(dir_entry_display); + continue; + } - // if ( $fname =~ m{ fonts [/] map [/] dvips [/] }xms ) { - // $map_dvips_found = 1; - // } - } + // if the path doesn't contain a man page... + if dir_entry_str.find("/man/").is_none() { + let pkg_name_s = format!("/{}/", real_pkg_name); + // ...then we want to have the package name in the path + if dir_entry_str.find(&pkg_name_s).is_none() { + e0028!(real_pkg_name, dir_entry_display); } - Err(e) => { - e0027!(dir_entry.path().display(), e); + } + + if dir_entry_str.ends_with(".map") { + map_files_found = true; + let re: Regex = Regex::new(r"fonts[/]map[/]dvips[/]").unwrap(); + if re.is_match(dir_entry_str) { + map_dvips_found = true; } } } @@ -765,7 +756,6 @@ fn check_tds_archive( } }); - tmp_dir.close()?; // now check if each package file is in the tds archive for (k, paths) in hashes.iter() { if !tds_hashes.contains_key(k) { @@ -773,8 +763,6 @@ fn check_tds_archive( e0026!(p); } } - - Ok(()) } fn get_extension_from_filename(filename: &str) -> Option<&str> { @@ -807,6 +795,21 @@ fn found_unwanted_filetype(fname: &str, ft: &FType) -> bool { } } +fn fix_perms(entry: &str) { + if !ARGS.correct_perms { + return; + } + + i0005!(entry); + let rc = run_cmd("/bin/chmod", &["-v", "ug=rwX,o=rX", entry]); + + if rc.status { + if let Some(op) = rc.output { + print!("{}", op); + } + } +} + fn check_and_correct_perms(dir_entry: &str, p: u32) { if owner_has(p, 5) || !others_have(p, 4) { e0002!(dir_entry, perms_to_string(p)); @@ -817,6 +820,21 @@ fn check_and_correct_perms(dir_entry: &str, p: u32) { }; } +// Sets permissions for a file or directory +// Sample invocation: set_perms("somfile", 0o644); +fn set_perms(entry: &str, p: u32) -> bool { + let ps = &format!("{:o}", p); + let rc = run_cmd("chmod", &["-v", ps, entry]); + + if rc.status { + if let Some(op) = rc.output { + print!("{}", op); + } + true + } else { + false + } +} #[derive(Debug, Clone, PartialEq)] enum FileKind { @@ -825,7 +843,6 @@ enum FileKind { Symlink(String), } - impl Display for FileKind { fn fmt(&self, f: &mut ::std::fmt::Formatter) -> Result<(), ::std::fmt::Error> { match *self { @@ -836,8 +853,6 @@ impl Display for FileKind { } } - - #[derive(Debug, Clone, PartialEq)] enum ReadmeKind { No, @@ -845,15 +860,17 @@ enum ReadmeKind { Symlink(String), } -fn check_package(root: &str, tds_zip: &Option<String>) -> Option<Hashes> { - //let mut urls = HashMap::default(); //: <String,UrlStatus> - +fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> { let mut lcnames: HashMap<PathBuf, Vec<(PathBuf, FileKind)>> = HashMap::default(); let mut doublenames: HashMap<PathBuf, Vec<PathBuf>> = HashMap::default(); let mut inodes = HashSet::default(); + let ut = Utils::new(utils::CheckType::PACKAGE); + + + i0002!(root); // This hash contains all package file names. // @@ -862,185 +879,162 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<Hashes> { // String: the file name without any directory part // ReadmeKind: is it a certain README, file or symlink? // A special case of a README file is a file with has a different name but - // was poinded to by a symlink. Example: README --> README.rst - let mut file_names: HashMap<PathBuf, (Metadata, String, ReadmeKind)> = HashMap::default(); - { - let mut readme_found = false; - - let root_absolute = PathBuf::from(root) - .canonicalize() - .unwrap() - .to_string_lossy() - .to_string(); - for dir_entry in WalkDir::new(root).follow_links(false) { - match dir_entry { - Ok(dir_entry) => { - let dir_entry_str = match dir_entry.path().to_str() { - Some(d) => d, - None => { - e0031!(dir_entry.path().to_string_lossy()); - continue; - } - }; - let meta = match dir_entry.metadata() { - Ok(meta) => meta, - Err(e) => { - e0023!(e); - continue; - } - }; - - if !check_inode(&mut inodes, &meta) { + // was pointed to by a symlink. Example: README --> README.rst + let mut file_names: FileNamesHashMap = HashMap::default(); + let mut readme_found = false; + + let root_absolute = PathBuf::from(root) + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(); + + for dir_entry in WalkDir::new(root).follow_links(false) { + match dir_entry { + Ok(dir_entry) => { + let dir_entry_str = match dir_entry.path().to_str() { + Some(d) => d, + None => { + e0031!(dir_entry.path().to_string_lossy()); continue; } + }; + let meta = match dir_entry.metadata() { + Ok(meta) => meta, + Err(e) => { + e0023!(e); + continue; + } + }; - // this is the file_name without the directory part - // unwrap() is ok here as we covered potential UTF-8 related errors - // above in the definition of dir_entry_str - let file_name = dir_entry.file_name().to_str().unwrap().to_string(); + if !check_inode(&mut inodes, &meta) { + continue; + } - let ft = get_filetype(&dir_entry); + // this is the file_name without the directory part + // unwrap() is ok here as we covered potential UTF-8 related errors + // above in the definition of dir_entry_str + let file_name = dir_entry.file_name().to_str().unwrap().to_string(); - if found_unwanted_filetype(dir_entry_str, &ft) { - continue; - } + // we check for weird stuff like socket files aso. + let ft = get_filetype(&dir_entry); + if found_unwanted_filetype(&dir_entry_str, &ft) { + continue; + } - if let Some((offset, ch)) = filename_has_bad_chars(&dir_entry) { - e0001!(ch, dir_entry_str, offset); - } + ut.filename_has_bad_chars(&dir_entry, dir_entry_str); - // 1. dealing with symlinks - if ft == FType::Symlink { - match get_symlink(&dir_entry) { - Ok(None) => { - e0010!(dir_entry_str); - continue; - } - Err(e) => { - e0027!(dir_entry_str, e); + // 1. dealing with symlinks + if ft == FType::Symlink { + match get_symlink(&dir_entry) { + Ok(None) => { + e0010!(&dir_entry_str); + continue; + } + Err(e) => { + e0027!(&dir_entry_str, e); + continue; + } + Ok(Some(p)) => { + let pd: String = + p.canonicalize().unwrap().to_string_lossy().to_string(); + if !pd.starts_with(&root_absolute) { + e0030!(&dir_entry_str, p.display()); continue; } - Ok(Some(p)) => { - let pd: String = - p.canonicalize().unwrap().to_string_lossy().to_string(); - if !pd.starts_with(&root_absolute) { - e0030!(dir_entry_str, p.display()); - continue; - } - let lc_dir_entry_str = dir_entry_str.to_lowercase(); - if let Some(_dir_name) = filename(dir_entry_str) { - let lcnref = &mut lcnames; - lcnref - .entry(PathBuf::from(lc_dir_entry_str)) - .or_insert_with(Vec::new) - .push(( - PathBuf::from(dir_entry_str), - //FileKind::Symlink(dir_entry_str.into()), - FileKind::Symlink(pd.clone()), - )); - } - if is_readme(&dir_entry) { - readme_found = true; - file_names.insert( - p, - ( - meta, - file_name, - ReadmeKind::Symlink(dir_entry_str.to_string()), - ), - ); - } - continue; + let lc_dir_entry_str = dir_entry_str.to_lowercase(); + if let Some(_dir_name) = filename(&dir_entry_str) { + let lcnref = &mut lcnames; + lcnref + .entry(PathBuf::from(lc_dir_entry_str)) + .or_insert_with(Vec::new) + .push(( + PathBuf::from(&dir_entry_str), + //FileKind::Symlink(&dir_entry_str.into()), + FileKind::Symlink(pd.clone()), + )); } + if is_readme(&dir_entry) { + readme_found = true; + file_names.insert( + p, + ( + meta, + file_name, + ReadmeKind::Symlink(dir_entry_str.to_string()), + ), + ); + } + continue; } } + } - let p = get_perms(&dir_entry.path()); + let p = get_perms(&dir_entry.path()); - // 2. dealing with directories - if ft == FType::Directory { - let lc_dir_entry_str = dir_entry_str.to_lowercase(); - if let Some(_dir_name) = filename(dir_entry_str) { - let lcnref = &mut lcnames; - - lcnref - .entry(PathBuf::from(lc_dir_entry_str)) - .or_insert_with(Vec::new) - .push((PathBuf::from(dir_entry_str), FileKind::Directory)); - } + // 2. dealing with directories + if ft == FType::Directory { - if !owner_has(p, 5) || !others_have(p, 5) { - e0011!(dir_entry_str, perms_to_string(p)); - if ARGS.correct_perms { - i0005!(&dir_entry_str); - set_perms(&dir_entry_str, 0o775); - } - } + let lc_dir_entry_str = dir_entry_str.to_lowercase(); + if let Some(_dir_name) = filename(&dir_entry_str) { + let lcnref = &mut lcnames; - match is_empty_directory(&dir_entry) { - Ok(true) => { - e0004!(dir_entry_str); - } - Err(e) => { - e0027!(dir_entry_str, e); - } - Ok(false) => (), - } - if is_hidden_directory(&dir_entry) { - e0006!(dir_entry_str); - } + lcnref + .entry(PathBuf::from(lc_dir_entry_str)) + .or_insert_with(Vec::new) + .push((PathBuf::from(&dir_entry_str), FileKind::Directory)); + } - if is_unwanted_directory(&dir_entry.file_name().to_str().unwrap()) { - e0018!(dir_entry_str); + if !owner_has(p, 5) || !others_have(p, 5) { + e0011!(&dir_entry_str, perms_to_string(p)); + if ARGS.correct_perms { + i0005!(&dir_entry_str); + set_perms(&dir_entry_str, 0o775); } - continue; } - // 3. dealing with regular files - if is_hidden(&file_name) { - e0007!(dir_entry_str); - } - if is_temporary_file(&dir_entry_str) { - e0008!(dir_entry_str); - } + ut.check_for_empty_directory(&dir_entry_str, &dir_entry_str); + ut.check_for_hidden_directory(&file_name, &dir_entry_str); + ut.is_unwanted_directory(&file_name, dir_entry_str); + continue; + } - if let Some(file_name) = filename(dir_entry_str) { - let doubleref = &mut doublenames; + // 3. dealing with regular files + ut.check_for_hidden_file(&file_name, &dir_entry_str); + ut.check_for_temporary_file(&dir_entry_str); - doubleref - .entry(PathBuf::from(file_name)) - .or_insert_with(Vec::new) - .push(PathBuf::from(dir_entry_str)); - } + // if is_temporary_file(&dir_entry_str) { + // e0008!(&dir_entry_str); + // } - // detecting large files in package directory - - if is_readme(&dir_entry) { - // We want to deal with README files only if they are - // in the root directory of the package. - let f = format!( - "{}{}{}", - root, - // we have to pay attention if `root` ends already with '/' - if root.ends_with('/') { "" } else { "/" }, - &file_name - ); + if let Some(file_name) = filename(&dir_entry_str) { + let doubleref = &mut doublenames; - if dir_entry_str == f { - readme_found = true; - file_names.insert( - dir_entry.path().to_path_buf(), - (meta, file_name.clone(), ReadmeKind::Yes), - ); - } else { - file_names.entry(dir_entry.path().to_path_buf()).or_insert(( - meta, - file_name.clone(), - ReadmeKind::No, - )); - } + doubleref + .entry(PathBuf::from(file_name)) + .or_insert_with(Vec::new) + .push(PathBuf::from(&dir_entry_str)); + } + + if is_readme(&dir_entry) { + // We want to deal with README files only if they are + // in the root directory of the package. + let f = format!( + "{}{}{}", + root, + // we have to pay attention if `root` ends already with '/' + if root.ends_with('/') { "" } else { "/" }, + &file_name + ); + + if dir_entry_str == f { + readme_found = true; + file_names.insert( + dir_entry.path().to_path_buf(), + (meta, file_name.clone(), ReadmeKind::Yes), + ); } else { file_names.entry(dir_entry.path().to_path_buf()).or_insert(( meta, @@ -1048,168 +1042,203 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<Hashes> { ReadmeKind::No, )); } + } else { + file_names.entry(dir_entry.path().to_path_buf()).or_insert(( + meta, + file_name.clone(), + ReadmeKind::No, + )); + } - let lc_dir_entry_str = dir_entry_str.to_lowercase(); - - let lcnref = &mut lcnames; + let lc_dir_entry_str = dir_entry_str.to_lowercase(); - lcnref - .entry(PathBuf::from(lc_dir_entry_str)) - .or_insert_with(Vec::new) - .push((PathBuf::from(dir_entry_str), FileKind::File)); + let lcnref = &mut lcnames; - } + lcnref + .entry(PathBuf::from(lc_dir_entry_str)) + .or_insert_with(Vec::new) + .push((PathBuf::from(&dir_entry_str), FileKind::File)); + } - Err(e) => { - eprintln!("{}", e); - } + Err(e) => { + eprintln!("{}", e); } } + } - if !readme_found { - e0009!(); - } + if !readme_found { + e0009!(); } let lc = LinkCheck::new(4, false); let mut detective = filemagic::Filetype::new(); - let mut sizes = HashMap::default(); - let mut generated: HashMap<PathBuf, PathBuf> = HashMap::default(); + let mut sizes: SizesHashMap = HashMap::default(); + let mut generated: GeneratedHashMap = HashMap::default(); - { - // Processing a single file entry, with the "sizes" hashmap collecting - // same-size files. Entries are either Found::One or Found::Multiple, - // so that we can submit the first file's path as a hashing job when the - // first duplicate is found. Hashing each file is submitted as a job to - // the pool. - let mut process = |fsize, path: &PathBuf| { - let sizeref = &mut sizes; + // Processing a single file entry, with the "sizes" hashmap collecting + // same-size files. Entries are either Found::One or Found::Multiple, + // so that we can submit the first file's path as a hashing job when the + // first duplicate is found. Hashing each file is submitted as a job to + // the pool. + let mut process = |fsize, path: &PathBuf| { + let sizeref = &mut sizes; - let path = path.clone(); - sizeref.entry(fsize).or_insert_with(Vec::new).push(path); + let path = path.clone(); + sizeref.entry(fsize).or_insert_with(Vec::new).push(path); + }; + for (path, (meta, _file_name, is_readme)) in file_names.iter() { + //println!("DEBUG: path {:?}, file_name {:?}, is_readme: {} ", &path, &file_name, is_readme); + let dir_entry_str = match path.to_str() { + Some(d) => d, + None => { + e0031!(&path.to_string_lossy()); + continue; + } }; - for (path, (meta, _file_name, is_readme)) in file_names.iter() { - //println!("DEBUG: path {:?}, file_name {:?}, is_readme: {} ", &path, &file_name, is_readme); - let dir_entry_str = match path.to_str() { - Some(d) => d, - None => { - e0031!(&path.to_string_lossy()); - continue; - } - }; + let fsize = meta.len(); + ut.check_filesize(fsize, dir_entry_str); - let fsize = meta.len(); + let p = get_perms(&path); + if !owner_has(p, 4) { + e0002!(&dir_entry_str, perms_to_string(p)); + fix_perms(&dir_entry_str); + continue; + } + let ftr = detective.analyze(&dir_entry_str); + //println!(">>> {:?}", ftr); + // we ignore errors from filetype recognition + if ftr.is_err() { + continue; + } - // if it is an empty file we don't need - // to check further - if fsize == 0 { - e0005!(dir_entry_str); - continue; - } - if fsize > 40 * 1024 * 1024 { - w0005!(dir_entry_str, fsize / 1024 / 1024); - continue; - } + let ft = ftr.unwrap(); - let p = get_perms(&path); - if !owner_has(p, 4) { - e0002!(dir_entry_str, perms_to_string(p)); - if ARGS.correct_perms { - fix_perms(&dir_entry_str); - } + // DEBUG !readme_symlinked.contains(&dir_entry_str) + if ReadmeKind::No != *is_readme { + if !check_readme(&dir_entry_str, is_readme, &ft) { continue; } - let ftr = detective.analyze(&dir_entry_str); - //println!(">>> {:?}", ftr); - // we ignore errors from filetype recognition - if ftr.is_err() { - continue; + if ARGS.urlcheck { + lc.check_urls(&dir_entry_str); } + } - let ft = ftr.unwrap(); - - // DEBUG !readme_symlinked.contains(dir_entry_str) - if ReadmeKind::No != *is_readme { - if !check_readme(dir_entry_str, is_readme, &ft) { - continue; - } - if ARGS.urlcheck { - lc.check_urls(dir_entry_str); + match ft { + filemagic::Mimetype::Text(_) => { + check_and_correct_perms(&dir_entry_str, p); + let fext = get_extension_from_filename(&dir_entry_str); + if fext == Some("ins") || fext == Some("dtx") { + check_generated_files(&dir_entry_str, &mut generated); } - } - match ft { - filemagic::Mimetype::Text | filemagic::Mimetype::TextCRLF => { - check_and_correct_perms(dir_entry_str, p); - let fext = get_extension_from_filename(&dir_entry_str); - if fext == Some("ins") || fext == Some("dtx") { - check_generated_files(&dir_entry_str, &mut generated); - } - match fext { - Some("bat") | Some("cmd") | - Some("nsh") | Some("reg") => (), - Some(_) | None => { - if ft == filemagic::Mimetype::TextCRLF { - e0012!(dir_entry_str); - if ARGS.correct_crlf { - fix_crlf(dir_entry_str); + match fext { + // deal with Windows files + Some("bat") | Some("cmd") | Some("nsh") | Some("reg") => { + match ft { + filemagic::Mimetype::Text(LineEnding::CRLF) => (), + filemagic::Mimetype::Text(LineEnding::CR) => + { + e0037!(&dir_entry_str); + if ARGS.correct_le { + make_crlf(&dir_entry_str); } - } + }, + filemagic::Mimetype::Text(LineEnding::MIXED(0, 0, 0) ) => (), + filemagic::Mimetype::Text(LineEnding::MIXED(cr, lf, crlf) ) => + { + e0038!(&dir_entry_str, cr, lf, crlf); + if ARGS.correct_le { + fix_inconsistent_le(&dir_entry_str); + } + }, + filemagic::Mimetype::Text(LineEnding::LF) => + { + w0008!(&dir_entry_str); + }, + fmm => println!("Should not occur: {} has {:?}", dir_entry_str, fmm), } - }; + }, + Some(_) | None => { + match ft { + filemagic::Mimetype::Text(LineEnding::CRLF) => + { + e0012!(&dir_entry_str); + if ARGS.correct_le { + fix_inconsistent_le(&dir_entry_str); + } + }, + filemagic::Mimetype::Text(LineEnding::CR) => + { + e0037!(&dir_entry_str); + if ARGS.correct_le { + fix_inconsistent_le(&dir_entry_str); + } + }, + filemagic::Mimetype::Text(LineEnding::MIXED(0, 0, 0) ) => (), + filemagic::Mimetype::Text(LineEnding::MIXED(cr, lf, crlf) ) => + { + //println!(">>>{}: {:?} {},{},{}", &dir_entry_str, ft, x, y, z); + e0038!(&dir_entry_str, cr, lf, crlf); + if ARGS.correct_le { + fix_inconsistent_le(&dir_entry_str); + } + }, + filemagic::Mimetype::Text(LineEnding::LF) => (), + fmm => println!("Should not occur: {} has {:?}", dir_entry_str, fmm), + } + }, } + }, - filemagic::Mimetype::BOM(b) => { - //println!("{}: {} with BOM detected", dir_entry_str, b.as_ref()); - w0004!(dir_entry_str, b.as_ref()); - check_and_correct_perms(dir_entry_str, p); - } - filemagic::Mimetype::Binary - | filemagic::Mimetype::Script - | filemagic::Mimetype::ScriptCRLF => { - if !owner_has(p, 4) || !others_have(p, 4) { - e0002!(dir_entry_str, perms_to_string(p)); + filemagic::Mimetype::BOM(b) => { + //println!("{}: {} with BOM detected", dir_entry_str, b.as_ref()); + w0004!(&dir_entry_str, b.as_ref()); + check_and_correct_perms(&dir_entry_str, p); + } + filemagic::Mimetype::Binary + | filemagic::Mimetype::Script(_) => { + if !owner_has(p, 4) || !others_have(p, 4) { + e0002!(&dir_entry_str, perms_to_string(p)); + }; + fix_perms(&dir_entry_str); + } + filemagic::Mimetype::Pdf => { + check_and_correct_perms(&dir_entry_str, p); + let ret = is_pdf_ok(&dir_entry_str); + if !ret.status { + e0017!(&dir_entry_str); + if let Some(output) = ret.output { + println!("{}", &output); }; - if ARGS.correct_perms { - fix_perms(&dir_entry_str); - } - } - filemagic::Mimetype::Pdf => { - check_and_correct_perms(dir_entry_str, p); - let ret = is_pdf_ok(&dir_entry_str); - if !ret.status { - e0017!(dir_entry_str); - if let Some(output) = ret.output { - println!("{}", &output); - }; - } - } - - filemagic::Mimetype::Archive | filemagic::Mimetype::Zip => { - if dir_entry_str.ends_with(".tds.zip") { - e0035!(dir_entry_str); - } else { - w0001!(dir_entry_str); - } - check_and_correct_perms(dir_entry_str, p); } - - filemagic::Mimetype::Data => check_and_correct_perms(dir_entry_str, p), - _ => continue, } - if others_match(p, 0) { - e0002!(dir_entry_str, perms_to_string(p)); - if ARGS.correct_perms { - i0005!(&dir_entry_str); - set_perms(&dir_entry_str, 0o664); + + filemagic::Mimetype::Archive | filemagic::Mimetype::Zip => { + if dir_entry_str.ends_with(".tds.zip") { + e0035!(&dir_entry_str); + } else { + w0001!(&dir_entry_str); } + check_and_correct_perms(&dir_entry_str, p); } - if !(ARGS.ignore_dupes && tds_zip.is_none()) { - process(fsize, &path); + + filemagic::Mimetype::Data => check_and_correct_perms(&dir_entry_str, p), + filemagic::Mimetype::Zerofile => check_and_correct_perms(&dir_entry_str, p), + _ => continue, + } + + if others_match(p, 0) { + e0002!(&dir_entry_str, perms_to_string(p)); + if ARGS.correct_perms { + i0005!(&dir_entry_str); + set_perms(&dir_entry_str, 0o664); } } + if !(ARGS.ignore_dupes && tds_zip.is_none()) { + process(fsize, &path); + } } print_casefolding(&lcnames); @@ -1259,7 +1288,6 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<Hashes> { } fn print_casefolding(hashes: &HashMap<PathBuf, Vec<(PathBuf, FileKind)>>) { - for (k, eles) in hashes.iter() { //println!("pcf: {:?}, {:?}", k, &eles); if eles.len() == 1 { @@ -1274,12 +1302,14 @@ fn print_casefolding(hashes: &HashMap<PathBuf, Vec<(PathBuf, FileKind)>>) { } } -fn print_generated(hashes: &HashMap<PathBuf, Vec<PathBuf>>, generated: &HashMap<PathBuf, PathBuf>) { +fn print_generated(hashes: &HashMap<PathBuf, Vec<PathBuf>>, generated: &GeneratedHashMap) { for (k, gen) in generated.iter() { - if hashes.contains_key(k) { - let v = &hashes[k]; + let mut path = PathBuf::new(); + path.push(k); + if hashes.contains_key(&path) { + let v = &hashes[&path]; for fname in v { - e0019!(fname.to_str().unwrap(), gen.to_str().unwrap()); + e0019!(fname.to_str().unwrap(), gen.as_str()); } } } @@ -1317,3 +1347,5 @@ fn show_tmp_endings() { println!("{:23} {}", t, c); } } + + |