summaryrefslogtreecommitdiff
path: root/support/pkgcheck/src/main.rs
diff options
context:
space:
mode:
Diffstat (limited to 'support/pkgcheck/src/main.rs')
-rw-r--r--support/pkgcheck/src/main.rs926
1 files changed, 479 insertions, 447 deletions
diff --git a/support/pkgcheck/src/main.rs b/support/pkgcheck/src/main.rs
index 6fb4136f91..88b9323692 100644
--- a/support/pkgcheck/src/main.rs
+++ b/support/pkgcheck/src/main.rs
@@ -1,4 +1,3 @@
-extern crate pest;
#[macro_use]
extern crate pest_derive;
@@ -8,18 +7,19 @@ mod gparser;
mod messages;
use messages::{explains, explains_all};
-#[macro_use]
-mod utils;
mod linkcheck;
+mod utils;
mod recode;
-use recode::crlf2lf;
+use recode::{wrong_line_endings2lf, wrong_line_endings2crlf};
mod filemagic;
+use filemagic::{LineEnding};
use linkcheck::LinkCheck;
use std::fmt;
use std::fmt::Display;
+
use std::str;
use utils::*;
@@ -27,7 +27,6 @@ use std::os::unix::fs::MetadataExt;
use scoped_threadpool::Pool;
-
use tempdir::TempDir;
#[macro_use]
extern crate lazy_static;
@@ -63,6 +62,15 @@ fn err(path: &PathBuf, err: &io::Error) {
type HashSender = Sender<(u64, PathBuf, Vec<u8>)>;
//type DupeSender = Sender<(u64, Vec<PathBuf>)>;
+// SizesHashMap contains
+// - file sizes
+// - and a vector of file names having that size
+type SizesHashMap = HashMap<u64, Vec<PathBuf>>;
+
+type GeneratedHashMap = HashMap<String, String>;
+
+type FileNamesHashMap = HashMap<PathBuf, (Metadata, String, ReadmeKind)>;
+
const BLOCKSIZE: usize = 4096;
fn hash_file_inner(path: &PathBuf) -> io::Result<Vec<u8>> {
@@ -87,48 +95,29 @@ fn hash_file(fsize: u64, path: PathBuf, tx: &HashSender) {
}
}
-fn fix_perms(entry: &str) {
- i0005!(entry);
- let rc = run_cmd("/bin/chmod", &["-v", "ug=rwX,o=rX", entry]);
- if rc.status {
- if let Some(op) = rc.output {
- print!("{}", op);
+// returns false if an error occured
+fn fix_inconsistent_le(fname: &str) -> bool {
+ i0004!(fname);
+ match wrong_line_endings2lf(fname) {
+ Ok(_) => {
+ i0007!(fname, "LF");
+ true
+ }
+ Err(e) => {
+ e0027!(fname, e);
+ false
}
}
}
-fn filename_has_bad_chars(entry: &DirEntry) -> Option<(usize, char)> {
- let s = entry.file_name().to_str().unwrap();
-
- for (i, c) in s.char_indices() {
- match c {
- '\x00'..='\x1f' => return Some((i, c)),
- ' ' | '!' | '"' | '#' | '$' | '%' | '&' | '\'' => return Some((i, c)),
- '(' | ')' | '*' => return Some((i, c)),
- '+' | ',' | '-' | '.' => (),
- '0'..='9' => (),
- ':' | ';' | '<' | '=' | '>' | '?' | '@' => return Some((i, c)),
- 'A'..='Z' => (),
- '[' | '\\' | ']' | '^' => return Some((i, c)),
- '_' => (),
- '`' => return Some((i, c)),
- 'a'..='z' => (),
- '{' | '|' | '}' | '~' => return Some((i, c)),
- '/' => return None,
- _ => return Some((i, c)),
- };
- }
- None
-}
-// returns false if CRLF, otherwise true
-fn fix_crlf(fname: &str) -> bool {
+// returns false if an error occured
+fn make_crlf(fname: &str) -> bool {
i0004!(fname);
- //run_cmd("/usr/bin/fromdos", &["-p", "-e", "-v", fname]).status
- match crlf2lf(fname) {
+ match wrong_line_endings2crlf(fname) {
Ok(_) => {
- i0007!(fname);
+ i0007!(fname, "CRLF");
true
}
Err(e) => {
@@ -157,7 +146,7 @@ fn check_readme(dir_entry: &str, is_readme: &ReadmeKind, ft: &filemagic::Mimetyp
e0029!(msg_name, b.as_ref());
return false;
}
- filemagic::Mimetype::Text | filemagic::Mimetype::TextCRLF => match File::open(dir_entry) {
+ filemagic::Mimetype::Text(_le) => match File::open(dir_entry) {
Ok(f) => {
if !check_readme_inner(&msg_name, &f) {
return false;
@@ -216,8 +205,8 @@ struct Args {
ignore_dupes: bool,
#[structopt(short = "v", long = "verbose", help = "Verbose operation?")]
verbose: bool,
- #[structopt(short = "L", long = "correct-crlf", help = "Correct CRLF line endings")]
- correct_crlf: bool,
+ #[structopt(short = "L", long = "correct-le", help = "Correct line endings")]
+ correct_le: bool,
#[structopt(short = "C", long = "correct-perms", help = "Correct permissions")]
correct_perms: bool,
#[structopt(long = "no-colors", help = "Don't display messages in color")]
@@ -309,7 +298,7 @@ impl DupPath {
}
}
-type Hashes = HashMap<(u64, Vec<u8>), DupPath>;
+type DupHashes = HashMap<(u64, Vec<u8>), DupPath>;
fn main() {
match &ARGS.explain {
@@ -336,11 +325,18 @@ fn main() {
process::exit(1);
}
Some(d) => {
- if !exists_dir(&d) {
- f0002!(d);
+ // make sure the given directory ends with a '/' (slash)
+ let ds: String = if d.ends_with('/') {
+ d.to_string()
+ } else {
+ let d_s = d.to_string();
+ d_s + "/"
+ };
+ if !exists_dir(&ds) {
+ f0002!(&ds);
process::exit(1);
};
- d
+ ds
}
};
@@ -365,16 +361,11 @@ fn main() {
}
};
- if let Some(hashes) = check_package(pkg_dir, tds_zip) {
+ if let Some(hashes) = check_package(&pkg_dir, tds_zip) {
if tds_zip.is_some() {
if let Some(pn) = pkg_name {
if let Some(s) = ARGS.tds_zip.as_ref() {
- match check_tds_archive(&pn, s, &hashes) {
- Ok(_) => (),
- Err(e) => {
- e0027!(s, e);
- }
- }
+ check_tds_archive(&pn, s, &hashes);
}
}
}
@@ -387,7 +378,7 @@ fn main() {
}
}
-fn print_duplicates(hashes: &Hashes) {
+fn print_duplicates(hashes: &DupHashes) {
let mut total_dupes = 0;
let mut total_files = 0;
let mut total_size = 0;
@@ -471,7 +462,7 @@ fn get_filetype(entry: &DirEntry) -> FType {
// read file into buffer[u8]
// then convert into string
//
-fn check_generated_files(entry: &str, generated: &mut HashMap<PathBuf, PathBuf>) {
+fn check_generated_files(entry: &str, generated: &mut GeneratedHashMap) {
// unwrap() is ok here as we only call this function for files,
// specifically .ins or .dtx files
let entry_fname = filename(entry).unwrap().to_string();
@@ -508,9 +499,12 @@ fn check_generated_files(entry: &str, generated: &mut HashMap<PathBuf, PathBuf>)
fname.to_string()
};
- let pb = PathBuf::from(fname_x);
-
- generated.entry(pb).or_insert_with(|| PathBuf::from(entry));
+ // If the filename in the generate statement contains a path component
+ // we ignore it so that a generated file will be reported even if it is
+ // in a different place in the package directory
+ if let Some(pb) = utils::filename(&fname_x) {
+ generated.entry(pb.to_string()).or_insert_with(|| entry.to_string());
+ }
}
};
}
@@ -542,7 +536,7 @@ fn check_tds_archive_name(tds_zip: &Option<String>) -> Option<String> {
let mut pname = String::from(utils::basename(&tz));
let plen = pname.len();
pname.truncate(plen - 8);
- Some(pname.to_string())
+ Some(pname)
}
}
}
@@ -564,11 +558,7 @@ fn unzip_tds_archive(tds_zip: &str, tmp_dir: &str) {
}
}
-fn check_tds_archive(
- pkg_name: &str,
- tds_zip: &str,
- hashes: &HashMap<(u64, Vec<u8>), DupPath>,
-) -> Result<(), io::Error> {
+fn check_tds_archive(pkg_name: &str, tds_zip: &str, hashes: &DupHashes) {
i0003!(tds_zip);
let dir_entry = Path::new(tds_zip);
@@ -581,77 +571,48 @@ fn check_tds_archive(
}
};
+ let ut = Utils::new(utils::CheckType::TDS);
+
// We have a discrepancy between
// TDS zip archive name: babel-base.tds.zip
// and package name: babel
// which yields a false test when checking for the package name
- // the path names of the TDS zip archive files.
+ // the path names of the TDS zip archive files.
// Therefore, we correct here.
// latex-tools, latex-graphics, latex-amsmath, latex-base => latex
- //let mut pkg_replacements: HashMap<String,String> = HashMap::default();
- //pkg_replacements.insert("babel-base".to_string(), "babel".to_string());
- let mut pkg_replacements: HashMap<&str,&str> = HashMap::default();
+ // latex-tools-dev, latex-graphics-dev, latex-amsmath-dev, latex-base-dev => latex-dev
+ let mut pkg_replacements: HashMap<&str, &str> = HashMap::default();
pkg_replacements.insert("babel-base", "babel");
pkg_replacements.insert("latex-base", "latex");
pkg_replacements.insert("latex-tools", "latex");
pkg_replacements.insert("latex-graphics", "latex");
pkg_replacements.insert("latex-amsmath", "latex");
- let real_pkg_name = if let Some(real_name) = pkg_replacements.get(pkg_name) {
+ pkg_replacements.insert("latex-base-dev", "latex-dev");
+ pkg_replacements.insert("latex-tools-dev", "latex-dev");
+ pkg_replacements.insert("latex-graphics-dev", "latex-dev");
+ pkg_replacements.insert("latex-amsmath-dev", "latex-dev");
+ let real_pkg_name = if let Some(real_name) = pkg_replacements.get(pkg_name) {
real_name
} else {
pkg_name
};
- let tmp_dir = TempDir::new("pkgcheck")?;
- let tmp_dir_offset = tmp_dir.path().to_str().unwrap().len() + 1;
- unzip_tds_archive(tds_zip, tmp_dir.path().to_str().unwrap());
-
- let tds_toplevel_dirs: HashSet<String> = [
- "tex", "fonts", "metafont", "metapost", "bibtex", "scripts", "doc", "source",
- ]
- .iter()
- .map(|&s| s.to_string())
- .collect();
-
- for entry in fs::read_dir(tmp_dir.path())? {
- let entry = entry?;
- let path = entry.path();
-
- match path.file_stem() {
- None => (),
- Some(file_stem) => {
- let fs = file_stem.to_str().expect("bad file name");
- if !tds_toplevel_dirs.contains(fs) {
- if path.is_dir() {
- e0020!(fs);
- } else {
- e0034!(fs);
- }
- }
- }
+ let tmp_dir = match TempDir::new("pkgcheck") {
+ Ok(tdir) => tdir,
+ Err(e) => {
+ f0007!(e);
+ process::exit(1);
}
+ };
- // if path.is_dir() {
- // match path.file_stem() {
- // None => (),
- // Some(file_stem) => {
- // let fs = file_stem.to_str().expect("bad file name");
- // if !tds_toplevel_dirs.contains(fs) {
- // e0020!(fs);
- // }
- // }
- // }
- // } else {
- // println!("mist");
- // e0020!(path.display());
- // }
- }
+ let tmp_dir_offset = tmp_dir.path().to_str().unwrap().len() + 1;
+ unzip_tds_archive(tds_zip, tmp_dir.path().to_str().unwrap());
// in order to compare the package files with the content of the
// tds zip archive we need to checksum the files in the tds zip
// archive.
- let mut sizes = HashMap::default();
+ let mut sizes: SizesHashMap = HashMap::default();
let mut pool = Pool::new(num_cpus::get() as u32 + 1);
{
// Processing a single file entry, with the "sizes" hashmap collecting
@@ -672,63 +633,93 @@ fn check_tds_archive(
for dir_entry in WalkDir::new(tmp_dir.path().to_str().unwrap()).follow_links(false) {
match dir_entry {
Ok(dir_entry) => {
- //let dir_entry_str = dir_entry.path().to_str().unwrap();
+ let dir_entry_str = match dir_entry.path().to_str() {
+ Some(d) => d,
+ None => {
+ e0031!(dir_entry.path().to_string_lossy());
+ continue;
+ }
+ };
+
+ // this is the file_name without the directory part
+ // unwrap() is ok here as we covered potential UTF-8 related errors
+ // above in the definition of dir_entry_str
+ let file_name = dir_entry.file_name().to_str().unwrap().to_string();
+
+ let meta = match dir_entry.metadata() {
+ Ok(meta) => meta,
+ Err(e) => {
+ e0027!(dir_entry.path().display(), e);
+ continue;
+ }
+ };
let ft = get_filetype(&dir_entry);
+ if let FType::Error(e) = ft {
+ e0023!(e);
+ continue;
+ }
+
+ let dir_entry_display = if dir_entry.depth() == 0 {
+ &dir_entry_str[tmp_dir_offset - 1..]
+ } else {
+ &dir_entry_str[tmp_dir_offset..]
+ };
+
if ft == FType::Directory {
+ ut.check_for_empty_directory(&dir_entry_str, &dir_entry_display);
+ ut.check_for_hidden_directory(&file_name, &dir_entry_display);
+ ut.is_unwanted_directory(&file_name, dir_entry_str);
continue;
}
- if let FType::Error(e) = ft {
- e0023!(e);
+ // We don't want to have other than directories in the top level
+ // directory of a TDS zip archive
+ if dir_entry.depth() == 1 && ft == FType::Regular {
+ e0034!(dir_entry_display);
continue;
}
- match dir_entry.metadata() {
- Ok(meta) => {
- let fsize = meta.len();
- process(fsize, &dir_entry);
+ ut.check_for_temporary_file(&dir_entry_display);
- let dir_entry_str = match dir_entry.path().to_str() {
- Some(d) => d,
- None => {
- e0031!(dir_entry.path().to_string_lossy());
- continue;
- }
- };
- let dir_entry_display = &dir_entry_str[tmp_dir_offset..];
+ // The LaTeX team provides the file `.tex` as a file with an empty name
+ // in order to make `\input\relax` work (explained by David Carlisle)
+ // Therefore, we don't call check_for_hidden_file() in this case
+ match (pkg_name, dir_entry_display) {
+ ("latex-tools", "tex/latex/tools/.tex") => (),
+ ("latex-tools-dev", "tex/latex-dev/tools/.tex") => (),
+ (_, _) => ut.check_for_hidden_file(&file_name, &dir_entry_display)
+ };
- if is_temporary_file(&dir_entry_str) {
- e0008!(dir_entry_display);
- }
- if fsize > 40 * 1024 * 1024 {
- w0006!(dir_entry_display, fsize / 1024 / 1024);
- }
+ let fsize = meta.len();
+ process(fsize, &dir_entry);
+ ut.check_filesize(fsize, &dir_entry_display);
- // if the path doesn't contain a man page...
- if dir_entry_str.find("/man/").is_none() {
- let pkg_name_s = format!("/{}/", real_pkg_name);
- // ...then we want to have the package name in the path
- if dir_entry_str.find(&pkg_name_s).is_none() {
- e0028!(real_pkg_name, dir_entry_display);
- }
- }
-
- if dir_entry_str.ends_with(".map") {
- map_files_found = true;
- let re: Regex = Regex::new(r"fonts[/]map[/]dvips[/]").unwrap();
- if re.is_match(dir_entry_str) {
- map_dvips_found = true;
- }
+ // if we encounter a .dtx or .ins file we check
+ // that it is in a subdirectory of either source/ or doc/
+ if (dir_entry_str.ends_with(".dtx") || dir_entry_str.ends_with(".ins"))
+ && !(dir_entry_display.starts_with("source/")
+ || dir_entry_display.starts_with("doc/"))
+ {
+ e0036!(dir_entry_display);
+ continue;
+ }
- // if ( $fname =~ m{ fonts [/] map [/] dvips [/] }xms ) {
- // $map_dvips_found = 1;
- // }
- }
+ // if the path doesn't contain a man page...
+ if dir_entry_str.find("/man/").is_none() {
+ let pkg_name_s = format!("/{}/", real_pkg_name);
+ // ...then we want to have the package name in the path
+ if dir_entry_str.find(&pkg_name_s).is_none() {
+ e0028!(real_pkg_name, dir_entry_display);
}
- Err(e) => {
- e0027!(dir_entry.path().display(), e);
+ }
+
+ if dir_entry_str.ends_with(".map") {
+ map_files_found = true;
+ let re: Regex = Regex::new(r"fonts[/]map[/]dvips[/]").unwrap();
+ if re.is_match(dir_entry_str) {
+ map_dvips_found = true;
}
}
}
@@ -765,7 +756,6 @@ fn check_tds_archive(
}
});
- tmp_dir.close()?;
// now check if each package file is in the tds archive
for (k, paths) in hashes.iter() {
if !tds_hashes.contains_key(k) {
@@ -773,8 +763,6 @@ fn check_tds_archive(
e0026!(p);
}
}
-
- Ok(())
}
fn get_extension_from_filename(filename: &str) -> Option<&str> {
@@ -807,6 +795,21 @@ fn found_unwanted_filetype(fname: &str, ft: &FType) -> bool {
}
}
+fn fix_perms(entry: &str) {
+ if !ARGS.correct_perms {
+ return;
+ }
+
+ i0005!(entry);
+ let rc = run_cmd("/bin/chmod", &["-v", "ug=rwX,o=rX", entry]);
+
+ if rc.status {
+ if let Some(op) = rc.output {
+ print!("{}", op);
+ }
+ }
+}
+
fn check_and_correct_perms(dir_entry: &str, p: u32) {
if owner_has(p, 5) || !others_have(p, 4) {
e0002!(dir_entry, perms_to_string(p));
@@ -817,6 +820,21 @@ fn check_and_correct_perms(dir_entry: &str, p: u32) {
};
}
+// Sets permissions for a file or directory
+// Sample invocation: set_perms("somfile", 0o644);
+fn set_perms(entry: &str, p: u32) -> bool {
+ let ps = &format!("{:o}", p);
+ let rc = run_cmd("chmod", &["-v", ps, entry]);
+
+ if rc.status {
+ if let Some(op) = rc.output {
+ print!("{}", op);
+ }
+ true
+ } else {
+ false
+ }
+}
#[derive(Debug, Clone, PartialEq)]
enum FileKind {
@@ -825,7 +843,6 @@ enum FileKind {
Symlink(String),
}
-
impl Display for FileKind {
fn fmt(&self, f: &mut ::std::fmt::Formatter) -> Result<(), ::std::fmt::Error> {
match *self {
@@ -836,8 +853,6 @@ impl Display for FileKind {
}
}
-
-
#[derive(Debug, Clone, PartialEq)]
enum ReadmeKind {
No,
@@ -845,15 +860,17 @@ enum ReadmeKind {
Symlink(String),
}
-fn check_package(root: &str, tds_zip: &Option<String>) -> Option<Hashes> {
- //let mut urls = HashMap::default(); //: <String,UrlStatus>
-
+fn check_package(root: &str, tds_zip: &Option<String>) -> Option<DupHashes> {
let mut lcnames: HashMap<PathBuf, Vec<(PathBuf, FileKind)>> = HashMap::default();
let mut doublenames: HashMap<PathBuf, Vec<PathBuf>> = HashMap::default();
let mut inodes = HashSet::default();
+ let ut = Utils::new(utils::CheckType::PACKAGE);
+
+
+
i0002!(root);
// This hash contains all package file names.
//
@@ -862,185 +879,162 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<Hashes> {
// String: the file name without any directory part
// ReadmeKind: is it a certain README, file or symlink?
// A special case of a README file is a file with has a different name but
- // was poinded to by a symlink. Example: README --> README.rst
- let mut file_names: HashMap<PathBuf, (Metadata, String, ReadmeKind)> = HashMap::default();
- {
- let mut readme_found = false;
-
- let root_absolute = PathBuf::from(root)
- .canonicalize()
- .unwrap()
- .to_string_lossy()
- .to_string();
- for dir_entry in WalkDir::new(root).follow_links(false) {
- match dir_entry {
- Ok(dir_entry) => {
- let dir_entry_str = match dir_entry.path().to_str() {
- Some(d) => d,
- None => {
- e0031!(dir_entry.path().to_string_lossy());
- continue;
- }
- };
- let meta = match dir_entry.metadata() {
- Ok(meta) => meta,
- Err(e) => {
- e0023!(e);
- continue;
- }
- };
-
- if !check_inode(&mut inodes, &meta) {
+ // was pointed to by a symlink. Example: README --> README.rst
+ let mut file_names: FileNamesHashMap = HashMap::default();
+ let mut readme_found = false;
+
+ let root_absolute = PathBuf::from(root)
+ .canonicalize()
+ .unwrap()
+ .to_string_lossy()
+ .to_string();
+
+ for dir_entry in WalkDir::new(root).follow_links(false) {
+ match dir_entry {
+ Ok(dir_entry) => {
+ let dir_entry_str = match dir_entry.path().to_str() {
+ Some(d) => d,
+ None => {
+ e0031!(dir_entry.path().to_string_lossy());
continue;
}
+ };
+ let meta = match dir_entry.metadata() {
+ Ok(meta) => meta,
+ Err(e) => {
+ e0023!(e);
+ continue;
+ }
+ };
- // this is the file_name without the directory part
- // unwrap() is ok here as we covered potential UTF-8 related errors
- // above in the definition of dir_entry_str
- let file_name = dir_entry.file_name().to_str().unwrap().to_string();
+ if !check_inode(&mut inodes, &meta) {
+ continue;
+ }
- let ft = get_filetype(&dir_entry);
+ // this is the file_name without the directory part
+ // unwrap() is ok here as we covered potential UTF-8 related errors
+ // above in the definition of dir_entry_str
+ let file_name = dir_entry.file_name().to_str().unwrap().to_string();
- if found_unwanted_filetype(dir_entry_str, &ft) {
- continue;
- }
+ // we check for weird stuff like socket files aso.
+ let ft = get_filetype(&dir_entry);
+ if found_unwanted_filetype(&dir_entry_str, &ft) {
+ continue;
+ }
- if let Some((offset, ch)) = filename_has_bad_chars(&dir_entry) {
- e0001!(ch, dir_entry_str, offset);
- }
+ ut.filename_has_bad_chars(&dir_entry, dir_entry_str);
- // 1. dealing with symlinks
- if ft == FType::Symlink {
- match get_symlink(&dir_entry) {
- Ok(None) => {
- e0010!(dir_entry_str);
- continue;
- }
- Err(e) => {
- e0027!(dir_entry_str, e);
+ // 1. dealing with symlinks
+ if ft == FType::Symlink {
+ match get_symlink(&dir_entry) {
+ Ok(None) => {
+ e0010!(&dir_entry_str);
+ continue;
+ }
+ Err(e) => {
+ e0027!(&dir_entry_str, e);
+ continue;
+ }
+ Ok(Some(p)) => {
+ let pd: String =
+ p.canonicalize().unwrap().to_string_lossy().to_string();
+ if !pd.starts_with(&root_absolute) {
+ e0030!(&dir_entry_str, p.display());
continue;
}
- Ok(Some(p)) => {
- let pd: String =
- p.canonicalize().unwrap().to_string_lossy().to_string();
- if !pd.starts_with(&root_absolute) {
- e0030!(dir_entry_str, p.display());
- continue;
- }
- let lc_dir_entry_str = dir_entry_str.to_lowercase();
- if let Some(_dir_name) = filename(dir_entry_str) {
- let lcnref = &mut lcnames;
- lcnref
- .entry(PathBuf::from(lc_dir_entry_str))
- .or_insert_with(Vec::new)
- .push((
- PathBuf::from(dir_entry_str),
- //FileKind::Symlink(dir_entry_str.into()),
- FileKind::Symlink(pd.clone()),
- ));
- }
- if is_readme(&dir_entry) {
- readme_found = true;
- file_names.insert(
- p,
- (
- meta,
- file_name,
- ReadmeKind::Symlink(dir_entry_str.to_string()),
- ),
- );
- }
- continue;
+ let lc_dir_entry_str = dir_entry_str.to_lowercase();
+ if let Some(_dir_name) = filename(&dir_entry_str) {
+ let lcnref = &mut lcnames;
+ lcnref
+ .entry(PathBuf::from(lc_dir_entry_str))
+ .or_insert_with(Vec::new)
+ .push((
+ PathBuf::from(&dir_entry_str),
+ //FileKind::Symlink(&dir_entry_str.into()),
+ FileKind::Symlink(pd.clone()),
+ ));
}
+ if is_readme(&dir_entry) {
+ readme_found = true;
+ file_names.insert(
+ p,
+ (
+ meta,
+ file_name,
+ ReadmeKind::Symlink(dir_entry_str.to_string()),
+ ),
+ );
+ }
+ continue;
}
}
+ }
- let p = get_perms(&dir_entry.path());
+ let p = get_perms(&dir_entry.path());
- // 2. dealing with directories
- if ft == FType::Directory {
- let lc_dir_entry_str = dir_entry_str.to_lowercase();
- if let Some(_dir_name) = filename(dir_entry_str) {
- let lcnref = &mut lcnames;
-
- lcnref
- .entry(PathBuf::from(lc_dir_entry_str))
- .or_insert_with(Vec::new)
- .push((PathBuf::from(dir_entry_str), FileKind::Directory));
- }
+ // 2. dealing with directories
+ if ft == FType::Directory {
- if !owner_has(p, 5) || !others_have(p, 5) {
- e0011!(dir_entry_str, perms_to_string(p));
- if ARGS.correct_perms {
- i0005!(&dir_entry_str);
- set_perms(&dir_entry_str, 0o775);
- }
- }
+ let lc_dir_entry_str = dir_entry_str.to_lowercase();
+ if let Some(_dir_name) = filename(&dir_entry_str) {
+ let lcnref = &mut lcnames;
- match is_empty_directory(&dir_entry) {
- Ok(true) => {
- e0004!(dir_entry_str);
- }
- Err(e) => {
- e0027!(dir_entry_str, e);
- }
- Ok(false) => (),
- }
- if is_hidden_directory(&dir_entry) {
- e0006!(dir_entry_str);
- }
+ lcnref
+ .entry(PathBuf::from(lc_dir_entry_str))
+ .or_insert_with(Vec::new)
+ .push((PathBuf::from(&dir_entry_str), FileKind::Directory));
+ }
- if is_unwanted_directory(&dir_entry.file_name().to_str().unwrap()) {
- e0018!(dir_entry_str);
+ if !owner_has(p, 5) || !others_have(p, 5) {
+ e0011!(&dir_entry_str, perms_to_string(p));
+ if ARGS.correct_perms {
+ i0005!(&dir_entry_str);
+ set_perms(&dir_entry_str, 0o775);
}
- continue;
}
- // 3. dealing with regular files
- if is_hidden(&file_name) {
- e0007!(dir_entry_str);
- }
- if is_temporary_file(&dir_entry_str) {
- e0008!(dir_entry_str);
- }
+ ut.check_for_empty_directory(&dir_entry_str, &dir_entry_str);
+ ut.check_for_hidden_directory(&file_name, &dir_entry_str);
+ ut.is_unwanted_directory(&file_name, dir_entry_str);
+ continue;
+ }
- if let Some(file_name) = filename(dir_entry_str) {
- let doubleref = &mut doublenames;
+ // 3. dealing with regular files
+ ut.check_for_hidden_file(&file_name, &dir_entry_str);
+ ut.check_for_temporary_file(&dir_entry_str);
- doubleref
- .entry(PathBuf::from(file_name))
- .or_insert_with(Vec::new)
- .push(PathBuf::from(dir_entry_str));
- }
+ // if is_temporary_file(&dir_entry_str) {
+ // e0008!(&dir_entry_str);
+ // }
- // detecting large files in package directory
-
- if is_readme(&dir_entry) {
- // We want to deal with README files only if they are
- // in the root directory of the package.
- let f = format!(
- "{}{}{}",
- root,
- // we have to pay attention if `root` ends already with '/'
- if root.ends_with('/') { "" } else { "/" },
- &file_name
- );
+ if let Some(file_name) = filename(&dir_entry_str) {
+ let doubleref = &mut doublenames;
- if dir_entry_str == f {
- readme_found = true;
- file_names.insert(
- dir_entry.path().to_path_buf(),
- (meta, file_name.clone(), ReadmeKind::Yes),
- );
- } else {
- file_names.entry(dir_entry.path().to_path_buf()).or_insert((
- meta,
- file_name.clone(),
- ReadmeKind::No,
- ));
- }
+ doubleref
+ .entry(PathBuf::from(file_name))
+ .or_insert_with(Vec::new)
+ .push(PathBuf::from(&dir_entry_str));
+ }
+
+ if is_readme(&dir_entry) {
+ // We want to deal with README files only if they are
+ // in the root directory of the package.
+ let f = format!(
+ "{}{}{}",
+ root,
+ // we have to pay attention if `root` ends already with '/'
+ if root.ends_with('/') { "" } else { "/" },
+ &file_name
+ );
+
+ if dir_entry_str == f {
+ readme_found = true;
+ file_names.insert(
+ dir_entry.path().to_path_buf(),
+ (meta, file_name.clone(), ReadmeKind::Yes),
+ );
} else {
file_names.entry(dir_entry.path().to_path_buf()).or_insert((
meta,
@@ -1048,168 +1042,203 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<Hashes> {
ReadmeKind::No,
));
}
+ } else {
+ file_names.entry(dir_entry.path().to_path_buf()).or_insert((
+ meta,
+ file_name.clone(),
+ ReadmeKind::No,
+ ));
+ }
- let lc_dir_entry_str = dir_entry_str.to_lowercase();
-
- let lcnref = &mut lcnames;
+ let lc_dir_entry_str = dir_entry_str.to_lowercase();
- lcnref
- .entry(PathBuf::from(lc_dir_entry_str))
- .or_insert_with(Vec::new)
- .push((PathBuf::from(dir_entry_str), FileKind::File));
+ let lcnref = &mut lcnames;
- }
+ lcnref
+ .entry(PathBuf::from(lc_dir_entry_str))
+ .or_insert_with(Vec::new)
+ .push((PathBuf::from(&dir_entry_str), FileKind::File));
+ }
- Err(e) => {
- eprintln!("{}", e);
- }
+ Err(e) => {
+ eprintln!("{}", e);
}
}
+ }
- if !readme_found {
- e0009!();
- }
+ if !readme_found {
+ e0009!();
}
let lc = LinkCheck::new(4, false);
let mut detective = filemagic::Filetype::new();
- let mut sizes = HashMap::default();
- let mut generated: HashMap<PathBuf, PathBuf> = HashMap::default();
+ let mut sizes: SizesHashMap = HashMap::default();
+ let mut generated: GeneratedHashMap = HashMap::default();
- {
- // Processing a single file entry, with the "sizes" hashmap collecting
- // same-size files. Entries are either Found::One or Found::Multiple,
- // so that we can submit the first file's path as a hashing job when the
- // first duplicate is found. Hashing each file is submitted as a job to
- // the pool.
- let mut process = |fsize, path: &PathBuf| {
- let sizeref = &mut sizes;
+ // Processing a single file entry, with the "sizes" hashmap collecting
+ // same-size files. Entries are either Found::One or Found::Multiple,
+ // so that we can submit the first file's path as a hashing job when the
+ // first duplicate is found. Hashing each file is submitted as a job to
+ // the pool.
+ let mut process = |fsize, path: &PathBuf| {
+ let sizeref = &mut sizes;
- let path = path.clone();
- sizeref.entry(fsize).or_insert_with(Vec::new).push(path);
+ let path = path.clone();
+ sizeref.entry(fsize).or_insert_with(Vec::new).push(path);
+ };
+ for (path, (meta, _file_name, is_readme)) in file_names.iter() {
+ //println!("DEBUG: path {:?}, file_name {:?}, is_readme: {} ", &path, &file_name, is_readme);
+ let dir_entry_str = match path.to_str() {
+ Some(d) => d,
+ None => {
+ e0031!(&path.to_string_lossy());
+ continue;
+ }
};
- for (path, (meta, _file_name, is_readme)) in file_names.iter() {
- //println!("DEBUG: path {:?}, file_name {:?}, is_readme: {} ", &path, &file_name, is_readme);
- let dir_entry_str = match path.to_str() {
- Some(d) => d,
- None => {
- e0031!(&path.to_string_lossy());
- continue;
- }
- };
+ let fsize = meta.len();
+ ut.check_filesize(fsize, dir_entry_str);
- let fsize = meta.len();
+ let p = get_perms(&path);
+ if !owner_has(p, 4) {
+ e0002!(&dir_entry_str, perms_to_string(p));
+ fix_perms(&dir_entry_str);
+ continue;
+ }
+ let ftr = detective.analyze(&dir_entry_str);
+ //println!(">>> {:?}", ftr);
+ // we ignore errors from filetype recognition
+ if ftr.is_err() {
+ continue;
+ }
- // if it is an empty file we don't need
- // to check further
- if fsize == 0 {
- e0005!(dir_entry_str);
- continue;
- }
- if fsize > 40 * 1024 * 1024 {
- w0005!(dir_entry_str, fsize / 1024 / 1024);
- continue;
- }
+ let ft = ftr.unwrap();
- let p = get_perms(&path);
- if !owner_has(p, 4) {
- e0002!(dir_entry_str, perms_to_string(p));
- if ARGS.correct_perms {
- fix_perms(&dir_entry_str);
- }
+ // DEBUG !readme_symlinked.contains(&dir_entry_str)
+ if ReadmeKind::No != *is_readme {
+ if !check_readme(&dir_entry_str, is_readme, &ft) {
continue;
}
- let ftr = detective.analyze(&dir_entry_str);
- //println!(">>> {:?}", ftr);
- // we ignore errors from filetype recognition
- if ftr.is_err() {
- continue;
+ if ARGS.urlcheck {
+ lc.check_urls(&dir_entry_str);
}
+ }
- let ft = ftr.unwrap();
-
- // DEBUG !readme_symlinked.contains(dir_entry_str)
- if ReadmeKind::No != *is_readme {
- if !check_readme(dir_entry_str, is_readme, &ft) {
- continue;
- }
- if ARGS.urlcheck {
- lc.check_urls(dir_entry_str);
+ match ft {
+ filemagic::Mimetype::Text(_) => {
+ check_and_correct_perms(&dir_entry_str, p);
+ let fext = get_extension_from_filename(&dir_entry_str);
+ if fext == Some("ins") || fext == Some("dtx") {
+ check_generated_files(&dir_entry_str, &mut generated);
}
- }
- match ft {
- filemagic::Mimetype::Text | filemagic::Mimetype::TextCRLF => {
- check_and_correct_perms(dir_entry_str, p);
- let fext = get_extension_from_filename(&dir_entry_str);
- if fext == Some("ins") || fext == Some("dtx") {
- check_generated_files(&dir_entry_str, &mut generated);
- }
- match fext {
- Some("bat") | Some("cmd") |
- Some("nsh") | Some("reg") => (),
- Some(_) | None => {
- if ft == filemagic::Mimetype::TextCRLF {
- e0012!(dir_entry_str);
- if ARGS.correct_crlf {
- fix_crlf(dir_entry_str);
+ match fext {
+ // deal with Windows files
+ Some("bat") | Some("cmd") | Some("nsh") | Some("reg") => {
+ match ft {
+ filemagic::Mimetype::Text(LineEnding::CRLF) => (),
+ filemagic::Mimetype::Text(LineEnding::CR) =>
+ {
+ e0037!(&dir_entry_str);
+ if ARGS.correct_le {
+ make_crlf(&dir_entry_str);
}
- }
+ },
+ filemagic::Mimetype::Text(LineEnding::MIXED(0, 0, 0) ) => (),
+ filemagic::Mimetype::Text(LineEnding::MIXED(cr, lf, crlf) ) =>
+ {
+ e0038!(&dir_entry_str, cr, lf, crlf);
+ if ARGS.correct_le {
+ fix_inconsistent_le(&dir_entry_str);
+ }
+ },
+ filemagic::Mimetype::Text(LineEnding::LF) =>
+ {
+ w0008!(&dir_entry_str);
+ },
+ fmm => println!("Should not occur: {} has {:?}", dir_entry_str, fmm),
}
- };
+ },
+ Some(_) | None => {
+ match ft {
+ filemagic::Mimetype::Text(LineEnding::CRLF) =>
+ {
+ e0012!(&dir_entry_str);
+ if ARGS.correct_le {
+ fix_inconsistent_le(&dir_entry_str);
+ }
+ },
+ filemagic::Mimetype::Text(LineEnding::CR) =>
+ {
+ e0037!(&dir_entry_str);
+ if ARGS.correct_le {
+ fix_inconsistent_le(&dir_entry_str);
+ }
+ },
+ filemagic::Mimetype::Text(LineEnding::MIXED(0, 0, 0) ) => (),
+ filemagic::Mimetype::Text(LineEnding::MIXED(cr, lf, crlf) ) =>
+ {
+ //println!(">>>{}: {:?} {},{},{}", &dir_entry_str, ft, x, y, z);
+ e0038!(&dir_entry_str, cr, lf, crlf);
+ if ARGS.correct_le {
+ fix_inconsistent_le(&dir_entry_str);
+ }
+ },
+ filemagic::Mimetype::Text(LineEnding::LF) => (),
+ fmm => println!("Should not occur: {} has {:?}", dir_entry_str, fmm),
+ }
+ },
}
+ },
- filemagic::Mimetype::BOM(b) => {
- //println!("{}: {} with BOM detected", dir_entry_str, b.as_ref());
- w0004!(dir_entry_str, b.as_ref());
- check_and_correct_perms(dir_entry_str, p);
- }
- filemagic::Mimetype::Binary
- | filemagic::Mimetype::Script
- | filemagic::Mimetype::ScriptCRLF => {
- if !owner_has(p, 4) || !others_have(p, 4) {
- e0002!(dir_entry_str, perms_to_string(p));
+ filemagic::Mimetype::BOM(b) => {
+ //println!("{}: {} with BOM detected", dir_entry_str, b.as_ref());
+ w0004!(&dir_entry_str, b.as_ref());
+ check_and_correct_perms(&dir_entry_str, p);
+ }
+ filemagic::Mimetype::Binary
+ | filemagic::Mimetype::Script(_) => {
+ if !owner_has(p, 4) || !others_have(p, 4) {
+ e0002!(&dir_entry_str, perms_to_string(p));
+ };
+ fix_perms(&dir_entry_str);
+ }
+ filemagic::Mimetype::Pdf => {
+ check_and_correct_perms(&dir_entry_str, p);
+ let ret = is_pdf_ok(&dir_entry_str);
+ if !ret.status {
+ e0017!(&dir_entry_str);
+ if let Some(output) = ret.output {
+ println!("{}", &output);
};
- if ARGS.correct_perms {
- fix_perms(&dir_entry_str);
- }
- }
- filemagic::Mimetype::Pdf => {
- check_and_correct_perms(dir_entry_str, p);
- let ret = is_pdf_ok(&dir_entry_str);
- if !ret.status {
- e0017!(dir_entry_str);
- if let Some(output) = ret.output {
- println!("{}", &output);
- };
- }
- }
-
- filemagic::Mimetype::Archive | filemagic::Mimetype::Zip => {
- if dir_entry_str.ends_with(".tds.zip") {
- e0035!(dir_entry_str);
- } else {
- w0001!(dir_entry_str);
- }
- check_and_correct_perms(dir_entry_str, p);
}
-
- filemagic::Mimetype::Data => check_and_correct_perms(dir_entry_str, p),
- _ => continue,
}
- if others_match(p, 0) {
- e0002!(dir_entry_str, perms_to_string(p));
- if ARGS.correct_perms {
- i0005!(&dir_entry_str);
- set_perms(&dir_entry_str, 0o664);
+
+ filemagic::Mimetype::Archive | filemagic::Mimetype::Zip => {
+ if dir_entry_str.ends_with(".tds.zip") {
+ e0035!(&dir_entry_str);
+ } else {
+ w0001!(&dir_entry_str);
}
+ check_and_correct_perms(&dir_entry_str, p);
}
- if !(ARGS.ignore_dupes && tds_zip.is_none()) {
- process(fsize, &path);
+
+ filemagic::Mimetype::Data => check_and_correct_perms(&dir_entry_str, p),
+ filemagic::Mimetype::Zerofile => check_and_correct_perms(&dir_entry_str, p),
+ _ => continue,
+ }
+
+ if others_match(p, 0) {
+ e0002!(&dir_entry_str, perms_to_string(p));
+ if ARGS.correct_perms {
+ i0005!(&dir_entry_str);
+ set_perms(&dir_entry_str, 0o664);
}
}
+ if !(ARGS.ignore_dupes && tds_zip.is_none()) {
+ process(fsize, &path);
+ }
}
print_casefolding(&lcnames);
@@ -1259,7 +1288,6 @@ fn check_package(root: &str, tds_zip: &Option<String>) -> Option<Hashes> {
}
fn print_casefolding(hashes: &HashMap<PathBuf, Vec<(PathBuf, FileKind)>>) {
-
for (k, eles) in hashes.iter() {
//println!("pcf: {:?}, {:?}", k, &eles);
if eles.len() == 1 {
@@ -1274,12 +1302,14 @@ fn print_casefolding(hashes: &HashMap<PathBuf, Vec<(PathBuf, FileKind)>>) {
}
}
-fn print_generated(hashes: &HashMap<PathBuf, Vec<PathBuf>>, generated: &HashMap<PathBuf, PathBuf>) {
+fn print_generated(hashes: &HashMap<PathBuf, Vec<PathBuf>>, generated: &GeneratedHashMap) {
for (k, gen) in generated.iter() {
- if hashes.contains_key(k) {
- let v = &hashes[k];
+ let mut path = PathBuf::new();
+ path.push(k);
+ if hashes.contains_key(&path) {
+ let v = &hashes[&path];
for fname in v {
- e0019!(fname.to_str().unwrap(), gen.to_str().unwrap());
+ e0019!(fname.to_str().unwrap(), gen.as_str());
}
}
}
@@ -1317,3 +1347,5 @@ fn show_tmp_endings() {
println!("{:23} {}", t, c);
}
}
+
+