From 9dfeafbb128537c1633d77071d8cc9b9f8c5a800 Mon Sep 17 00:00:00 2001 From: Norbert Preining Date: Tue, 27 Dec 2022 03:01:20 +0000 Subject: CTAN sync 202212270301 --- support/pkgcheck/CHANGES.md | 9 +++++ support/pkgcheck/Cargo.toml | 16 ++++----- support/pkgcheck/README.md | 1 + support/pkgcheck/bin/pkgcheck | Bin 7258856 -> 7266008 bytes support/pkgcheck/docs/fatald.tex | 20 ++++++----- support/pkgcheck/docs/pkgcheck.pdf | Bin 89196 -> 89893 bytes support/pkgcheck/docs/title.tex | 2 +- support/pkgcheck/docs/warningsd.tex | 10 +++++- support/pkgcheck/src/filemagic.rs | 12 +++---- support/pkgcheck/src/gparser.rs | 1 + support/pkgcheck/src/main.rs | 56 ++++++++++++++++++----------- support/pkgcheck/src/messages/fatald.rs | 2 +- support/pkgcheck/src/messages/mod.rs | 9 ++++- support/pkgcheck/src/messages/warningsd.rs | 11 ++++++ support/pkgcheck/src/recode.rs | 8 ++--- 15 files changed, 103 insertions(+), 54 deletions(-) (limited to 'support/pkgcheck') diff --git a/support/pkgcheck/CHANGES.md b/support/pkgcheck/CHANGES.md index ade233c38d..3f1414d6bf 100644 --- a/support/pkgcheck/CHANGES.md +++ b/support/pkgcheck/CHANGES.md @@ -6,6 +6,15 @@ # Changes +## 2022-12-26 (3.1.0) +### Added + - now checking for hard links which generate a warning message + - new message w0010: now hardlinks are detected and reported + - add armtex to exception list when checking path names in the TDS zip archive + - add latex-cyrillic to exception list when checking path names in the TDS zip archive +### Changed + - typo in F010 message + ## 2022-05-26 (3.0.0) ### Added - fatal messages Fnnnn go to stderr instead of stdout diff --git a/support/pkgcheck/Cargo.toml b/support/pkgcheck/Cargo.toml index f7120709d4..c765d91590 100644 --- a/support/pkgcheck/Cargo.toml +++ b/support/pkgcheck/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pkgcheck" -version = "3.0.0" +version = "3.1.0" authors = ["Manfred Lotz "] description = "Command-line tool to check packages uploaded to CTAN." license = "MIT/Apache-2.0" @@ -18,28 +18,28 @@ lto = true [dependencies] blake3 = "1.3" -clap = { version = "3.1.6", features = ["derive"] } -clap_complete = "3.1.1" +clap = { version = "4", features = ["derive"] } +clap_complete = "4" colored = "2" config = "0.13" escargot = "0.5" fern = "0.6" glob = "0.3" home = "0.5.3" -once_cell = "1.10.0" +once_cell = "1.13" #lazy_static = "1" -linkify = "0.8" +linkify = "0.9" log = "0.4" num_cpus = "1" openssl-probe = "0.1.4" -pest = "2.1" -pest_derive = "2.1" +pest = "2.3" +pest_derive = "2.3" regex = "1" reqwest = { version = "0.11", features = ["blocking", "gzip"] } rustc-hash = "1.1" scoped_threadpool = "0.1" serde = { version = "1.0", features = ["derive"] } -serde_yaml = "0.8" +serde_yaml = "0.9" tempfile = "3.1" threadpool = "1.8" tokio = { version = "1", features = ["full"] } diff --git a/support/pkgcheck/README.md b/support/pkgcheck/README.md index d014061799..834c1d96f9 100644 --- a/support/pkgcheck/README.md +++ b/support/pkgcheck/README.md @@ -5,6 +5,7 @@ Author: Manfred Lotz, License: Apache License, Version 2.0 or MIT License +Copyright (c) 2018 Manfred Lotz --- diff --git a/support/pkgcheck/bin/pkgcheck b/support/pkgcheck/bin/pkgcheck index 04da19b362..abaea81efc 100755 Binary files a/support/pkgcheck/bin/pkgcheck and b/support/pkgcheck/bin/pkgcheck differ diff --git a/support/pkgcheck/docs/fatald.tex b/support/pkgcheck/docs/fatald.tex index aa80e74e28..7a880fe775 100644 --- a/support/pkgcheck/docs/fatald.tex +++ b/support/pkgcheck/docs/fatald.tex @@ -27,8 +27,8 @@ Specify a valid TDS zip archive when calling \texttt{pkgcheck} \subsection{F0005 -\/- Bad file name for the zip archive}\label{f0005----bad-file-name-for-the-zip-archive}} -pkgcheck detected that the file name of the TDS zip archive doesn't end -with \texttt{.tds.zip} +pkgcheck detected that the file name of the TDS zip archive +doesn\textquotesingle t end with \texttt{.tds.zip} \hypertarget{f0006----unknown-error-code-specified-with-option--e-resp----explain-exiting}{% \subsection{F0006 -\/- Unknown error code specified with option -e resp. @@ -47,10 +47,12 @@ Make sure the temp directory is writable and/or the temp directory has enough space. \hypertarget{f0008----config-file--doesnt-exist-exiting}{% -\subsection{\texorpdfstring{F0008 -\/- Config file doesn't exist. -Exiting...}{F0008 -\/- Config file doesn't exist. Exiting...}}\label{f0008----config-file--doesnt-exist-exiting}} +\subsection{\texorpdfstring{F0008 -\/- Config file +doesn\textquotesingle t exist. +Exiting...}{F0008 -\/- Config file doesn\textquotesingle t exist. Exiting...}}\label{f0008----config-file--doesnt-exist-exiting}} -The config file specified at the command line doesn't exist. +The config file specified at the command line doesn\textquotesingle t +exist. \hypertarget{f0009----error-reading-config-file-config_file--exiting}{% \subsection{\texorpdfstring{F0009 -\/- Error reading config file @@ -59,10 +61,10 @@ Exiting...}{F0009 -\/- Error reading config file \textless config\_file\textgrea The config file \textless config\_file\textgreater{} could not be read. -\hypertarget{f0010----config-files-content-could-no-be-read-properly--exiting}{% -\subsection{\texorpdfstring{F0010 -\/- Config file's content could no be -read properly: -Exiting...}{F0010 -\/- Config file's content could no be read properly: Exiting...}}\label{f0010----config-files-content-could-no-be-read-properly--exiting}} +\hypertarget{f0010----config-files-content-could-not-be-read-properly--exiting}{% +\subsection{\texorpdfstring{F0010 -\/- Config file\textquotesingle s +content could not be read properly: +Exiting...}{F0010 -\/- Config file\textquotesingle s content could not be read properly: Exiting...}}\label{f0010----config-files-content-could-not-be-read-properly--exiting}} The content of the config has errors as given in . Please correct and rerun. diff --git a/support/pkgcheck/docs/pkgcheck.pdf b/support/pkgcheck/docs/pkgcheck.pdf index 29df6c8e4a..5d5259bab4 100644 Binary files a/support/pkgcheck/docs/pkgcheck.pdf and b/support/pkgcheck/docs/pkgcheck.pdf differ diff --git a/support/pkgcheck/docs/title.tex b/support/pkgcheck/docs/title.tex index 020feb2956..8dcca045d1 100644 --- a/support/pkgcheck/docs/title.tex +++ b/support/pkgcheck/docs/title.tex @@ -1 +1 @@ -\title{pkgcheck Utility, v3.0.0} +\title{pkgcheck Utility, v3.1.0} diff --git a/support/pkgcheck/docs/warningsd.tex b/support/pkgcheck/docs/warningsd.tex index b681972557..24667fc2f8 100644 --- a/support/pkgcheck/docs/warningsd.tex +++ b/support/pkgcheck/docs/warningsd.tex @@ -59,7 +59,8 @@ in the TDS zip archive. archive}\label{w0007----empty-directory-detected-in-the-tds-zip-archive}} Empty directories in a TDS zip archive are discouraged. As they usually -don't create errors in the distribution we issue a warning only. +don\textquotesingle t create errors in the distribution we issue a +warning only. \hypertarget{w0008----windows-file-has-unix-line-endings}{% \subsection{W0008 -\/- Windows file has Unix line @@ -90,3 +91,10 @@ This message can only show up if \texttt{pkgcheck} got called with \texttt{-\/-config}. Indicates that an entry in the \texttt{pkgcheck} config file does the same as the hard-coded entry. This helps to keep a clean config file. + +\hypertarget{w0010----hardlinks-detected-with-inode-}{% +\subsection{\texorpdfstring{W0010 -\/- Hardlinks detected with inode +}{W0010 -\/- Hardlinks detected with inode }}\label{w0010----hardlinks-detected-with-inode-}} + +Hardlinks found in the package directory tree. The inode number will be +displayed diff --git a/support/pkgcheck/src/filemagic.rs b/support/pkgcheck/src/filemagic.rs index f3a4bcc391..0785664c46 100644 --- a/support/pkgcheck/src/filemagic.rs +++ b/support/pkgcheck/src/filemagic.rs @@ -19,7 +19,7 @@ use unicode_bom::Bom; // - https://filesignatures.net/ // - https://github.com/7h3rAm/cigma/blob/master/cigma/magicbytes.json -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Eq)] pub enum LineEnding { Lf, Cr, @@ -27,7 +27,7 @@ pub enum LineEnding { Mixed(usize, usize, usize), } -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Eq)] pub enum Mimetype { Binary, Script(LineEnding), @@ -264,7 +264,7 @@ impl Filetype { // https://en.wikipedia.org/wiki/Executable_and_Linkable_Format fn is_binary(vec: &[u8]) -> Option { - let binary_re: RegexSet = RegexSet::new(&[ + let binary_re: RegexSet = RegexSet::new([ r"^(?-u)\x7FELF[\x01\x02][\x01\x02]\x01[\x00-\x11]", // Executable and Linkable Format (ELF) r"^(?-u)\x00\x00\x03\xF3", // AmigaOS loadseg()ble executable/binary r"^(?-u)MZ", // DOS MZ executable file format and its descendants (including NE and PE) @@ -288,7 +288,7 @@ fn is_archive(vec: &[u8]) -> Option { // - opendocument formats // - Word Open XML // Those we do not regard as archives - let special_zip: RegexSet = RegexSet::new(&[ + let special_zip: RegexSet = RegexSet::new([ r"^(?-u)PK\x03\x04.{20,}\x08\x00\x00\x00mimetypeapplication", r"^(?-u)PK\x03\x04\x14\x00\x06\x00", // Word Open XML (.docx) r"^(?-u)PK\x03\x04\x14\x00\x08\x00", // Java Jar file @@ -303,7 +303,7 @@ fn is_archive(vec: &[u8]) -> Option { return Some(Mimetype::Data); } - let archive_re: RegexSet = RegexSet::new(&[ + let archive_re: RegexSet = RegexSet::new([ r"^(?-u)\x37\x7A\xBC\xAF\x27\x1C", // 7zip r"^(?-u)\x1f\x8B", // gzip (.gz) r"^(?-u)\x1f\x9D", // LZW (.tar.Z) @@ -332,7 +332,7 @@ fn is_archive(vec: &[u8]) -> Option { return Some(Mimetype::Archive); } - let archive_re: RegexSet = RegexSet::new(&[ + let archive_re: RegexSet = RegexSet::new([ r"^(?-u)PK(\x03\x04|\x4c\x49\x54\x45|\x30\x30\x50|\x05\x06|\x07\x08)", // zip archive ]) .unwrap(); diff --git a/support/pkgcheck/src/gparser.rs b/support/pkgcheck/src/gparser.rs index 136f6f41b1..4c10ba9f7a 100644 --- a/support/pkgcheck/src/gparser.rs +++ b/support/pkgcheck/src/gparser.rs @@ -1,3 +1,4 @@ + use log::*; use pest::Parser; diff --git a/support/pkgcheck/src/main.rs b/support/pkgcheck/src/main.rs index f0766cdeba..9ff8390f9c 100644 --- a/support/pkgcheck/src/main.rs +++ b/support/pkgcheck/src/main.rs @@ -83,14 +83,14 @@ fn format_message(message: &Arguments, no_color: bool) -> Cow<'static, str> { } } -#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct TdsException { pub pkg: String, pub tpkg: String, } -#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct PathExceptions { pub tds_path_exceptions: Vec, @@ -122,19 +122,21 @@ fn get_config_file_name() -> Option { fn read_yaml_config() -> FxHashMap { let mut pkg_replacements: FxHashMap = FxHashMap::default(); for (p, q) in [ + ("armtex", "armenian"), ("babel-base", "babel"), - ("latex-base", "latex"), - ("latex-lab", "latex"), - ("latex-tools", "latex"), - ("latex-graphics", "latex"), ("latex-amsmath", "latex"), - ("latex-firstaid", "latex/firstaid"), + ("latex-amsmath-dev", "latex-dev"), + ("latex-base", "latex"), ("latex-base-dev", "latex-dev"), + ("latex-cyrillic", "cyrillic"), + ("latex-firstaid", "latex/firstaid"), + ("latex-firstaid-dev", "latex-dev/firstaid"), + ("latex-graphics", "latex"), + ("latex-graphics-dev", "latex-dev"), + ("latex-lab", "latex"), ("latex-lab-dev", "latex-dev"), + ("latex-tools", "latex"), ("latex-tools-dev", "latex-dev"), - ("latex-graphics-dev", "latex-dev"), - ("latex-amsmath-dev", "latex-dev"), - ("latex-firstaid-dev", "latex-dev/firstaid"), ("vntex-nonfree", "vntex"), ] { pkg_replacements.insert(p.to_string(), q.to_string()); @@ -209,7 +211,7 @@ const BLOCKSIZE: usize = 4096; fn hash_file_inner(path: &Path) -> io::Result> { let mut buf = [0u8; BLOCKSIZE]; - let mut fp = File::open(&path)?; + let mut fp = File::open(path)?; let mut digest = blake3::Hasher::new(); loop { match fp.read(&mut buf)? { @@ -357,7 +359,7 @@ struct Args { group = "only_one" )] explain_all: bool, - #[clap(long = "generate-completion", group = "only_one", arg_enum)] + #[clap(long = "generate-completion", group = "only_one", value_enum)] generator: Option, #[clap( long = "show-temp-endings", @@ -371,11 +373,11 @@ struct Args { config_file: Option, } -// We take care to avoid visiting a single inode twice, -// which takes care of (false positive) hardlinks. +// In the pas we took care to avoid visiting a single inode twice, which takes care of (false positive) hardlinks. +// Now we want to know if there is a hardlink in the package directory #[cfg(unix)] -fn check_inode(set: &mut FxHashSet<(u64, u64)>, meta: &Metadata) -> bool { - set.insert((get_devno(meta), meta.ino())) +fn check_inode(set: &mut FxHashMap<(u64, u64), Vec>, filename: &str, meta: &Metadata) { + set.entry((get_devno(meta), meta.ino())).or_insert_with(Vec::new).push(filename.to_string()); } #[cfg(not(unix))] @@ -616,7 +618,7 @@ fn check_generated_files(entry: &str, generated: &mut GeneratedHashMap) { // the name of the .ins resp. .dtx without extension let entry_base = &entry_fname[0..entry_fname.len() - 4]; - let fhdl = File::open(&entry); + let fhdl = File::open(entry); match fhdl { Ok(mut f) => { let mut buf = Vec::new(); @@ -1076,7 +1078,7 @@ fn check_package(root: &str, tds_zip: &Option) -> Option { let mut doublenames: FxHashMap> = FxHashMap::default(); - let mut inodes = FxHashSet::default(); + let mut inodes = FxHashMap::default(); let ut = Utils::new(utils::CheckType::Package); @@ -1116,9 +1118,7 @@ fn check_package(root: &str, tds_zip: &Option) -> Option { } }; - if !check_inode(&mut inodes, &meta) { - continue; - } + check_inode(&mut inodes, dir_entry_str, &meta); // this is the file_name without the directory part // unwrap() is ok here as we covered potential UTF-8 related errors @@ -1441,6 +1441,7 @@ fn check_package(root: &str, tds_zip: &Option) -> Option { print_casefolding(&lcnames); print_generated(&doublenames, &generated); + print_hardlinks(&inodes); if !ARGS.ignore_same_named { print_doublenames(&doublenames); } @@ -1487,6 +1488,19 @@ fn check_package(root: &str, tds_zip: &Option) -> Option { Some(hashes) } +fn print_hardlinks(hashes: &FxHashMap<(u64, u64), Vec>) { + for ((_devid,inode), eles) in hashes.iter() { + if eles.len() > 1 { + w0010!(inode); + for hfile in eles.iter() { + info!(" >>> {}", &hfile); + } + + } + + } +} + fn print_casefolding(hashes: &FxHashMap>) { for (k, eles) in hashes.iter() { //println!("pcf: {:?}, {:?}", k, &eles); diff --git a/support/pkgcheck/src/messages/fatald.rs b/support/pkgcheck/src/messages/fatald.rs index ca81ea7d2f..0558612c72 100644 --- a/support/pkgcheck/src/messages/fatald.rs +++ b/support/pkgcheck/src/messages/fatald.rs @@ -100,7 +100,7 @@ The config file could not be read. pub fn f0010d() { error!( r#" -F0010 -- Config file's content could no be read properly: Exiting... +F0010 -- Config file's content could not be read properly: Exiting... The content of the config has errors as given in . Please correct and rerun. diff --git a/support/pkgcheck/src/messages/mod.rs b/support/pkgcheck/src/messages/mod.rs index 3de39bdb81..721f27ff7b 100644 --- a/support/pkgcheck/src/messages/mod.rs +++ b/support/pkgcheck/src/messages/mod.rs @@ -123,7 +123,7 @@ macro_rules! f0010 { ($fmt:expr) => {{ error_occurred!(); eprintln!( - "{} Config file's content could no be read properly: {} Exiting...", + "{} Config file's content could not be read properly: {} Exiting...", fatal!("F0010"), $fmt ); }}; @@ -576,6 +576,11 @@ macro_rules! w0009 { }}; } +macro_rules! w0010 { + ($fmt:expr) => {{ + warn!("{} Hardlinks detected with inode={}", "W0010", $fmt,); + }}; +} macro_rules! i0002 { ($fmt:expr) => {{ @@ -717,6 +722,7 @@ pub fn explains(err: &str) { "W0007" => w0007d(), "W0008" => w0008d(), "W0009" => w0009d(), + "W0010" => w0010d(), e => eprintln!( "{} Unknown error code `{}` specified with option -e resp. --explain. Exiting...", @@ -799,4 +805,5 @@ pub fn explains_all() { explains("W0007"); explains("W0008"); explains("W0009"); + explains("W0010"); } diff --git a/support/pkgcheck/src/messages/warningsd.rs b/support/pkgcheck/src/messages/warningsd.rs index 44a6ad6d4e..6fe37835cf 100644 --- a/support/pkgcheck/src/messages/warningsd.rs +++ b/support/pkgcheck/src/messages/warningsd.rs @@ -120,3 +120,14 @@ hard-coded entry. This helps to keep a clean config file. ) } +pub fn w0010d() { + warn!( + r#" +W0010 -- Hardlinks detected with inode + +Hardlinks found in the package directory tree. The inode number will be +displayed +"# + ) +} + diff --git a/support/pkgcheck/src/recode.rs b/support/pkgcheck/src/recode.rs index efb8b932b6..827da9587b 100644 --- a/support/pkgcheck/src/recode.rs +++ b/support/pkgcheck/src/recode.rs @@ -64,9 +64,7 @@ pub fn wrong_line_endings2crlf(fname: &str) -> Result<(), io::Error> { let mut buffer: Vec = Vec::new(); // read the whole file - if let Err(e) = hdl_in.read_to_end(&mut buffer) { - return Err(e); - }; + hdl_in.read_to_end(&mut buffer)?; let another_vec: Vec = fix_line_endings2crlf(&buffer); @@ -94,9 +92,7 @@ pub fn wrong_line_endings2lf(fname: &str) -> Result<(), io::Error> { let mut buffer: Vec = Vec::new(); // read the whole file - if let Err(e) = hdl_in.read_to_end(&mut buffer) { - return Err(e); - }; + hdl_in.read_to_end(&mut buffer)?; let another_vec: Vec = fix_line_endings2lf(&buffer); -- cgit v1.2.3