diff options
Diffstat (limited to 'support/pkgcheck/src/linkcheck.rs')
-rw-r--r-- | support/pkgcheck/src/linkcheck.rs | 259 |
1 files changed, 259 insertions, 0 deletions
diff --git a/support/pkgcheck/src/linkcheck.rs b/support/pkgcheck/src/linkcheck.rs new file mode 100644 index 0000000000..60b993e960 --- /dev/null +++ b/support/pkgcheck/src/linkcheck.rs @@ -0,0 +1,259 @@ +use threadpool::ThreadPool; + +use colored::*; +use linkify::{LinkFinder, LinkKind}; + +use std::fs::File; +use std::io::prelude::*; + +//use reqwest::Client; +use openssl_probe; +use reqwest; +use reqwest::header; +use std::time::Duration; +//use std::thread; +use std::sync::Arc; +use std::sync::Mutex; + +use std::sync::atomic::Ordering; + +use std::collections::HashMap; +use std::collections::HashSet; + +enum UrlStatus { + Unknown, + UrlOk, + UrlError(String), +} + +struct HashVal { + paths: HashSet<String>, + status: UrlStatus, +} + +type UrlHash = HashMap<String, HashVal>; + +pub struct LinkCheck { + pool: Mutex<ThreadPool>, + urlhash: Arc<Mutex<UrlHash>>, + print_all: bool, +} + +impl LinkCheck { + pub fn new(num_threads: usize, print_all: bool) -> LinkCheck { + openssl_probe::init_ssl_cert_env_vars(); + let pool = Mutex::new(ThreadPool::new(num_threads)); + LinkCheck { + pool, + urlhash: Arc::new(Mutex::new(HashMap::default())), + print_all, + } + } + + pub fn check_urls(&self, fname: &str) { + let print_all = self.print_all; + if let Some(links) = get_links(fname) { + for l in links { + let urlhash = self.urlhash.clone(); + let fname_s = String::from(fname); + self.pool.lock().unwrap().execute(move || { + check_link(&l, &fname_s, &urlhash, print_all); + }); + } + } + } +} + +impl Drop for LinkCheck { + fn drop(&mut self) { + //println!("Now dropping ..."); + let pool = self.pool.lock().unwrap(); + pool.join(); + } +} + +fn check_link(url: &str, fname: &str, urlhash: &Arc<Mutex<UrlHash>>, print_all: bool) { + let url = String::from(url); + + let mut run_check_link = false; + + // It is very important to keep the lock for the urlhash + // only for a short period of time + // + // If we don't find the url in the urlhash then + // we set `run_check_link` to `true` so that we will + // check the url + { + let f = String::from(fname); + + let mut urlhash = urlhash.lock().unwrap(); + if !urlhash.contains_key(&url) { + let mut hs = HashSet::default(); + hs.insert(f.clone()); + let url1 = url.clone(); + + urlhash.insert( + url1, + HashVal { + status: UrlStatus::Unknown, + paths: hs, + }, + ); + run_check_link = true; + } else if let Some(hs) = urlhash.get_mut(&url) { + match &hs.status { + UrlStatus::Unknown => { + hs.paths.insert(f); + } + UrlStatus::UrlOk => { + if print_all { + print_ok(no_colors!(), &url, &f); + }; + } + UrlStatus::UrlError(e) => { + e0022!(f, e); + } + } + } + } + + // + if run_check_link { + match check_link_inner(&url, true) { + UrlStatus::UrlOk => { + let mut urlhash = urlhash.lock().unwrap(); + if let Some(mut hs) = urlhash.get_mut(&url) { + if print_all { + for p in hs.paths.iter() { + print_ok(no_colors!(), &url, p); + } + } + hs.status = UrlStatus::UrlOk; + } + } + UrlStatus::UrlError(e) => { + let mut urlhash = urlhash.lock().unwrap(); + if let Some(mut hs) = urlhash.get_mut(&url) { + for p in hs.paths.iter() { + e0022!(p, e); + } + hs.status = UrlStatus::UrlError(e); + } + } + _ => (), + } + } +} + +fn get_links(fname: &str) -> Option<Vec<String>> { + let fhdl = File::open(fname); + match fhdl { + Ok(mut f) => { + let mut buf = Vec::new(); + + match f.read_to_end(&mut buf) { + Ok(_bytes_read) => { + return get_links_inner(&String::from_utf8_lossy(&buf)); + } + Err(e) => println!("Error reading file {}: {:?}", fname, e), + } + } + Err(e) => println!("Error opening file {}: {}", fname, e), + } + + None +} + +// retrieves links in a string and then checks those links +fn get_links_inner(s: &str) -> Option<Vec<String>> { + let mut finder = LinkFinder::new(); + finder.kinds(&[LinkKind::Url]); + let links: Vec<_> = finder.links(s).collect(); + let result: Vec<&str> = links.iter().map(|e| e.as_str()).collect(); + + let mut links = vec![]; + for r in result { + if !r.starts_with("http://") && !r.starts_with("https://") && !r.starts_with("ftp://") { + continue; + } + // This is a workaround to prevent URLs ending with ` + if r.ends_with('`') { + links.push(String::from(&r[..r.len()-1])); + } else { + links.push(String::from(r)); + } + + } + if !links.is_empty() { + Some(links) + } else { + None + } +} + +fn check_link_inner(l: &str, head: bool) -> UrlStatus { + let mut headers = header::HeaderMap::new(); + headers.insert( + header::USER_AGENT, + header::HeaderValue::from_static( + "Mozilla/5.0 (X11; Linux i686; rv:64.0) Gecko/20100101 Firefox/64.0", + ), + ); + + let default_policy = reqwest::RedirectPolicy::default(); + let policy = reqwest::RedirectPolicy::custom(move |attempt| { + if attempt.url().host_str() == Some("127.0.0.1") { + attempt.stop() + } else { + default_policy.redirect(attempt) + } + }); + + let cb = reqwest::Client::builder() + .gzip(true) + .redirect(policy) + .default_headers(headers) + .timeout(Duration::from_secs(7)) + .build() + .unwrap(); + // let url: Url = + // match l.parse() { + // Ok(url) => url, + // Err(e) => { println!("Error: {:?}", e); panic!("Scheiss"); } + // }; + let resp = if head { + cb.head(l).send() + } else { + cb.get(l).send() + }; + match resp { + Ok(s) => { + if s.status().is_informational() + || s.status().is_success() + || s.status().is_redirection() + { + return UrlStatus::UrlOk; + } + + if head { + check_link_inner(l, false) + } else { + let e = format!("{}: {}", l, s.status()); + UrlStatus::UrlError(e) + } + } + Err(e) => { + let e = format!("{}", e); + UrlStatus::UrlError(e) + } + } +} + +fn print_ok(no_colors: bool, url: &str, f: &str) { + if no_colors { + println!("✔ {} in {}", &url, f); + } else { + // println!("✔ {} in {}", &url, f); + println!("{} {} in {}", "✔".bright_green().bold(), url, f); + } +} |