diff options
author | Norbert Preining <norbert@preining.info> | 2019-09-02 13:46:59 +0900 |
---|---|---|
committer | Norbert Preining <norbert@preining.info> | 2019-09-02 13:46:59 +0900 |
commit | e0c6872cf40896c7be36b11dcc744620f10adf1d (patch) | |
tree | 60335e10d2f4354b0674ec22d7b53f0f8abee672 /support/pkgcheck/src/filemagic.rs |
Initial commit
Diffstat (limited to 'support/pkgcheck/src/filemagic.rs')
-rw-r--r-- | support/pkgcheck/src/filemagic.rs | 422 |
1 files changed, 422 insertions, 0 deletions
diff --git a/support/pkgcheck/src/filemagic.rs b/support/pkgcheck/src/filemagic.rs new file mode 100644 index 0000000000..40edd18351 --- /dev/null +++ b/support/pkgcheck/src/filemagic.rs @@ -0,0 +1,422 @@ +#![allow(clippy::trivial_regex)] + +use regex::bytes::Regex; +use regex::bytes::RegexSet; + +use std::fs; +use std::fs::File; +use std::io; +use std::io::prelude::*; +use std::path::Path; + +use std::os::unix::fs::FileTypeExt; + +use unicode_bom::Bom; + +// +// File signatures links +// - https://asecuritysite.com/forensics/magic +// - https://filesignatures.net/ +// - https://github.com/7h3rAm/cigma/blob/master/cigma/magicbytes.json + +#[derive(Debug, PartialEq)] +pub enum Mimetype { + Binary, + Script, + ScriptCRLF, + Pdf, + Archive, + Zip, + Text, + TextCRLF, + Data, + Unknown, + BlockDevice, + CharDevice, + Directory, + Symlink, + Fifo, + Socket, + Zerofile, + VeryShort, + BOM(Bom), +} + +pub struct Filetype { + buffer: Vec<u8>, +} + +fn is_binary_data(vec: &[u8], len: usize) -> bool { + for v in vec.iter().take(len) { + if *v <= 8 { + return true; + } + } + + false +} + +fn _is_crlf(buffer: &[u8], len: usize) -> bool { + let mut cr = 0; + let mut lf = 0; + + const CR: u8 = 0x0d; // 13 + const LF: u8 = 0x0a; // 10 + + for c in buffer.iter().take(len) { + if *c == LF { + lf += 1; + } else if *c == CR { + cr += 1; + } + } + + let diff: i32 = cr - lf; + if cr > 0 && diff == 0 { + return true; + } + + //println!("cr: {}, lf: {}", cr, lf); + // Heuristics: we accept if only a few lines are not CRLF + match (cr, lf) { + (0, _lf) => return false, + (_cr, 0) => return true, + (cr, _lf) => { + if cr > 500 && diff.abs() < 3 { + return true; + } + } + } + + false +} + +fn is_crlf(buffer: &[u8], len: usize) -> bool { + let mut seen_cr = false; + let mut n_crlf = 0; + let mut n_lf = 0; + let mut n_cr = 0; + + const CR: u8 = 0x0d; // CR 0x0D 13 \r + const LF: u8 = 0x0a; // LF 0x0A 10 \n + + for c in buffer.iter().take(len) { + if *c == LF { + if seen_cr { + n_crlf += 1; + } else { + n_lf += 1; + } + } else if seen_cr { + n_cr += 1; + } + + seen_cr = *c == CR; + } + + // println!("cr: {}, lf: {}, crlf: {}", n_cr, n_lf, n_crlf); + // if (n_crlf == 0 && n_cr == 0 && n_nel == 0 && n_lf == 0) + // --> no line terminators + if n_crlf == 0 && n_cr == 0 && n_lf == 0 { + return false; + } + + if n_crlf > 0 || n_cr > 0 { + return true; + } + + false +} + +impl Filetype { + pub fn new() -> Self { + Filetype { + buffer: vec![0; 1024 * 1024], + } + } + + pub fn analyze(&mut self, fname: &str) -> Result<Mimetype, io::Error> { + // Result<Err,Mimetype> { + let path = Path::new(fname); + + if let Some(ft) = get_filetype(path) { + return Ok(ft); + } + + let metadata = fs::symlink_metadata(fname)?; + let file_length: usize = metadata.len() as usize; + + if file_length == 0 { + return Ok(Mimetype::Zerofile); + } + + if metadata.len() == 1 { + return Ok(Mimetype::VeryShort); + } + + let mut hdl_in = File::open(path)?; + + let mut bytes_read: usize = hdl_in.read(&mut self.buffer[0..262])?; + + // PostScript signatures + // - %!PS-Adobe-1.0, %!PS-Adobe-2.0, %!PS-Adobe-3.0, %!PS-Adobe-3.1 + // - %! and a line feed + if bytes_read >= 4 && &self.buffer[0..4] == b"%!PS" { + return Ok(Mimetype::Data); + } + + // - %!\r\n%%BoundingBox: + let re: Regex = Regex::new(r"^(?-u)%!(\x0d\x0a|\x0A)%%BoundingBox").unwrap(); + if bytes_read >= 20 && re.is_match(&self.buffer) { + return Ok(Mimetype::Data); + } + + if bytes_read >= 4 && &self.buffer[0..4] == b"%PDF" { + return Ok(Mimetype::Pdf); + } + + if bytes_read >= 5 && &self.buffer[0..5] == b"<?php" { + return Ok(Mimetype::Script); + } + + // rtf document + if bytes_read >= 6 && &self.buffer[0..6] == b"\x7B\x5C\x72\x74\x66\x31" { + return Ok(Mimetype::Data); + } + + // ZOO archive http://fileformats.archiveteam.org/wiki/ZOO + if bytes_read >= 60 && &self.buffer[20..24] == b"\xDC\xA7\xC4\xFD" { + return Ok(Mimetype::Archive); + } + + let bom: Bom = Bom::from(&self.buffer[0..]); + + if bom != Bom::Null { + return Ok(Mimetype::BOM(bom)); + } + + if is_binary_data(&self.buffer, bytes_read) { + match analyze_binary(&self.buffer) { + Some(Mimetype::Zip) => { + if fname.ends_with(".cdy") { + return Ok(Mimetype::Data); + } else { + return Ok(Mimetype::Zip); + } + } + Some(mt) => return Ok(mt), + None => return Ok(Mimetype::Unknown), + } + } + + // https://en.wikipedia.org/wiki/BinHex + if bytes_read >= 200 + && self + .buffer + .starts_with(b"(This file must be converted with BinHex 4.0)") + { + return Ok(Mimetype::Binary); + } + + if bytes_read < file_length { + if let Ok(rb) = hdl_in.read(&mut self.buffer[262..]) { + bytes_read += rb + } + } + + let crlf = is_crlf(&self.buffer, bytes_read); + + if bytes_read >= 5 && (self.buffer.starts_with(b"#!") || self.buffer.starts_with(b"<?php")) + { + if crlf { + return Ok(Mimetype::ScriptCRLF); + } else { + return Ok(Mimetype::Script); + } + } + + let is_script = Regex::new(r"\.(csh|ksh|py|pl|sh|tcsh)$") + .unwrap() + .is_match(fname.as_bytes()); + + match (crlf, is_script) { + (false, false) => Ok(Mimetype::Text), + (true, false) => Ok(Mimetype::TextCRLF), + (false, true) => Ok(Mimetype::Script), + (true, true) => Ok(Mimetype::ScriptCRLF), + } + } +} + +// https://en.wikipedia.org/wiki/Executable_and_Linkable_Format +fn is_binary(vec: &[u8]) -> Option<Mimetype> { + let binary_re: RegexSet = RegexSet::new(&[ + r"^(?-u)\x7FELF[\x01\x02][\x01\x02]\x01[\x00-\x11]", // Executable and Linkable Format (ELF) + r"^(?-u)\x00\x00\x03\xF3", // AmigaOS loadseg()ble executable/binary + r"^(?-u)MZ", // DOS MZ executable file format and its descendants (including NE and PE) + r"^(?-u)\x64 \x65\x78\x0A\x30\x33\x35\x00", // Dalvik's executable + r"^(?-u)#[!]", // script executable + r"^(?-u)(\xCE|\xCF)\xFA\xED\xFE", // Mach-O binary + r"^(?-u)\x1B\x4C\x75\x61", // Lua bytecode + ]) + .unwrap(); + + if binary_re.is_match(vec) { + return Some(Mimetype::Binary); + } + None +} + +// https://github.com/7h3rAm/cigma/blob/master/cigma/magicbytes.json +// https://en.wikipedia.org/wiki/List_of_file_signatures +fn is_archive(vec: &[u8]) -> Option<Mimetype> { + // we first have to catch zip files with mimetype formats + // - opendocument formats + // - Word Open XML + // Those we do not regard as archives + let special_zip: RegexSet = RegexSet::new(&[ + r"^(?-u)PK\x03\x04.{20,}\x08\x00\x00\x00mimetypeapplication", + r"^(?-u)PK\x03\x04\x14\x00\x06\x00", // Word Open XML (.docx) + r"^(?-u)PK\x03\x04\x14\x00\x08\x00", // Java Jar file + r"^(?-u)PK\x03\x04\x14\x00\x08\x08", // Java Jar file + r"^(?-u)PK\x03\x04\x0A.*?META-INF", // Java Jar file + r"^(?-u)PK\x03\x04.*?META-INF", // Java Jar file + r"^(?-u)PK\x03\x04\x0A.*?\x56\x92\x48\x4F\xEF", // Java Jar file + ]) + .unwrap(); + + if special_zip.is_match(vec) { + return Some(Mimetype::Data); + } + + let archive_re: RegexSet = RegexSet::new(&[ + r"^(?-u)\x37\x7A\xBC\xAF\x27\x1C", // 7zip + r"^(?-u)\x1f\x8B", // gzip (.gz) + r"^(?-u)\x1f\x9D", // LZW (.tar.Z) + r"^(?-u)\x1f\xA0", // LZH (.tar.Z) + r"^(?-u)\xFD\x37\x7A\x58\x5A\x00\x00", // XZ comp. utility using LZMA2 compression (.xz) + r"^(?-u)\x4D\x53\x43\x46", // Microsoft cabinet (.cab) + r"^(?-u)\x42\x5A\x68", // bzip2 + r"^(?-u)\x5A\x57\x53", // lzma + r"^(?-u)\x5D\x00\x00(\x01|\x02|\x04|\x08|\x10|\x20|\x40|\x80)\x00", // lzma + r"^(?-u)\x5D\x00\x00\x00\x01", // lzma + r"^(?-u)(SIT!|SITD|STi0|StuffIt)", // SIT / stuffit (macintosh related) + r"^(?-u)\x4D\x5A", // DOS MZ executable format, but found in zip archives + r"^(?-u)\x52\x61\x72\x21\x1A\x07\x00", // RAR archive version 1.50 onwards + r"^(?-u)\x52\x61\x72\x21\x1A\x07\x01\x00", // RAR archive version 5.0 onwards + // https://en.wikipedia.org/wiki/LHA_(file_format) + r"^(?-u)..-lh[0124567d]", // LHarc (canonical LZH) + r"^(?-u)..-lh[89abce]", // LHarc (Joe Jared extensions) + r"^(?-u)..-lhx", // LHarc (UNLHA32 extensions) + r"^(?-u)..-(pc1|pm0|pm1|pm2|pms)", // LHarc (PMarc extensions) + r"^(?-u)..-lz[s234578]", // LHarc (LArc extensions) + r"^(?-u)\x53\x5a\x44\x44\x88\xf0\x27\x33", // RAR archive version 5.0 onwards + ]) + .unwrap(); + + if archive_re.is_match(vec) { + return Some(Mimetype::Archive); + } + + let archive_re: RegexSet = RegexSet::new(&[ + r"^(?-u)PK(\x03\x04|\x4c\x49\x54\x45|\x30\x30\x50|\x05\x06|\x07\x08)", // zip archive + ]) + .unwrap(); + if archive_re.is_match(vec) { + return Some(Mimetype::Zip); + } + + None +} + +fn analyze_binary(vec: &[u8]) -> Option<Mimetype> { + let rc = is_binary(vec); + if rc.is_some() { + return rc; + } + + let rc = is_archive(vec); + if rc.is_some() { + return rc; + } + + Some(Mimetype::Data) +} + +fn get_filetype(entry: &Path) -> Option<Mimetype> { + match entry.symlink_metadata() { + Ok(mt) => { + let ft = mt.file_type(); + if ft.is_symlink() { + return Some(Mimetype::Symlink); + } + if ft.is_dir() { + return Some(Mimetype::Directory); + } + if ft.is_block_device() { + return Some(Mimetype::BlockDevice); + } + if ft.is_char_device() { + return Some(Mimetype::CharDevice); + } + if ft.is_fifo() { + return Some(Mimetype::Fifo); + } + if ft.is_socket() { + return Some(Mimetype::Socket); + } + None + } + Err(_e) => None, + } +} + +#[test] +fn test_filetype() { + let mut ft = Filetype::new(); + + assert!(ft.analyze("tests_filemagic/zerofile").ok() == Some(Mimetype::Zerofile)); + assert!(ft.analyze("tests_filemagic/a_small_file").ok() == Some(Mimetype::VeryShort)); + assert!(ft.analyze("/dev/null").ok() == Some(Mimetype::CharDevice)); + assert!(ft.analyze("tests_filemagic/").ok() == Some(Mimetype::Directory)); + assert!(ft.analyze("tests_filemagic/zerofile_symlink").ok() == Some(Mimetype::Symlink)); + + assert!(ft.analyze("tests_filemagic/some.pdf").ok() == Some(Mimetype::Pdf)); + + // This file is a pdf but has lines starting with % before the pdf signature shows up + // The unix `file` command) says: data + // analyze() says TextCRLF + //assert!(ft.analyze("tests_filemagic/musterlogo.pdf").ok() == Some(Mimetype::Script)); + + assert!(ft.analyze("tests_filemagic/x.pl").ok() == Some(Mimetype::Script)); + assert!(ft.analyze("tests_filemagic/main.php").ok() == Some(Mimetype::Script)); + + assert!(ft.analyze("tests_filemagic/test.7z").ok() == Some(Mimetype::Archive)); + assert!(ft.analyze("tests_filemagic/x.tgz").ok() == Some(Mimetype::Archive)); + assert!(ft.analyze("tests_filemagic/test.pdf.xz").ok() == Some(Mimetype::Archive)); + assert!(ft.analyze("tests_filemagic/swebib.cab").ok() == Some(Mimetype::Archive)); + assert!(ft.analyze("tests_filemagic/test.tar.bz2").ok() == Some(Mimetype::Archive)); + assert!(ft.analyze("tests_filemagic/PIE.rar").ok() == Some(Mimetype::Archive)); + assert!(ft.analyze("tests_filemagic/infozip-os390.tar.Z").ok() == Some(Mimetype::Archive)); + assert!(ft.analyze("tests_filemagic/bla.lha").ok() == Some(Mimetype::Archive)); + assert!(ft.analyze("tests_filemagic/dvi.zoo").ok() == Some(Mimetype::Archive)); + assert!(ft.analyze("tests_filemagic/rsfs-oztex.sit").ok() == Some(Mimetype::Archive)); + + assert!(ft.analyze("tests_filemagic/empty.zip").ok() == Some(Mimetype::Zip)); + + assert!(ft.analyze("tests_filemagic/README").ok() == Some(Mimetype::Text)); + // assert!(ft.analyze("tests_filemagic/README1").ok() == Some(Mimetype::Text)); + + assert!(ft.analyze("tests_filemagic/cp").ok() == Some(Mimetype::Binary)); + assert!(ft.analyze("tests_filemagic/cheq-f.sit-hqx").ok() == Some(Mimetype::Binary)); + assert!(ft.analyze("tests_filemagic/MuchMore").ok() == Some(Mimetype::Binary)); + + assert!(ft.analyze("tests_filemagic/support.ps").ok() == Some(Mimetype::Data)); + assert!(ft.analyze("tests_filemagic/rosette.eps").ok() == Some(Mimetype::Data)); + assert!(ft.analyze("tests_filemagic/eutest.ps").ok() == Some(Mimetype::Data)); + // assert!(ft.analyze("tests_filemagic/NORMAL.PS").ok() == Some(Mimetype::Data)); + assert!(ft.analyze("tests_filemagic/chap5.rtf").ok() == Some(Mimetype::Data)); + assert!(ft.analyze("tests_filemagic/commons-math.jar").ok() == Some(Mimetype::Data)); + + assert!(ft.analyze("tests_filemagic/8stbu11h.htm").ok() == Some(Mimetype::TextCRLF)); +} |