CodeMirror/czkawka
1
0
Fork 0
mirror of synced 2024-04-29 01:52:39 +12:00
Code Issues Projects Releases Wiki Activity

Begin of working with finding biggest files

Browse source
This commit is contained in:
Rafał Mikrut 2020-09-25 21:05:47 +02:00
parent 45e0bde981
commit d17f2d3ef1
5 changed files with 585 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
Instruction.md Normal file
View file

@ -0,0 +1,12 @@
# Instruction
## GUI
## CLI

569
czkawka_core/src/big_file.rs Normal file
View file

@ -0,0 +1,569 @@
use crate::common::{Common, Messages};
use std::collections::BTreeMap;
use std::fs;
use std::fs::Metadata;
use std::path::Path;
use std::time::SystemTime;
#[derive(Clone)]
pub struct FileEntry {
pub path: String,
pub size: u64,
pub created_date: SystemTime,
pub modified_date: SystemTime,
}
/// Info struck with helpful information's about results
pub struct Info {
pub number_of_checked_files: usize,
pub number_of_checked_folders: usize,
pub number_of_ignored_files: usize,
pub number_of_ignored_things: usize,
pub taken_space: u64,
}
impl Info {
pub fn new() -> Info {
Info {
number_of_checked_files: 0,
number_of_checked_folders: 0,
number_of_ignored_files: 0,
number_of_ignored_things: 0,
taken_space: 0,
}
}
}
impl Default for Info {
fn default() -> Self {
Self::new()
}
}
/// Struct with required information's to work
pub struct BigFile {
text_messages: Messages,
information: Info,
big_files: BTreeMap<u64, Vec<FileEntry>>,
excluded_items: Vec<String>,
excluded_directories: Vec<String>,
included_directories: Vec<String>,
allowed_extensions: Vec<String>,
recursive_search: bool,
number_of_files_to_check: usize,
}
impl BigFile {
pub fn new() -> BigFile {
BigFile {
text_messages: Default::default(),
information: Info::new(),
big_files: Default::default(),
excluded_items: vec![],
excluded_directories: vec![],
included_directories: vec![],
allowed_extensions: vec![],
recursive_search: false,
number_of_files_to_check: 50,
}
}
pub fn get_text_messages(&self) -> &Messages {
&self.text_messages
}
pub fn get_information(&self) -> &Info {
&self.information
}
pub fn find_big_files(&mut self) {
self.optimize_directories();
self.debug_print();
}
/// List of allowed extensions, only files with this extensions will be checking if are duplicates
pub fn set_allowed_extensions(&mut self, mut allowed_extensions: String) {
let start_time: SystemTime = SystemTime::now();
if allowed_extensions.is_empty() {
return;
}
allowed_extensions = allowed_extensions.replace("IMAGE", "jpg,kra,gif,png,bmp,tiff,webp,hdr,svg");
allowed_extensions = allowed_extensions.replace("VIDEO", "mp4,flv,mkv,webm,vob,ogv,gifv,avi,mov,wmv,mpg,m4v,m4p,mpeg,3gp");
allowed_extensions = allowed_extensions.replace("MUSIC", "mp3,flac,ogg,tta,wma,webm");
allowed_extensions = allowed_extensions.replace("TEXT", "txt,doc,docx,odt,rtf");
let extensions: Vec<String> = allowed_extensions.split(',').map(String::from).collect();
for mut extension in extensions {
if extension == "" || extension.replace('.', "").trim() == "" {
continue;
}
if extension.starts_with('.') {
extension = extension[1..].to_string();
}
if extension[1..].contains('.') {
self.text_messages.warnings.push(".".to_string() + extension.as_str() + " is not valid extension(valid extension doesn't have dot inside)");
continue;
}
if !self.allowed_extensions.contains(&extension.trim().to_string()) {
self.allowed_extensions.push(extension.trim().to_string());
}
}
if self.allowed_extensions.is_empty() {
self.text_messages.messages.push("No valid extensions were provided, so allowing all extensions by default.".to_string());
}
Common::print_time(start_time, SystemTime::now(), "set_allowed_extensions".to_string());
}
fn look_for_big_files(&mut self) {
let start_time: SystemTime = SystemTime::now();
let mut folders_to_check: Vec<String> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
// Add root folders for finding
for id in &self.included_directories {
folders_to_check.push(id.to_string());
}
self.information.number_of_checked_folders += folders_to_check.len();
let mut current_folder: String;
let mut next_folder: String;
while !folders_to_check.is_empty() {
current_folder = folders_to_check.pop().unwrap();
let read_dir = match fs::read_dir(&current_folder) {
Ok(t) => t,
Err(_) => {
self.text_messages.warnings.push("Cannot open dir ".to_string() + current_folder.as_str());
continue;
} // Permissions denied
};
for entry in read_dir {
let entry_data = match entry {
Ok(t) => t,
Err(_) => {
self.text_messages.warnings.push("Cannot read entry in dir ".to_string() + current_folder.as_str());
continue;
} //Permissions denied
};
let metadata: Metadata = match entry_data.metadata() {
Ok(t) => t,
Err(_) => {
self.text_messages.warnings.push("Cannot read metadata in dir ".to_string() + current_folder.as_str());
continue;
} //Permissions denied
};
if metadata.is_dir() {
self.information.number_of_checked_folders += 1;
// if entry_data.file_name().into_string().is_err() { // Probably this can be removed, if crash still will be happens, then uncomment this line
// self.text_messages.warnings.push("Cannot read folder name in dir ".to_string() + current_folder.as_str());
// continue; // Permissions denied
// }
if !self.recursive_search {
continue;
}
let mut is_excluded_dir = false;
next_folder = "".to_owned() + &current_folder + &entry_data.file_name().into_string().unwrap() + "/";
for ed in &self.excluded_directories {
if next_folder == *ed {
is_excluded_dir = true;
break;
}
}
if !is_excluded_dir {
let mut found_expression: bool = false;
for expression in &self.excluded_items {
if Common::regex_check(expression, &next_folder) {
found_expression = true;
break;
}
}
if found_expression {
break;
}
folders_to_check.push(next_folder);
}
} else if metadata.is_file() {
let mut have_valid_extension: bool;
let file_name_lowercase: String = entry_data.file_name().into_string().unwrap().to_lowercase();
// Checking allowed extensions
if !self.allowed_extensions.is_empty() {
have_valid_extension = false;
for i in &self.allowed_extensions {
if file_name_lowercase.ends_with((".".to_string() + i.to_lowercase().as_str()).as_str()) {
have_valid_extension = true;
break;
}
}
} else {
have_valid_extension = true;
}
// Checking files
if have_valid_extension {
let current_file_name = "".to_owned() + &current_folder + &entry_data.file_name().into_string().unwrap();
// Checking expressions
let mut found_expression: bool = false;
for expression in &self.excluded_items {
if Common::regex_check(expression, &current_file_name) {
found_expression = true;
break;
}
}
if found_expression {
break;
}
// Creating new file entry
let fe: FileEntry = FileEntry {
path: current_file_name.clone(),
size: metadata.len(),
created_date: match metadata.created() {
Ok(t) => t,
Err(_) => {
self.text_messages.warnings.push("Unable to get creation date from file ".to_string() + current_file_name.as_str());
SystemTime::now()
} // Permissions Denied
},
modified_date: match metadata.modified() {
Ok(t) => t,
Err(_) => {
self.text_messages.warnings.push("Unable to get modification date from file ".to_string() + current_file_name.as_str());
SystemTime::now()
} // Permissions Denied
},
};
self.big_files.entry(metadata.len()).or_insert_with(Vec::new);
self.big_files.get_mut(&metadata.len()).unwrap().push(fe);
self.information.number_of_checked_files += 1;
} else {
self.information.number_of_ignored_files += 1;
}
} else {
// Probably this is symbolic links so we are free to ignore this
self.information.number_of_ignored_things += 1;
}
}
}
//
let mut new_map: BTreeMap<u64, Vec<FileEntry>> = Default::default();
let mut number_of_files: usize = 0;
for (size, vector) in &self.big_files {
if number_of_files < self.number_of_files_to_check {
for file in vector {
if number_of_files < self.number_of_files_to_check {
new_map.entry(*size).or_insert_with(Vec::new);
new_map.get_mut(size).unwrap().push(file.clone());
number_of_files += 1;
} else {
break;
}
}
} else {
break;
}
}
self.big_files = new_map;
Common::print_time(start_time, SystemTime::now(), "look_for_big_files".to_string());
}
/// Debug print
#[allow(dead_code)]
#[allow(unreachable_code)]
fn debug_print(&self) {
#[cfg(not(debug_assertions))]
{
return;
}
println!("---------------DEBUG PRINT---------------");
// println!("Number of all checked folders - {}", self.information.number_of_checked_folders);
// println!("Number of empty folders - {}", self.information.number_of_empty_folders);
// println!("Included directories - {:?}", self.included_directories);
println!("-----------------------------------------");
}
pub fn set_number_of_files_to_check(&mut self, number_of_files_to_check: usize) {
self.number_of_files_to_check = number_of_files_to_check;
}
/// Setting excluded items which needs to contains * wildcrard
/// Are a lot of slower than absolute path, so it should be used to heavy
pub fn set_excluded_items(&mut self, mut excluded_items: String) {
let start_time: SystemTime = SystemTime::now();
if excluded_items.is_empty() {
return;
}
excluded_items = excluded_items.replace("\"", "");
let expressions: Vec<String> = excluded_items.split(',').map(String::from).collect();
let mut checked_expressions: Vec<String> = Vec::new();
for expression in expressions {
let expression: String = expression.trim().to_string();
if expression == "" {
continue;
}
if expression == "DEFAULT" {
// TODO add more files by default
checked_expressions.push("*/.git/*".to_string());
continue;
}
if !expression.contains('*') {
self.text_messages.warnings.push("Excluded Items Warning: Wildcard * is required in expression, ignoring ".to_string() + expression.as_str());
continue;
}
checked_expressions.push(expression);
}
self.excluded_items = checked_expressions;
Common::print_time(start_time, SystemTime::now(), "set_excluded_items".to_string());
}
/// Remove unused entries when included or excluded overlaps with each other or are duplicated etc.
fn optimize_directories(&mut self) -> bool {
let start_time: SystemTime = SystemTime::now();
let mut optimized_included: Vec<String> = Vec::<String>::new();
let mut optimized_excluded: Vec<String> = Vec::<String>::new();
// Remove duplicated entries like: "/", "/"
self.excluded_directories.sort();
self.included_directories.sort();
self.excluded_directories.dedup();
self.included_directories.dedup();
// Optimize for duplicated included directories - "/", "/home". "/home/Pulpit" to "/"
if self.recursive_search {
// This is only point which can't be done when recursive search is disabled.
let mut is_inside: bool;
for ed_checked in &self.excluded_directories {
is_inside = false;
for ed_help in &self.excluded_directories {
if ed_checked == ed_help {
// We checking same element
continue;
}
if ed_checked.starts_with(ed_help) {
is_inside = true;
break;
}
}
if !is_inside {
optimized_excluded.push(ed_checked.to_string());
}
}
for id_checked in &self.included_directories {
is_inside = false;
for id_help in &self.included_directories {
if id_checked == id_help {
// We checking same element
continue;
}
if id_checked.starts_with(id_help) {
is_inside = true;
break;
}
}
if !is_inside {
optimized_included.push(id_checked.to_string());
}
}
self.included_directories = optimized_included;
optimized_included = Vec::<String>::new();
self.excluded_directories = optimized_excluded;
optimized_excluded = Vec::<String>::new();
}
// Remove include directories which are inside any exclude directory
for id in &self.included_directories {
let mut is_inside: bool = false;
for ed in &self.excluded_directories {
if id.starts_with(ed) {
is_inside = true;
break;
}
}
if !is_inside {
optimized_included.push(id.to_string());
}
}
self.included_directories = optimized_included;
optimized_included = Vec::<String>::new();
// Remove non existed directories
for id in &self.included_directories {
let path = Path::new(id);
if path.exists() {
optimized_included.push(id.to_string());
}
}
for ed in &self.excluded_directories {
let path = Path::new(ed);
if path.exists() {
optimized_excluded.push(ed.to_string());
}
}
self.included_directories = optimized_included;
// optimized_included = Vec::<String>::new();
self.excluded_directories = optimized_excluded;
optimized_excluded = Vec::<String>::new();
// Excluded paths must are inside include path, because
for ed in &self.excluded_directories {
let mut is_inside: bool = false;
for id in &self.included_directories {
if ed.starts_with(id) {
is_inside = true;
break;
}
}
if is_inside {
optimized_excluded.push(ed.to_string());
}
}
self.excluded_directories = optimized_excluded;
// optimized_excluded = Vec::<String>::new();
if self.included_directories.is_empty() {
self.text_messages.errors.push("Optimize Directories ERROR: Excluded directories overlaps all included directories.".to_string());
return false;
}
// Not needed, but better is to have sorted everything
self.excluded_directories.sort();
self.included_directories.sort();
Common::print_time(start_time, SystemTime::now(), "optimize_directories".to_string());
true
}
/// Setting include directories, at least one must be provided
pub fn set_include_directory(&mut self, mut include_directory: String) -> bool {
let start_time: SystemTime = SystemTime::now();
if include_directory.is_empty() {
self.text_messages.errors.push("At least one directory must be provided".to_string());
return false;
}
include_directory = include_directory.replace("\"", "");
let directories: Vec<String> = include_directory.split(',').map(String::from).collect();
let mut checked_directories: Vec<String> = Vec::new();
for directory in directories {
let directory: String = directory.trim().to_string();
if directory == "" {
continue;
}
if directory.contains('*') {
self.text_messages.warnings.push("Include Directory Warning: Wildcards in path are not supported, ignoring ".to_string() + directory.as_str());
continue;
}
if !directory.starts_with('/') {
self.text_messages.warnings.push("Include Directory Warning: Relative path are not supported, ignoring ".to_string() + directory.as_str());
continue;
}
if !Path::new(&directory).exists() {
self.text_messages.warnings.push("Include Directory Warning: Provided folder path must exits, ignoring ".to_string() + directory.as_str());
continue;
}
if !Path::new(&directory).is_dir() {
self.text_messages.warnings.push("Include Directory Warning: Provided path must point at the directory, ignoring ".to_string() + directory.as_str());
continue;
}
// directory must end with /, due to possibility of incorrect assumption, that e.g. /home/rafal is top folder to /home/rafalinho
if !directory.ends_with('/') {
checked_directories.push(directory + "/");
} else {
checked_directories.push(directory);
}
}
if checked_directories.is_empty() {
self.text_messages.errors.push("Include Directory ERROR: Not found even one correct path to include which is required.".to_string());
return false;
}
self.included_directories = checked_directories;
Common::print_time(start_time, SystemTime::now(), "set_include_directory".to_string());
true
}
/// Setting absolute path to exclude
pub fn set_exclude_directory(&mut self, mut exclude_directory: String) {
let start_time: SystemTime = SystemTime::now();
if exclude_directory.is_empty() {
return;
}
exclude_directory = exclude_directory.replace("\"", "");
let directories: Vec<String> = exclude_directory.split(',').map(String::from).collect();
let mut checked_directories: Vec<String> = Vec::new();
for directory in directories {
let directory: String = directory.trim().to_string();
if directory == "" {
continue;
}
if directory == "/" {
self.text_messages.errors.push("Exclude Directory ERROR: Excluding / is pointless, because it means that no files will be scanned.".to_string());
break;
}
if directory.contains('*') {
self.text_messages.warnings.push("Exclude Directory Warning: Wildcards in path are not supported, ignoring ".to_string() + directory.as_str());
continue;
}
if !directory.starts_with('/') {
self.text_messages.warnings.push("Exclude Directory Warning: Relative path are not supported, ignoring ".to_string() + directory.as_str());
continue;
}
if !Path::new(&directory).exists() {
self.text_messages.warnings.push("Exclude Directory Warning: Provided folder path must exits, ignoring ".to_string() + directory.as_str());
continue;
}
if !Path::new(&directory).is_dir() {
self.text_messages.warnings.push("Exclude Directory Warning: Provided path must point at the directory, ignoring ".to_string() + directory.as_str());
continue;
}
// directory must end with /, due to possibility of incorrect assumption, that e.g. /home/rafal is top folder to /home/rafalinho
if !directory.ends_with('/') {
checked_directories.push(directory.trim().to_string() + "/");
} else {
checked_directories.push(directory.trim().to_string());
}
}
self.excluded_directories = checked_directories;
Common::print_time(start_time, SystemTime::now(), "set_exclude_directory".to_string());
}
}
impl Default for BigFile {
fn default() -> Self {
Self::new()
}
}

1
czkawka_core/src/common.rs
View file

@ -109,6 +109,7 @@ pub struct Messages {
pub warnings: Vec<String>,
pub errors: Vec<String>,
}
impl Messages {
pub fn new() -> Messages {
Messages {

2
czkawka_core/src/duplicate.rs
View file

@ -187,6 +187,8 @@ impl DuplicateFinder {
self.excluded_items = checked_expressions;
Common::print_time(start_time, SystemTime::now(), "set_excluded_items".to_string());
}
/// List of allowed extensions, only files with this extensions will be checking if are duplicates
pub fn set_allowed_extensions(&mut self, mut allowed_extensions: String) {
let start_time: SystemTime = SystemTime::now();
if allowed_extensions.is_empty() {

1
czkawka_core/src/lib.rs
View file

@ -1,3 +1,4 @@
pub mod big_file;
pub mod common;
pub mod duplicate;
pub mod empty_folder;