2020-09-26 07:05:47 +12:00
use std ::collections ::BTreeMap ;
2023-05-03 08:37:12 +12:00
use std ::fs ;
use std ::fs ::{ DirEntry , File , Metadata } ;
2021-01-09 23:52:43 +13:00
use std ::io ::{ BufWriter , Write } ;
2023-05-03 08:37:12 +12:00
use std ::path ::{ Path , PathBuf } ;
2023-05-08 06:54:05 +12:00
2023-05-03 08:37:12 +12:00
use std ::sync ::atomic ::{ AtomicUsize , Ordering } ;
2020-12-02 22:25:27 +13:00
use std ::sync ::Arc ;
2021-11-28 08:49:20 +13:00
use crossbeam_channel ::Receiver ;
2023-05-03 08:37:12 +12:00
use futures ::channel ::mpsc ::UnboundedSender ;
2023-07-10 18:36:03 +12:00
use humansize ::{ format_size , BINARY } ;
2023-10-02 02:20:58 +13:00
use log ::{ debug , info } ;
2021-12-18 07:29:37 +13:00
use rayon ::prelude ::* ;
2021-11-28 08:49:20 +13:00
2023-05-03 08:37:12 +12:00
use crate ::common ::{ check_folder_children , prepare_thread_handler_common , send_info_and_wait_for_ending_all_threads , split_path } ;
2023-05-11 07:27:41 +12:00
use crate ::common_dir_traversal ::{ common_get_entry_data_metadata , common_read_dir , get_lowercase_name , get_modified_time , CheckingMethod , ProgressData , ToolType } ;
2021-11-28 08:49:20 +13:00
use crate ::common_directory ::Directories ;
use crate ::common_extensions ::Extensions ;
use crate ::common_items ::ExcludedItems ;
use crate ::common_messages ::Messages ;
use crate ::common_traits ::{ DebugPrint , PrintResults , SaveResults } ;
2020-09-26 07:05:47 +12:00
#[ derive(Clone) ]
pub struct FileEntry {
2020-10-15 05:41:37 +13:00
pub path : PathBuf ,
2020-09-26 07:05:47 +12:00
pub size : u64 ,
2020-10-01 17:53:10 +13:00
pub modified_date : u64 ,
2020-09-26 07:05:47 +12:00
}
2022-06-05 07:20:21 +12:00
#[ derive(Copy, Clone, Eq, PartialEq) ]
pub enum SearchMode {
BiggestFiles ,
SmallestFiles ,
}
2023-02-19 22:21:14 +13:00
#[ derive(Eq, PartialEq, Clone, Debug, Copy) ]
2020-11-01 07:19:33 +13:00
pub enum DeleteMethod {
None ,
Delete ,
}
2021-11-28 08:57:10 +13:00
2020-09-26 07:05:47 +12:00
/// Info struck with helpful information's about results
2020-10-10 09:32:08 +13:00
#[ derive(Default) ]
2020-09-26 07:05:47 +12:00
pub struct Info {
2020-09-26 23:48:53 +12:00
pub number_of_real_files : usize ,
2020-09-26 07:05:47 +12:00
}
2020-10-10 09:32:08 +13:00
impl Info {
2023-01-29 06:54:02 +13:00
#[ must_use ]
2020-10-10 09:32:08 +13:00
pub fn new ( ) -> Self {
Default ::default ( )
2020-09-26 07:05:47 +12:00
}
}
/// Struct with required information's to work
pub struct BigFile {
2023-05-11 07:27:41 +12:00
tool_type : ToolType ,
2020-09-26 07:05:47 +12:00
text_messages : Messages ,
information : Info ,
2022-06-05 07:20:21 +12:00
big_files : Vec < ( u64 , FileEntry ) > ,
2020-09-27 03:52:13 +13:00
excluded_items : ExcludedItems ,
directories : Directories ,
allowed_extensions : Extensions ,
2020-09-26 07:05:47 +12:00
recursive_search : bool ,
number_of_files_to_check : usize ,
2020-11-01 07:19:33 +13:00
delete_method : DeleteMethod ,
2020-10-10 22:15:20 +13:00
stopped_search : bool ,
2022-06-05 07:20:21 +12:00
search_mode : SearchMode ,
2020-09-26 07:05:47 +12:00
}
impl BigFile {
2023-01-29 06:54:02 +13:00
#[ must_use ]
2020-10-10 09:32:08 +13:00
pub fn new ( ) -> Self {
Self {
2023-05-11 07:27:41 +12:00
tool_type : ToolType ::BigFile ,
2020-09-26 07:05:47 +12:00
text_messages : Default ::default ( ) ,
information : Info ::new ( ) ,
big_files : Default ::default ( ) ,
2020-09-27 03:52:13 +13:00
excluded_items : ExcludedItems ::new ( ) ,
directories : Directories ::new ( ) ,
allowed_extensions : Extensions ::new ( ) ,
2020-09-26 23:48:53 +12:00
recursive_search : true ,
2020-09-26 07:05:47 +12:00
number_of_files_to_check : 50 ,
2020-11-01 07:19:33 +13:00
delete_method : DeleteMethod ::None ,
2020-10-10 22:15:20 +13:00
stopped_search : false ,
2022-07-29 04:56:55 +12:00
search_mode : SearchMode ::BiggestFiles ,
2020-09-26 07:05:47 +12:00
}
}
2020-09-27 06:51:28 +13:00
2023-05-03 08:37:12 +12:00
pub fn find_big_files ( & mut self , stop_receiver : Option < & Receiver < ( ) > > , progress_sender : Option < & UnboundedSender < ProgressData > > ) {
2023-10-02 02:20:58 +13:00
info! ( " Starting finding big files " ) ;
2020-09-27 06:51:28 +13:00
self . optimize_directories ( ) ;
2020-12-02 22:25:27 +13:00
if ! self . look_for_big_files ( stop_receiver , progress_sender ) {
2020-10-10 22:15:20 +13:00
self . stopped_search = true ;
return ;
}
2020-11-01 07:19:33 +13:00
self . delete_files ( ) ;
2020-09-27 06:51:28 +13:00
self . debug_print ( ) ;
}
2023-01-29 06:54:02 +13:00
#[ must_use ]
2020-10-10 22:15:20 +13:00
pub fn get_stopped_search ( & self ) -> bool {
self . stopped_search
}
2020-09-27 06:51:28 +13:00
2022-06-05 07:20:21 +12:00
pub fn set_search_mode ( & mut self , search_mode : SearchMode ) {
self . search_mode = search_mode ;
}
2023-01-29 06:54:02 +13:00
#[ must_use ]
2022-06-05 07:20:21 +12:00
pub const fn get_big_files ( & self ) -> & Vec < ( u64 , FileEntry ) > {
2020-10-02 06:26:25 +13:00
& self . big_files
}
2023-01-29 06:54:02 +13:00
#[ must_use ]
2020-10-10 09:32:08 +13:00
pub const fn get_text_messages ( & self ) -> & Messages {
2020-09-26 07:05:47 +12:00
& self . text_messages
}
2023-01-29 06:54:02 +13:00
#[ must_use ]
2020-10-10 09:32:08 +13:00
pub const fn get_information ( & self ) -> & Info {
2020-09-26 07:05:47 +12:00
& self . information
}
2020-11-01 07:19:33 +13:00
pub fn set_delete_method ( & mut self , delete_method : DeleteMethod ) {
self . delete_method = delete_method ;
}
2020-09-26 23:48:53 +12:00
pub fn set_recursive_search ( & mut self , recursive_search : bool ) {
self . recursive_search = recursive_search ;
}
2022-04-24 06:21:46 +12:00
#[ cfg(target_family = " unix " ) ]
pub fn set_exclude_other_filesystems ( & mut self , exclude_other_filesystems : bool ) {
self . directories . set_exclude_other_filesystems ( exclude_other_filesystems ) ;
}
2022-07-20 05:09:52 +12:00
#[ cfg(not(target_family = " unix " )) ]
pub fn set_exclude_other_filesystems ( & mut self , _exclude_other_filesystems : bool ) { }
2022-04-24 06:21:46 +12:00
2020-09-26 07:05:47 +12:00
/// List of allowed extensions, only files with this extensions will be checking if are duplicates
2020-09-27 03:52:13 +13:00
pub fn set_allowed_extensions ( & mut self , allowed_extensions : String ) {
self . allowed_extensions . set_allowed_extensions ( allowed_extensions , & mut self . text_messages ) ;
2020-09-26 07:05:47 +12:00
}
2023-05-03 08:37:12 +12:00
fn look_for_big_files ( & mut self , stop_receiver : Option < & Receiver < ( ) > > , progress_sender : Option < & UnboundedSender < ProgressData > > ) -> bool {
2023-10-02 02:20:58 +13:00
debug! ( " look_for_big_files - start " ) ;
2020-10-15 05:41:37 +13:00
let mut folders_to_check : Vec < PathBuf > = Vec ::with_capacity ( 1024 * 2 ) ; // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
2022-06-05 07:20:21 +12:00
let mut old_map : BTreeMap < u64 , Vec < FileEntry > > = Default ::default ( ) ;
2020-09-26 07:05:47 +12:00
// Add root folders for finding
2020-09-27 03:52:13 +13:00
for id in & self . directories . included_directories {
2020-10-15 05:41:37 +13:00
folders_to_check . push ( id . clone ( ) ) ;
2020-09-26 07:05:47 +12:00
}
2023-05-11 07:27:41 +12:00
let ( progress_thread_handle , progress_thread_run , atomic_counter , _check_was_stopped ) =
prepare_thread_handler_common ( progress_sender , 0 , 0 , 0 , CheckingMethod ::None , self . tool_type ) ;
2020-12-02 22:25:27 +13:00
2020-09-26 07:05:47 +12:00
while ! folders_to_check . is_empty ( ) {
2020-11-05 06:15:06 +13:00
if stop_receiver . is_some ( ) & & stop_receiver . unwrap ( ) . try_recv ( ) . is_ok ( ) {
2023-05-03 08:37:12 +12:00
send_info_and_wait_for_ending_all_threads ( & progress_thread_run , progress_thread_handle ) ;
2020-10-10 22:15:20 +13:00
return false ;
}
2020-12-02 22:25:27 +13:00
2021-12-18 07:29:37 +13:00
let segments : Vec < _ > = folders_to_check
. par_iter ( )
. map ( | current_folder | {
let mut dir_result = vec! [ ] ;
let mut warnings = vec! [ ] ;
let mut fe_result = vec! [ ] ;
2023-05-03 08:37:12 +12:00
let Some ( read_dir ) = common_read_dir ( current_folder , & mut warnings ) else {
return ( dir_result , warnings , fe_result ) ;
2021-12-18 07:29:37 +13:00
} ;
2020-09-26 07:05:47 +12:00
2021-12-18 07:29:37 +13:00
// Check every sub folder/file/link etc.
2023-05-03 08:37:12 +12:00
for entry in read_dir {
let Some ( ( entry_data , metadata ) ) = common_get_entry_data_metadata ( & entry , & mut warnings , current_folder ) else {
continue ;
2021-12-18 07:29:37 +13:00
} ;
2023-05-03 08:37:12 +12:00
2021-12-18 07:29:37 +13:00
if metadata . is_dir ( ) {
2023-05-03 08:37:12 +12:00
check_folder_children (
& mut dir_result ,
& mut warnings ,
current_folder ,
entry_data ,
self . recursive_search ,
& self . directories ,
& self . excluded_items ,
) ;
2021-12-18 07:29:37 +13:00
} else if metadata . is_file ( ) {
2023-05-03 08:37:12 +12:00
self . collect_file_entry ( & atomic_counter , & metadata , entry_data , & mut fe_result , & mut warnings , current_folder ) ;
2021-12-18 07:29:37 +13:00
}
}
( dir_result , warnings , fe_result )
} )
. collect ( ) ;
// Advance the frontier
folders_to_check . clear ( ) ;
// Process collected data
for ( segment , warnings , fe_result ) in segments {
folders_to_check . extend ( segment ) ;
self . text_messages . warnings . extend ( warnings ) ;
for ( size , fe ) in fe_result {
2023-09-16 05:04:05 +12:00
old_map . entry ( size ) . or_default ( ) . push ( fe ) ;
2020-09-26 07:05:47 +12:00
}
}
}
2023-05-03 08:37:12 +12:00
send_info_and_wait_for_ending_all_threads ( & progress_thread_run , progress_thread_handle ) ;
self . extract_n_biggest_files ( old_map ) ;
2023-10-02 02:20:58 +13:00
debug! ( " look_for_big_files - end " ) ;
2023-05-03 08:37:12 +12:00
true
}
2020-12-02 22:25:27 +13:00
2023-05-03 08:37:12 +12:00
pub fn collect_file_entry (
& self ,
atomic_counter : & Arc < AtomicUsize > ,
metadata : & Metadata ,
entry_data : & DirEntry ,
fe_result : & mut Vec < ( u64 , FileEntry ) > ,
warnings : & mut Vec < String > ,
current_folder : & Path ,
) {
atomic_counter . fetch_add ( 1 , Ordering ::Relaxed ) ;
if metadata . len ( ) = = 0 {
return ;
}
2020-09-26 07:05:47 +12:00
2023-05-03 08:37:12 +12:00
let Some ( file_name_lowercase ) = get_lowercase_name ( entry_data , warnings ) else {
return ;
} ;
if ! self . allowed_extensions . matches_filename ( & file_name_lowercase ) {
return ;
}
let current_file_name = current_folder . join ( entry_data . file_name ( ) ) ;
if self . excluded_items . is_excluded ( & current_file_name ) {
return ;
}
let fe : FileEntry = FileEntry {
path : current_file_name . clone ( ) ,
size : metadata . len ( ) ,
modified_date : get_modified_time ( metadata , warnings , & current_file_name , false ) ,
} ;
fe_result . push ( ( fe . size , fe ) ) ;
}
pub fn extract_n_biggest_files ( & mut self , old_map : BTreeMap < u64 , Vec < FileEntry > > ) {
2023-10-02 02:20:58 +13:00
debug! ( " extract_n_biggest_files - start " ) ;
2022-06-05 07:20:21 +12:00
let iter : Box < dyn Iterator < Item = _ > > ;
if self . search_mode = = SearchMode ::SmallestFiles {
iter = Box ::new ( old_map . into_iter ( ) ) ;
} else {
iter = Box ::new ( old_map . into_iter ( ) . rev ( ) ) ;
}
for ( size , mut vector ) in iter {
2020-09-26 23:48:53 +12:00
if self . information . number_of_real_files < self . number_of_files_to_check {
2022-06-05 07:20:21 +12:00
if vector . len ( ) > 1 {
2022-07-25 06:48:02 +12:00
vector . sort_unstable_by_key ( | e | {
2022-06-05 07:20:21 +12:00
let t = split_path ( e . path . as_path ( ) ) ;
( t . 0 , t . 1 )
} ) ;
}
2020-09-26 07:05:47 +12:00
for file in vector {
2020-09-26 23:48:53 +12:00
if self . information . number_of_real_files < self . number_of_files_to_check {
2022-06-05 07:20:21 +12:00
self . big_files . push ( ( size , file ) ) ;
2020-09-26 23:48:53 +12:00
self . information . number_of_real_files + = 1 ;
2020-09-26 07:05:47 +12:00
} else {
break ;
}
}
} else {
break ;
}
}
2023-10-02 02:20:58 +13:00
debug! ( " extract_n_biggest_files - end " ) ;
2020-09-26 07:05:47 +12:00
}
2020-09-27 03:52:13 +13:00
pub fn set_number_of_files_to_check ( & mut self , number_of_files_to_check : usize ) {
self . number_of_files_to_check = number_of_files_to_check ;
}
2020-10-07 21:34:15 +13:00
/// Setting excluded items which needs to contains * wildcard
2020-09-27 03:52:13 +13:00
/// Are a lot of slower than absolute path, so it should be used to heavy
2021-01-11 08:44:10 +13:00
pub fn set_excluded_items ( & mut self , excluded_items : Vec < String > ) {
2020-09-27 03:52:13 +13:00
self . excluded_items . set_excluded_items ( excluded_items , & mut self . text_messages ) ;
}
fn optimize_directories ( & mut self ) {
self . directories . optimize_directories ( self . recursive_search , & mut self . text_messages ) ;
}
2021-01-11 08:44:10 +13:00
pub fn set_included_directory ( & mut self , included_directory : Vec < PathBuf > ) {
2020-09-27 03:52:13 +13:00
self . directories . set_included_directory ( included_directory , & mut self . text_messages ) ;
}
2021-01-11 08:44:10 +13:00
pub fn set_excluded_directory ( & mut self , excluded_directory : Vec < PathBuf > ) {
2020-09-27 03:52:13 +13:00
self . directories . set_excluded_directory ( excluded_directory , & mut self . text_messages ) ;
}
2020-11-01 07:19:33 +13:00
/// Function to delete files, from filed Vector
fn delete_files ( & mut self ) {
match self . delete_method {
DeleteMethod ::Delete = > {
2022-06-05 07:20:21 +12:00
for ( _ , file_entry ) in & self . big_files {
2023-02-19 22:21:14 +13:00
if fs ::remove_file ( & file_entry . path ) . is_err ( ) {
2022-06-05 07:20:21 +12:00
self . text_messages . warnings . push ( file_entry . path . display ( ) . to_string ( ) ) ;
2020-11-01 07:19:33 +13:00
}
}
}
DeleteMethod ::None = > {
//Just do nothing
}
}
}
2020-09-27 03:52:13 +13:00
}
impl Default for BigFile {
fn default ( ) -> Self {
Self ::new ( )
}
}
impl DebugPrint for BigFile {
2020-09-26 07:05:47 +12:00
#[ allow(dead_code) ]
#[ allow(unreachable_code) ]
2020-09-26 23:48:53 +12:00
/// Debugging printing - only available on debug build
2020-09-26 07:05:47 +12:00
fn debug_print ( & self ) {
#[ cfg(not(debug_assertions)) ]
{
return ;
}
println! ( " ---------------DEBUG PRINT--------------- " ) ;
2020-09-26 23:48:53 +12:00
println! ( " ### Information's " ) ;
println! ( " Errors size - {} " , self . text_messages . errors . len ( ) ) ;
println! ( " Warnings size - {} " , self . text_messages . warnings . len ( ) ) ;
println! ( " Messages size - {} " , self . text_messages . messages . len ( ) ) ;
println! ( " ### Other " ) ;
println! ( " Big files size {} in {} groups " , self . information . number_of_real_files , self . big_files . len ( ) ) ;
2020-09-27 03:52:13 +13:00
println! ( " Excluded items - {:?} " , self . excluded_items . items ) ;
println! ( " Included directories - {:?} " , self . directories . included_directories ) ;
println! ( " Excluded directories - {:?} " , self . directories . excluded_directories ) ;
2021-11-23 23:10:24 +13:00
println! ( " Recursive search - {} " , self . recursive_search ) ;
2022-04-24 06:21:46 +12:00
#[ cfg(target_family = " unix " ) ]
2022-06-01 03:52:55 +12:00
println! ( " Skip other filesystems - {} " , self . directories . exclude_other_filesystems ( ) ) ;
2020-09-26 23:48:53 +12:00
println! ( " Number of files to check - {:?} " , self . number_of_files_to_check ) ;
2020-09-26 07:05:47 +12:00
println! ( " ----------------------------------------- " ) ;
}
2020-09-27 03:52:13 +13:00
}
2020-10-15 05:41:37 +13:00
2020-09-27 03:52:13 +13:00
impl SaveResults for BigFile {
/// Saving results to provided file
fn save_results_to_file ( & mut self , file_name : & str ) -> bool {
let file_name : String = match file_name {
" " = > " results.txt " . to_string ( ) ,
k = > k . to_string ( ) ,
} ;
2020-09-26 07:05:47 +12:00
2021-01-09 23:52:43 +13:00
let file_handler = match File ::create ( & file_name ) {
2020-09-27 03:52:13 +13:00
Ok ( t ) = > t ,
2021-11-15 03:53:55 +13:00
Err ( e ) = > {
2022-12-21 20:44:26 +13:00
self . text_messages . errors . push ( format! ( " Failed to create file {file_name} , reason {e} " ) ) ;
2020-09-27 03:52:13 +13:00
return false ;
2020-09-26 07:05:47 +12:00
}
2020-09-27 03:52:13 +13:00
} ;
2021-01-09 23:52:43 +13:00
let mut writer = BufWriter ::new ( file_handler ) ;
2020-09-26 07:05:47 +12:00
2021-11-15 03:53:55 +13:00
if let Err ( e ) = writeln! (
2021-01-09 23:52:43 +13:00
writer ,
2020-10-15 05:41:37 +13:00
" Results of searching {:?} with excluded directories {:?} and excluded items {:?} " ,
self . directories . included_directories , self . directories . excluded_directories , self . excluded_items . items
2021-11-15 03:53:55 +13:00
) {
2022-12-21 20:44:26 +13:00
self . text_messages . errors . push ( format! ( " Failed to save results to file {file_name} , reason {e} " ) ) ;
2020-10-15 05:41:37 +13:00
return false ;
2020-09-26 07:05:47 +12:00
}
2020-09-27 03:52:13 +13:00
if self . information . number_of_real_files ! = 0 {
2022-07-31 18:45:19 +12:00
if self . search_mode = = SearchMode ::BiggestFiles {
write! ( writer , " {} the biggest files. \n \n " , self . information . number_of_real_files ) . unwrap ( ) ;
} else {
write! ( writer , " {} the smallest files. \n \n " , self . information . number_of_real_files ) . unwrap ( ) ;
}
2023-01-29 06:54:02 +13:00
for ( size , file_entry ) in & self . big_files {
2022-09-14 08:18:07 +12:00
writeln! ( writer , " {} ({}) - {} " , format_size ( * size , BINARY ) , size , file_entry . path . display ( ) ) . unwrap ( ) ;
2020-09-26 07:05:47 +12:00
}
2020-09-27 03:52:13 +13:00
} else {
2021-01-09 23:52:43 +13:00
write! ( writer , " Not found any files. " ) . unwrap ( ) ;
2020-09-26 07:05:47 +12:00
}
2023-05-08 06:54:05 +12:00
2020-09-26 07:05:47 +12:00
true
}
2020-09-27 03:52:13 +13:00
}
2020-10-15 05:41:37 +13:00
2020-09-27 03:52:13 +13:00
impl PrintResults for BigFile {
fn print_results ( & self ) {
2022-07-31 18:45:19 +12:00
if self . search_mode = = SearchMode ::BiggestFiles {
println! ( " {} the biggest files. \n \n " , self . information . number_of_real_files ) ;
} else {
println! ( " {} the smallest files. \n \n " , self . information . number_of_real_files ) ;
}
2023-01-29 06:54:02 +13:00
for ( size , file_entry ) in & self . big_files {
2022-09-14 08:18:07 +12:00
println! ( " {} ( {} ) - {} " , format_size ( * size , BINARY ) , size , file_entry . path . display ( ) ) ;
2020-09-26 07:05:47 +12:00
}
}
}