2022-01-06 03:20:21 +13:00
use std ::collections ::{ BTreeSet , HashMap , HashSet } ;
2023-05-03 08:37:12 +12:00
use std ::fs ::{ DirEntry , File , Metadata } ;
2020-10-15 08:10:27 +13:00
use std ::io ::Write ;
2020-12-27 22:56:26 +13:00
use std ::io ::* ;
2023-05-03 08:37:12 +12:00
use std ::mem ;
2021-12-30 01:43:38 +13:00
use std ::panic ;
2020-12-27 22:56:26 +13:00
use std ::path ::{ Path , PathBuf } ;
2020-12-02 22:25:27 +13:00
use std ::sync ::atomic ::{ AtomicBool , AtomicUsize , Ordering } ;
use std ::sync ::Arc ;
2023-05-03 08:37:12 +12:00
use std ::time ::SystemTime ;
2020-12-02 22:25:27 +13:00
2021-11-28 08:49:20 +13:00
use bk_tree ::BKTree ;
use crossbeam_channel ::Receiver ;
2023-05-03 08:37:12 +12:00
use futures ::channel ::mpsc ::UnboundedSender ;
2022-09-14 08:18:07 +12:00
use humansize ::format_size ;
use humansize ::BINARY ;
2021-11-28 08:49:20 +13:00
use image ::GenericImageView ;
2022-04-03 02:11:28 +12:00
use image_hasher ::{ FilterType , HashAlg , HasherConfig } ;
2021-11-28 08:49:20 +13:00
use rayon ::prelude ::* ;
2021-12-26 10:23:18 +13:00
use serde ::{ Deserialize , Serialize } ;
2021-11-28 08:49:20 +13:00
2022-06-09 07:42:51 +12:00
#[ cfg(feature = " heif " ) ]
use crate ::common ::get_dynamic_image_from_heic ;
2022-07-28 17:29:50 +12:00
use crate ::common ::{
2023-05-03 08:37:12 +12:00
check_folder_children , create_crash_message , get_dynamic_image_from_raw_image , get_number_of_threads , open_cache_folder , prepare_thread_handler_common ,
2023-05-08 06:54:05 +12:00
send_info_and_wait_for_ending_all_threads , HEIC_EXTENSIONS , IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS , RAW_IMAGE_EXTENSIONS ,
2022-07-28 17:29:50 +12:00
} ;
2023-05-03 08:37:12 +12:00
use crate ::common_dir_traversal ::{ common_get_entry_data_metadata , common_read_dir , get_lowercase_name , get_modified_time , CheckingMethod , ProgressData } ;
2021-11-28 08:49:20 +13:00
use crate ::common_directory ::Directories ;
2021-12-18 07:29:37 +13:00
use crate ::common_extensions ::Extensions ;
2021-11-28 08:49:20 +13:00
use crate ::common_items ::ExcludedItems ;
use crate ::common_messages ::Messages ;
2023-05-08 06:54:05 +12:00
use crate ::common_traits ::{ DebugPrint , PrintResults , ResultEntry , SaveResults } ;
2022-01-20 10:35:07 +13:00
use crate ::flc ;
2023-05-03 08:37:12 +12:00
type ImHash = Vec < u8 > ;
2021-11-28 08:49:20 +13:00
2021-12-03 03:33:06 +13:00
pub const SIMILAR_VALUES : [ [ u32 ; 6 ] ; 4 ] = [
2022-07-31 18:45:19 +12:00
[ 1 , 2 , 5 , 7 , 14 , 20 ] , // 8
2021-12-03 03:33:06 +13:00
[ 2 , 5 , 15 , 30 , 40 , 40 ] , // 16
[ 4 , 10 , 20 , 40 , 40 , 40 ] , // 32
[ 6 , 20 , 40 , 40 , 40 , 40 ] , // 64
2021-11-19 18:35:26 +13:00
] ;
2021-12-26 10:23:18 +13:00
#[ derive(Clone, Debug, Serialize, Deserialize) ]
2020-10-15 08:10:27 +13:00
pub struct FileEntry {
pub path : PathBuf ,
pub size : u64 ,
2020-10-16 04:51:47 +13:00
pub dimensions : String ,
2020-10-15 08:10:27 +13:00
pub modified_date : u64 ,
2023-05-03 08:37:12 +12:00
pub hash : ImHash ,
2022-07-03 07:30:59 +12:00
pub similarity : u32 ,
2020-10-15 08:10:27 +13:00
}
2023-05-08 06:54:05 +12:00
impl ResultEntry for FileEntry {
fn get_path ( & self ) -> & Path {
& self . path
}
}
2020-10-15 08:10:27 +13:00
2022-01-15 09:14:13 +13:00
/// Used by CLI tool when we cannot use directly values
2023-02-19 22:21:14 +13:00
#[ derive(Clone, Debug, Copy) ]
2021-11-18 23:23:17 +13:00
pub enum SimilarityPreset {
2022-07-31 18:45:19 +12:00
Original ,
2021-11-18 23:23:17 +13:00
VeryHigh ,
High ,
Medium ,
Small ,
VerySmall ,
Minimal ,
None ,
}
2020-10-16 04:43:59 +13:00
/// Distance metric to use with the BK-tree.
struct Hamming ;
2023-05-03 08:37:12 +12:00
impl bk_tree ::Metric < ImHash > for Hamming {
fn distance ( & self , a : & ImHash , b : & ImHash ) -> u32 {
2021-08-07 02:40:43 +12:00
hamming ::distance_fast ( a , b ) . unwrap ( ) as u32
2021-07-22 07:48:21 +12:00
}
2023-05-03 08:37:12 +12:00
fn threshold_distance ( & self , a : & ImHash , b : & ImHash , _threshold : u32 ) -> Option < u32 > {
2021-08-07 02:40:43 +12:00
Some ( self . distance ( a , b ) )
}
2020-10-16 04:43:59 +13:00
}
2020-10-15 08:10:27 +13:00
/// Struct to store most basics info about all folder
pub struct SimilarImages {
information : Info ,
text_messages : Messages ,
directories : Directories ,
2021-12-18 07:29:37 +13:00
allowed_extensions : Extensions ,
2020-10-15 08:10:27 +13:00
excluded_items : ExcludedItems ,
2023-05-03 08:37:12 +12:00
bktree : BKTree < ImHash , Hamming > ,
2020-11-08 04:26:40 +13:00
similar_vectors : Vec < Vec < FileEntry > > ,
2021-12-24 21:18:55 +13:00
similar_referenced_vectors : Vec < ( FileEntry , Vec < FileEntry > ) > ,
2020-10-15 08:10:27 +13:00
recursive_search : bool ,
minimal_file_size : u64 ,
2021-08-07 09:23:11 +12:00
maximal_file_size : u64 ,
2023-05-03 08:37:12 +12:00
image_hashes : HashMap < ImHash , Vec < FileEntry > > ,
// Hashmap with image hashes and Vector with names of files
2020-10-15 08:10:27 +13:00
stopped_search : bool ,
2022-07-03 07:30:59 +12:00
similarity : u32 ,
2022-01-06 03:20:21 +13:00
images_to_check : HashMap < String , FileEntry > ,
2021-11-18 23:23:17 +13:00
hash_size : u8 ,
hash_alg : HashAlg ,
image_filter : FilterType ,
2021-03-05 00:09:53 +13:00
use_cache : bool ,
2021-12-01 00:45:09 +13:00
delete_outdated_cache : bool ,
2021-12-09 11:08:05 +13:00
exclude_images_with_same_size : bool ,
2021-12-24 21:18:55 +13:00
use_reference_folders : bool ,
2022-01-06 10:47:27 +13:00
save_also_as_json : bool ,
2020-10-15 08:10:27 +13:00
}
/// Info struck with helpful information's about results
#[ derive(Default) ]
pub struct Info {
2021-12-24 21:18:55 +13:00
pub number_of_duplicates : usize ,
pub number_of_groups : u64 ,
2020-10-15 08:10:27 +13:00
}
2021-11-28 08:57:10 +13:00
2020-10-15 08:10:27 +13:00
impl Info {
2023-01-29 06:54:02 +13:00
#[ must_use ]
2020-10-15 08:10:27 +13:00
pub fn new ( ) -> Self {
Default ::default ( )
}
}
2023-01-29 06:54:02 +13:00
/// Method implementation for `EmptyFolder`
2020-10-15 08:10:27 +13:00
impl SimilarImages {
/// New function providing basics values
2023-01-29 06:54:02 +13:00
#[ must_use ]
2020-10-15 08:10:27 +13:00
pub fn new ( ) -> Self {
Self {
information : Default ::default ( ) ,
text_messages : Messages ::new ( ) ,
directories : Directories ::new ( ) ,
excluded_items : Default ::default ( ) ,
2021-12-18 07:29:37 +13:00
allowed_extensions : Extensions ::new ( ) ,
2020-10-16 04:43:59 +13:00
bktree : BKTree ::new ( Hamming ) ,
2020-10-15 08:10:27 +13:00
similar_vectors : vec ! [ ] ,
2021-12-24 21:18:55 +13:00
similar_referenced_vectors : Default ::default ( ) ,
2020-10-15 08:10:27 +13:00
recursive_search : true ,
minimal_file_size : 1024 * 16 , // 16 KB should be enough to exclude too small images from search
2021-08-07 09:23:11 +12:00
maximal_file_size : u64 ::MAX ,
2020-10-15 08:10:27 +13:00
image_hashes : Default ::default ( ) ,
stopped_search : false ,
2022-07-03 07:30:59 +12:00
similarity : 0 ,
2020-12-27 22:56:26 +13:00
images_to_check : Default ::default ( ) ,
2021-11-18 23:23:17 +13:00
hash_size : 8 ,
hash_alg : HashAlg ::Gradient ,
image_filter : FilterType ::Lanczos3 ,
2021-03-05 00:09:53 +13:00
use_cache : true ,
2021-12-01 00:45:09 +13:00
delete_outdated_cache : true ,
2021-12-09 11:08:05 +13:00
exclude_images_with_same_size : false ,
2021-12-24 21:18:55 +13:00
use_reference_folders : false ,
2022-01-06 10:47:27 +13:00
save_also_as_json : false ,
2020-10-15 08:10:27 +13:00
}
}
2021-11-18 23:23:17 +13:00
pub fn set_hash_size ( & mut self , hash_size : u8 ) {
self . hash_size = match hash_size {
2021-12-03 03:33:06 +13:00
8 | 16 | 32 | 64 = > hash_size ,
2021-11-18 23:23:17 +13:00
e = > {
2022-12-21 20:44:26 +13:00
panic! ( " Invalid value of hash size {e} " ) ;
2021-11-18 23:23:17 +13:00
}
}
}
2021-12-01 00:45:09 +13:00
pub fn set_delete_outdated_cache ( & mut self , delete_outdated_cache : bool ) {
self . delete_outdated_cache = delete_outdated_cache ;
}
2021-12-09 11:08:05 +13:00
pub fn set_exclude_images_with_same_size ( & mut self , exclude_images_with_same_size : bool ) {
self . exclude_images_with_same_size = exclude_images_with_same_size ;
}
2021-11-18 23:23:17 +13:00
pub fn set_hash_alg ( & mut self , hash_alg : HashAlg ) {
self . hash_alg = hash_alg ;
}
pub fn set_image_filter ( & mut self , image_filter : FilterType ) {
self . image_filter = image_filter ;
}
2022-01-06 10:47:27 +13:00
pub fn set_save_also_as_json ( & mut self , save_also_as_json : bool ) {
self . save_also_as_json = save_also_as_json ;
}
2021-12-29 09:20:48 +13:00
2023-01-29 06:54:02 +13:00
#[ must_use ]
2020-10-15 08:10:27 +13:00
pub fn get_stopped_search ( & self ) -> bool {
self . stopped_search
}
2023-01-29 06:54:02 +13:00
#[ must_use ]
2020-10-15 08:10:27 +13:00
pub const fn get_text_messages ( & self ) -> & Messages {
& self . text_messages
}
2023-01-29 06:54:02 +13:00
#[ must_use ]
2020-11-08 04:26:40 +13:00
pub const fn get_similar_images ( & self ) -> & Vec < Vec < FileEntry > > {
2020-10-15 20:04:02 +13:00
& self . similar_vectors
}
2023-01-29 06:54:02 +13:00
#[ must_use ]
2021-12-24 21:18:55 +13:00
pub fn get_similar_images_referenced ( & self ) -> & Vec < ( FileEntry , Vec < FileEntry > ) > {
& self . similar_referenced_vectors
}
2023-01-29 06:54:02 +13:00
#[ must_use ]
2021-12-24 21:18:55 +13:00
pub fn get_use_reference ( & self ) -> bool {
self . use_reference_folders
}
2023-01-29 06:54:02 +13:00
#[ must_use ]
2020-10-15 08:10:27 +13:00
pub const fn get_information ( & self ) -> & Info {
& self . information
}
2021-03-05 00:09:53 +13:00
pub fn set_use_cache ( & mut self , use_cache : bool ) {
self . use_cache = use_cache ;
}
2020-10-15 08:10:27 +13:00
pub fn set_recursive_search ( & mut self , recursive_search : bool ) {
self . recursive_search = recursive_search ;
}
2022-04-24 06:21:46 +12:00
#[ cfg(target_family = " unix " ) ]
pub fn set_exclude_other_filesystems ( & mut self , exclude_other_filesystems : bool ) {
self . directories . set_exclude_other_filesystems ( exclude_other_filesystems ) ;
}
2022-07-20 05:09:52 +12:00
#[ cfg(not(target_family = " unix " )) ]
pub fn set_exclude_other_filesystems ( & mut self , _exclude_other_filesystems : bool ) { }
2022-04-24 06:21:46 +12:00
2021-12-18 07:29:37 +13:00
pub fn set_allowed_extensions ( & mut self , allowed_extensions : String ) {
self . allowed_extensions . set_allowed_extensions ( allowed_extensions , & mut self . text_messages ) ;
}
2020-10-15 08:10:27 +13:00
pub fn set_minimal_file_size ( & mut self , minimal_file_size : u64 ) {
self . minimal_file_size = match minimal_file_size {
0 = > 1 ,
t = > t ,
} ;
}
2021-08-07 09:23:11 +12:00
pub fn set_maximal_file_size ( & mut self , maximal_file_size : u64 ) {
self . maximal_file_size = match maximal_file_size {
0 = > 1 ,
t = > t ,
} ;
}
2022-07-03 07:30:59 +12:00
pub fn set_similarity ( & mut self , similarity : u32 ) {
2020-11-08 04:26:40 +13:00
self . similarity = similarity ;
}
2020-10-15 08:10:27 +13:00
/// Public function used by CLI to search for empty folders
2023-05-03 08:37:12 +12:00
pub fn find_similar_images ( & mut self , stop_receiver : Option < & Receiver < ( ) > > , progress_sender : Option < & UnboundedSender < ProgressData > > ) {
2020-10-15 08:10:27 +13:00
self . directories . optimize_directories ( true , & mut self . text_messages ) ;
2021-12-24 21:18:55 +13:00
self . use_reference_folders = ! self . directories . reference_directories . is_empty ( ) ;
2020-12-02 22:25:27 +13:00
if ! self . check_for_similar_images ( stop_receiver , progress_sender ) {
2020-10-15 08:10:27 +13:00
self . stopped_search = true ;
return ;
}
2021-12-29 08:20:24 +13:00
if ! self . hash_images ( stop_receiver , progress_sender ) {
self . stopped_search = true ;
return ;
}
if ! self . find_similar_hashes ( stop_receiver , progress_sender ) {
2020-11-08 04:26:40 +13:00
self . stopped_search = true ;
return ;
}
2020-10-15 08:10:27 +13:00
// if self.delete_folders {
// self.delete_empty_folders();
// }
self . debug_print ( ) ;
}
// pub fn set_delete_folder(&mut self, delete_folder: bool) {
// self.delete_folders = delete_folder;
// }
/// Function to check if folder are empty.
2023-01-29 06:54:02 +13:00
/// Parameter `initial_checking` for second check before deleting to be sure that checked folder is still empty
2023-05-03 08:37:12 +12:00
fn check_for_similar_images ( & mut self , stop_receiver : Option < & Receiver < ( ) > > , progress_sender : Option < & UnboundedSender < ProgressData > > ) -> bool {
2020-10-15 08:10:27 +13:00
let mut folders_to_check : Vec < PathBuf > = Vec ::with_capacity ( 1024 * 2 ) ; // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
2022-01-01 10:34:24 +13:00
if ! self . allowed_extensions . using_custom_extensions ( ) {
2022-05-17 04:23:07 +12:00
self . allowed_extensions . extend_allowed_extensions ( IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS ) ;
self . allowed_extensions . extend_allowed_extensions ( RAW_IMAGE_EXTENSIONS ) ;
2022-06-09 07:42:51 +12:00
#[ cfg(feature = " heif " ) ]
self . allowed_extensions . extend_allowed_extensions ( HEIC_EXTENSIONS ) ;
2022-05-24 05:04:28 +12:00
} else {
self . allowed_extensions
2022-06-09 07:42:51 +12:00
. validate_allowed_extensions ( & [ IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS , RAW_IMAGE_EXTENSIONS , HEIC_EXTENSIONS ] . concat ( ) ) ;
2022-05-24 05:04:28 +12:00
if ! self . allowed_extensions . using_custom_extensions ( ) {
return true ;
}
2022-01-01 10:34:24 +13:00
}
2021-12-18 07:29:37 +13:00
2020-10-15 08:10:27 +13:00
// Add root folders for finding
for id in & self . directories . included_directories {
folders_to_check . push ( id . clone ( ) ) ;
}
2023-05-08 06:54:05 +12:00
let ( progress_thread_handle , progress_thread_run , atomic_counter , _check_was_stopped ) = prepare_thread_handler_common ( progress_sender , 0 , 2 , 0 , CheckingMethod ::None ) ;
2020-12-02 22:25:27 +13:00
2020-10-15 08:10:27 +13:00
while ! folders_to_check . is_empty ( ) {
2020-11-05 06:15:06 +13:00
if stop_receiver . is_some ( ) & & stop_receiver . unwrap ( ) . try_recv ( ) . is_ok ( ) {
2023-05-03 08:37:12 +12:00
send_info_and_wait_for_ending_all_threads ( & progress_thread_run , progress_thread_handle ) ;
2020-10-15 08:10:27 +13:00
return false ;
}
2021-12-18 07:29:37 +13:00
let segments : Vec < _ > = folders_to_check
. par_iter ( )
. map ( | current_folder | {
let mut dir_result = vec! [ ] ;
let mut warnings = vec! [ ] ;
let mut fe_result = vec! [ ] ;
2023-05-03 08:37:12 +12:00
let Some ( read_dir ) = common_read_dir ( current_folder , & mut warnings ) else {
return ( dir_result , warnings , fe_result ) ;
2021-12-18 07:29:37 +13:00
} ;
2023-05-03 08:37:12 +12:00
for entry in read_dir {
let Some ( ( entry_data , metadata ) ) = common_get_entry_data_metadata ( & entry , & mut warnings , current_folder ) else {
continue ;
2021-12-18 07:29:37 +13:00
} ;
2023-05-03 08:37:12 +12:00
2021-12-18 07:29:37 +13:00
if metadata . is_dir ( ) {
2023-05-03 08:37:12 +12:00
check_folder_children (
& mut dir_result ,
& mut warnings ,
current_folder ,
entry_data ,
self . recursive_search ,
& self . directories ,
& self . excluded_items ,
) ;
2021-12-18 07:29:37 +13:00
} else if metadata . is_file ( ) {
2023-05-03 08:37:12 +12:00
atomic_counter . fetch_add ( 1 , Ordering ::Relaxed ) ;
self . add_file_entry ( & metadata , current_folder , entry_data , & mut fe_result , & mut warnings ) ;
2021-12-18 07:29:37 +13:00
}
2020-10-15 08:10:27 +13:00
}
2021-12-18 07:29:37 +13:00
( dir_result , warnings , fe_result )
} )
. collect ( ) ;
// Advance the frontier
folders_to_check . clear ( ) ;
// Process collected data
for ( segment , warnings , fe_result ) in segments {
folders_to_check . extend ( segment ) ;
self . text_messages . warnings . extend ( warnings ) ;
for ( name , fe ) in fe_result {
self . images_to_check . insert ( name , fe ) ;
2020-10-15 08:10:27 +13:00
}
}
}
2021-12-18 07:29:37 +13:00
2023-05-03 08:37:12 +12:00
send_info_and_wait_for_ending_all_threads ( & progress_thread_run , progress_thread_handle ) ;
2023-05-08 06:54:05 +12:00
2020-11-08 04:26:40 +13:00
true
}
2020-10-15 08:10:27 +13:00
2023-05-03 08:37:12 +12:00
fn add_file_entry ( & self , metadata : & Metadata , current_folder : & Path , entry_data : & DirEntry , fe_result : & mut Vec < ( String , FileEntry ) > , warnings : & mut Vec < String > ) {
let Some ( file_name_lowercase ) = get_lowercase_name ( entry_data , warnings ) else {
return ;
} ;
2020-12-27 22:56:26 +13:00
2023-05-03 08:37:12 +12:00
if ! self . allowed_extensions . matches_filename ( & file_name_lowercase ) {
return ;
}
// Checking files
if ( self . minimal_file_size ..= self . maximal_file_size ) . contains ( & metadata . len ( ) ) {
let current_file_name = current_folder . join ( entry_data . file_name ( ) ) ;
if self . excluded_items . is_excluded ( & current_file_name ) {
return ;
}
let fe : FileEntry = FileEntry {
path : current_file_name . clone ( ) ,
size : metadata . len ( ) ,
dimensions : String ::new ( ) ,
modified_date : get_modified_time ( metadata , warnings , & current_file_name , false ) ,
hash : Vec ::new ( ) ,
similarity : 0 ,
} ;
2020-10-15 08:10:27 +13:00
2023-05-03 08:37:12 +12:00
fe_result . push ( ( current_file_name . to_string_lossy ( ) . to_string ( ) , fe ) ) ;
}
}
fn hash_images_load_cache ( & mut self ) -> ( HashMap < String , FileEntry > , HashMap < String , FileEntry > , HashMap < String , FileEntry > ) {
2021-03-05 00:09:53 +13:00
let loaded_hash_map ;
2020-12-27 22:56:26 +13:00
2022-01-06 03:20:21 +13:00
let mut records_already_cached : HashMap < String , FileEntry > = Default ::default ( ) ;
let mut non_cached_files_to_check : HashMap < String , FileEntry > = Default ::default ( ) ;
2021-03-05 00:09:53 +13:00
if self . use_cache {
2021-12-01 00:45:09 +13:00
loaded_hash_map = match load_hashes_from_file ( & mut self . text_messages , self . delete_outdated_cache , self . hash_size , self . hash_alg , self . image_filter ) {
2021-03-05 00:09:53 +13:00
Some ( t ) = > t ,
None = > Default ::default ( ) ,
} ;
for ( name , file_entry ) in & self . images_to_check {
if ! loaded_hash_map . contains_key ( name ) {
// If loaded data doesn't contains current image info
2021-01-15 00:17:15 +13:00
non_cached_files_to_check . insert ( name . clone ( ) , file_entry . clone ( ) ) ;
2020-12-27 22:56:26 +13:00
} else {
2023-05-08 06:54:05 +12:00
let loaded_item = loaded_hash_map . get ( name ) . unwrap ( ) ;
if file_entry . size ! = loaded_item . size | | file_entry . modified_date ! = loaded_item . modified_date {
// When size or modification date of image changed, then it is clear that is different image
non_cached_files_to_check . insert ( name . clone ( ) , file_entry . clone ( ) ) ;
} else {
// Checking may be omitted when already there is entry with same size and modification date
records_already_cached . insert ( name . clone ( ) , loaded_item . clone ( ) ) ;
}
2020-12-27 22:56:26 +13:00
}
}
2021-03-05 00:09:53 +13:00
} else {
loaded_hash_map = Default ::default ( ) ;
mem ::swap ( & mut self . images_to_check , & mut non_cached_files_to_check ) ;
2020-12-27 22:56:26 +13:00
}
2023-05-03 08:37:12 +12:00
( loaded_hash_map , records_already_cached , non_cached_files_to_check )
}
// Cache algorithm:
// - Load data from file
2023-05-08 06:54:05 +12:00
// - Remove from data to search, already loaded entries from cache(size and modified date must match)
2023-05-03 08:37:12 +12:00
// - Check hash of files which doesn't have saved entry
// - Join already read hashes with hashes which were read from file
// - Join all hashes and save it to file
fn hash_images ( & mut self , stop_receiver : Option < & Receiver < ( ) > > , progress_sender : Option < & UnboundedSender < ProgressData > > ) -> bool {
let ( loaded_hash_map , records_already_cached , non_cached_files_to_check ) = self . hash_images_load_cache ( ) ;
2020-12-27 22:56:26 +13:00
2023-05-08 06:54:05 +12:00
let ( progress_thread_handle , progress_thread_run , atomic_counter , check_was_stopped ) =
prepare_thread_handler_common ( progress_sender , 1 , 2 , non_cached_files_to_check . len ( ) , CheckingMethod ::None ) ;
2023-05-03 08:37:12 +12:00
let mut vec_file_entry : Vec < ( FileEntry , ImHash ) > = non_cached_files_to_check
2021-12-30 01:43:38 +13:00
. into_par_iter ( )
2023-05-03 08:37:12 +12:00
. map ( | ( _s , file_entry ) | {
atomic_counter . fetch_add ( 1 , Ordering ::Relaxed ) ;
2020-11-08 19:41:29 +13:00
if stop_receiver . is_some ( ) & & stop_receiver . unwrap ( ) . try_recv ( ) . is_ok ( ) {
2022-05-10 05:40:35 +12:00
check_was_stopped . store ( true , Ordering ::Relaxed ) ;
2020-11-08 19:41:29 +13:00
return None ;
}
2023-05-03 08:37:12 +12:00
Some ( Some ( self . collect_image_file_entry ( file_entry ) ) )
2020-11-08 19:41:29 +13:00
} )
. while_some ( )
2023-01-29 06:54:02 +13:00
. filter ( Option ::is_some )
. map ( Option ::unwrap )
2023-05-03 08:37:12 +12:00
. collect ::< Vec < ( FileEntry , ImHash ) > > ( ) ;
2020-11-22 23:03:15 +13:00
2023-05-03 08:37:12 +12:00
send_info_and_wait_for_ending_all_threads ( & progress_thread_run , progress_thread_handle ) ;
2020-12-02 22:25:27 +13:00
2020-12-27 22:56:26 +13:00
// Just connect loaded results with already calculated hashes
2023-05-08 06:54:05 +12:00
for file_entry in records_already_cached . into_values ( ) {
2020-12-27 22:56:26 +13:00
vec_file_entry . push ( ( file_entry . clone ( ) , file_entry . hash ) ) ;
}
2021-12-01 00:45:09 +13:00
// All valid entries are used to create bktree used to check for hash similarity
2020-12-27 22:56:26 +13:00
for ( file_entry , buf ) in & vec_file_entry {
2021-11-30 01:53:09 +13:00
// Only use to comparing, non broken hashes(all 0 or 255 hashes means that algorithm fails to decode them because e.g. contains a log of alpha channel)
2021-12-30 01:43:38 +13:00
if ! ( buf . is_empty ( ) | | buf . iter ( ) . all ( | e | * e = = 0 ) | | buf . iter ( ) . all ( | e | * e = = 255 ) ) {
2022-06-05 18:01:17 +12:00
self . image_hashes . entry ( buf . clone ( ) ) . or_insert_with ( Vec ::< FileEntry > ::new ) . push ( file_entry . clone ( ) ) ;
2021-11-30 01:53:09 +13:00
}
2020-11-08 19:41:29 +13:00
}
2021-03-05 00:09:53 +13:00
if self . use_cache {
// Must save all results to file, old loaded from file with all currently counted results
2022-01-06 03:20:21 +13:00
let mut all_results : HashMap < String , FileEntry > = loaded_hash_map ;
2021-03-05 00:09:53 +13:00
for ( file_entry , _hash ) in vec_file_entry {
all_results . insert ( file_entry . path . to_string_lossy ( ) . to_string ( ) , file_entry ) ;
}
2022-01-06 10:47:27 +13:00
save_hashes_to_file (
& all_results ,
& mut self . text_messages ,
self . save_also_as_json ,
self . hash_size ,
self . hash_alg ,
self . image_filter ,
) ;
2020-12-27 22:56:26 +13:00
}
2022-05-10 05:40:35 +12:00
// Break if stop was clicked after saving to cache
if check_was_stopped . load ( Ordering ::Relaxed ) {
return false ;
}
2021-12-29 08:20:24 +13:00
true
}
2023-05-03 08:37:12 +12:00
fn collect_image_file_entry ( & self , mut file_entry : FileEntry ) -> ( FileEntry , ImHash ) {
let file_name_lowercase = file_entry . path . to_string_lossy ( ) . to_lowercase ( ) ;
let image ;
#[ allow(clippy::never_loop) ] // Required to implement nice if/else
' krztyna : loop {
if RAW_IMAGE_EXTENSIONS . iter ( ) . any ( | e | file_name_lowercase . ends_with ( e ) ) {
image = match get_dynamic_image_from_raw_image ( & file_entry . path ) {
Some ( t ) = > t ,
None = > return ( file_entry , Vec ::new ( ) ) ,
} ;
break 'krztyna ;
}
#[ cfg(feature = " heif " ) ]
if HEIC_EXTENSIONS . iter ( ) . any ( | e | file_name_lowercase . ends_with ( e ) ) {
image = match get_dynamic_image_from_heic ( & file_entry . path . to_string_lossy ( ) ) {
Ok ( t ) = > t ,
Err ( _ ) = > {
return ( file_entry , Vec ::new ( ) ) ;
}
} ;
break 'krztyna ;
}
// Normal image extension, when any other fail, not using if/else
let result = panic ::catch_unwind ( | | {
match image ::open ( file_entry . path . clone ( ) ) {
Ok ( t ) = > Ok ( t ) ,
// Err(_inspected) => return Some(None), // Something is wrong with image,
// For broken images empty hash is used, because without it will try to resecan files each time when it is called(missing cache file is responsible for it)
// This may cause problems(very rarely), when e.g. file was not available due lack of permissions, but it is available now
Err ( _inspected ) = > Err ( ( ) ) ,
}
} ) ;
// If image crashed during opening, we just skip checking its hash and go on
if let Ok ( image_result ) = result {
if let Ok ( image2 ) = image_result {
image = image2 ;
} else {
return ( file_entry , Vec ::new ( ) ) ;
}
} else {
let message = create_crash_message ( " Image-rs " , & file_entry . path . to_string_lossy ( ) , " https://github.com/image-rs/image/issues " ) ;
println! ( " {message} " ) ;
return ( file_entry , Vec ::new ( ) ) ;
}
break 'krztyna ;
}
let dimensions = image . dimensions ( ) ;
file_entry . dimensions = format! ( " {} x {} " , dimensions . 0 , dimensions . 1 ) ;
let hasher_config = HasherConfig ::new ( )
. hash_size ( self . hash_size as u32 , self . hash_size as u32 )
. hash_alg ( self . hash_alg )
. resize_filter ( self . image_filter ) ;
let hasher = hasher_config . to_hasher ( ) ;
let hash = hasher . hash_image ( & image ) ;
let buf : ImHash = hash . as_bytes ( ) . to_vec ( ) ;
file_entry . hash = buf . clone ( ) ;
( file_entry , buf )
}
fn compare_hashes (
& self ,
hashes_to_check : & [ ImHash ] ,
atomic_counter : & Arc < AtomicUsize > ,
stop_receiver : Option < & Receiver < ( ) > > ,
check_was_stopped : & AtomicBool ,
tolerance : u32 ,
hashes_with_multiple_images : & HashSet < ImHash > ,
all_hashed_images : & HashMap < ImHash , Vec < FileEntry > > ,
) -> Option < ( HashMap < ImHash , u32 > , HashMap < ImHash , ( ImHash , u32 ) > ) > {
let mut hashes_parents : HashMap < ImHash , u32 > = Default ::default ( ) ; // Hashes used as parent (hash, children_number_of_hash)
let mut hashes_similarity : HashMap < ImHash , ( ImHash , u32 ) > = Default ::default ( ) ; // Hashes used as child, (parent_hash, similarity)
// Sprawdź czy hash nie jest użyty jako master gdzie indziej
// Jeśli tak to przejdź do sprawdzania kolejnego elementu
// Zweryfikuj czy sprawdzany element ma rodzica
// Jeśli ma to sprawdź czy similarity nowego rodzica jest mniejsze niż starego
// // Jeśli tak to zmniejsz ilość dzieci starego rodzica, dodaj ilość dzieci w nowym rodzicu i podmień rekord hashes_similarity
// // Jeśli nie to dodaj nowy rekord w hashes_similarity jak i hashes_parents z liczbą dzieci równą 1
for ( index , hash_to_check ) in hashes_to_check . iter ( ) . enumerate ( ) {
// Don't check for user stop too often
// Also don't add too often data to atomic variable
const CYCLES_COUNTER : usize = 0b11_1111 ;
if ( ( index & CYCLES_COUNTER ) = = CYCLES_COUNTER ) & & index ! = 0 {
atomic_counter . fetch_add ( CYCLES_COUNTER , Ordering ::Relaxed ) ;
if stop_receiver . is_some ( ) & & stop_receiver . unwrap ( ) . try_recv ( ) . is_ok ( ) {
check_was_stopped . store ( true , Ordering ::Relaxed ) ;
return None ;
}
}
hashes_parents . insert ( ( * hash_to_check ) . clone ( ) , 0 ) ;
let mut found_items = self
. bktree
. find ( hash_to_check , tolerance )
. filter ( | ( similarity , _hash ) | if self . use_reference_folders { true } else { * similarity ! = 0 } )
. collect ::< Vec < _ > > ( ) ;
found_items . sort_unstable_by_key ( | f | f . 0 ) ;
for ( similarity , compared_hash ) in found_items {
image_to_check (
& mut hashes_parents ,
& mut hashes_similarity ,
hashes_with_multiple_images ,
hash_to_check ,
compared_hash ,
similarity ,
) ;
}
}
debug_check_for_duplicated_things ( self . use_reference_folders , & hashes_parents , & hashes_similarity , all_hashed_images , " BEFORE " ) ;
Some ( ( hashes_parents , hashes_similarity ) )
}
fn chunk_hashes ( & mut self , all_hashed_images : & HashMap < ImHash , Vec < FileEntry > > , all_hashes : & Vec < ImHash > ) -> ( Vec < Vec < ImHash > > , HashSet < ImHash > ) {
let mut hashes_with_multiple_images : HashSet < ImHash > = Default ::default ( ) ; // Fast way to check if hash have multiple images
let mut files_from_referenced_folders : HashMap < ImHash , Vec < FileEntry > > = HashMap ::new ( ) ;
let mut normal_files : HashMap < ImHash , Vec < FileEntry > > = HashMap ::new ( ) ;
let number_of_processors = get_number_of_threads ( ) ;
let chunk_size ;
let mut initial_hashes : Vec < ImHash > = Vec ::new ( ) ;
let mut additional_chunk_to_check : Vec < ImHash > = Default ::default ( ) ;
let mut chunks : Vec < Vec < ImHash > > ;
if self . use_reference_folders {
let reference_directories = self . directories . reference_directories . clone ( ) ;
all_hashed_images . clone ( ) . into_iter ( ) . for_each ( | ( hash , vec_file_entry ) | {
for file_entry in vec_file_entry {
if reference_directories . iter ( ) . any ( | e | file_entry . path . starts_with ( e ) ) {
files_from_referenced_folders . entry ( hash . clone ( ) ) . or_insert_with ( Vec ::new ) . push ( file_entry ) ;
} else {
normal_files . entry ( hash . clone ( ) ) . or_insert_with ( Vec ::new ) . push ( file_entry ) ;
}
}
} ) ;
for ( hash , vec_files ) in normal_files {
if vec_files . len ( ) > = 2 {
hashes_with_multiple_images . insert ( hash . clone ( ) ) ;
}
self . bktree . add ( hash ) ;
}
for ( hash , vec_files ) in files_from_referenced_folders {
if vec_files . len ( ) > = 2 {
hashes_with_multiple_images . insert ( hash . clone ( ) ) ;
}
initial_hashes . push ( hash ) ;
}
chunk_size = initial_hashes . len ( ) / number_of_processors ;
chunks = if chunk_size > 0 {
initial_hashes . chunks ( chunk_size ) . map ( < [ std ::vec ::Vec < u8 > ] > ::to_vec ) . collect ::< Vec < _ > > ( )
} else {
vec! [ initial_hashes ]
} ;
} else {
for ( hash , vec_files ) in all_hashed_images {
if vec_files . len ( ) > = 2 {
additional_chunk_to_check . push ( hash . clone ( ) ) ;
hashes_with_multiple_images . insert ( hash . clone ( ) ) ;
} else {
self . bktree . add ( hash . clone ( ) ) ;
}
}
chunk_size = all_hashes . len ( ) / number_of_processors ;
chunks = if chunk_size > 0 {
all_hashes . chunks ( chunk_size ) . map ( < [ Vec < u8 > ] > ::to_vec ) . collect ::< Vec < _ > > ( )
} else {
vec! [ all_hashes . clone ( ) ]
} ;
chunks . push ( additional_chunk_to_check ) ;
}
( chunks , hashes_with_multiple_images )
}
fn collect_hash_compare_result (
& self ,
hashes_parents : HashMap < ImHash , u32 > ,
hashes_with_multiple_images : & HashSet < ImHash > ,
all_hashed_images : & HashMap < ImHash , Vec < FileEntry > > ,
collected_similar_images : & mut HashMap < ImHash , Vec < FileEntry > > ,
hashes_similarity : HashMap < ImHash , ( ImHash , u32 ) > ,
) {
if self . use_reference_folders {
// This is same step as without reference folders, but also checks if children are inside/outside reference directories, because may happen, that one file is inside reference folder and other outside
// Collecting results to vector
for ( parent_hash , child_number ) in hashes_parents {
// If hash contains other hasher OR multiple images are available for checked hash
if child_number > 0 | | hashes_with_multiple_images . contains ( & parent_hash ) {
let vec_fe = all_hashed_images
. get ( & parent_hash )
. unwrap ( )
. iter ( )
. filter ( | e | is_in_reference_folder ( & self . directories . reference_directories , & e . path ) )
. cloned ( )
. collect ( ) ;
collected_similar_images . insert ( parent_hash . clone ( ) , vec_fe ) ;
}
}
for ( child_hash , ( parent_hash , similarity ) ) in hashes_similarity {
let mut vec_fe : Vec < _ > = all_hashed_images
. get ( & child_hash )
. unwrap ( )
. iter ( )
. filter ( | e | ! is_in_reference_folder ( & self . directories . reference_directories , & e . path ) )
. cloned ( )
. collect ( ) ;
for mut fe in & mut vec_fe {
fe . similarity = similarity ;
}
collected_similar_images . get_mut ( & parent_hash ) . unwrap ( ) . append ( & mut vec_fe ) ;
}
} else {
// Collecting results to vector
for ( parent_hash , child_number ) in hashes_parents {
// If hash contains other hasher OR multiple images are available for checked hash
if child_number > 0 | | hashes_with_multiple_images . contains ( & parent_hash ) {
let vec_fe = all_hashed_images . get ( & parent_hash ) . unwrap ( ) . clone ( ) ;
collected_similar_images . insert ( parent_hash . clone ( ) , vec_fe ) ;
}
}
for ( child_hash , ( parent_hash , similarity ) ) in hashes_similarity {
let mut vec_fe = all_hashed_images . get ( & child_hash ) . unwrap ( ) . clone ( ) ;
for mut fe in & mut vec_fe {
fe . similarity = similarity ;
}
collected_similar_images . get_mut ( & parent_hash ) . unwrap ( ) . append ( & mut vec_fe ) ;
}
}
}
fn check_for_duplicate_hashes (
& self ,
parts : Vec < ( HashMap < ImHash , u32 > , HashMap < ImHash , ( ImHash , u32 ) > ) > ,
hashes_with_multiple_images : & HashSet < ImHash > ,
all_hashed_images : & HashMap < ImHash , Vec < FileEntry > > ,
collected_similar_images : & mut HashMap < ImHash , Vec < FileEntry > > ,
) {
let mut hashes_parents : HashMap < ImHash , u32 > = Default ::default ( ) ;
let mut hashes_similarity : HashMap < ImHash , ( ImHash , u32 ) > = Default ::default ( ) ;
let mut iter = parts . into_iter ( ) ;
// At start fill arrays with first item
// Normal algorithm would do exactly same thing, but slower, one record after one
if let Some ( ( first_hashes_parents , first_hashes_similarity ) ) = iter . next ( ) {
hashes_parents = first_hashes_parents ;
hashes_similarity = first_hashes_similarity ;
}
for ( partial_hashes_with_parents , partial_hashes_with_similarity ) in iter {
for ( parent_hash , _child_number ) in partial_hashes_with_parents {
if ! hashes_parents . contains_key ( & parent_hash ) & & ! hashes_similarity . contains_key ( & parent_hash ) {
hashes_parents . insert ( parent_hash , 0 ) ;
}
}
for ( hash_to_check , ( compared_hash , similarity ) ) in partial_hashes_with_similarity {
image_to_check (
& mut hashes_parents ,
& mut hashes_similarity ,
hashes_with_multiple_images ,
& hash_to_check ,
& compared_hash ,
similarity ,
) ;
}
}
debug_check_for_duplicated_things ( self . use_reference_folders , & hashes_parents , & hashes_similarity , all_hashed_images , " LATTER " ) ;
// Just simple check if all original hashes with multiple entries are available in end results
let original_hashes_at_start = hashes_with_multiple_images . len ( ) ;
let original_hashes_in_end_results = hashes_parents
. iter ( )
. filter ( | ( parent_hash , _child_number ) | hashes_with_multiple_images . contains ( * parent_hash ) )
. count ( ) ;
if ! self . use_reference_folders {
assert_eq! ( original_hashes_at_start , original_hashes_in_end_results ) ;
}
self . collect_hash_compare_result ( hashes_parents , hashes_with_multiple_images , all_hashed_images , collected_similar_images , hashes_similarity ) ;
}
2020-11-09 04:19:40 +13:00
2023-05-03 08:37:12 +12:00
fn find_similar_hashes ( & mut self , stop_receiver : Option < & Receiver < ( ) > > , progress_sender : Option < & UnboundedSender < ProgressData > > ) -> bool {
2022-07-10 05:18:26 +12:00
if self . image_hashes . is_empty ( ) {
return true ;
}
2022-07-03 07:30:59 +12:00
let tolerance = self . similarity ;
2020-11-08 04:26:40 +13:00
2021-12-29 08:20:24 +13:00
// Results
2023-05-03 08:37:12 +12:00
let mut collected_similar_images : HashMap < ImHash , Vec < FileEntry > > = Default ::default ( ) ;
2021-08-07 02:40:43 +12:00
2023-05-08 06:54:05 +12:00
let all_hashed_images = mem ::take ( & mut self . image_hashes ) ;
2021-12-29 08:20:24 +13:00
2023-05-03 08:37:12 +12:00
let all_hashes : Vec < _ > = all_hashed_images . clone ( ) . into_keys ( ) . collect ( ) ;
2021-12-29 08:20:24 +13:00
2022-07-03 07:30:59 +12:00
// Checking entries with tolerance 0 is really easy and fast, because only entries with same hashes needs to be checked
if tolerance = = 0 {
for ( hash , vec_file_entry ) in all_hashed_images . clone ( ) {
if vec_file_entry . len ( ) > = 2 {
collected_similar_images . insert ( hash , vec_file_entry ) ;
}
2021-12-29 08:20:24 +13:00
}
2022-07-03 07:30:59 +12:00
} else {
2023-05-08 06:54:05 +12:00
let ( progress_thread_handle , progress_thread_run , atomic_counter , check_was_stopped ) =
prepare_thread_handler_common ( progress_sender , 2 , 2 , all_hashes . len ( ) , CheckingMethod ::None ) ;
2021-08-07 02:40:43 +12:00
2022-08-30 07:37:31 +12:00
// Don't use hashes with multiple images in bktree, because they will always be master of group and cannot be find by other hashes
2021-12-26 10:23:18 +13:00
2023-05-03 08:37:12 +12:00
let ( chunks , hashes_with_multiple_images ) = self . chunk_hashes ( & all_hashed_images , & all_hashes ) ;
2022-07-03 07:30:59 +12:00
let parts : Vec < _ > = chunks
. into_par_iter ( )
. map ( | hashes_to_check | {
2023-05-03 08:37:12 +12:00
self . compare_hashes (
& hashes_to_check ,
& atomic_counter ,
stop_receiver ,
& check_was_stopped ,
tolerance ,
& hashes_with_multiple_images ,
& all_hashed_images ,
)
2022-07-03 07:30:59 +12:00
} )
. while_some ( )
. collect ( ) ;
2021-12-29 09:20:48 +13:00
2023-05-03 08:37:12 +12:00
send_info_and_wait_for_ending_all_threads ( & progress_thread_run , progress_thread_handle ) ;
2021-12-29 09:20:48 +13:00
2022-07-03 07:30:59 +12:00
if check_was_stopped . load ( Ordering ::Relaxed ) {
return false ;
}
2022-01-06 03:20:21 +13:00
2023-05-03 08:37:12 +12:00
self . check_for_duplicate_hashes ( parts , & hashes_with_multiple_images , & all_hashed_images , & mut collected_similar_images ) ;
}
2021-12-29 09:20:48 +13:00
2023-05-03 08:37:12 +12:00
self . verify_duplicated_items ( & collected_similar_images ) ;
2022-07-03 07:30:59 +12:00
2023-05-03 08:37:12 +12:00
self . similar_vectors = collected_similar_images . into_values ( ) . collect ( ) ;
2022-07-03 07:30:59 +12:00
2023-05-03 08:37:12 +12:00
self . exclude_items_with_same_size ( ) ;
2021-12-29 09:20:48 +13:00
2023-05-03 08:37:12 +12:00
self . check_for_reference_folders ( ) ;
2022-09-14 08:18:07 +12:00
2023-05-03 08:37:12 +12:00
if self . use_reference_folders {
for ( _fe , vector ) in & self . similar_referenced_vectors {
self . information . number_of_duplicates + = vector . len ( ) ;
self . information . number_of_groups + = 1 ;
2020-10-15 08:10:27 +13:00
}
2023-05-03 08:37:12 +12:00
} else {
for vector in & self . similar_vectors {
self . information . number_of_duplicates + = vector . len ( ) - 1 ;
self . information . number_of_groups + = 1 ;
2022-01-06 03:20:21 +13:00
}
}
2020-10-15 08:10:27 +13:00
2023-05-08 06:54:05 +12:00
// Clean unused data to save ram
2023-05-03 08:37:12 +12:00
self . image_hashes = Default ::default ( ) ;
self . images_to_check = Default ::default ( ) ;
self . bktree = BKTree ::new ( Hamming ) ;
true
}
fn exclude_items_with_same_size ( & mut self ) {
2021-12-14 08:13:49 +13:00
if self . exclude_images_with_same_size {
2023-05-08 06:54:05 +12:00
for vec_file_entry in mem ::take ( & mut self . similar_vectors ) {
2021-12-14 08:13:49 +13:00
let mut bt_sizes : BTreeSet < u64 > = Default ::default ( ) ;
let mut vec_values = Vec ::new ( ) ;
for file_entry in vec_file_entry {
if ! bt_sizes . contains ( & file_entry . size ) {
bt_sizes . insert ( file_entry . size ) ;
vec_values . push ( file_entry ) ;
}
}
if vec_values . len ( ) > 1 {
self . similar_vectors . push ( vec_values ) ;
}
}
}
2023-05-03 08:37:12 +12:00
}
2021-12-14 08:13:49 +13:00
2023-05-03 08:37:12 +12:00
fn check_for_reference_folders ( & mut self ) {
2021-12-24 21:18:55 +13:00
if self . use_reference_folders {
2023-05-08 06:54:05 +12:00
self . similar_referenced_vectors = mem ::take ( & mut self . similar_vectors )
2021-12-24 21:18:55 +13:00
. into_iter ( )
. filter_map ( | vec_file_entry | {
2023-05-08 06:54:05 +12:00
let ( mut files_from_referenced_folders , normal_files ) : ( Vec < _ > , Vec < _ > ) =
vec_file_entry . into_iter ( ) . partition ( | e | self . directories . is_in_referenced_directory ( e . get_path ( ) ) ) ;
2021-12-24 21:18:55 +13:00
if files_from_referenced_folders . is_empty ( ) | | normal_files . is_empty ( ) {
None
} else {
Some ( ( files_from_referenced_folders . pop ( ) . unwrap ( ) , normal_files ) )
}
} )
. collect ::< Vec < ( FileEntry , Vec < FileEntry > ) > > ( ) ;
}
2023-05-03 08:37:12 +12:00
}
2021-12-24 21:18:55 +13:00
2023-05-03 08:37:12 +12:00
#[ allow(dead_code) ]
#[ allow(unreachable_code) ]
#[ allow(unused_variables) ]
pub fn verify_duplicated_items ( & self , collected_similar_images : & HashMap < ImHash , Vec < FileEntry > > ) {
#[ cfg(not(debug_assertions)) ]
return ;
// Validating if group contains duplicated results
let mut result_hashset : HashSet < String > = Default ::default ( ) ;
let mut found = false ;
for vec_file_entry in collected_similar_images . values ( ) {
if vec_file_entry . is_empty ( ) {
println! ( " Empty group " ) ;
found = true ;
continue ;
2021-12-24 21:18:55 +13:00
}
2023-05-03 08:37:12 +12:00
if vec_file_entry . len ( ) = = 1 {
println! ( " Single Element {vec_file_entry:?} " ) ;
found = true ;
continue ;
}
for file_entry in vec_file_entry {
let st = file_entry . path . to_string_lossy ( ) . to_string ( ) ;
if result_hashset . contains ( & st ) {
found = true ;
println! ( " Duplicated Element {st} " ) ;
} else {
result_hashset . insert ( st ) ;
}
2021-12-24 21:18:55 +13:00
}
}
2023-05-03 08:37:12 +12:00
assert! ( ! found , " Found Invalid entries, verify errors before " ) ; // TODO crashes with empty result with reference folder, verify why
2020-10-15 08:10:27 +13:00
}
/// Set included dir which needs to be relative, exists etc.
2021-01-11 08:44:10 +13:00
pub fn set_included_directory ( & mut self , included_directory : Vec < PathBuf > ) {
2020-10-15 08:10:27 +13:00
self . directories . set_included_directory ( included_directory , & mut self . text_messages ) ;
}
2021-12-24 21:18:55 +13:00
pub fn set_reference_directory ( & mut self , reference_directory : Vec < PathBuf > ) {
self . directories . set_reference_directory ( reference_directory ) ;
}
2021-01-11 08:44:10 +13:00
pub fn set_excluded_directory ( & mut self , excluded_directory : Vec < PathBuf > ) {
2020-10-15 08:10:27 +13:00
self . directories . set_excluded_directory ( excluded_directory , & mut self . text_messages ) ;
}
2021-01-11 08:44:10 +13:00
pub fn set_excluded_items ( & mut self , excluded_items : Vec < String > ) {
2020-10-15 08:10:27 +13:00
self . excluded_items . set_excluded_items ( excluded_items , & mut self . text_messages ) ;
}
}
2021-11-28 08:57:10 +13:00
2022-08-30 07:37:31 +12:00
fn image_to_check < ' a > (
2023-05-03 08:37:12 +12:00
hashes_parents : & mut HashMap < ImHash , u32 > ,
hashes_similarity : & mut HashMap < ImHash , ( ImHash , u32 ) > ,
hashes_with_multiple_images : & HashSet < ImHash > ,
hash_to_check : & ' a ImHash ,
compared_hash : & ' a ImHash ,
2022-08-30 07:37:31 +12:00
similarity : u32 ,
) {
if let Some ( children_number ) = hashes_parents . get ( compared_hash ) {
if * children_number > 0 | | hashes_with_multiple_images . contains ( compared_hash ) {
return ;
}
}
// If there is already record, with smaller sensitivity, then replace it
let mut need_to_add = false ;
let mut need_to_check = false ;
// TODO consider to replace variables from above with closures
// If current checked hash, have parent, first we must check if similarity between them is lower than checked item
if let Some ( ( current_parent_hash , current_similarity_with_parent ) ) = hashes_similarity . get ( hash_to_check ) {
if * current_similarity_with_parent > similarity {
need_to_check = true ;
* hashes_parents . get_mut ( current_parent_hash ) . unwrap ( ) - = 1 ;
hashes_similarity . remove ( hash_to_check ) . unwrap ( ) ;
}
} else {
need_to_check = true ;
}
if need_to_check {
if let Some ( ( other_parent_hash , other_similarity ) ) = hashes_similarity . get ( compared_hash ) {
if * other_similarity > similarity {
need_to_add = true ;
* hashes_parents . get_mut ( other_parent_hash ) . unwrap ( ) - = 1 ;
}
}
// But when there is no record, just add it
else {
2023-01-29 06:54:02 +13:00
need_to_add = true ;
2022-08-30 07:37:31 +12:00
}
}
if need_to_add {
2023-05-03 08:37:12 +12:00
hashes_similarity . insert ( compared_hash . clone ( ) , ( hash_to_check . clone ( ) , similarity ) ) ;
2022-08-30 07:37:31 +12:00
if let Some ( number_of_children ) = hashes_parents . get_mut ( hash_to_check ) {
* number_of_children + = 1 ;
} else {
2023-05-03 08:37:12 +12:00
hashes_parents . insert ( hash_to_check . clone ( ) , 1 ) ;
2022-08-30 07:37:31 +12:00
}
}
}
2022-09-14 08:18:07 +12:00
fn is_in_reference_folder ( reference_directories : & [ PathBuf ] , path : & Path ) -> bool {
2022-11-24 08:23:17 +13:00
reference_directories . iter ( ) . any ( | e | path . starts_with ( e ) )
2022-09-14 08:18:07 +12:00
}
2020-10-15 08:10:27 +13:00
impl Default for SimilarImages {
fn default ( ) -> Self {
Self ::new ( )
}
}
impl DebugPrint for SimilarImages {
#[ allow(dead_code) ]
#[ allow(unreachable_code) ]
fn debug_print ( & self ) {
#[ cfg(not(debug_assertions)) ]
{
return ;
}
println! ( " ---------------DEBUG PRINT--------------- " ) ;
println! ( " Included directories - {:?} " , self . directories . included_directories ) ;
println! ( " ----------------------------------------- " ) ;
}
}
2021-11-28 08:57:10 +13:00
2020-10-15 08:10:27 +13:00
impl SaveResults for SimilarImages {
fn save_results_to_file ( & mut self , file_name : & str ) -> bool {
let file_name : String = match file_name {
" " = > " results.txt " . to_string ( ) ,
k = > k . to_string ( ) ,
} ;
2021-01-09 23:52:43 +13:00
let file_handler = match File ::create ( & file_name ) {
2020-10-15 08:10:27 +13:00
Ok ( t ) = > t ,
2021-11-15 03:53:55 +13:00
Err ( e ) = > {
2022-12-21 20:44:26 +13:00
self . text_messages . errors . push ( format! ( " Failed to create file {file_name} , reason {e} " ) ) ;
2020-10-15 08:10:27 +13:00
return false ;
}
} ;
2021-01-09 23:52:43 +13:00
let mut writer = BufWriter ::new ( file_handler ) ;
2020-10-15 08:10:27 +13:00
2021-11-15 03:53:55 +13:00
if let Err ( e ) = writeln! (
2021-01-09 23:52:43 +13:00
writer ,
2020-10-15 08:10:27 +13:00
" Results of searching {:?} with excluded directories {:?} and excluded items {:?} " ,
self . directories . included_directories , self . directories . excluded_directories , self . excluded_items . items
2021-11-15 03:53:55 +13:00
) {
2022-12-21 20:44:26 +13:00
self . text_messages . errors . push ( format! ( " Failed to save results to file {file_name} , reason {e} " ) ) ;
2020-10-15 08:10:27 +13:00
return false ;
}
if ! self . similar_vectors . is_empty ( ) {
2021-01-09 23:52:43 +13:00
write! ( writer , " {} images which have similar friends \n \n " , self . similar_vectors . len ( ) ) . unwrap ( ) ;
2020-10-15 08:10:27 +13:00
2023-01-29 06:54:02 +13:00
for struct_similar in & self . similar_vectors {
2021-02-10 05:29:49 +13:00
writeln! ( writer , " Found {} images which have similar friends " , self . similar_vectors . len ( ) ) . unwrap ( ) ;
for file_entry in struct_similar {
writeln! (
writer ,
" {} - {} - {} - {} " ,
file_entry . path . display ( ) ,
file_entry . dimensions ,
2022-09-14 08:18:07 +12:00
format_size ( file_entry . size , BINARY ) ,
2021-11-18 23:23:17 +13:00
get_string_from_similarity ( & file_entry . similarity , self . hash_size )
2021-02-10 05:29:49 +13:00
)
. unwrap ( ) ;
}
writeln! ( writer ) . unwrap ( ) ;
}
2020-10-15 08:10:27 +13:00
} else {
2021-01-09 23:52:43 +13:00
write! ( writer , " Not found any similar images. " ) . unwrap ( ) ;
2020-10-15 08:10:27 +13:00
}
2020-12-27 22:56:26 +13:00
2020-10-15 08:10:27 +13:00
true
}
}
2021-11-28 08:57:10 +13:00
2020-10-15 08:10:27 +13:00
impl PrintResults for SimilarImages {
fn print_results ( & self ) {
if ! self . similar_vectors . is_empty ( ) {
println! ( " Found {} images which have similar friends " , self . similar_vectors . len ( ) ) ;
2020-11-08 04:26:40 +13:00
for vec_file_entry in & self . similar_vectors {
for file_entry in vec_file_entry {
println! (
" {} - {} - {} - {} " ,
file_entry . path . display ( ) ,
file_entry . dimensions ,
2022-09-14 08:18:07 +12:00
format_size ( file_entry . size , BINARY ) ,
2021-11-18 23:23:17 +13:00
get_string_from_similarity ( & file_entry . similarity , self . hash_size )
2020-11-08 04:26:40 +13:00
) ;
}
println! ( ) ;
}
2020-10-15 08:10:27 +13:00
}
}
}
2022-01-06 10:47:27 +13:00
pub fn save_hashes_to_file (
hashmap : & HashMap < String , FileEntry > ,
text_messages : & mut Messages ,
save_also_as_json : bool ,
hash_size : u8 ,
hash_alg : HashAlg ,
image_filter : FilterType ,
) {
if let Some ( ( ( file_handler , cache_file ) , ( file_handler_json , cache_file_json ) ) ) =
open_cache_folder ( & get_cache_file ( & hash_size , & hash_alg , & image_filter ) , true , save_also_as_json , & mut text_messages . warnings )
{
{
let writer = BufWriter ::new ( file_handler . unwrap ( ) ) ; // Unwrap because cannot fail here
if let Err ( e ) = bincode ::serialize_into ( writer , hashmap ) {
2021-12-22 06:44:20 +13:00
text_messages
2022-01-06 10:47:27 +13:00
. warnings
. push ( format! ( " Cannot write data to cache file {} , reason {} " , cache_file . display ( ) , e ) ) ;
2020-12-27 22:56:26 +13:00
return ;
}
}
2022-01-06 10:47:27 +13:00
if save_also_as_json {
if let Some ( file_handler_json ) = file_handler_json {
let writer = BufWriter ::new ( file_handler_json ) ;
if let Err ( e ) = serde_json ::to_writer ( writer , hashmap ) {
text_messages
. warnings
. push ( format! ( " Cannot write data to cache file {} , reason {} " , cache_file_json . display ( ) , e ) ) ;
return ;
}
}
2021-12-26 10:23:18 +13:00
}
text_messages . messages . push ( format! ( " Properly saved to file {} cache entries. " , hashmap . len ( ) ) ) ;
2020-12-27 22:56:26 +13:00
}
}
2021-11-28 08:57:10 +13:00
2021-12-22 06:44:20 +13:00
pub fn load_hashes_from_file (
text_messages : & mut Messages ,
delete_outdated_cache : bool ,
hash_size : u8 ,
hash_alg : HashAlg ,
image_filter : FilterType ,
2022-01-06 03:20:21 +13:00
) -> Option < HashMap < String , FileEntry > > {
2022-01-06 10:47:27 +13:00
if let Some ( ( ( file_handler , cache_file ) , ( file_handler_json , cache_file_json ) ) ) =
open_cache_folder ( & get_cache_file ( & hash_size , & hash_alg , & image_filter ) , false , true , & mut text_messages . warnings )
{
let mut hashmap_loaded_entries : HashMap < String , FileEntry > ;
if let Some ( file_handler ) = file_handler {
let reader = BufReader ::new ( file_handler ) ;
hashmap_loaded_entries = match bincode ::deserialize_from ( reader ) {
Ok ( t ) = > t ,
Err ( e ) = > {
text_messages
. warnings
. push ( format! ( " Failed to load data from cache file {} , reason {} " , cache_file . display ( ) , e ) ) ;
return None ;
}
} ;
} else {
let reader = BufReader ::new ( file_handler_json . unwrap ( ) ) ; // Unwrap cannot fail, because at least one file must be valid
hashmap_loaded_entries = match serde_json ::from_reader ( reader ) {
Ok ( t ) = > t ,
Err ( e ) = > {
text_messages
. warnings
. push ( format! ( " Failed to load data from cache file {} , reason {} " , cache_file_json . display ( ) , e ) ) ;
return None ;
}
} ;
}
2021-12-26 10:23:18 +13:00
// Don't load cache data if destination file not exists
if delete_outdated_cache {
hashmap_loaded_entries . retain ( | src_path , _file_entry | Path ::new ( src_path ) . exists ( ) ) ;
2020-12-27 22:56:26 +13:00
}
2021-12-26 10:23:18 +13:00
text_messages . messages . push ( format! ( " Properly loaded {} cache entries. " , hashmap_loaded_entries . len ( ) ) ) ;
2020-12-27 22:56:26 +13:00
return Some ( hashmap_loaded_entries ) ;
}
None
}
2021-11-18 23:23:17 +13:00
fn get_cache_file ( hash_size : & u8 , hash_alg : & HashAlg , image_filter : & FilterType ) -> String {
2021-12-22 06:44:20 +13:00
format! (
2022-07-03 07:30:59 +12:00
" cache_similar_images_{}_{}_{}_50.bin " ,
2021-12-22 06:44:20 +13:00
hash_size ,
convert_algorithm_to_string ( hash_alg ) ,
2021-12-26 10:23:18 +13:00
convert_filters_to_string ( image_filter ) ,
2021-12-22 06:44:20 +13:00
)
2021-11-18 23:23:17 +13:00
}
2023-01-29 06:54:02 +13:00
#[ must_use ]
2022-07-03 07:30:59 +12:00
pub fn get_string_from_similarity ( similarity : & u32 , hash_size : u8 ) -> String {
2021-11-19 18:35:26 +13:00
let index_preset = match hash_size {
2021-12-03 03:33:06 +13:00
8 = > 0 ,
16 = > 1 ,
32 = > 2 ,
64 = > 3 ,
2021-11-19 18:35:26 +13:00
_ = > panic! ( ) ,
} ;
2022-07-06 05:58:57 +12:00
if * similarity = = 0 {
flc! ( " core_similarity_original " )
} else if * similarity < = SIMILAR_VALUES [ index_preset ] [ 0 ] {
2022-07-03 07:30:59 +12:00
flc! ( " core_similarity_very_high " )
} else if * similarity < = SIMILAR_VALUES [ index_preset ] [ 1 ] {
flc! ( " core_similarity_high " )
} else if * similarity < = SIMILAR_VALUES [ index_preset ] [ 2 ] {
flc! ( " core_similarity_medium " )
} else if * similarity < = SIMILAR_VALUES [ index_preset ] [ 3 ] {
flc! ( " core_similarity_small " )
} else if * similarity < = SIMILAR_VALUES [ index_preset ] [ 4 ] {
flc! ( " core_similarity_very_small " )
} else if * similarity < = SIMILAR_VALUES [ index_preset ] [ 5 ] {
flc! ( " core_similarity_minimal " )
} else {
panic! ( ) ;
2021-11-18 23:23:17 +13:00
}
}
2023-01-29 06:54:02 +13:00
#[ must_use ]
2022-07-03 07:30:59 +12:00
pub fn return_similarity_from_similarity_preset ( similarity_preset : & SimilarityPreset , hash_size : u8 ) -> u32 {
2021-11-19 18:35:26 +13:00
let index_preset = match hash_size {
2021-12-03 03:33:06 +13:00
8 = > 0 ,
16 = > 1 ,
32 = > 2 ,
64 = > 3 ,
2021-11-18 23:23:17 +13:00
_ = > panic! ( ) ,
2021-11-19 18:35:26 +13:00
} ;
match similarity_preset {
2022-07-31 18:45:19 +12:00
SimilarityPreset ::Original = > 0 ,
2022-07-03 07:30:59 +12:00
SimilarityPreset ::VeryHigh = > SIMILAR_VALUES [ index_preset ] [ 0 ] ,
SimilarityPreset ::High = > SIMILAR_VALUES [ index_preset ] [ 1 ] ,
SimilarityPreset ::Medium = > SIMILAR_VALUES [ index_preset ] [ 2 ] ,
SimilarityPreset ::Small = > SIMILAR_VALUES [ index_preset ] [ 3 ] ,
SimilarityPreset ::VerySmall = > SIMILAR_VALUES [ index_preset ] [ 4 ] ,
SimilarityPreset ::Minimal = > SIMILAR_VALUES [ index_preset ] [ 5 ] ,
2021-11-19 18:35:26 +13:00
SimilarityPreset ::None = > panic! ( " " ) ,
2021-11-18 23:23:17 +13:00
}
}
fn convert_filters_to_string ( image_filter : & FilterType ) -> String {
match image_filter {
FilterType ::Lanczos3 = > " Lanczos3 " ,
FilterType ::Nearest = > " Nearest " ,
FilterType ::Triangle = > " Triangle " ,
FilterType ::Gaussian = > " Gaussian " ,
FilterType ::CatmullRom = > " CatmullRom " ,
}
. to_string ( )
}
fn convert_algorithm_to_string ( hash_alg : & HashAlg ) -> String {
match hash_alg {
HashAlg ::Mean = > " Mean " ,
HashAlg ::Gradient = > " Gradient " ,
HashAlg ::Blockhash = > " Blockhash " ,
HashAlg ::VertGradient = > " VertGradient " ,
HashAlg ::DoubleGradient = > " DoubleGradient " ,
}
. to_string ( )
}
pub fn test_image_conversion_speed ( ) {
let file_name : & str = " test.jpg " ;
let file_path = Path ::new ( file_name ) ;
match image ::open ( file_path ) {
Ok ( img_open ) = > {
for alg in [ HashAlg ::Blockhash , HashAlg ::Gradient , HashAlg ::DoubleGradient , HashAlg ::VertGradient , HashAlg ::Mean ] {
2021-12-22 06:44:20 +13:00
for filter in [
FilterType ::Lanczos3 ,
FilterType ::CatmullRom ,
FilterType ::Gaussian ,
FilterType ::Nearest ,
FilterType ::Triangle ,
] {
2021-12-03 03:33:06 +13:00
for size in [ 8 , 16 , 32 , 64 ] {
2021-11-18 23:23:17 +13:00
let hasher_config = HasherConfig ::new ( ) . hash_alg ( alg ) . resize_filter ( filter ) . hash_size ( size , size ) ;
let start = SystemTime ::now ( ) ;
let hasher = hasher_config . to_hasher ( ) ;
let _hash = hasher . hash_image ( & img_open ) ;
let end = SystemTime ::now ( ) ;
println! ( " {:?} us {:?} {:?} {} x {} " , end . duration_since ( start ) . unwrap ( ) . as_micros ( ) , alg , filter , size , size ) ;
}
}
}
}
Err ( e ) = > {
println! (
" Failed to open test file {}, reason {} " ,
match file_path . canonicalize ( ) {
Ok ( t ) = > t . to_string_lossy ( ) . to_string ( ) ,
Err ( _inspected ) = > file_name . to_string ( ) ,
} ,
e
) ;
2021-07-25 08:00:39 +12:00
}
}
}
2022-07-03 07:30:59 +12:00
#[ allow(dead_code) ]
2023-05-03 08:37:12 +12:00
#[ allow(unreachable_code) ]
#[ allow(unused_variables) ]
2022-07-06 05:58:57 +12:00
// Function to validate if after first check there are any duplicated entries
// E.g. /a.jpg is used also as master and similar image which is forbidden, because may
// cause accidentally delete more pictures that user wanted
2022-07-03 07:30:59 +12:00
fn debug_check_for_duplicated_things (
2023-05-03 08:37:12 +12:00
use_reference_folders : bool ,
hashes_parents : & HashMap < ImHash , u32 > ,
hashes_similarity : & HashMap < ImHash , ( ImHash , u32 ) > ,
all_hashed_images : & HashMap < ImHash , Vec < FileEntry > > ,
2022-07-03 07:30:59 +12:00
numm : & str ,
) {
2023-05-03 08:37:12 +12:00
#[ cfg(not(debug_assertions)) ]
return ;
if use_reference_folders {
return ;
}
2022-08-30 07:37:31 +12:00
let mut found_broken_thing = false ;
2022-07-03 07:30:59 +12:00
let mut hashmap_hashes : HashSet < _ > = Default ::default ( ) ;
let mut hashmap_names : HashSet < _ > = Default ::default ( ) ;
2023-01-29 06:54:02 +13:00
for ( hash , number_of_children ) in hashes_parents {
2022-07-03 07:30:59 +12:00
if * number_of_children > 0 {
2023-05-03 08:37:12 +12:00
if hashmap_hashes . contains ( hash ) {
println! ( " ------1--HASH-- {} {:?} " , numm , all_hashed_images . get ( hash ) . unwrap ( ) ) ;
2022-08-30 07:37:31 +12:00
found_broken_thing = true ;
2022-07-03 07:30:59 +12:00
}
2023-01-29 06:54:02 +13:00
hashmap_hashes . insert ( ( * hash ) . clone ( ) ) ;
2022-07-03 07:30:59 +12:00
2023-05-03 08:37:12 +12:00
for i in all_hashed_images . get ( hash ) . unwrap ( ) {
2022-07-03 07:30:59 +12:00
let name = i . path . to_string_lossy ( ) . to_string ( ) ;
if hashmap_names . contains ( & name ) {
2022-12-21 20:44:26 +13:00
println! ( " ------1--NAME-- {numm} {name:?} " ) ;
2022-08-30 07:37:31 +12:00
found_broken_thing = true ;
2022-07-03 07:30:59 +12:00
}
hashmap_names . insert ( name ) ;
}
}
}
for hash in hashes_similarity . keys ( ) {
2023-05-03 08:37:12 +12:00
if hashmap_hashes . contains ( hash ) {
println! ( " ------2--HASH-- {} {:?} " , numm , all_hashed_images . get ( hash ) . unwrap ( ) ) ;
2022-08-30 07:37:31 +12:00
found_broken_thing = true ;
2022-07-03 07:30:59 +12:00
}
2023-01-29 06:54:02 +13:00
hashmap_hashes . insert ( ( * hash ) . clone ( ) ) ;
2022-07-03 07:30:59 +12:00
2023-05-03 08:37:12 +12:00
for i in all_hashed_images . get ( hash ) . unwrap ( ) {
2022-07-03 07:30:59 +12:00
let name = i . path . to_string_lossy ( ) . to_string ( ) ;
if hashmap_names . contains ( & name ) {
2022-12-21 20:44:26 +13:00
println! ( " ------2--NAME-- {numm} {name:?} " ) ;
2022-08-30 07:37:31 +12:00
found_broken_thing = true ;
2022-07-03 07:30:59 +12:00
}
hashmap_names . insert ( name ) ;
}
}
2022-08-30 07:37:31 +12:00
2023-05-03 08:37:12 +12:00
assert! ( ! found_broken_thing ) ;
2022-07-03 07:30:59 +12:00
}