2023-10-08 05:04:17 +13:00
use std ::collections ::{ BTreeMap , BTreeSet , HashMap , HashSet } ;
2023-10-11 07:54:41 +13:00
use std ::io ::Write ;
2020-12-27 22:56:26 +13:00
use std ::path ::{ Path , PathBuf } ;
2023-06-10 08:11:47 +12:00
use std ::sync ::atomic ::Ordering ;
2023-05-03 08:37:12 +12:00
use std ::time ::SystemTime ;
2023-07-10 18:36:03 +12:00
use std ::{ mem , panic } ;
2020-12-02 22:25:27 +13:00
2021-11-28 08:49:20 +13:00
use bk_tree ::BKTree ;
2023-12-04 00:06:42 +13:00
use crossbeam_channel ::{ Receiver , Sender } ;
2023-10-11 07:54:41 +13:00
use fun_time ::fun_time ;
2023-07-10 18:36:03 +12:00
use humansize ::{ format_size , BINARY } ;
2021-11-28 08:49:20 +13:00
use image ::GenericImageView ;
2022-04-03 02:11:28 +12:00
use image_hasher ::{ FilterType , HashAlg , HasherConfig } ;
2023-10-11 07:54:41 +13:00
use log ::debug ;
2021-11-28 08:49:20 +13:00
use rayon ::prelude ::* ;
2021-12-26 10:23:18 +13:00
use serde ::{ Deserialize , Serialize } ;
2021-11-28 08:49:20 +13:00
2022-06-09 07:42:51 +12:00
#[ cfg(feature = " heif " ) ]
use crate ::common ::get_dynamic_image_from_heic ;
2022-07-28 17:29:50 +12:00
use crate ::common ::{
2023-12-17 11:21:09 +13:00
check_if_stop_received , create_crash_message , delete_files_custom , get_dynamic_image_from_raw_image , prepare_thread_handler_common , send_info_and_wait_for_ending_all_threads ,
HEIC_EXTENSIONS , IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS , RAW_IMAGE_EXTENSIONS ,
2022-07-28 17:29:50 +12:00
} ;
2023-10-08 05:04:17 +13:00
use crate ::common_cache ::{ get_similar_images_cache_file , load_cache_from_file_generalized_by_path , save_cache_to_file_generalized } ;
2023-12-17 11:21:09 +13:00
use crate ::common_dir_traversal ::{ CheckingMethod , DirTraversalBuilder , DirTraversalResult , FileEntry , ProgressData , ToolType } ;
2023-10-14 07:33:17 +13:00
use crate ::common_tool ::{ CommonData , CommonToolData , DeleteMethod } ;
2023-10-11 07:54:41 +13:00
use crate ::common_traits ::{ DebugPrint , PrintResults , ResultEntry } ;
2022-01-20 10:35:07 +13:00
use crate ::flc ;
2023-05-03 08:37:12 +12:00
type ImHash = Vec < u8 > ;
2021-11-28 08:49:20 +13:00
2021-12-03 03:33:06 +13:00
pub const SIMILAR_VALUES : [ [ u32 ; 6 ] ; 4 ] = [
2022-07-31 18:45:19 +12:00
[ 1 , 2 , 5 , 7 , 14 , 20 ] , // 8
2021-12-03 03:33:06 +13:00
[ 2 , 5 , 15 , 30 , 40 , 40 ] , // 16
[ 4 , 10 , 20 , 40 , 40 , 40 ] , // 32
[ 6 , 20 , 40 , 40 , 40 , 40 ] , // 64
2021-11-19 18:35:26 +13:00
] ;
2021-12-26 10:23:18 +13:00
#[ derive(Clone, Debug, Serialize, Deserialize) ]
2023-12-17 11:21:09 +13:00
pub struct ImagesEntry {
2020-10-15 08:10:27 +13:00
pub path : PathBuf ,
pub size : u64 ,
2024-02-11 11:57:52 +13:00
pub width : u32 ,
pub height : u32 ,
2020-10-15 08:10:27 +13:00
pub modified_date : u64 ,
2023-05-03 08:37:12 +12:00
pub hash : ImHash ,
2022-07-03 07:30:59 +12:00
pub similarity : u32 ,
2023-12-17 11:21:09 +13:00
pub image_type : ImageType ,
2020-10-15 08:10:27 +13:00
}
2023-10-05 19:06:47 +13:00
2023-12-17 11:21:09 +13:00
impl ResultEntry for ImagesEntry {
2023-05-08 06:54:05 +12:00
fn get_path ( & self ) -> & Path {
& self . path
}
2023-10-08 05:04:17 +13:00
fn get_modified_date ( & self ) -> u64 {
self . modified_date
}
fn get_size ( & self ) -> u64 {
self . size
}
2023-05-08 06:54:05 +12:00
}
2023-12-17 11:21:09 +13:00
impl FileEntry {
fn into_images_entry ( self ) -> ImagesEntry {
ImagesEntry {
size : self . size ,
path : self . path ,
modified_date : self . modified_date ,
2024-02-11 11:57:52 +13:00
width : 0 ,
height : 0 ,
2023-12-17 11:21:09 +13:00
hash : Vec ::new ( ) ,
similarity : 0 ,
image_type : ImageType ::Unknown ,
}
}
}
#[ derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq) ]
pub enum ImageType {
Normal ,
Raw ,
Heic ,
Unknown ,
}
const MAX_IMAGE_STAGE : u8 = 2 ;
2020-10-15 08:10:27 +13:00
2023-02-19 22:21:14 +13:00
#[ derive(Clone, Debug, Copy) ]
2021-11-18 23:23:17 +13:00
pub enum SimilarityPreset {
2022-07-31 18:45:19 +12:00
Original ,
2021-11-18 23:23:17 +13:00
VeryHigh ,
High ,
Medium ,
Small ,
VerySmall ,
Minimal ,
None ,
}
2020-10-16 04:43:59 +13:00
struct Hamming ;
2023-05-03 08:37:12 +12:00
impl bk_tree ::Metric < ImHash > for Hamming {
fn distance ( & self , a : & ImHash , b : & ImHash ) -> u32 {
2021-08-07 02:40:43 +12:00
hamming ::distance_fast ( a , b ) . unwrap ( ) as u32
2021-07-22 07:48:21 +12:00
}
2023-05-03 08:37:12 +12:00
fn threshold_distance ( & self , a : & ImHash , b : & ImHash , _threshold : u32 ) -> Option < u32 > {
2021-08-07 02:40:43 +12:00
Some ( self . distance ( a , b ) )
}
2020-10-16 04:43:59 +13:00
}
2020-10-15 08:10:27 +13:00
pub struct SimilarImages {
2023-10-05 19:06:47 +13:00
common_data : CommonToolData ,
2020-10-15 08:10:27 +13:00
information : Info ,
2023-05-03 08:37:12 +12:00
bktree : BKTree < ImHash , Hamming > ,
2023-12-17 11:21:09 +13:00
similar_vectors : Vec < Vec < ImagesEntry > > ,
similar_referenced_vectors : Vec < ( ImagesEntry , Vec < ImagesEntry > ) > ,
2023-05-03 08:37:12 +12:00
// Hashmap with image hashes and Vector with names of files
2023-12-17 11:21:09 +13:00
image_hashes : HashMap < ImHash , Vec < ImagesEntry > > ,
2022-07-03 07:30:59 +12:00
similarity : u32 ,
2023-12-17 11:21:09 +13:00
images_to_check : BTreeMap < String , ImagesEntry > ,
hash_size : u8 ,
2021-11-18 23:23:17 +13:00
hash_alg : HashAlg ,
image_filter : FilterType ,
2021-12-09 11:08:05 +13:00
exclude_images_with_same_size : bool ,
2020-10-15 08:10:27 +13:00
}
#[ derive(Default) ]
pub struct Info {
2021-12-24 21:18:55 +13:00
pub number_of_duplicates : usize ,
pub number_of_groups : u64 ,
2020-10-15 08:10:27 +13:00
}
2021-11-28 08:57:10 +13:00
2020-10-15 08:10:27 +13:00
impl SimilarImages {
pub fn new ( ) -> Self {
Self {
2023-10-05 19:06:47 +13:00
common_data : CommonToolData ::new ( ToolType ::SimilarImages ) ,
2020-10-15 08:10:27 +13:00
information : Default ::default ( ) ,
2020-10-16 04:43:59 +13:00
bktree : BKTree ::new ( Hamming ) ,
2020-10-15 08:10:27 +13:00
similar_vectors : vec ! [ ] ,
2021-12-24 21:18:55 +13:00
similar_referenced_vectors : Default ::default ( ) ,
2020-10-15 08:10:27 +13:00
image_hashes : Default ::default ( ) ,
2022-07-03 07:30:59 +12:00
similarity : 0 ,
2020-12-27 22:56:26 +13:00
images_to_check : Default ::default ( ) ,
2021-11-18 23:23:17 +13:00
hash_size : 8 ,
hash_alg : HashAlg ::Gradient ,
image_filter : FilterType ::Lanczos3 ,
2021-12-09 11:08:05 +13:00
exclude_images_with_same_size : false ,
2021-11-18 23:23:17 +13:00
}
}
2023-10-15 04:48:57 +13:00
#[ fun_time(message = " find_similar_images " , level = " info " ) ]
2023-12-04 00:06:42 +13:00
pub fn find_similar_images ( & mut self , stop_receiver : Option < & Receiver < ( ) > > , progress_sender : Option < & Sender < ProgressData > > ) {
2024-01-18 11:35:02 +13:00
self . prepare_items ( ) ;
2023-10-05 19:06:47 +13:00
self . common_data . use_reference_folders = ! self . common_data . directories . reference_directories . is_empty ( ) ;
2020-12-02 22:25:27 +13:00
if ! self . check_for_similar_images ( stop_receiver , progress_sender ) {
2023-10-05 19:06:47 +13:00
self . common_data . stopped_search = true ;
2020-10-15 08:10:27 +13:00
return ;
}
2021-12-29 08:20:24 +13:00
if ! self . hash_images ( stop_receiver , progress_sender ) {
2023-10-05 19:06:47 +13:00
self . common_data . stopped_search = true ;
2021-12-29 08:20:24 +13:00
return ;
}
if ! self . find_similar_hashes ( stop_receiver , progress_sender ) {
2023-10-05 19:06:47 +13:00
self . common_data . stopped_search = true ;
2020-11-08 04:26:40 +13:00
return ;
}
2023-10-14 07:33:17 +13:00
self . delete_files ( ) ;
2020-10-15 08:10:27 +13:00
self . debug_print ( ) ;
}
2023-12-17 11:21:09 +13:00
// #[fun_time(message = "check_for_similar_images", level = "debug")]
2023-12-04 00:06:42 +13:00
fn check_for_similar_images ( & mut self , stop_receiver : Option < & Receiver < ( ) > > , progress_sender : Option < & Sender < ProgressData > > ) -> bool {
2023-12-17 11:21:09 +13:00
if cfg! ( feature = " heif " ) {
self . common_data
2024-01-18 11:35:02 +13:00
. extensions
. set_and_validate_allowed_extensions ( & [ IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS , RAW_IMAGE_EXTENSIONS , HEIC_EXTENSIONS ] . concat ( ) ) ;
2022-05-24 05:04:28 +12:00
} else {
2023-10-05 19:06:47 +13:00
self . common_data
2024-01-18 11:35:02 +13:00
. extensions
. set_and_validate_allowed_extensions ( & [ IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS , RAW_IMAGE_EXTENSIONS ] . concat ( ) ) ;
2022-01-01 10:34:24 +13:00
}
2024-01-18 11:35:02 +13:00
if ! self . common_data . extensions . set_any_extensions ( ) {
2023-12-17 11:21:09 +13:00
return true ;
2020-10-15 08:10:27 +13:00
}
2023-12-17 11:21:09 +13:00
let normal_image_extensions = IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS . iter ( ) . collect ::< HashSet < _ > > ( ) ;
let raw_image_extensions = RAW_IMAGE_EXTENSIONS . iter ( ) . collect ::< HashSet < _ > > ( ) ;
let heic_extensions = HEIC_EXTENSIONS . iter ( ) . collect ::< HashSet < _ > > ( ) ;
let result = DirTraversalBuilder ::new ( )
. group_by ( | _fe | ( ) )
. stop_receiver ( stop_receiver )
. progress_sender ( progress_sender )
. common_data ( & self . common_data )
. max_stage ( MAX_IMAGE_STAGE )
. build ( )
. run ( ) ;
match result {
DirTraversalResult ::SuccessFiles { grouped_file_entries , warnings } = > {
self . images_to_check = grouped_file_entries
. into_values ( )
. flatten ( )
. map ( | fe | {
let fe_str = fe . path . to_string_lossy ( ) . to_string ( ) ;
let extension_lowercase = fe . path . extension ( ) . unwrap_or_default ( ) . to_string_lossy ( ) . to_lowercase ( ) ;
let mut image_entry = fe . into_images_entry ( ) ;
if normal_image_extensions . contains ( & extension_lowercase . as_str ( ) ) {
image_entry . image_type = ImageType ::Normal ;
} else if raw_image_extensions . contains ( & extension_lowercase . as_str ( ) ) {
image_entry . image_type = ImageType ::Raw ;
} else if heic_extensions . contains ( & extension_lowercase . as_str ( ) ) {
image_entry . image_type = ImageType ::Heic ;
2021-12-18 07:29:37 +13:00
}
2023-12-17 11:21:09 +13:00
( fe_str , image_entry )
} )
. collect ( ) ;
2023-10-05 19:06:47 +13:00
self . common_data . text_messages . warnings . extend ( warnings ) ;
2023-12-17 11:21:09 +13:00
debug! ( " check_files - Found {} image files. " , self . images_to_check . len ( ) ) ;
true
2020-10-15 08:10:27 +13:00
}
2023-12-17 11:21:09 +13:00
DirTraversalResult ::Stopped = > false ,
2023-05-03 08:37:12 +12:00
}
}
2023-10-15 04:48:57 +13:00
#[ fun_time(message = " hash_images_load_cache " , level = " debug " ) ]
2023-12-17 11:21:09 +13:00
fn hash_images_load_cache ( & mut self ) -> ( BTreeMap < String , ImagesEntry > , BTreeMap < String , ImagesEntry > , BTreeMap < String , ImagesEntry > ) {
2021-03-05 00:09:53 +13:00
let loaded_hash_map ;
2020-12-27 22:56:26 +13:00
2023-12-17 11:21:09 +13:00
let mut records_already_cached : BTreeMap < String , ImagesEntry > = Default ::default ( ) ;
let mut non_cached_files_to_check : BTreeMap < String , ImagesEntry > = Default ::default ( ) ;
2021-03-05 00:09:53 +13:00
2023-10-05 19:06:47 +13:00
if self . common_data . use_cache {
2023-12-17 11:21:09 +13:00
let ( messages , loaded_items ) = load_cache_from_file_generalized_by_path ::< ImagesEntry > (
2023-10-08 05:04:17 +13:00
& get_similar_images_cache_file ( & self . hash_size , & self . hash_alg , & self . image_filter ) ,
self . get_delete_outdated_cache ( ) ,
& self . images_to_check ,
) ;
self . get_text_messages_mut ( ) . extend_with_another_messages ( messages ) ;
loaded_hash_map = loaded_items . unwrap_or_default ( ) ;
2021-03-05 00:09:53 +13:00
2023-10-11 07:54:41 +13:00
debug! ( " hash_images-load_cache - starting calculating diff " ) ;
2023-10-08 05:04:17 +13:00
for ( name , file_entry ) in mem ::take ( & mut self . images_to_check ) {
if let Some ( cached_file_entry ) = loaded_hash_map . get ( & name ) {
2023-12-08 07:38:41 +13:00
records_already_cached . insert ( name , cached_file_entry . clone ( ) ) ;
2020-12-27 22:56:26 +13:00
} else {
2023-10-08 05:04:17 +13:00
non_cached_files_to_check . insert ( name , file_entry ) ;
2020-12-27 22:56:26 +13:00
}
}
2023-10-11 07:54:41 +13:00
debug! (
" hash_images_load_cache - completed diff between loaded and prechecked files, {}({}) - non cached, {}({}) - already cached " ,
non_cached_files_to_check . len ( ) ,
format_size ( non_cached_files_to_check . values ( ) . map ( | e | e . size ) . sum ::< u64 > ( ) , BINARY ) ,
records_already_cached . len ( ) ,
format_size ( records_already_cached . values ( ) . map ( | e | e . size ) . sum ::< u64 > ( ) , BINARY ) ,
) ;
2021-03-05 00:09:53 +13:00
} else {
loaded_hash_map = Default ::default ( ) ;
mem ::swap ( & mut self . images_to_check , & mut non_cached_files_to_check ) ;
2020-12-27 22:56:26 +13:00
}
2023-05-03 08:37:12 +12:00
( loaded_hash_map , records_already_cached , non_cached_files_to_check )
}
// Cache algorithm:
// - Load data from file
2023-05-08 06:54:05 +12:00
// - Remove from data to search, already loaded entries from cache(size and modified date must match)
2023-05-03 08:37:12 +12:00
// - Check hash of files which doesn't have saved entry
// - Join already read hashes with hashes which were read from file
// - Join all hashes and save it to file
2023-10-15 04:48:57 +13:00
#[ fun_time(message = " hash_images " , level = " debug " ) ]
2023-12-04 00:06:42 +13:00
fn hash_images ( & mut self , stop_receiver : Option < & Receiver < ( ) > > , progress_sender : Option < & Sender < ProgressData > > ) -> bool {
2023-05-03 08:37:12 +12:00
let ( loaded_hash_map , records_already_cached , non_cached_files_to_check ) = self . hash_images_load_cache ( ) ;
2020-12-27 22:56:26 +13:00
2023-05-08 06:54:05 +12:00
let ( progress_thread_handle , progress_thread_run , atomic_counter , check_was_stopped ) =
2023-10-05 19:06:47 +13:00
prepare_thread_handler_common ( progress_sender , 1 , 2 , non_cached_files_to_check . len ( ) , CheckingMethod ::None , self . common_data . tool_type ) ;
2023-05-03 08:37:12 +12:00
2023-10-05 19:06:47 +13:00
debug! ( " hash_images - start hashing images " ) ;
2023-12-17 11:21:09 +13:00
let mut vec_file_entry : Vec < ImagesEntry > = non_cached_files_to_check
2021-12-30 01:43:38 +13:00
. into_par_iter ( )
2023-12-17 11:21:09 +13:00
. map ( | ( _s , mut file_entry ) | {
2023-05-03 08:37:12 +12:00
atomic_counter . fetch_add ( 1 , Ordering ::Relaxed ) ;
2023-10-15 04:48:57 +13:00
if check_if_stop_received ( stop_receiver ) {
2022-05-10 05:40:35 +12:00
check_was_stopped . store ( true , Ordering ::Relaxed ) ;
2020-11-08 19:41:29 +13:00
return None ;
}
2023-12-17 11:21:09 +13:00
self . collect_image_file_entry ( & mut file_entry ) ;
Some ( Some ( file_entry ) )
2020-11-08 19:41:29 +13:00
} )
. while_some ( )
2023-12-17 11:21:09 +13:00
. filter_map ( | e | e )
. collect ::< Vec < ImagesEntry > > ( ) ;
2024-02-04 08:18:24 +13:00
debug! ( " hash_images - end hashing {} images " , vec_file_entry . len ( ) ) ;
2020-11-22 23:03:15 +13:00
2023-05-03 08:37:12 +12:00
send_info_and_wait_for_ending_all_threads ( & progress_thread_run , progress_thread_handle ) ;
2020-12-02 22:25:27 +13:00
2020-12-27 22:56:26 +13:00
// Just connect loaded results with already calculated hashes
2023-05-08 06:54:05 +12:00
for file_entry in records_already_cached . into_values ( ) {
2023-12-17 11:21:09 +13:00
vec_file_entry . push ( file_entry ) ;
2020-12-27 22:56:26 +13:00
}
2021-12-01 00:45:09 +13:00
// All valid entries are used to create bktree used to check for hash similarity
2023-12-17 11:21:09 +13:00
for file_entry in & vec_file_entry {
// Only use to comparing, non broken hashes(all 0 or 255 hashes means that algorithm fails to decode them because e.g. contains a lot of alpha channel)
if ! ( file_entry . hash . is_empty ( ) | | file_entry . hash . iter ( ) . all ( | e | * e = = 0 ) | | file_entry . hash . iter ( ) . all ( | e | * e = = 255 ) ) {
self . image_hashes . entry ( file_entry . hash . clone ( ) ) . or_default ( ) . push ( file_entry . clone ( ) ) ;
2021-11-30 01:53:09 +13:00
}
2020-11-08 19:41:29 +13:00
}
2023-10-05 19:06:47 +13:00
self . save_to_cache ( vec_file_entry , loaded_hash_map ) ;
// Break if stop was clicked after saving to cache
if check_was_stopped . load ( Ordering ::Relaxed ) {
return false ;
}
true
}
2023-10-15 04:48:57 +13:00
#[ fun_time(message = " save_to_cache " , level = " debug " ) ]
2023-12-17 11:21:09 +13:00
fn save_to_cache ( & mut self , vec_file_entry : Vec < ImagesEntry > , loaded_hash_map : BTreeMap < String , ImagesEntry > ) {
2023-10-05 19:06:47 +13:00
if self . common_data . use_cache {
2021-03-05 00:09:53 +13:00
// Must save all results to file, old loaded from file with all currently counted results
2023-12-17 11:21:09 +13:00
let mut all_results : BTreeMap < String , ImagesEntry > = loaded_hash_map ;
for file_entry in vec_file_entry {
2021-03-05 00:09:53 +13:00
all_results . insert ( file_entry . path . to_string_lossy ( ) . to_string ( ) , file_entry ) ;
}
2023-10-08 05:04:17 +13:00
let messages = save_cache_to_file_generalized (
& get_similar_images_cache_file ( & self . hash_size , & self . hash_alg , & self . image_filter ) ,
2022-01-06 10:47:27 +13:00
& all_results ,
2023-10-05 19:06:47 +13:00
self . common_data . save_also_as_json ,
2023-10-08 05:04:17 +13:00
0 ,
2022-01-06 10:47:27 +13:00
) ;
2023-10-08 05:04:17 +13:00
self . get_text_messages_mut ( ) . extend_with_another_messages ( messages ) ;
2020-12-27 22:56:26 +13:00
}
2021-12-29 08:20:24 +13:00
}
2023-10-05 19:06:47 +13:00
2023-12-17 11:21:09 +13:00
fn collect_image_file_entry ( & self , file_entry : & mut ImagesEntry ) {
let img ;
2023-05-03 08:37:12 +12:00
2023-12-17 11:21:09 +13:00
if file_entry . image_type = = ImageType ::Heic {
2023-05-03 08:37:12 +12:00
#[ cfg(feature = " heif " ) ]
2023-12-17 11:21:09 +13:00
{
img = match get_dynamic_image_from_heic ( & file_entry . path . to_string_lossy ( ) ) {
2023-05-03 08:37:12 +12:00
Ok ( t ) = > t ,
Err ( _ ) = > {
2023-12-17 11:21:09 +13:00
return ;
2023-05-03 08:37:12 +12:00
}
} ;
}
2023-12-17 11:21:09 +13:00
#[ cfg(not(feature = " heif " )) ]
{
if let Ok ( image_result ) = panic ::catch_unwind ( | | image ::open ( & file_entry . path ) ) {
if let Ok ( image2 ) = image_result {
img = image2 ;
} else {
return ;
}
2023-05-03 08:37:12 +12:00
} else {
2023-12-17 11:21:09 +13:00
let message = crate ::common ::create_crash_message ( " Image-rs " , & file_entry . path . to_string_lossy ( ) , " https://github.com/image-rs/image/issues " ) ;
println! ( " {message} " ) ;
return ;
}
}
} else {
match file_entry . image_type {
ImageType ::Normal | ImageType ::Heic = > {
if let Ok ( image_result ) = panic ::catch_unwind ( | | image ::open ( & file_entry . path ) ) {
if let Ok ( image2 ) = image_result {
img = image2 ;
} else {
return ;
}
} else {
let message = create_crash_message ( " Image-rs " , & file_entry . path . to_string_lossy ( ) , " https://github.com/image-rs/image/issues " ) ;
println! ( " {message} " ) ;
return ;
}
}
ImageType ::Raw = > {
img = match get_dynamic_image_from_raw_image ( & file_entry . path ) {
Some ( t ) = > t ,
None = > return ,
} ;
}
_ = > {
unreachable! ( ) ;
2023-05-03 08:37:12 +12:00
}
}
}
2023-12-17 11:21:09 +13:00
let dimensions = img . dimensions ( ) ;
2023-05-03 08:37:12 +12:00
2024-02-11 11:57:52 +13:00
file_entry . width = dimensions . 0 ;
file_entry . height = dimensions . 1 ;
2023-05-03 08:37:12 +12:00
let hasher_config = HasherConfig ::new ( )
. hash_size ( self . hash_size as u32 , self . hash_size as u32 )
. hash_alg ( self . hash_alg )
. resize_filter ( self . image_filter ) ;
let hasher = hasher_config . to_hasher ( ) ;
2023-12-17 11:21:09 +13:00
let hash = hasher . hash_image ( & img ) ;
file_entry . hash = hash . as_bytes ( ) . to_vec ( ) ;
2023-05-03 08:37:12 +12:00
}
2023-06-10 08:11:47 +12:00
// Split hashes at 2 parts, base hashes and hashes to compare, 3 argument is set of hashes with multiple images
2023-10-15 04:48:57 +13:00
#[ fun_time(message = " split_hashes " , level = " debug " ) ]
2023-12-17 11:21:09 +13:00
fn split_hashes ( & mut self , all_hashed_images : & HashMap < ImHash , Vec < ImagesEntry > > ) -> ( Vec < ImHash > , HashSet < ImHash > ) {
2023-06-10 08:11:47 +12:00
let hashes_with_multiple_images : HashSet < ImHash > = all_hashed_images
. iter ( )
. filter_map ( | ( hash , vec_file_entry ) | {
if vec_file_entry . len ( ) > = 2 {
return Some ( hash . clone ( ) ) ;
} ;
None
} )
. collect ( ) ;
let mut base_hashes = Vec ::new ( ) ; // Initial hashes
2023-10-05 19:06:47 +13:00
if self . common_data . use_reference_folders {
2023-12-17 11:21:09 +13:00
let mut files_from_referenced_folders : HashMap < ImHash , Vec < ImagesEntry > > = HashMap ::new ( ) ;
let mut normal_files : HashMap < ImHash , Vec < ImagesEntry > > = HashMap ::new ( ) ;
2023-06-10 08:11:47 +12:00
2023-05-03 08:37:12 +12:00
all_hashed_images . clone ( ) . into_iter ( ) . for_each ( | ( hash , vec_file_entry ) | {
for file_entry in vec_file_entry {
2023-10-05 19:06:47 +13:00
if is_in_reference_folder ( & self . common_data . directories . reference_directories , & file_entry . path ) {
files_from_referenced_folders . entry ( hash . clone ( ) ) . or_default ( ) . push ( file_entry ) ;
2023-05-03 08:37:12 +12:00
} else {
2023-10-05 19:06:47 +13:00
normal_files . entry ( hash . clone ( ) ) . or_default ( ) . push ( file_entry ) ;
2023-05-03 08:37:12 +12:00
}
}
} ) ;
2023-06-10 08:11:47 +12:00
for hash in normal_files . into_keys ( ) {
2023-05-03 08:37:12 +12:00
self . bktree . add ( hash ) ;
}
2023-06-10 08:11:47 +12:00
for hash in files_from_referenced_folders . into_keys ( ) {
base_hashes . push ( hash ) ;
}
2023-05-03 08:37:12 +12:00
} else {
2023-06-10 08:11:47 +12:00
for original_hash in all_hashed_images . keys ( ) {
self . bktree . add ( original_hash . clone ( ) ) ;
2023-05-03 08:37:12 +12:00
}
2023-06-10 08:11:47 +12:00
base_hashes = all_hashed_images . keys ( ) . cloned ( ) . collect ::< Vec < _ > > ( ) ;
2023-05-03 08:37:12 +12:00
}
2023-06-10 08:11:47 +12:00
( base_hashes , hashes_with_multiple_images )
2023-05-03 08:37:12 +12:00
}
2023-10-15 04:48:57 +13:00
#[ fun_time(message = " collect_hash_compare_result " , level = " debug " ) ]
2023-05-03 08:37:12 +12:00
fn collect_hash_compare_result (
& self ,
hashes_parents : HashMap < ImHash , u32 > ,
hashes_with_multiple_images : & HashSet < ImHash > ,
2023-12-17 11:21:09 +13:00
all_hashed_images : & HashMap < ImHash , Vec < ImagesEntry > > ,
collected_similar_images : & mut HashMap < ImHash , Vec < ImagesEntry > > ,
2023-05-03 08:37:12 +12:00
hashes_similarity : HashMap < ImHash , ( ImHash , u32 ) > ,
) {
2024-02-04 08:18:24 +13:00
// Collecting results to vector
for ( parent_hash , child_number ) in hashes_parents {
// If hash contains other hasher OR multiple images are available for checked hash
if child_number > 0 | | hashes_with_multiple_images . contains ( & parent_hash ) {
let vec_fe = all_hashed_images [ & parent_hash ] . clone ( ) ;
collected_similar_images . insert ( parent_hash . clone ( ) , vec_fe ) ;
2023-05-03 08:37:12 +12:00
}
2024-02-04 08:18:24 +13:00
}
2023-05-03 08:37:12 +12:00
2024-02-04 08:18:24 +13:00
for ( child_hash , ( parent_hash , similarity ) ) in hashes_similarity {
let mut vec_fe = all_hashed_images [ & child_hash ] . clone ( ) ;
for fe in & mut vec_fe {
fe . similarity = similarity ;
2023-05-03 08:37:12 +12:00
}
2024-02-04 08:18:24 +13:00
collected_similar_images . get_mut ( & parent_hash ) . unwrap ( ) . append ( & mut vec_fe ) ;
2023-05-03 08:37:12 +12:00
}
}
2023-10-15 04:48:57 +13:00
#[ fun_time(message = " compare_hashes_with_non_zero_tolerance " , level = " debug " ) ]
2023-06-10 08:11:47 +12:00
fn compare_hashes_with_non_zero_tolerance (
& mut self ,
2023-12-17 11:21:09 +13:00
all_hashed_images : & HashMap < ImHash , Vec < ImagesEntry > > ,
collected_similar_images : & mut HashMap < ImHash , Vec < ImagesEntry > > ,
2023-12-04 00:06:42 +13:00
progress_sender : Option < & Sender < ProgressData > > ,
2023-06-10 08:11:47 +12:00
stop_receiver : Option < & Receiver < ( ) > > ,
tolerance : u32 ,
) -> bool {
// Don't use hashes with multiple images in bktree, because they will always be master of group and cannot be find by other hashes
let ( base_hashes , hashes_with_multiple_images ) = self . split_hashes ( all_hashed_images ) ;
2023-05-03 08:37:12 +12:00
2023-06-10 08:11:47 +12:00
let ( progress_thread_handle , progress_thread_run , atomic_counter , check_was_stopped ) =
2023-10-05 19:06:47 +13:00
prepare_thread_handler_common ( progress_sender , 2 , 2 , base_hashes . len ( ) , CheckingMethod ::None , self . common_data . tool_type ) ;
2023-06-10 08:11:47 +12:00
let mut hashes_parents : HashMap < ImHash , u32 > = Default ::default ( ) ; // Hashes used as parent (hash, children_number_of_hash)
let mut hashes_similarity : HashMap < ImHash , ( ImHash , u32 ) > = Default ::default ( ) ; // Hashes used as child, (parent_hash, similarity)
2023-05-03 08:37:12 +12:00
2023-06-10 08:11:47 +12:00
// Check them in chunks, to decrease number of used memory
2023-06-11 18:51:21 +12:00
// println!();
2023-06-10 08:11:47 +12:00
let base_hashes_chunks = base_hashes . chunks ( 1000 ) ;
for chunk in base_hashes_chunks {
let partial_results = chunk
. into_par_iter ( )
. map ( | hash_to_check | {
atomic_counter . fetch_add ( 1 , Ordering ::Relaxed ) ;
2023-10-15 04:48:57 +13:00
if check_if_stop_received ( stop_receiver ) {
2023-06-10 08:11:47 +12:00
check_was_stopped . store ( true , Ordering ::Relaxed ) ;
return None ;
}
let mut found_items = self
. bktree
. find ( hash_to_check , tolerance )
. filter ( | ( similarity , compared_hash ) | {
* similarity ! = 0 & & ! hashes_parents . contains_key ( * compared_hash ) & & ! hashes_with_multiple_images . contains ( * compared_hash )
} )
. filter ( | ( similarity , compared_hash ) | {
if let Some ( ( _ , other_similarity_with_parent ) ) = hashes_similarity . get ( * compared_hash ) {
// If current hash is more similar to other hash than to current parent hash, then skip check earlier
// Because there is no way to be more similar to other hash than to current parent hash
if * similarity > = * other_similarity_with_parent {
return false ;
}
}
true
} )
. collect ::< Vec < _ > > ( ) ;
2023-12-08 07:38:41 +13:00
// Sort by tolerance
2023-06-10 08:11:47 +12:00
found_items . sort_unstable_by_key ( | f | f . 0 ) ;
Some ( ( hash_to_check , found_items ) )
} )
. while_some ( )
. filter ( | ( original_hash , vec_similar_hashes ) | ! vec_similar_hashes . is_empty ( ) | | hashes_with_multiple_images . contains ( * original_hash ) )
. collect ::< Vec < _ > > ( ) ;
2023-06-11 18:51:21 +12:00
// for (hash, vec) in &partial_results {
// println!("{hash:?} --- {:?}", vec.iter().map(|e| e.1).collect::<Vec<_>>());
// }
2023-06-10 08:11:47 +12:00
if check_was_stopped . load ( Ordering ::Relaxed ) {
send_info_and_wait_for_ending_all_threads ( & progress_thread_run , progress_thread_handle ) ;
return false ;
2023-05-03 08:37:12 +12:00
}
2023-06-10 08:11:47 +12:00
self . connect_results ( partial_results , & mut hashes_parents , & mut hashes_similarity , & hashes_with_multiple_images ) ;
2023-05-03 08:37:12 +12:00
}
2023-06-10 08:11:47 +12:00
send_info_and_wait_for_ending_all_threads ( & progress_thread_run , progress_thread_handle ) ;
2023-10-05 19:06:47 +13:00
debug_check_for_duplicated_things ( self . common_data . use_reference_folders , & hashes_parents , & hashes_similarity , all_hashed_images , " LATTER " ) ;
2023-06-10 08:11:47 +12:00
self . collect_hash_compare_result ( hashes_parents , & hashes_with_multiple_images , all_hashed_images , collected_similar_images , hashes_similarity ) ;
2023-05-03 08:37:12 +12:00
2023-06-10 08:11:47 +12:00
true
}
2023-05-03 08:37:12 +12:00
2023-10-15 04:48:57 +13:00
#[ fun_time(message = " connect_results " , level = " debug " ) ]
2023-06-10 08:11:47 +12:00
fn connect_results (
& self ,
partial_results : Vec < ( & ImHash , Vec < ( u32 , & ImHash ) > ) > ,
hashes_parents : & mut HashMap < ImHash , u32 > ,
hashes_similarity : & mut HashMap < ImHash , ( ImHash , u32 ) > ,
hashes_with_multiple_images : & HashSet < ImHash > ,
) {
for ( original_hash , vec_compared_hashes ) in partial_results {
let mut number_of_added_child_items = 0 ;
for ( similarity , compared_hash ) in vec_compared_hashes {
// If hash is already in results skip it
// This check duplicates check from bktree.find, but it is needed to because when iterating over elements, this structure can change
if hashes_parents . contains_key ( compared_hash ) {
continue ;
}
// If there is already record, with smaller sensitivity, then replace it
let mut need_to_add = false ;
let mut need_to_check = false ;
// TODO consider to replace variables from above with closures
// If current checked hash, have parent, first we must check if similarity between them is lower than checked item
if let Some ( ( current_parent_hash , current_similarity_with_parent ) ) = hashes_similarity . get ( original_hash ) {
if * current_similarity_with_parent > similarity {
need_to_check = true ;
* hashes_parents . get_mut ( current_parent_hash ) . unwrap ( ) - = 1 ;
if hashes_parents . get ( current_parent_hash ) = = Some ( & 0 ) & & ! hashes_with_multiple_images . contains ( current_parent_hash ) {
hashes_parents . remove ( current_parent_hash ) ;
}
hashes_similarity . remove ( original_hash ) . unwrap ( ) ;
}
} else {
need_to_check = true ;
}
if need_to_check {
if let Some ( ( other_parent_hash , other_similarity ) ) = hashes_similarity . get ( compared_hash ) {
if * other_similarity > similarity {
need_to_add = true ;
* hashes_parents . get_mut ( other_parent_hash ) . unwrap ( ) - = 1 ;
}
}
// But when there is no record, just add it
else {
need_to_add = true ;
}
}
if need_to_add {
hashes_similarity . insert ( compared_hash . clone ( ) , ( original_hash . clone ( ) , similarity ) ) ;
number_of_added_child_items + = 1 ;
}
}
if number_of_added_child_items > 0 | | hashes_with_multiple_images . contains ( original_hash ) {
hashes_parents . insert ( ( * original_hash ) . clone ( ) , number_of_added_child_items ) ;
}
}
2023-05-03 08:37:12 +12:00
}
2020-11-09 04:19:40 +13:00
2023-10-15 04:48:57 +13:00
#[ fun_time(message = " find_similar_hashes " , level = " debug " ) ]
2023-12-04 00:06:42 +13:00
fn find_similar_hashes ( & mut self , stop_receiver : Option < & Receiver < ( ) > > , progress_sender : Option < & Sender < ProgressData > > ) -> bool {
2022-07-10 05:18:26 +12:00
if self . image_hashes . is_empty ( ) {
return true ;
}
2022-07-03 07:30:59 +12:00
let tolerance = self . similarity ;
2020-11-08 04:26:40 +13:00
2021-12-29 08:20:24 +13:00
// Results
2023-12-17 11:21:09 +13:00
let mut collected_similar_images : HashMap < ImHash , Vec < ImagesEntry > > = Default ::default ( ) ;
2021-08-07 02:40:43 +12:00
2023-05-08 06:54:05 +12:00
let all_hashed_images = mem ::take ( & mut self . image_hashes ) ;
2021-12-29 08:20:24 +13:00
2022-07-03 07:30:59 +12:00
// Checking entries with tolerance 0 is really easy and fast, because only entries with same hashes needs to be checked
if tolerance = = 0 {
2023-06-10 08:11:47 +12:00
for ( hash , vec_file_entry ) in all_hashed_images {
2022-07-03 07:30:59 +12:00
if vec_file_entry . len ( ) > = 2 {
collected_similar_images . insert ( hash , vec_file_entry ) ;
}
2021-12-29 08:20:24 +13:00
}
2023-10-11 07:54:41 +13:00
} else if ! self . compare_hashes_with_non_zero_tolerance ( & all_hashed_images , & mut collected_similar_images , progress_sender , stop_receiver , tolerance ) {
return false ;
2023-05-03 08:37:12 +12:00
}
2021-12-29 09:20:48 +13:00
2023-05-03 08:37:12 +12:00
self . verify_duplicated_items ( & collected_similar_images ) ;
2022-07-03 07:30:59 +12:00
2023-06-10 08:11:47 +12:00
// Info about hashes is not needed anymore, so we drop this info
2023-05-03 08:37:12 +12:00
self . similar_vectors = collected_similar_images . into_values ( ) . collect ( ) ;
2022-07-03 07:30:59 +12:00
2023-05-03 08:37:12 +12:00
self . exclude_items_with_same_size ( ) ;
2021-12-29 09:20:48 +13:00
2023-06-10 08:11:47 +12:00
self . remove_multiple_records_from_reference_folders ( ) ;
2022-09-14 08:18:07 +12:00
2023-10-05 19:06:47 +13:00
if self . common_data . use_reference_folders {
2023-05-03 08:37:12 +12:00
for ( _fe , vector ) in & self . similar_referenced_vectors {
self . information . number_of_duplicates + = vector . len ( ) ;
self . information . number_of_groups + = 1 ;
2020-10-15 08:10:27 +13:00
}
2023-05-03 08:37:12 +12:00
} else {
for vector in & self . similar_vectors {
self . information . number_of_duplicates + = vector . len ( ) - 1 ;
self . information . number_of_groups + = 1 ;
2022-01-06 03:20:21 +13:00
}
}
2020-10-15 08:10:27 +13:00
2023-05-08 06:54:05 +12:00
// Clean unused data to save ram
2023-05-03 08:37:12 +12:00
self . image_hashes = Default ::default ( ) ;
self . images_to_check = Default ::default ( ) ;
self . bktree = BKTree ::new ( Hamming ) ;
true
}
2023-10-15 04:48:57 +13:00
#[ fun_time(message = " exclude_items_with_same_size " , level = " debug " ) ]
2023-05-03 08:37:12 +12:00
fn exclude_items_with_same_size ( & mut self ) {
2021-12-14 08:13:49 +13:00
if self . exclude_images_with_same_size {
2023-05-08 06:54:05 +12:00
for vec_file_entry in mem ::take ( & mut self . similar_vectors ) {
2021-12-14 08:13:49 +13:00
let mut bt_sizes : BTreeSet < u64 > = Default ::default ( ) ;
let mut vec_values = Vec ::new ( ) ;
for file_entry in vec_file_entry {
if ! bt_sizes . contains ( & file_entry . size ) {
bt_sizes . insert ( file_entry . size ) ;
vec_values . push ( file_entry ) ;
}
}
if vec_values . len ( ) > 1 {
self . similar_vectors . push ( vec_values ) ;
}
}
}
2023-05-03 08:37:12 +12:00
}
2021-12-14 08:13:49 +13:00
2023-10-15 04:48:57 +13:00
#[ fun_time(message = " remove_multiple_records_from_reference_folders " , level = " debug " ) ]
2023-06-10 08:11:47 +12:00
fn remove_multiple_records_from_reference_folders ( & mut self ) {
2023-10-05 19:06:47 +13:00
if self . common_data . use_reference_folders {
2023-05-08 06:54:05 +12:00
self . similar_referenced_vectors = mem ::take ( & mut self . similar_vectors )
2021-12-24 21:18:55 +13:00
. into_iter ( )
. filter_map ( | vec_file_entry | {
2023-10-05 19:06:47 +13:00
let ( mut files_from_referenced_folders , normal_files ) : ( Vec < _ > , Vec < _ > ) = vec_file_entry
. into_iter ( )
. partition ( | e | self . common_data . directories . is_in_referenced_directory ( e . get_path ( ) ) ) ;
2021-12-24 21:18:55 +13:00
if files_from_referenced_folders . is_empty ( ) | | normal_files . is_empty ( ) {
None
} else {
Some ( ( files_from_referenced_folders . pop ( ) . unwrap ( ) , normal_files ) )
}
} )
2023-12-17 11:21:09 +13:00
. collect ::< Vec < ( ImagesEntry , Vec < ImagesEntry > ) > > ( ) ;
2021-12-24 21:18:55 +13:00
}
2023-05-03 08:37:12 +12:00
}
2021-12-24 21:18:55 +13:00
2023-05-03 08:37:12 +12:00
#[ allow(unused_variables) ]
2023-06-10 08:11:47 +12:00
// TODO this probably not works good when reference folders are used
2023-12-17 11:21:09 +13:00
pub fn verify_duplicated_items ( & self , collected_similar_images : & HashMap < ImHash , Vec < ImagesEntry > > ) {
2023-10-11 07:54:41 +13:00
if ! cfg! ( debug_assertions ) {
return ;
}
2023-05-03 08:37:12 +12:00
// Validating if group contains duplicated results
let mut result_hashset : HashSet < String > = Default ::default ( ) ;
let mut found = false ;
2023-10-13 19:33:35 +13:00
2023-05-03 08:37:12 +12:00
for vec_file_entry in collected_similar_images . values ( ) {
if vec_file_entry . is_empty ( ) {
println! ( " Empty group " ) ;
found = true ;
continue ;
2021-12-24 21:18:55 +13:00
}
2023-05-03 08:37:12 +12:00
if vec_file_entry . len ( ) = = 1 {
println! ( " Single Element {vec_file_entry:?} " ) ;
found = true ;
continue ;
}
for file_entry in vec_file_entry {
let st = file_entry . path . to_string_lossy ( ) . to_string ( ) ;
if result_hashset . contains ( & st ) {
found = true ;
println! ( " Duplicated Element {st} " ) ;
} else {
result_hashset . insert ( st ) ;
}
2021-12-24 21:18:55 +13:00
}
}
2024-02-04 08:18:24 +13:00
assert! ( ! found , " Found Invalid entries, verify errors before " ) ;
2020-10-15 08:10:27 +13:00
}
2023-10-14 07:33:17 +13:00
fn delete_files ( & mut self ) {
if self . common_data . delete_method = = DeleteMethod ::None {
return ;
}
let vec_files = self . similar_vectors . iter ( ) . collect ::< Vec < _ > > ( ) ;
delete_files_custom ( & vec_files , & self . common_data . delete_method , & mut self . common_data . text_messages , self . common_data . dry_run ) ;
}
2020-10-15 08:10:27 +13:00
}
2021-11-28 08:57:10 +13:00
2022-09-14 08:18:07 +12:00
fn is_in_reference_folder ( reference_directories : & [ PathBuf ] , path : & Path ) -> bool {
2022-11-24 08:23:17 +13:00
reference_directories . iter ( ) . any ( | e | path . starts_with ( e ) )
2022-09-14 08:18:07 +12:00
}
2020-10-15 08:10:27 +13:00
impl Default for SimilarImages {
fn default ( ) -> Self {
Self ::new ( )
}
}
impl DebugPrint for SimilarImages {
fn debug_print ( & self ) {
2023-10-11 07:54:41 +13:00
if ! cfg! ( debug_assertions ) {
2020-10-15 08:10:27 +13:00
return ;
}
println! ( " ---------------DEBUG PRINT--------------- " ) ;
2023-10-05 19:06:47 +13:00
self . debug_print_common ( ) ;
2020-10-15 08:10:27 +13:00
println! ( " ----------------------------------------- " ) ;
}
}
2021-11-28 08:57:10 +13:00
2023-10-11 07:54:41 +13:00
impl PrintResults for SimilarImages {
fn write_results < T : Write > ( & self , writer : & mut T ) -> std ::io ::Result < ( ) > {
2020-10-15 08:10:27 +13:00
if ! self . similar_vectors . is_empty ( ) {
2023-10-11 07:54:41 +13:00
write! ( writer , " {} images which have similar friends \n \n " , self . similar_vectors . len ( ) ) ? ;
2020-10-15 08:10:27 +13:00
2023-01-29 06:54:02 +13:00
for struct_similar in & self . similar_vectors {
2023-10-11 07:54:41 +13:00
writeln! ( writer , " Found {} images which have similar friends " , struct_similar . len ( ) ) ? ;
2021-02-10 05:29:49 +13:00
for file_entry in struct_similar {
writeln! (
writer ,
2024-02-11 11:57:52 +13:00
" {:?} - {}x{} - {} - {} " ,
2023-12-08 07:38:41 +13:00
file_entry . path ,
2024-02-11 11:57:52 +13:00
file_entry . width ,
file_entry . height ,
2022-09-14 08:18:07 +12:00
format_size ( file_entry . size , BINARY ) ,
2021-11-18 23:23:17 +13:00
get_string_from_similarity ( & file_entry . similarity , self . hash_size )
2023-10-11 07:54:41 +13:00
) ? ;
2021-02-10 05:29:49 +13:00
}
2023-10-11 07:54:41 +13:00
writeln! ( writer ) ? ;
2021-02-10 05:29:49 +13:00
}
2023-10-13 05:48:46 +13:00
} else if ! self . similar_referenced_vectors . is_empty ( ) {
writeln! ( writer , " {} images which have similar friends \n \n " , self . similar_referenced_vectors . len ( ) ) ? ;
for ( file_entry , vec_file_entry ) in & self . similar_referenced_vectors {
writeln! ( writer , " Found {} images which have similar friends " , vec_file_entry . len ( ) ) ? ;
writeln! ( writer ) ? ;
writeln! (
writer ,
2024-02-11 11:57:52 +13:00
" {:?} - {}x{} - {} - {} " ,
2023-12-08 07:38:41 +13:00
file_entry . path ,
2024-02-11 11:57:52 +13:00
file_entry . width ,
file_entry . height ,
2023-10-13 05:48:46 +13:00
format_size ( file_entry . size , BINARY ) ,
get_string_from_similarity ( & file_entry . similarity , self . hash_size )
) ? ;
for file_entry in vec_file_entry {
writeln! (
writer ,
2024-02-11 11:57:52 +13:00
" {:?} - {}x{} - {} - {} " ,
2023-12-08 07:38:41 +13:00
file_entry . path ,
2024-02-11 11:57:52 +13:00
file_entry . width ,
file_entry . height ,
2023-10-13 05:48:46 +13:00
format_size ( file_entry . size , BINARY ) ,
get_string_from_similarity ( & file_entry . similarity , self . hash_size )
) ? ;
}
writeln! ( writer ) ? ;
}
2020-10-15 08:10:27 +13:00
} else {
2023-10-11 07:54:41 +13:00
write! ( writer , " Not found any similar images. " ) ? ;
2020-10-15 08:10:27 +13:00
}
2020-12-27 22:56:26 +13:00
2023-10-11 07:54:41 +13:00
Ok ( ( ) )
2020-10-15 08:10:27 +13:00
}
2023-10-13 05:48:46 +13:00
fn save_results_to_file_as_json ( & self , file_name : & str , pretty_print : bool ) -> std ::io ::Result < ( ) > {
if self . get_use_reference ( ) {
self . save_results_to_file_as_json_internal ( file_name , & self . similar_referenced_vectors , pretty_print )
} else {
self . save_results_to_file_as_json_internal ( file_name , & self . similar_vectors , pretty_print )
}
}
2020-10-15 08:10:27 +13:00
}
2022-07-03 07:30:59 +12:00
pub fn get_string_from_similarity ( similarity : & u32 , hash_size : u8 ) -> String {
2021-11-19 18:35:26 +13:00
let index_preset = match hash_size {
2021-12-03 03:33:06 +13:00
8 = > 0 ,
16 = > 1 ,
32 = > 2 ,
64 = > 3 ,
2023-06-10 08:11:47 +12:00
_ = > panic! ( " Invalid hash size {hash_size} " ) ,
2021-11-19 18:35:26 +13:00
} ;
2022-07-06 05:58:57 +12:00
if * similarity = = 0 {
flc! ( " core_similarity_original " )
} else if * similarity < = SIMILAR_VALUES [ index_preset ] [ 0 ] {
2022-07-03 07:30:59 +12:00
flc! ( " core_similarity_very_high " )
} else if * similarity < = SIMILAR_VALUES [ index_preset ] [ 1 ] {
flc! ( " core_similarity_high " )
} else if * similarity < = SIMILAR_VALUES [ index_preset ] [ 2 ] {
flc! ( " core_similarity_medium " )
} else if * similarity < = SIMILAR_VALUES [ index_preset ] [ 3 ] {
flc! ( " core_similarity_small " )
} else if * similarity < = SIMILAR_VALUES [ index_preset ] [ 4 ] {
flc! ( " core_similarity_very_small " )
} else if * similarity < = SIMILAR_VALUES [ index_preset ] [ 5 ] {
flc! ( " core_similarity_minimal " )
} else {
panic! ( ) ;
2021-11-18 23:23:17 +13:00
}
}
2022-07-03 07:30:59 +12:00
pub fn return_similarity_from_similarity_preset ( similarity_preset : & SimilarityPreset , hash_size : u8 ) -> u32 {
2021-11-19 18:35:26 +13:00
let index_preset = match hash_size {
2021-12-03 03:33:06 +13:00
8 = > 0 ,
16 = > 1 ,
32 = > 2 ,
64 = > 3 ,
2021-11-18 23:23:17 +13:00
_ = > panic! ( ) ,
2021-11-19 18:35:26 +13:00
} ;
match similarity_preset {
2022-07-31 18:45:19 +12:00
SimilarityPreset ::Original = > 0 ,
2022-07-03 07:30:59 +12:00
SimilarityPreset ::VeryHigh = > SIMILAR_VALUES [ index_preset ] [ 0 ] ,
SimilarityPreset ::High = > SIMILAR_VALUES [ index_preset ] [ 1 ] ,
SimilarityPreset ::Medium = > SIMILAR_VALUES [ index_preset ] [ 2 ] ,
SimilarityPreset ::Small = > SIMILAR_VALUES [ index_preset ] [ 3 ] ,
SimilarityPreset ::VerySmall = > SIMILAR_VALUES [ index_preset ] [ 4 ] ,
SimilarityPreset ::Minimal = > SIMILAR_VALUES [ index_preset ] [ 5 ] ,
2021-11-19 18:35:26 +13:00
SimilarityPreset ::None = > panic! ( " " ) ,
2021-11-18 23:23:17 +13:00
}
}
2023-10-08 05:04:17 +13:00
pub fn convert_filters_to_string ( image_filter : & FilterType ) -> String {
2021-11-18 23:23:17 +13:00
match image_filter {
FilterType ::Lanczos3 = > " Lanczos3 " ,
FilterType ::Nearest = > " Nearest " ,
FilterType ::Triangle = > " Triangle " ,
FilterType ::Gaussian = > " Gaussian " ,
FilterType ::CatmullRom = > " CatmullRom " ,
}
. to_string ( )
}
2023-10-08 05:04:17 +13:00
pub fn convert_algorithm_to_string ( hash_alg : & HashAlg ) -> String {
2021-11-18 23:23:17 +13:00
match hash_alg {
HashAlg ::Mean = > " Mean " ,
HashAlg ::Gradient = > " Gradient " ,
HashAlg ::Blockhash = > " Blockhash " ,
HashAlg ::VertGradient = > " VertGradient " ,
HashAlg ::DoubleGradient = > " DoubleGradient " ,
}
. to_string ( )
}
pub fn test_image_conversion_speed ( ) {
let file_name : & str = " test.jpg " ;
let file_path = Path ::new ( file_name ) ;
match image ::open ( file_path ) {
Ok ( img_open ) = > {
for alg in [ HashAlg ::Blockhash , HashAlg ::Gradient , HashAlg ::DoubleGradient , HashAlg ::VertGradient , HashAlg ::Mean ] {
2021-12-22 06:44:20 +13:00
for filter in [
FilterType ::Lanczos3 ,
FilterType ::CatmullRom ,
FilterType ::Gaussian ,
FilterType ::Nearest ,
FilterType ::Triangle ,
] {
2021-12-03 03:33:06 +13:00
for size in [ 8 , 16 , 32 , 64 ] {
2021-11-18 23:23:17 +13:00
let hasher_config = HasherConfig ::new ( ) . hash_alg ( alg ) . resize_filter ( filter ) . hash_size ( size , size ) ;
let start = SystemTime ::now ( ) ;
let hasher = hasher_config . to_hasher ( ) ;
let _hash = hasher . hash_image ( & img_open ) ;
let end = SystemTime ::now ( ) ;
println! ( " {:?} us {:?} {:?} {} x {} " , end . duration_since ( start ) . unwrap ( ) . as_micros ( ) , alg , filter , size , size ) ;
}
}
}
}
Err ( e ) = > {
println! (
" Failed to open test file {}, reason {} " ,
match file_path . canonicalize ( ) {
Ok ( t ) = > t . to_string_lossy ( ) . to_string ( ) ,
Err ( _inspected ) = > file_name . to_string ( ) ,
} ,
e
) ;
2021-07-25 08:00:39 +12:00
}
}
}
2022-07-03 07:30:59 +12:00
#[ allow(dead_code) ]
2023-05-03 08:37:12 +12:00
#[ allow(unreachable_code) ]
#[ allow(unused_variables) ]
2022-07-06 05:58:57 +12:00
// Function to validate if after first check there are any duplicated entries
// E.g. /a.jpg is used also as master and similar image which is forbidden, because may
// cause accidentally delete more pictures that user wanted
2022-07-03 07:30:59 +12:00
fn debug_check_for_duplicated_things (
2023-05-03 08:37:12 +12:00
use_reference_folders : bool ,
hashes_parents : & HashMap < ImHash , u32 > ,
hashes_similarity : & HashMap < ImHash , ( ImHash , u32 ) > ,
2023-12-17 11:21:09 +13:00
all_hashed_images : & HashMap < ImHash , Vec < ImagesEntry > > ,
2022-07-03 07:30:59 +12:00
numm : & str ,
) {
2023-10-11 07:54:41 +13:00
if ! cfg! ( debug_assertions ) {
return ;
}
2023-05-03 08:37:12 +12:00
if use_reference_folders {
return ;
}
2022-08-30 07:37:31 +12:00
let mut found_broken_thing = false ;
2022-07-03 07:30:59 +12:00
let mut hashmap_hashes : HashSet < _ > = Default ::default ( ) ;
let mut hashmap_names : HashSet < _ > = Default ::default ( ) ;
2023-01-29 06:54:02 +13:00
for ( hash , number_of_children ) in hashes_parents {
2022-07-03 07:30:59 +12:00
if * number_of_children > 0 {
2023-05-03 08:37:12 +12:00
if hashmap_hashes . contains ( hash ) {
2024-02-04 08:18:24 +13:00
println! ( " ------1--HASH-- {} {:?} " , numm , all_hashed_images [ hash ] ) ;
2022-08-30 07:37:31 +12:00
found_broken_thing = true ;
2022-07-03 07:30:59 +12:00
}
2023-01-29 06:54:02 +13:00
hashmap_hashes . insert ( ( * hash ) . clone ( ) ) ;
2022-07-03 07:30:59 +12:00
2024-02-04 08:18:24 +13:00
for i in & all_hashed_images [ hash ] {
2022-07-03 07:30:59 +12:00
let name = i . path . to_string_lossy ( ) . to_string ( ) ;
if hashmap_names . contains ( & name ) {
2022-12-21 20:44:26 +13:00
println! ( " ------1--NAME-- {numm} {name:?} " ) ;
2022-08-30 07:37:31 +12:00
found_broken_thing = true ;
2022-07-03 07:30:59 +12:00
}
hashmap_names . insert ( name ) ;
}
}
}
for hash in hashes_similarity . keys ( ) {
2023-05-03 08:37:12 +12:00
if hashmap_hashes . contains ( hash ) {
2024-02-04 08:18:24 +13:00
println! ( " ------2--HASH-- {} {:?} " , numm , all_hashed_images [ hash ] ) ;
2022-08-30 07:37:31 +12:00
found_broken_thing = true ;
2022-07-03 07:30:59 +12:00
}
2023-01-29 06:54:02 +13:00
hashmap_hashes . insert ( ( * hash ) . clone ( ) ) ;
2022-07-03 07:30:59 +12:00
2024-02-04 08:18:24 +13:00
for i in & all_hashed_images [ hash ] {
2022-07-03 07:30:59 +12:00
let name = i . path . to_string_lossy ( ) . to_string ( ) ;
if hashmap_names . contains ( & name ) {
2022-12-21 20:44:26 +13:00
println! ( " ------2--NAME-- {numm} {name:?} " ) ;
2022-08-30 07:37:31 +12:00
found_broken_thing = true ;
2022-07-03 07:30:59 +12:00
}
hashmap_names . insert ( name ) ;
}
}
2022-08-30 07:37:31 +12:00
2023-05-03 08:37:12 +12:00
assert! ( ! found_broken_thing ) ;
2022-07-03 07:30:59 +12:00
}
2023-06-10 08:11:47 +12:00
2023-10-05 19:06:47 +13:00
impl CommonData for SimilarImages {
fn get_cd ( & self ) -> & CommonToolData {
& self . common_data
}
fn get_cd_mut ( & mut self ) -> & mut CommonToolData {
& mut self . common_data
}
}
impl SimilarImages {
pub fn set_hash_size ( & mut self , hash_size : u8 ) {
self . hash_size = match hash_size {
8 | 16 | 32 | 64 = > hash_size ,
e = > {
panic! ( " Invalid value of hash size {e} " ) ;
}
}
}
pub fn set_exclude_images_with_same_size ( & mut self , exclude_images_with_same_size : bool ) {
self . exclude_images_with_same_size = exclude_images_with_same_size ;
}
pub fn set_hash_alg ( & mut self , hash_alg : HashAlg ) {
self . hash_alg = hash_alg ;
}
pub fn set_image_filter ( & mut self , image_filter : FilterType ) {
self . image_filter = image_filter ;
}
2023-12-17 11:21:09 +13:00
pub const fn get_similar_images ( & self ) -> & Vec < Vec < ImagesEntry > > {
2023-10-05 19:06:47 +13:00
& self . similar_vectors
}
2023-12-17 11:21:09 +13:00
pub fn get_similar_images_referenced ( & self ) -> & Vec < ( ImagesEntry , Vec < ImagesEntry > ) > {
2023-10-05 19:06:47 +13:00
& self . similar_referenced_vectors
}
pub fn get_use_reference ( & self ) -> bool {
self . common_data . use_reference_folders
}
pub const fn get_information ( & self ) -> & Info {
& self . information
}
pub fn set_similarity ( & mut self , similarity : u32 ) {
self . similarity = similarity ;
}
}
2023-06-10 08:11:47 +12:00
#[ cfg(test) ]
mod tests {
2023-06-11 18:51:21 +12:00
use std ::collections ::HashMap ;
2023-06-10 08:11:47 +12:00
use std ::path ::PathBuf ;
2023-10-05 19:06:47 +13:00
use bk_tree ::BKTree ;
2024-01-15 08:00:56 +13:00
use crate ::common_dir_traversal ::ToolType ;
2023-06-10 08:11:47 +12:00
use crate ::common_directory ::Directories ;
2023-10-05 19:06:47 +13:00
use crate ::common_tool ::CommonToolData ;
2023-12-17 11:21:09 +13:00
use crate ::similar_images ::{ Hamming , ImHash , ImageType , ImagesEntry , SimilarImages } ;
2023-06-10 08:11:47 +12:00
#[ test ]
fn test_compare_no_images ( ) {
2023-06-11 18:51:21 +12:00
for _ in 0 .. 100 {
let mut similar_images = SimilarImages ::default ( ) ;
similar_images . find_similar_images ( None , None ) ;
assert_eq! ( similar_images . get_similar_images ( ) . len ( ) , 0 ) ;
}
2023-06-10 08:11:47 +12:00
}
#[ test ]
fn test_compare_tolerance_0_normal_mode ( ) {
2023-06-11 18:51:21 +12:00
for _ in 0 .. 100 {
let mut similar_images = SimilarImages {
similarity : 0 ,
2023-12-04 00:06:42 +13:00
common_data : CommonToolData {
tool_type : ToolType ::SimilarImages ,
.. Default ::default ( )
} ,
2023-06-11 18:51:21 +12:00
.. Default ::default ( )
} ;
2023-06-10 08:11:47 +12:00
2023-06-11 18:51:21 +12:00
let fe1 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 ] , " abc.txt " ) ;
let fe2 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 ] , " bcd.txt " ) ;
let fe3 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 2 ] , " cde.txt " ) ;
let fe4 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 2 ] , " rrt.txt " ) ;
let fe5 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 2 ] , " bld.txt " ) ;
add_hashes ( & mut similar_images . image_hashes , vec! [ fe1 . clone ( ) , fe2 . clone ( ) , fe3 . clone ( ) , fe4 . clone ( ) , fe5 . clone ( ) ] ) ;
similar_images . find_similar_hashes ( None , None ) ;
assert_eq! ( similar_images . get_similar_images ( ) . len ( ) , 2 ) ;
let first_group = similar_images . get_similar_images ( ) [ 0 ] . iter ( ) . map ( | e | & e . path ) . collect ::< Vec < _ > > ( ) ;
let second_group = similar_images . get_similar_images ( ) [ 1 ] . iter ( ) . map ( | e | & e . path ) . collect ::< Vec < _ > > ( ) ;
// Initial order is not guaranteed, so we need to check both options
if similar_images . get_similar_images ( ) [ 0 ] [ 0 ] . hash = = fe1 . hash {
assert_eq! ( first_group , vec! [ & fe1 . path , & fe2 . path ] ) ;
assert_eq! ( second_group , vec! [ & fe3 . path , & fe4 . path , & fe5 . path ] ) ;
} else {
assert_eq! ( first_group , vec! [ & fe3 . path , & fe4 . path , & fe5 . path ] ) ;
assert_eq! ( second_group , vec! [ & fe1 . path , & fe2 . path ] ) ;
}
2023-06-10 08:11:47 +12:00
}
}
#[ test ]
fn test_simple_normal_one_group ( ) {
2023-06-11 18:51:21 +12:00
for _ in 0 .. 100 {
let mut similar_images = SimilarImages {
similarity : 1 ,
2023-12-04 00:06:42 +13:00
common_data : CommonToolData {
tool_type : ToolType ::SimilarImages ,
.. Default ::default ( )
} ,
2023-06-11 18:51:21 +12:00
.. Default ::default ( )
} ;
2023-06-10 08:11:47 +12:00
2023-06-11 18:51:21 +12:00
let fe1 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 ] , " abc.txt " ) ;
let fe2 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 ] , " bcd.txt " ) ;
2023-06-10 08:11:47 +12:00
2023-06-11 18:51:21 +12:00
add_hashes ( & mut similar_images . image_hashes , vec! [ fe1 , fe2 ] ) ;
2023-06-10 08:11:47 +12:00
2023-06-11 18:51:21 +12:00
similar_images . find_similar_hashes ( None , None ) ;
assert_eq! ( similar_images . get_similar_images ( ) . len ( ) , 1 ) ;
}
2023-06-10 08:11:47 +12:00
}
#[ test ]
fn test_simple_normal_one_group_extended ( ) {
2023-06-11 18:51:21 +12:00
for _ in 0 .. 100 {
let mut similar_images = SimilarImages {
similarity : 2 ,
2023-10-05 19:06:47 +13:00
common_data : CommonToolData {
use_reference_folders : false ,
2023-12-04 00:06:42 +13:00
tool_type : ToolType ::SimilarImages ,
2023-10-05 19:06:47 +13:00
.. Default ::default ( )
} ,
2023-06-11 18:51:21 +12:00
.. Default ::default ( )
} ;
2023-06-10 08:11:47 +12:00
2023-06-11 18:51:21 +12:00
let fe1 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 ] , " abc.txt " ) ;
let fe2 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 ] , " bcd.txt " ) ;
let fe3 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 2 ] , " rrd.txt " ) ;
2023-06-10 08:11:47 +12:00
2023-06-11 18:51:21 +12:00
add_hashes ( & mut similar_images . image_hashes , vec! [ fe1 , fe2 , fe3 ] ) ;
2023-06-10 08:11:47 +12:00
2023-06-11 18:51:21 +12:00
similar_images . find_similar_hashes ( None , None ) ;
assert_eq! ( similar_images . get_similar_images ( ) . len ( ) , 1 ) ;
assert_eq! ( similar_images . get_similar_images ( ) [ 0 ] . len ( ) , 3 ) ;
}
2023-06-10 08:11:47 +12:00
}
#[ test ]
fn test_simple_referenced_same_group ( ) {
2023-06-11 18:51:21 +12:00
for _ in 0 .. 100 {
let mut similar_images = SimilarImages {
similarity : 0 ,
2023-10-05 19:06:47 +13:00
common_data : CommonToolData {
2023-12-04 00:06:42 +13:00
tool_type : ToolType ::SimilarImages ,
2023-10-05 19:06:47 +13:00
directories : Directories {
reference_directories : vec ! [ PathBuf ::from ( " /home/rr/ " ) ] ,
.. Default ::default ( )
} ,
use_reference_folders : true ,
2023-06-11 18:51:21 +12:00
.. Default ::default ( )
} ,
2023-06-10 08:11:47 +12:00
.. Default ::default ( )
2023-06-11 18:51:21 +12:00
} ;
2023-06-10 08:11:47 +12:00
2023-06-11 18:51:21 +12:00
let fe1 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 ] , " /home/rr/abc.txt " ) ;
let fe2 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 ] , " /home/rr/bcd.txt " ) ;
2023-06-10 08:11:47 +12:00
2023-06-11 18:51:21 +12:00
add_hashes ( & mut similar_images . image_hashes , vec! [ fe1 , fe2 ] ) ;
2023-06-10 08:11:47 +12:00
2023-06-11 18:51:21 +12:00
similar_images . find_similar_hashes ( None , None ) ;
assert_eq! ( similar_images . get_similar_images ( ) . len ( ) , 0 ) ;
}
2023-06-10 08:11:47 +12:00
}
#[ test ]
fn test_simple_referenced_group_extended ( ) {
2023-06-11 18:51:21 +12:00
for _ in 0 .. 100 {
let mut similar_images = SimilarImages {
similarity : 0 ,
2023-10-05 19:06:47 +13:00
common_data : CommonToolData {
2023-12-04 00:06:42 +13:00
tool_type : ToolType ::SimilarImages ,
2023-10-05 19:06:47 +13:00
directories : Directories {
reference_directories : vec ! [ PathBuf ::from ( " /home/rr/ " ) ] ,
.. Default ::default ( )
} ,
use_reference_folders : true ,
2023-06-11 18:51:21 +12:00
.. Default ::default ( )
} ,
2023-06-10 08:11:47 +12:00
.. Default ::default ( )
2023-06-11 18:51:21 +12:00
} ;
2023-06-10 08:11:47 +12:00
2023-06-11 18:51:21 +12:00
let fe1 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 ] , " /home/rr/abc.txt " ) ;
let fe2 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 ] , " /home/kk/bcd.txt " ) ;
2023-06-10 08:11:47 +12:00
2023-06-11 18:51:21 +12:00
add_hashes ( & mut similar_images . image_hashes , vec! [ fe1 , fe2 ] ) ;
2023-06-10 08:11:47 +12:00
2023-06-11 18:51:21 +12:00
similar_images . find_similar_hashes ( None , None ) ;
assert_eq! ( similar_images . get_similar_images_referenced ( ) . len ( ) , 1 ) ;
assert_eq! ( similar_images . get_similar_images_referenced ( ) [ 0 ] . 1. len ( ) , 1 ) ;
}
2023-06-10 08:11:47 +12:00
}
#[ test ]
fn test_simple_referenced_group_extended2 ( ) {
2023-06-11 18:51:21 +12:00
for _ in 0 .. 100 {
let mut similar_images = SimilarImages {
similarity : 0 ,
2023-10-05 19:06:47 +13:00
common_data : CommonToolData {
2023-12-04 00:06:42 +13:00
tool_type : ToolType ::SimilarImages ,
2023-10-05 19:06:47 +13:00
directories : Directories {
reference_directories : vec ! [ PathBuf ::from ( " /home/rr/ " ) ] ,
.. Default ::default ( )
} ,
use_reference_folders : true ,
2023-06-11 18:51:21 +12:00
.. Default ::default ( )
} ,
2023-06-10 08:11:47 +12:00
.. Default ::default ( )
2023-06-11 18:51:21 +12:00
} ;
2023-06-10 08:11:47 +12:00
2023-06-11 18:51:21 +12:00
let fe1 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 ] , " /home/rr/abc.txt " ) ;
let fe2 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 ] , " /home/rr/abc2.txt " ) ;
let fe3 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 ] , " /home/kk/bcd.txt " ) ;
let fe4 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 ] , " /home/kk/bcd2.txt " ) ;
2023-06-10 08:11:47 +12:00
2023-06-11 18:51:21 +12:00
add_hashes ( & mut similar_images . image_hashes , vec! [ fe1 , fe2 , fe3 , fe4 ] ) ;
2023-06-10 08:11:47 +12:00
2023-06-11 18:51:21 +12:00
similar_images . find_similar_hashes ( None , None ) ;
let res = similar_images . get_similar_images_referenced ( ) ;
assert_eq! ( res . len ( ) , 1 ) ;
assert_eq! ( res [ 0 ] . 1. len ( ) , 2 ) ;
assert! ( res [ 0 ] . 1. iter ( ) . all ( | e | e . path . starts_with ( " /home/kk/ " ) ) ) ;
}
2023-06-10 08:11:47 +12:00
}
#[ test ]
fn test_simple_normal_too_small_similarity ( ) {
2023-06-11 18:51:21 +12:00
for _ in 0 .. 100 {
2023-06-10 08:11:47 +12:00
let mut similar_images = SimilarImages {
similarity : 1 ,
2023-10-05 19:06:47 +13:00
common_data : CommonToolData {
2023-12-04 00:06:42 +13:00
tool_type : ToolType ::SimilarImages ,
2023-10-05 19:06:47 +13:00
use_reference_folders : false ,
.. Default ::default ( )
} ,
2023-06-10 08:11:47 +12:00
.. Default ::default ( )
} ;
let fe1 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 0b00001 ] , " abc.txt " ) ;
let fe2 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 0b00100 ] , " bcd.txt " ) ;
let fe3 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 0b10000 ] , " rrd.txt " ) ;
2023-06-11 18:51:21 +12:00
add_hashes ( & mut similar_images . image_hashes , vec! [ fe1 , fe2 , fe3 ] ) ;
2023-06-10 08:11:47 +12:00
similar_images . find_similar_hashes ( None , None ) ;
let res = similar_images . get_similar_images ( ) ;
assert! ( res . is_empty ( ) ) ;
}
}
#[ test ]
fn test_simple_normal_union_of_similarity ( ) {
for _ in 0 .. 100 {
let mut similar_images = SimilarImages {
similarity : 4 ,
2023-10-05 19:06:47 +13:00
common_data : CommonToolData {
2023-12-04 00:06:42 +13:00
tool_type : ToolType ::SimilarImages ,
2023-10-05 19:06:47 +13:00
use_reference_folders : false ,
.. Default ::default ( )
} ,
2023-06-10 08:11:47 +12:00
.. Default ::default ( )
} ;
let fe1 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 0b0000_0001 ] , " abc.txt " ) ;
let fe2 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 0b0000_1111 ] , " bcd.txt " ) ;
let fe3 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 0b0111_1111 ] , " rrd.txt " ) ;
2023-06-11 18:51:21 +12:00
add_hashes ( & mut similar_images . image_hashes , vec! [ fe1 , fe2 , fe3 ] ) ;
2023-06-10 08:11:47 +12:00
similar_images . find_similar_hashes ( None , None ) ;
let res = similar_images . get_similar_images ( ) ;
assert_eq! ( res . len ( ) , 1 ) ;
let mut path = res [ 0 ] . iter ( ) . map ( | e | e . path . to_string_lossy ( ) . to_string ( ) ) . collect ::< Vec < _ > > ( ) ;
path . sort ( ) ;
if res [ 0 ] . len ( ) = = 3 {
assert_eq! ( path , vec! [ " abc.txt " . to_string ( ) , " bcd.txt " . to_string ( ) , " rrd.txt " . to_string ( ) ] ) ;
} else if res [ 0 ] . len ( ) = = 2 {
assert! ( path = = vec! [ " abc.txt " . to_string ( ) , " bcd.txt " . to_string ( ) ] | | path = = vec! [ " bcd.txt " . to_string ( ) , " rrd.txt " . to_string ( ) ] ) ;
} else {
panic! ( " Invalid number of items " ) ;
}
}
}
2023-10-05 19:06:47 +13:00
2023-06-11 18:51:21 +12:00
#[ test ]
fn test_reference_similarity_only_one ( ) {
for _ in 0 .. 100 {
let mut similar_images = SimilarImages {
similarity : 1 ,
2023-10-05 19:06:47 +13:00
common_data : CommonToolData {
2023-12-04 00:06:42 +13:00
tool_type : ToolType ::SimilarImages ,
2023-10-05 19:06:47 +13:00
directories : Directories {
reference_directories : vec ! [ PathBuf ::from ( " /home/rr/ " ) ] ,
.. Default ::default ( )
} ,
use_reference_folders : true ,
2023-06-11 18:51:21 +12:00
.. Default ::default ( )
} ,
.. Default ::default ( )
} ;
let fe1 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 0b0001 ] , " /home/rr/abc.txt " ) ;
let fe2 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 0b0011 ] , " /home/kk/bcd.txt " ) ;
add_hashes ( & mut similar_images . image_hashes , vec! [ fe1 , fe2 ] ) ;
similar_images . find_similar_hashes ( None , None ) ;
let res = similar_images . get_similar_images_referenced ( ) ;
assert_eq! ( res . len ( ) , 1 ) ;
assert_eq! ( res [ 0 ] . 1. len ( ) , 1 ) ;
assert_eq! ( res [ 0 ] . 0. path , PathBuf ::from ( " /home/rr/abc.txt " ) ) ;
assert_eq! ( res [ 0 ] . 1 [ 0 ] . path , PathBuf ::from ( " /home/kk/bcd.txt " ) ) ;
}
}
2023-10-05 19:06:47 +13:00
2023-06-11 18:51:21 +12:00
#[ test ]
fn test_reference_too_small_similarity ( ) {
for _ in 0 .. 100 {
let mut similar_images = SimilarImages {
similarity : 1 ,
2023-10-05 19:06:47 +13:00
common_data : CommonToolData {
2023-12-04 00:06:42 +13:00
tool_type : ToolType ::SimilarImages ,
2023-10-05 19:06:47 +13:00
directories : Directories {
reference_directories : vec ! [ PathBuf ::from ( " /home/rr/ " ) ] ,
.. Default ::default ( )
} ,
use_reference_folders : true ,
2023-06-11 18:51:21 +12:00
.. Default ::default ( )
} ,
.. Default ::default ( )
} ;
let fe1 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 0b0001 ] , " /home/rr/abc.txt " ) ;
let fe2 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 0b0010 ] , " /home/kk/bcd.txt " ) ;
add_hashes ( & mut similar_images . image_hashes , vec! [ fe1 , fe2 ] ) ;
similar_images . find_similar_hashes ( None , None ) ;
let res = similar_images . get_similar_images_referenced ( ) ;
assert_eq! ( res . len ( ) , 0 ) ;
}
}
#[ test ]
fn test_reference_minimal ( ) {
for _ in 0 .. 100 {
let mut similar_images = SimilarImages {
similarity : 1 ,
2023-10-05 19:06:47 +13:00
common_data : CommonToolData {
2023-12-04 00:06:42 +13:00
tool_type : ToolType ::SimilarImages ,
2023-10-05 19:06:47 +13:00
directories : Directories {
reference_directories : vec ! [ PathBuf ::from ( " /home/rr/ " ) ] ,
.. Default ::default ( )
} ,
use_reference_folders : true ,
2023-06-11 18:51:21 +12:00
.. Default ::default ( )
} ,
.. Default ::default ( )
} ;
let fe1 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 0b0001 ] , " /home/rr/abc.txt " ) ;
let fe2 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 0b0011 ] , " /home/kk/bcd.txt " ) ;
let fe3 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 0b0100 ] , " /home/kk/bcd2.txt " ) ;
let fe4 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 0b1100 ] , " /home/rr/krkr.txt " ) ;
add_hashes ( & mut similar_images . image_hashes , vec! [ fe1 , fe2 , fe3 , fe4 ] ) ;
similar_images . find_similar_hashes ( None , None ) ;
let res = similar_images . get_similar_images_referenced ( ) ;
assert_eq! ( res . len ( ) , 2 ) ;
assert_eq! ( res [ 0 ] . 1. len ( ) , 1 ) ;
assert_eq! ( res [ 1 ] . 1. len ( ) , 1 ) ;
if res [ 0 ] . 1 [ 0 ] . path = = PathBuf ::from ( " /home/kk/bcd.txt " ) {
assert_eq! ( res [ 0 ] . 0. path , PathBuf ::from ( " /home/rr/abc.txt " ) ) ;
assert_eq! ( res [ 1 ] . 0. path , PathBuf ::from ( " /home/rr/krkr.txt " ) ) ;
} else if res [ 0 ] . 1 [ 0 ] . path = = PathBuf ::from ( " /home/kk/bcd2.txt " ) {
assert_eq! ( res [ 0 ] . 0. path , PathBuf ::from ( " /home/rr/krkr.txt " ) ) ;
assert_eq! ( res [ 1 ] . 0. path , PathBuf ::from ( " /home/rr/abc.txt " ) ) ;
}
}
}
2024-02-04 08:18:24 +13:00
#[ test ]
fn test_reference_same ( ) {
for _ in 0 .. 100 {
let mut similar_images = SimilarImages {
similarity : 1 ,
common_data : CommonToolData {
tool_type : ToolType ::SimilarImages ,
directories : Directories {
reference_directories : vec ! [ PathBuf ::from ( " /home/rr/ " ) ] ,
.. Default ::default ( )
} ,
use_reference_folders : true ,
.. Default ::default ( )
} ,
.. Default ::default ( )
} ;
let fe1 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 ] , " /home/rr/abc.txt " ) ;
let fe2 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 ] , " /home/kk/bcd.txt " ) ;
add_hashes ( & mut similar_images . image_hashes , vec! [ fe1 , fe2 ] ) ;
similar_images . find_similar_hashes ( None , None ) ;
let res = similar_images . get_similar_images_referenced ( ) ;
assert_eq! ( res . len ( ) , 1 ) ;
assert_eq! ( res [ 0 ] . 1. len ( ) , 1 ) ;
}
}
2023-06-11 18:51:21 +12:00
#[ test ]
fn test_reference_union ( ) {
for _ in 0 .. 100 {
let mut similar_images = SimilarImages {
similarity : 10 ,
2023-10-05 19:06:47 +13:00
common_data : CommonToolData {
2023-12-04 00:06:42 +13:00
tool_type : ToolType ::SimilarImages ,
2023-10-05 19:06:47 +13:00
directories : Directories {
reference_directories : vec ! [ PathBuf ::from ( " /home/rr/ " ) ] ,
.. Default ::default ( )
} ,
use_reference_folders : true ,
2023-06-11 18:51:21 +12:00
.. Default ::default ( )
} ,
.. Default ::default ( )
} ;
let fe0 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 0b1000 ] , " /home/rr/abc2.txt " ) ;
let fe1 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 0b0001 ] , " /home/rr/abc.txt " ) ;
let fe2 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 0b1110 ] , " /home/kk/bcd.txt " ) ;
let fe3 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 0b0100 ] , " /home/kk/bcd2.txt " ) ;
let fe4 = create_random_file_entry ( vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 0b1100 ] , " /home/rr/krkr.txt " ) ;
add_hashes ( & mut similar_images . image_hashes , vec! [ fe0 , fe1 , fe2 , fe3 , fe4 ] ) ;
similar_images . find_similar_hashes ( None , None ) ;
let res = similar_images . get_similar_images_referenced ( ) ;
assert_eq! ( res . len ( ) , 1 ) ;
assert_eq! ( res [ 0 ] . 1. len ( ) , 2 ) ;
assert_eq! ( res [ 0 ] . 0. path , PathBuf ::from ( " /home/rr/krkr.txt " ) ) ;
}
}
2023-06-10 08:11:47 +12:00
#[ test ]
fn test_tolerance ( ) {
// This test not really tests anything, but shows that current hamming distance works
// in bits instead of bytes
// I tried to make it work in bytes, but it was terrible, so Hamming should be really Ok
let fe1 = vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 ] ;
let fe2 = vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 2 ] ;
let mut bktree = BKTree ::new ( Hamming ) ;
bktree . add ( fe1 ) ;
let ( similarity , _hash ) = bktree . find ( & fe2 , 100 ) . next ( ) . unwrap ( ) ;
assert_eq! ( similarity , 2 ) ;
let fe1 = vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 ] ;
let fe2 = vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 3 ] ;
let mut bktree = BKTree ::new ( Hamming ) ;
bktree . add ( fe1 ) ;
let ( similarity , _hash ) = bktree . find ( & fe2 , 100 ) . next ( ) . unwrap ( ) ;
assert_eq! ( similarity , 1 ) ;
2023-06-11 18:51:21 +12:00
let fe1 = vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 0b0000_0000 ] ;
let fe2 = vec! [ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 0b0000_1000 ] ;
let mut bktree = BKTree ::new ( Hamming ) ;
bktree . add ( fe1 ) ;
let ( similarity , _hash ) = bktree . find ( & fe2 , 100 ) . next ( ) . unwrap ( ) ;
assert_eq! ( similarity , 1 ) ;
}
2023-12-17 11:21:09 +13:00
fn add_hashes ( hashmap : & mut HashMap < ImHash , Vec < ImagesEntry > > , file_entries : Vec < ImagesEntry > ) {
2023-06-11 18:51:21 +12:00
for fe in file_entries {
2023-10-05 19:06:47 +13:00
hashmap . entry ( fe . hash . clone ( ) ) . or_default ( ) . push ( fe ) ;
2023-06-11 18:51:21 +12:00
}
2023-06-10 08:11:47 +12:00
}
2023-12-17 11:21:09 +13:00
fn create_random_file_entry ( hash : Vec < u8 > , name : & str ) -> ImagesEntry {
ImagesEntry {
2023-06-10 08:11:47 +12:00
path : PathBuf ::from ( name . to_string ( ) ) ,
size : 0 ,
2024-02-11 11:57:52 +13:00
width : 100 ,
height : 100 ,
2023-06-10 08:11:47 +12:00
modified_date : 0 ,
hash ,
similarity : 0 ,
2023-12-17 11:21:09 +13:00
image_type : ImageType ::Normal ,
2023-06-10 08:11:47 +12:00
}
}
}