From d1c66fda1b248f6d8fc1ab3a5361804d2bb69f1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mikrut?= <41945903+qarmin@users.noreply.github.com> Date: Sat, 2 Jul 2022 21:30:59 +0200 Subject: [PATCH] Change image hash compare algorithm and add multithreading (#762) * New image compare algorithm * Par iter * Ending words --- Cargo.lock | 115 ++-- czkawka_core/Cargo.toml | 4 +- czkawka_core/src/similar_images.rs | 536 +++++++++++------- czkawka_core/src/similar_videos.rs | 2 +- czkawka_gui/i18n/en/czkawka_gui.ftl | 8 - czkawka_gui/src/compute_results.rs | 12 +- .../connect_things/connect_button_search.rs | 6 +- .../connect_similar_image_size_change.rs | 6 +- .../src/gui_structs/gui_main_notebook.rs | 20 +- czkawka_gui/src/saving_loading.rs | 42 +- czkawka_gui/ui/about_dialog.ui | 2 +- czkawka_gui/ui/compare_images.ui | 2 +- czkawka_gui/ui/czkawka.cmb | 6 +- czkawka_gui/ui/main_window.ui | 9 +- czkawka_gui/ui/popover_right_click.ui | 2 +- czkawka_gui/ui/popover_select.ui | 2 +- czkawka_gui/ui/progress.ui | 2 +- czkawka_gui/ui/settings.ui | 2 +- 18 files changed, 415 insertions(+), 363 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3d2d242..0c5192c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -68,9 +68,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.57" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" +checksum = "bb07d2053ccdbe10e2af2995a2f116c1330396493dc1269f6a91d0ae82e19704" [[package]] name = "arrayref" @@ -254,9 +254,9 @@ dependencies = [ [[package]] name = "cairo-rs" -version = "0.15.11" +version = "0.15.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62be3562254e90c1c6050a72aa638f6315593e98c5cdaba9017cedbabf0a5dee" +checksum = "c76ee391b03d35510d9fa917357c7f1855bd9a6659c95a1b392e33f49b3369bc" dependencies = [ "bitflags", "cairo-sys-rs", @@ -284,9 +284,9 @@ checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" [[package]] name = "cfb" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25f6cc832d96f9961bfe67da666f1fb96387305d385db5222cbb7bce21121016" +checksum = "19ff6ea9647f5b4fb422a553d5b6fe1b398986a6e4f458d1219eb77e0e6e0606" dependencies = [ "byteorder", "fnv", @@ -341,9 +341,9 @@ dependencies = [ [[package]] name = "clap" -version = "3.2.5" +version = "3.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53da17d37dba964b9b3ecb5c5a1f193a2762c700e6829201e645b9381c99dc7" +checksum = "190814073e85d238f31ff738fcb0bf6910cedeb73376c87cd69291028966fd83" dependencies = [ "atty", "bitflags", @@ -358,9 +358,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "3.2.5" +version = "3.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c11d40217d16aee8508cc8e5fde8b4ff24639758608e5374e731b53f85749fb9" +checksum = "759bf187376e1afa7b85b959e6a664a3e7a95203415dba952ad19139e798f902" dependencies = [ "heck", "proc-macro-error", @@ -371,9 +371,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.2.2" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5538cd660450ebeb4234cfecf8f2284b844ffc4c50531e66d584ad5b91293613" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" dependencies = [ "os_str_bytes", ] @@ -445,9 +445,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.9" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ff1f980957787286a554052d03c7aee98d99cc32e09f6d45f0a814133c87978" +checksum = "7d82ee10ce34d7bc12c2122495e7593a9c41347ecdd64185af4ecf72cb1a7f83" dependencies = [ "cfg-if", "once_cell", @@ -498,6 +498,7 @@ dependencies = [ "libheif-rs", "lofty", "mime_guess", + "num_cpus", "once_cell", "pdf", "rawloader", @@ -626,9 +627,9 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "either" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +checksum = "3f107b87b6afc2a64fd13cac55fe06d6c8859f12d4b14cbcdd2c67d0976781be" [[package]] name = "encoding_rs" @@ -990,9 +991,9 @@ dependencies = [ [[package]] name = "gif" -version = "0.11.3" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3a7187e78088aead22ceedeee99779455b23fc231fe13ec443f99bb71694e5b" +checksum = "3edd93c6756b4dfaf2709eafcc345ba2636565295c198a9cfbf75fa5e3e00b06" dependencies = [ "color_quant", "weezl", @@ -1000,9 +1001,9 @@ dependencies = [ [[package]] name = "gio" -version = "0.15.11" +version = "0.15.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f132be35e05d9662b9fa0fee3f349c6621f7782e0105917f4cc73c1bf47eceb" +checksum = "68fdbc90312d462781a395f7a16d96a2b379bb6ef8cd6310a2df272771c4283b" dependencies = [ "bitflags", "futures-channel", @@ -1030,9 +1031,9 @@ dependencies = [ [[package]] name = "glib" -version = "0.15.11" +version = "0.15.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd124026a2fa8c33a3d17a3fe59c103f2d9fa5bd92c19e029e037736729abeab" +checksum = "edb0306fbad0ab5428b0ca674a23893db909a98582969c9b537be4ced78c505d" dependencies = [ "bitflags", "futures-channel", @@ -1362,7 +1363,7 @@ dependencies = [ "gif", "jpeg-decoder 0.2.6", "num-iter", - "num-rational 0.4.0", + "num-rational 0.4.1", "num-traits", "png 0.17.5", "scoped_threadpool", @@ -1404,9 +1405,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c6392766afd7964e2531940894cffe4bd8d7d17dbc3c1c4857040fd4b33bdb3" +checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" dependencies = [ "autocfg", "hashbrown", @@ -1540,9 +1541,9 @@ dependencies = [ [[package]] name = "linked-hash-map" -version = "0.5.4" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "locale_config" @@ -1569,9 +1570,9 @@ dependencies = [ [[package]] name = "lofty" -version = "0.6.3" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e09702a8eff21fa1cf105d189d8eea3609d7074ab1503d7c49d5b37b7403a65" +checksum = "3f4e4309226629ef3486925c6a1b346241abeb30758e307b3ca9cfd66587bf4f" dependencies = [ "base64 0.13.0", "byteorder", @@ -1732,9 +1733,9 @@ dependencies = [ [[package]] name = "num-rational" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d41702bd167c2df5520b384281bc111a4b5efcf7fbc4c9c222c815b07e0a6a6a" +checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" dependencies = [ "autocfg", "num-integer", @@ -1980,18 +1981,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e" +checksum = "78203e83c48cffbe01e4a2d35d566ca4de445d79a85372fc64e378bfc812a260" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb" +checksum = "710faf75e1b33345361201d36d04e98ac1ed8909151a017ed384700836104c74" dependencies = [ "proc-macro2", "quote", @@ -2091,9 +2092,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.39" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f" +checksum = "dd96a1e8ed2596c337f8eae5f24924ec83f5ad5ab21ea8e455d3566c69fbcaf7" dependencies = [ "unicode-ident", ] @@ -2109,9 +2110,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.18" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" +checksum = "3bcdf212e9776fbcb2d23ab029360416bb1706b1aea2d1a5ba002727cbcab804" dependencies = [ "proc-macro2", ] @@ -2286,7 +2287,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.10", + "semver 1.0.12", ] [[package]] @@ -2378,9 +2379,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.10" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a41d061efea015927ac527063765e73601444cdc344ba855bc7bd44578b25e1c" +checksum = "a2333e6df6d6598f2b1974829f853c2b4c5f4a6e503c10af918081aa6f8564e1" [[package]] name = "semver-parser" @@ -2393,18 +2394,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.137" +version = "1.0.138" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" +checksum = "1578c6245786b9d168c5447eeacfb96856573ca56c9d68fdcf394be134882a47" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.137" +version = "1.0.138" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be" +checksum = "023e9b1467aef8a10fb88f25611870ada9800ef7e22afce356bb0d2387b6f27c" dependencies = [ "proc-macro2", "quote", @@ -2413,9 +2414,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.81" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" +checksum = "82c2c1fdcd807d1098552c5b9a36e425e42e9fbd7c6a37a8425f390f781f7fa7" dependencies = [ "itoa", "ryu", @@ -2477,9 +2478,9 @@ checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" [[package]] name = "smallvec" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" +checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" [[package]] name = "snafu" @@ -2713,9 +2714,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.96" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0748dd251e24453cb8717f0354206b91557e4ec8703673a4b30208f2abaf1ebf" +checksum = "c50aef8a904de4c23c788f104b7dddc7d6f79c647c7c8ce4cc8f73eb0ca773dd" dependencies = [ "proc-macro2", "quote", @@ -2828,9 +2829,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd" +checksum = "72c91f41dcb2f096c05f0873d667dceec1087ce5bcf984ec8ffb19acddbb3217" dependencies = [ "itoa", "libc", @@ -2969,9 +2970,9 @@ checksum = "5bd2fe26506023ed7b5e1e315add59d6f584c621d037f9368fea9cfb988f368c" [[package]] name = "unicode-normalization" -version = "0.1.19" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" +checksum = "854cbdc4f7bc6ae19c820d44abdc3277ac3e1b2b93db20a636825d9322fb60e6" dependencies = [ "tinyvec", ] @@ -3254,5 +3255,5 @@ dependencies = [ "hmac", "pbkdf2", "sha1", - "time 0.3.9", + "time 0.3.11", ] diff --git a/czkawka_core/Cargo.toml b/czkawka_core/Cargo.toml index fced438..3d2cf9a 100644 --- a/czkawka_core/Cargo.toml +++ b/czkawka_core/Cargo.toml @@ -25,7 +25,7 @@ hamming = "0.1.3" # Needed by same music bitflags = "1.3.2" -lofty="0.6.2" +lofty="0.7.0" # Futures - needed by async progress sender futures = "0.3.21" @@ -65,6 +65,8 @@ imagepipe = "0.5.0" mime_guess = "2.0.4" infer = "0.8.0" +num_cpus = "1.13.1" + libheif-rs = { version = "0.15.0", optional = true } anyhow = { version = "1.0.57", optional = true } diff --git a/czkawka_core/src/similar_images.rs b/czkawka_core/src/similar_images.rs index a933f93..4baa430 100644 --- a/czkawka_core/src/similar_images.rs +++ b/czkawka_core/src/similar_images.rs @@ -45,11 +45,6 @@ pub struct ProgressData { pub images_to_check: usize, } -#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Debug, Serialize, Deserialize)] -pub enum Similarity { - Similar(u32), -} - #[derive(Clone, Debug, Serialize, Deserialize)] pub struct FileEntry { pub path: PathBuf, @@ -57,7 +52,7 @@ pub struct FileEntry { pub dimensions: String, pub modified_date: u64, pub hash: Vec, - pub similarity: Similarity, + pub similarity: u32, } /// Used by CLI tool when we cannot use directly values @@ -100,7 +95,7 @@ pub struct SimilarImages { maximal_file_size: u64, image_hashes: HashMap, Vec>, // Hashmap with image hashes and Vector with names of files stopped_search: bool, - similarity: Similarity, + similarity: u32, images_to_check: HashMap, hash_size: u8, hash_alg: HashAlg, @@ -109,7 +104,6 @@ pub struct SimilarImages { delete_outdated_cache: bool, exclude_images_with_same_size: bool, use_reference_folders: bool, - fast_comparing: bool, save_also_as_json: bool, } @@ -144,7 +138,7 @@ impl SimilarImages { maximal_file_size: u64::MAX, image_hashes: Default::default(), stopped_search: false, - similarity: Similarity::Similar(1), + similarity: 0, images_to_check: Default::default(), hash_size: 8, hash_alg: HashAlg::Gradient, @@ -153,7 +147,6 @@ impl SimilarImages { delete_outdated_cache: true, exclude_images_with_same_size: false, use_reference_folders: false, - fast_comparing: false, save_also_as_json: false, } } @@ -183,9 +176,6 @@ impl SimilarImages { self.image_filter = image_filter; } - pub fn set_fast_comparing(&mut self, fast_comparing: bool) { - self.fast_comparing = fast_comparing; - } pub fn set_save_also_as_json(&mut self, save_also_as_json: bool) { self.save_also_as_json = save_also_as_json; } @@ -243,7 +233,7 @@ impl SimilarImages { t => t, }; } - pub fn set_similarity(&mut self, similarity: Similarity) { + pub fn set_similarity(&mut self, similarity: u32) { self.similarity = similarity; } @@ -310,7 +300,7 @@ impl SimilarImages { progress_send .unbounded_send(ProgressData { current_stage: 0, - max_stage: 2, + max_stage: 3, images_checked: atomic_file_counter.load(Ordering::Relaxed) as usize, images_to_check: 0, }) @@ -448,7 +438,7 @@ impl SimilarImages { }, hash: Vec::new(), - similarity: Similarity::Similar(0), + similarity: 0, }; fe_result.push((current_file_name.to_string_lossy().to_string(), fe)); @@ -540,7 +530,7 @@ impl SimilarImages { progress_send .unbounded_send(ProgressData { current_stage: 1, - max_stage: 2, + max_stage: 3, images_checked: atomic_file_counter.load(Ordering::Relaxed) as usize, images_to_check, }) @@ -684,195 +674,268 @@ impl SimilarImages { fn find_similar_hashes(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { let hash_map_modification = SystemTime::now(); - let Similarity::Similar(similarity) = self.similarity; + let tolerance = self.similarity; // Results let mut collected_similar_images: HashMap, Vec> = Default::default(); - let mut temp_hashes = Default::default(); - mem::swap(&mut temp_hashes, &mut self.image_hashes); + let mut all_hashed_images = Default::default(); + mem::swap(&mut all_hashed_images, &mut self.image_hashes); - let mut this_time_check_hashes; // Temporary variable which - let mut master_of_group: HashSet> = Default::default(); // Hashes which are "master of groups", + let all_hashes: Vec<_> = all_hashed_images.keys().collect(); - let mut all_hashes_to_check: HashMap, Vec> = temp_hashes.clone(); // List of all hashes, which are or can be master of group - let mut available_hashes: HashMap, Vec> = Default::default(); // List of hashes which can be used as similar images - for (hash, vec_file_entry) in temp_hashes { - // There exists 2 or more images with same hash - if vec_file_entry.len() >= 2 { - master_of_group.insert(hash.clone()); - collected_similar_images.insert(hash, vec_file_entry); - } else { - self.bktree.add(hash.clone()); - available_hashes.insert(hash, vec_file_entry); + // Checking entries with tolerance 0 is really easy and fast, because only entries with same hashes needs to be checked + if tolerance == 0 { + for (hash, vec_file_entry) in all_hashed_images.clone() { + if vec_file_entry.len() >= 2 { + collected_similar_images.insert(hash, vec_file_entry); + } } - } - - //// PROGRESS THREAD START - let progress_thread_run = Arc::new(AtomicBool::new(true)); - - let atomic_mode_counter = Arc::new(AtomicUsize::new(0)); - - let progress_thread_handle = if let Some(progress_sender) = progress_sender { - let progress_send = progress_sender.clone(); - let progress_thread_run = progress_thread_run.clone(); - let atomic_mode_counter = atomic_mode_counter.clone(); - let all_images = match self.fast_comparing { - false => similarity as usize * available_hashes.len(), - true => available_hashes.len(), - }; - thread::spawn(move || loop { - progress_send - .unbounded_send(ProgressData { - current_stage: 2, - max_stage: 2, - images_checked: atomic_mode_counter.load(Ordering::Relaxed) as usize, - images_to_check: all_images, - }) - .unwrap(); - if !progress_thread_run.load(Ordering::Relaxed) { - break; - } - sleep(Duration::from_millis(LOOP_DURATION as u64)); - }) } else { - thread::spawn(|| {}) - }; - //// PROGRESS THREAD END - if similarity >= 1 { - if self.fast_comparing { - this_time_check_hashes = all_hashes_to_check.clone(); + //// PROGRESS THREAD START + let check_was_stopped = AtomicBool::new(false); // Used for breaking from GUI and ending check thread + let progress_thread_run = Arc::new(AtomicBool::new(true)); + let atomic_mode_counter = Arc::new(AtomicUsize::new(0)); - if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); - return false; - } - - for (hash, mut vec_file_entry) in this_time_check_hashes.into_iter() { - atomic_mode_counter.fetch_add(1, Ordering::Relaxed); - - // It is not available, because in same iteration, was already taken out - if !all_hashes_to_check.contains_key(&hash) { - continue; + let progress_thread_handle = if let Some(progress_sender) = progress_sender { + let progress_send = progress_sender.clone(); + let progress_thread_run = progress_thread_run.clone(); + let atomic_mode_counter = atomic_mode_counter.clone(); + let all_combinations_to_check = all_hashes.len(); + thread::spawn(move || loop { + progress_send + .unbounded_send(ProgressData { + current_stage: 2, + max_stage: 2, + images_checked: atomic_mode_counter.load(Ordering::Relaxed) as usize, + images_to_check: all_combinations_to_check, + }) + .unwrap(); + if !progress_thread_run.load(Ordering::Relaxed) { + break; } - - // Finds hashes with specific distance to original one - let vector_with_found_similar_hashes = self - .bktree - .find(&hash, similarity) - .filter(|(similarity, hash)| *similarity != 0 && available_hashes.contains_key(*hash)) - .collect::>(); - - // Not found any hash with specific distance - if vector_with_found_similar_hashes.is_empty() { - continue; - } - - // Current checked hash isn't in any group of similarity, so we create one, because found similar images - if !master_of_group.contains(&hash) { - master_of_group.insert(hash.clone()); - collected_similar_images.insert(hash.clone(), Vec::new()); - let _ = available_hashes.remove(&hash); // Cannot be used anymore as non master - - collected_similar_images.get_mut(&hash).unwrap().append(&mut vec_file_entry); - - // This shouldn't be executed too much times, so it should be quite fast to check this - if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); - return false; - } - } - - vector_with_found_similar_hashes.iter().for_each(|(similarity, other_hash)| { - let _ = all_hashes_to_check.remove(*other_hash); // Cannot be used anymore as master record - let mut vec_fe = available_hashes.remove(*other_hash).unwrap(); - for fe in &mut vec_fe { - fe.similarity = Similarity::Similar(*similarity) - } - - collected_similar_images.get_mut(&hash).unwrap().append(&mut vec_fe); - }); - } + sleep(Duration::from_millis(LOOP_DURATION as u64)); + }) } else { - for current_similarity in 1..=similarity { - this_time_check_hashes = all_hashes_to_check.clone(); + thread::spawn(|| {}) + }; + //// PROGRESS THREAD END - if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); - return false; - } + for hash in &all_hashes { + self.bktree.add(hash.to_vec()); + } - for (hash, mut vec_file_entry) in this_time_check_hashes.into_iter() { - atomic_mode_counter.fetch_add(1, Ordering::Relaxed); + let number_of_processors = num_cpus::get(); + let chunks: Vec<_> = all_hashes.chunks(all_hashes.len() / number_of_processors).collect(); - // It is not available, because in same iteration, was already taken out - if !all_hashes_to_check.contains_key(&hash) { - continue; + let parts: Vec<_> = chunks + .into_par_iter() + .map(|hashes_to_check| { + let mut hashes_parents: HashMap<&Vec, u32> = Default::default(); // Hash used as parent, childrens + let mut hashes_similarity: HashMap<&Vec, (&Vec, u32)> = Default::default(); // Hash used as child, (parent_hash,similarity) + + // Sprawdź czy hash nie jest użyty jako master gdzie indziej + // Jeśli tak to przejdź do sprawdzania kolejnego elementu + // Zweryfikuj czy sprawdzany element ma rodzica + // Jeśli ma to sprawdź czy similarity nowego rodzica jest mniejsze niż starego + // // Jeśli tak to zmniejsz ilość dzieci starego rodzica, dodaj ilość dzieci w nowym rodzicu i podmień rekord hashes_similarity + // // Jeśli nie to dodaj nowy rekord w hashes_similarity jak i hashes_parents z liczbą dzieci równą 1 + + for (index, hash_to_check) in hashes_to_check.iter().enumerate() { + // Don't check for user stop too often + // Also don't add too ofter data to variables + const CYCLES_COUNTER: usize = 50; + if index % CYCLES_COUNTER == 0 && index != 0 { + atomic_mode_counter.fetch_add(CYCLES_COUNTER, Ordering::Relaxed); + if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { + check_was_stopped.store(true, Ordering::Relaxed); + return None; + } } - // Finds hashes with specific distance to original one - let vector_with_found_similar_hashes = self + let mut found_items = self .bktree - .find(&hash, similarity) - .filter(|(similarity, hash)| (*similarity == current_similarity) && available_hashes.contains_key(*hash)) + .find(hash_to_check, tolerance) + .filter(|(similarity, _hash)| *similarity != 0) .collect::>(); - // Not found any hash with specific distance - if vector_with_found_similar_hashes.is_empty() { - continue; - } + found_items.sort_unstable_by_key(|f| f.0); - // Current checked hash isn't in any group of similarity, so we create one, because found similar images - if !master_of_group.contains(&hash) { - master_of_group.insert(hash.clone()); - collected_similar_images.insert(hash.clone(), Vec::new()); - let _ = available_hashes.remove(&hash); // Cannot be used anymore as non master - - collected_similar_images.get_mut(&hash).unwrap().append(&mut vec_file_entry); - - // This shouldn't be executed too much times, so it should be quite fast to check this - if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); - return false; - } - } - - vector_with_found_similar_hashes.iter().for_each(|(similarity, other_hash)| { - let _ = all_hashes_to_check.remove(*other_hash); // Cannot be used anymore as master record - let mut vec_fe = available_hashes.remove(*other_hash).unwrap(); - for fe in &mut vec_fe { - fe.similarity = Similarity::Similar(*similarity) + for (similarity, other_hash) in found_items { + // SSSTART + // Cannot use hash if already is used as master record(have more than 0 children) + if let Some(children_number) = hashes_parents.get(other_hash) { + if *children_number > 0 { + continue; + } } - collected_similar_images.get_mut(&hash).unwrap().append(&mut vec_fe); - }); + // If there is already record, with smaller sensitivity, then replace it + let mut need_to_add = false; + let mut need_to_check = false; + + // TODO replace variables from above with closures + // If current checked hash, have parent, first we must check if similarity between them is lower than checked item + if let Some((current_parent_hash, current_similarity_with_parent)) = hashes_similarity.get(hash_to_check) { + if *current_similarity_with_parent > similarity { + need_to_check = true; + + *hashes_parents.get_mut(current_parent_hash).unwrap() -= 1; + hashes_similarity.remove(hash_to_check).unwrap(); + } + } else { + need_to_check = true; + } + + if need_to_check { + if let Some((other_parent_hash, other_similarity)) = hashes_similarity.get(other_hash) { + if *other_similarity > similarity { + need_to_add = true; + *hashes_parents.get_mut(other_parent_hash).unwrap() -= 1; + } + } + // But when there is no record, just add it + else { + need_to_add = true + } + } + + if need_to_add { + hashes_similarity.insert(other_hash, (hash_to_check, similarity)); + + if let Some(number_of_children) = hashes_parents.get_mut(hash_to_check) { + *number_of_children += 1; + } else { + hashes_parents.insert(hash_to_check, 1); + } + } + // ENND + } } + + #[cfg(debug_assertions)] + debug_check_for_duplicated_things(hashes_parents.clone(), hashes_similarity.clone(), all_hashed_images.clone(), "BEFORE"); + + Some((hashes_parents, hashes_similarity)) + }) + .while_some() + .collect(); + + // End thread which send info to gui + progress_thread_run.store(false, Ordering::Relaxed); + progress_thread_handle.join().unwrap(); + + if check_was_stopped.load(Ordering::Relaxed) { + return false; + } + + { + let mut hashes_parents: HashMap<&Vec, u32> = Default::default(); + let mut hashes_similarity: HashMap<&Vec, (&Vec, u32)> = Default::default(); + let mut iter = parts.into_iter(); + // At start fill arrays with first item + // Normal algorithm would do exactly same thing, but slower, one record after one + if let Some((first_hashes_parents, first_hashes_similarity)) = iter.next() { + hashes_parents = first_hashes_parents; + hashes_similarity = first_hashes_similarity; + } + + for (_partial_hashes_with_parents, partial_hashes_with_similarity) in iter { + for (hash_to_check, (other_hash, similarity)) in partial_hashes_with_similarity { + // SSSTART + // Cannot use hash if already is used as master record(have more than 0 children) + if let Some(children_number) = hashes_parents.get(other_hash) { + if *children_number > 0 { + continue; + } + } + + // If there is already record, with smaller sensitivity, then replace it + let mut need_to_add = false; + let mut need_to_check = false; + + // TODO replace variables from above with closures + // If current checked hash, have parent, first we must check if similarity between them is lower than checked item + if let Some((current_parent_hash, current_similarity_with_parent)) = hashes_similarity.get(hash_to_check) { + if *current_similarity_with_parent > similarity { + need_to_check = true; + + *hashes_parents.get_mut(current_parent_hash).unwrap() -= 1; + hashes_similarity.remove(hash_to_check).unwrap(); + } + } else { + need_to_check = true; + } + + if need_to_check { + if let Some((other_parent_hash, other_similarity)) = hashes_similarity.get(other_hash) { + if *other_similarity > similarity { + need_to_add = true; + *hashes_parents.get_mut(other_parent_hash).unwrap() -= 1; + } + } + // But when there is no record, just add it + else { + need_to_add = true + } + } + + if need_to_add { + hashes_similarity.insert(other_hash, (hash_to_check, similarity)); + + if let Some(number_of_children) = hashes_parents.get_mut(hash_to_check) { + *number_of_children += 1; + } else { + hashes_parents.insert(hash_to_check, 1); + } + } + // ENND + } + } + + #[cfg(debug_assertions)] + debug_check_for_duplicated_things(hashes_parents.clone(), hashes_similarity.clone(), all_hashed_images.clone(), "LATTER"); + + // Collecting results + + for (parent_hash, child_number) in hashes_parents { + if child_number > 0 { + let vec_fe = all_hashed_images.get(parent_hash).unwrap().clone(); + collected_similar_images.insert(parent_hash.clone(), vec_fe); + } + } + + for (child_hash, (parent_hash, similarity)) in hashes_similarity { + let mut vec_fe = all_hashed_images.get(child_hash).unwrap().clone(); + for mut fe in &mut vec_fe { + fe.similarity = similarity; + } + collected_similar_images.get_mut(parent_hash).unwrap().append(&mut vec_fe); } } } - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); - // Validating if group contains duplicated results #[cfg(debug_assertions)] { let mut result_hashset: HashSet = Default::default(); let mut found = false; for (_hash, vec_file_entry) in collected_similar_images.iter() { + if vec_file_entry.is_empty() { + println!("Empty Element {:?}", vec_file_entry); + found = true; + continue; + } + if vec_file_entry.len() == 1 { + println!("Single Element {:?}", vec_file_entry); + found = true; + continue; + } for file_entry in vec_file_entry { let st = file_entry.path.to_string_lossy().to_string(); if result_hashset.contains(&st) { found = true; - println!("Invalid Element {}", st); + println!("Duplicated Element {}", st); } else { result_hashset.insert(st); } @@ -1148,14 +1211,14 @@ pub fn load_hashes_from_file( fn get_cache_file(hash_size: &u8, hash_alg: &HashAlg, image_filter: &FilterType) -> String { format!( - "cache_similar_images_{}_{}_{}.bin", + "cache_similar_images_{}_{}_{}_50.bin", hash_size, convert_algorithm_to_string(hash_alg), convert_filters_to_string(image_filter), ) } -pub fn get_string_from_similarity(similarity: &Similarity, hash_size: u8) -> String { +pub fn get_string_from_similarity(similarity: &u32, hash_size: u8) -> String { let index_preset = match hash_size { 8 => 0, 16 => 1, @@ -1164,52 +1227,44 @@ pub fn get_string_from_similarity(similarity: &Similarity, hash_size: u8) -> Str _ => panic!(), }; - match similarity { - // Similarity::None => { - // panic!() - // } - Similarity::Similar(h) => { - // #[cfg(debug_assertions)] - // { - // if *h <= SIMILAR_VALUES[index_preset][0] { - // format!("{} {}", flc!("core_similarity_very_high"), *h) - // } else if *h <= SIMILAR_VALUES[index_preset][1] { - // format!("{} {}", flc!("core_similarity_high"), *h) - // } else if *h <= SIMILAR_VALUES[index_preset][2] { - // format!("{} {}", flc!("core_similarity_medium"), *h) - // } else if *h <= SIMILAR_VALUES[index_preset][3] { - // format!("{} {}", flc!("core_similarity_small"), *h) - // } else if *h <= SIMILAR_VALUES[index_preset][4] { - // format!("{} {}", flc!("core_similarity_very_small"), *h) - // } else if *h <= SIMILAR_VALUES[index_preset][5] { - // format!("{} {}", flc!("core_similarity_minimal"), *h) - // } else { - // panic!(); - // } - // } - // #[cfg(not(debug_assertions))] - { - if *h <= SIMILAR_VALUES[index_preset][0] { - flc!("core_similarity_very_high") - } else if *h <= SIMILAR_VALUES[index_preset][1] { - flc!("core_similarity_high") - } else if *h <= SIMILAR_VALUES[index_preset][2] { - flc!("core_similarity_medium") - } else if *h <= SIMILAR_VALUES[index_preset][3] { - flc!("core_similarity_small") - } else if *h <= SIMILAR_VALUES[index_preset][4] { - flc!("core_similarity_very_small") - } else if *h <= SIMILAR_VALUES[index_preset][5] { - flc!("core_similarity_minimal") - } else { - panic!(); - } - } - } + // #[cfg(debug_assertions)] + // { + // if *similarity <= SIMILAR_VALUES[index_preset][0] { + // format!("{} {}", flc!("core_similarity_very_high"), *similarity) + // } else if *similarity <= SIMILAR_VALUES[index_preset][1] { + // format!("{} {}", flc!("core_similarity_high"), *similarity) + // } else if *similarity <= SIMILAR_VALUES[index_preset][2] { + // format!("{} {}", flc!("core_similarity_medium"), *similarity) + // } else if *similarity <= SIMILAR_VALUES[index_preset][3] { + // format!("{} {}", flc!("core_similarity_small"), *similarity) + // } else if *similarity <= SIMILAR_VALUES[index_preset][4] { + // format!("{} {}", flc!("core_similarity_very_small"), *similarity) + // } else if *similarity <= SIMILAR_VALUES[index_preset][5] { + // format!("{} {}", flc!("core_similarity_minimal"), *similarity) + // } else { + // panic!(); + // } + // } + // #[cfg(not(debug_assertions))] + + if *similarity <= SIMILAR_VALUES[index_preset][0] { + flc!("core_similarity_very_high") + } else if *similarity <= SIMILAR_VALUES[index_preset][1] { + flc!("core_similarity_high") + } else if *similarity <= SIMILAR_VALUES[index_preset][2] { + flc!("core_similarity_medium") + } else if *similarity <= SIMILAR_VALUES[index_preset][3] { + flc!("core_similarity_small") + } else if *similarity <= SIMILAR_VALUES[index_preset][4] { + flc!("core_similarity_very_small") + } else if *similarity <= SIMILAR_VALUES[index_preset][5] { + flc!("core_similarity_minimal") + } else { + panic!(); } } -pub fn return_similarity_from_similarity_preset(similarity_preset: &SimilarityPreset, hash_size: u8) -> Similarity { +pub fn return_similarity_from_similarity_preset(similarity_preset: &SimilarityPreset, hash_size: u8) -> u32 { let index_preset = match hash_size { 8 => 0, 16 => 1, @@ -1218,12 +1273,12 @@ pub fn return_similarity_from_similarity_preset(similarity_preset: &SimilarityPr _ => panic!(), }; match similarity_preset { - SimilarityPreset::VeryHigh => Similarity::Similar(SIMILAR_VALUES[index_preset][0]), - SimilarityPreset::High => Similarity::Similar(SIMILAR_VALUES[index_preset][1]), - SimilarityPreset::Medium => Similarity::Similar(SIMILAR_VALUES[index_preset][2]), - SimilarityPreset::Small => Similarity::Similar(SIMILAR_VALUES[index_preset][3]), - SimilarityPreset::VerySmall => Similarity::Similar(SIMILAR_VALUES[index_preset][4]), - SimilarityPreset::Minimal => Similarity::Similar(SIMILAR_VALUES[index_preset][5]), + SimilarityPreset::VeryHigh => SIMILAR_VALUES[index_preset][0], + SimilarityPreset::High => SIMILAR_VALUES[index_preset][1], + SimilarityPreset::Medium => SIMILAR_VALUES[index_preset][2], + SimilarityPreset::Small => SIMILAR_VALUES[index_preset][3], + SimilarityPreset::VerySmall => SIMILAR_VALUES[index_preset][4], + SimilarityPreset::Minimal => SIMILAR_VALUES[index_preset][5], SimilarityPreset::None => panic!(""), } } @@ -1290,3 +1345,44 @@ pub fn test_image_conversion_speed() { } } } + +#[allow(dead_code)] +fn debug_check_for_duplicated_things( + hashes_parents: HashMap<&Vec, u32>, + hashes_similarity: HashMap<&Vec, (&Vec, u32)>, + all_hashed_images: HashMap, Vec>, + numm: &str, +) { + let mut hashmap_hashes: HashSet<_> = Default::default(); + let mut hashmap_names: HashSet<_> = Default::default(); + for (hash, number_of_children) in &hashes_parents { + if *number_of_children > 0 { + if hashmap_hashes.contains(*hash) { + println!("------1--HASH--{} {:?}", numm, all_hashed_images.get(*hash).unwrap()); + } + hashmap_hashes.insert(hash.to_vec()); + + for i in all_hashed_images.get(*hash).unwrap() { + let name = i.path.to_string_lossy().to_string(); + if hashmap_names.contains(&name) { + println!("------1--NAME--{} {:?}", numm, name); + } + hashmap_names.insert(name); + } + } + } + for hash in hashes_similarity.keys() { + if hashmap_hashes.contains(*hash) { + println!("------2--HASH--{} {:?}", numm, all_hashed_images.get(*hash).unwrap()); + } + hashmap_hashes.insert(hash.to_vec()); + + for i in all_hashed_images.get(*hash).unwrap() { + let name = i.path.to_string_lossy().to_string(); + if hashmap_names.contains(&name) { + println!("------2--NAME--{} {:?}", numm, name); + } + hashmap_names.insert(name); + } + } +} diff --git a/czkawka_core/src/similar_videos.rs b/czkawka_core/src/similar_videos.rs index 39591a0..372ace7 100644 --- a/czkawka_core/src/similar_videos.rs +++ b/czkawka_core/src/similar_videos.rs @@ -803,7 +803,7 @@ pub fn load_hashes_from_file(text_messages: &mut Messages, delete_outdated_cache } fn get_cache_file() -> String { - "cache_similar_videos.bin".to_string() + "cache_similar_videos_50.bin".to_string() } pub fn check_if_ffmpeg_is_installed() -> bool { diff --git a/czkawka_gui/i18n/en/czkawka_gui.ftl b/czkawka_gui/i18n/en/czkawka_gui.ftl index 31baf01..72b5407 100644 --- a/czkawka_gui/i18n/en/czkawka_gui.ftl +++ b/czkawka_gui/i18n/en/czkawka_gui.ftl @@ -66,14 +66,6 @@ big_files_mode_label = Checked files big_files_mode_smallest_combo_box = The Smallest big_files_mode_biggest_combo_box = The Biggest -main_notebook_image_fast_compare = Fast compare -main_notebook_image_fast_compare_tooltip = - Speedup searching and comparing hashes. - - As opposed to normal mode - where each hash is compared to each other x times (where x is the similarity the user chose) - in this mode, exactly one comparison will be used. - - This option is recommended when comparing >10000 images with non 0 (Very High) similarity. - main_notebook_duplicates = Duplicate Files main_notebook_empty_directories = Empty Directories main_notebook_big_files = Big Files diff --git a/czkawka_gui/src/compute_results.rs b/czkawka_gui/src/compute_results.rs index 1149b19..0274d5b 100644 --- a/czkawka_gui/src/compute_results.rs +++ b/czkawka_gui/src/compute_results.rs @@ -818,10 +818,8 @@ pub fn connect_compute_results(gui_data: &GuiData, glib_stop_receiver: Receiver< // Sort let vec_file_entry = if vec_file_entry.len() >= 2 { let mut vec_file_entry = vec_file_entry.clone(); - vec_file_entry.sort_by_key(|e| { - let t = split_path(e.path.as_path()); - (t.0, t.1) - }); + // Use comparison by similarity, because it is more important that path here + vec_file_entry.sort_unstable_by_key(|e| e.similarity); vec_file_entry } else { vec_file_entry.clone() @@ -882,10 +880,8 @@ pub fn connect_compute_results(gui_data: &GuiData, glib_stop_receiver: Receiver< // Sort let vec_file_entry = if vec_file_entry.len() >= 2 { let mut vec_file_entry = vec_file_entry.clone(); - vec_file_entry.sort_by_key(|e| { - let t = split_path(e.path.as_path()); - (t.0, t.1) - }); + // Use comparsion by similarity, because it is more important that path here + vec_file_entry.sort_unstable_by_key(|e| e.similarity); vec_file_entry } else { vec_file_entry.clone() diff --git a/czkawka_gui/src/connect_things/connect_button_search.rs b/czkawka_gui/src/connect_things/connect_button_search.rs index 112c624..1e3d568 100644 --- a/czkawka_gui/src/connect_things/connect_button_search.rs +++ b/czkawka_gui/src/connect_things/connect_button_search.rs @@ -113,7 +113,6 @@ pub fn connect_button_search( let button_settings = gui_data.header.button_settings.clone(); let button_app_info = gui_data.header.button_app_info.clone(); let check_button_music_approximate_comparison = gui_data.main_notebook.check_button_music_approximate_comparison.clone(); - let check_button_image_fast_compare = gui_data.main_notebook.check_button_image_fast_compare.clone(); let check_button_settings_save_also_json = gui_data.settings.check_button_settings_save_also_json.clone(); buttons_search_clone.connect_clicked(move |_| { @@ -326,12 +325,10 @@ pub fn connect_button_search( let ignore_same_size = check_button_image_ignore_same_size.is_active(); - let similarity = similar_images::Similarity::Similar(scale_similarity_similar_images.value() as u32); + let similarity = scale_similarity_similar_images.value() as u32; let delete_outdated_cache = check_button_settings_similar_images_delete_outdated_cache.is_active(); - let fast_compare = check_button_image_fast_compare.is_active(); - let futures_sender_similar_images = futures_sender_similar_images.clone(); // Find similar images thread::spawn(move || { @@ -352,7 +349,6 @@ pub fn connect_button_search( sf.set_allowed_extensions(allowed_extensions); sf.set_delete_outdated_cache(delete_outdated_cache); sf.set_exclude_images_with_same_size(ignore_same_size); - sf.set_fast_comparing(fast_compare); sf.set_save_also_as_json(save_also_as_json); sf.find_similar_images(Some(&stop_receiver), Some(&futures_sender_similar_images)); let _ = glib_stop_sender.send(Message::SimilarImages(sf)); diff --git a/czkawka_gui/src/connect_things/connect_similar_image_size_change.rs b/czkawka_gui/src/connect_things/connect_similar_image_size_change.rs index 9b185b5..82e8003 100644 --- a/czkawka_gui/src/connect_things/connect_similar_image_size_change.rs +++ b/czkawka_gui/src/connect_things/connect_similar_image_size_change.rs @@ -1,13 +1,13 @@ use gtk4::prelude::*; -use czkawka_core::similar_images::{get_string_from_similarity, Similarity, SIMILAR_VALUES}; +use czkawka_core::similar_images::{get_string_from_similarity, SIMILAR_VALUES}; use crate::gui_structs::gui_data::GuiData; use crate::help_combo_box::IMAGES_HASH_SIZE_COMBO_BOX; pub fn connect_similar_image_size_change(gui_data: &GuiData) { let label_similar_images_minimal_similarity = gui_data.main_notebook.label_similar_images_minimal_similarity.clone(); - label_similar_images_minimal_similarity.set_text(&get_string_from_similarity(&Similarity::Similar(SIMILAR_VALUES[0][5]), 8)); + label_similar_images_minimal_similarity.set_text(&get_string_from_similarity(&SIMILAR_VALUES[0][5], 8)); let combo_box_image_hash_size = gui_data.main_notebook.combo_box_image_hash_size.clone(); let label_similar_images_minimal_similarity = gui_data.main_notebook.label_similar_images_minimal_similarity.clone(); @@ -26,6 +26,6 @@ pub fn connect_similar_image_size_change(gui_data: &GuiData) { scale_similarity_similar_images.set_range(0_f64, SIMILAR_VALUES[index][5] as f64); scale_similarity_similar_images.set_fill_level(SIMILAR_VALUES[index][5] as f64); - label_similar_images_minimal_similarity.set_text(&get_string_from_similarity(&Similarity::Similar(SIMILAR_VALUES[index][5]), hash_size as u8)); + label_similar_images_minimal_similarity.set_text(&get_string_from_similarity(&SIMILAR_VALUES[index][5], hash_size as u8)); }); } diff --git a/czkawka_gui/src/gui_structs/gui_main_notebook.rs b/czkawka_gui/src/gui_structs/gui_main_notebook.rs index 301692b..6c2e425 100644 --- a/czkawka_gui/src/gui_structs/gui_main_notebook.rs +++ b/czkawka_gui/src/gui_structs/gui_main_notebook.rs @@ -4,7 +4,7 @@ use gtk4::{Builder, CheckButton, ComboBoxText, Entry, EventControllerKey, Gestur use czkawka_core::common_dir_traversal::CheckingMethod; use czkawka_core::localizer_core::{fnc_get_similarity_minimal, fnc_get_similarity_very_high}; -use czkawka_core::similar_images::{get_string_from_similarity, Similarity, SIMILAR_VALUES}; +use czkawka_core::similar_images::{get_string_from_similarity, SIMILAR_VALUES}; use crate::flg; use crate::help_combo_box::{BIG_FILES_CHECK_METHOD_COMBO_BOX, DUPLICATES_CHECK_METHOD_COMBO_BOX, IMAGES_HASH_SIZE_COMBO_BOX}; @@ -94,8 +94,6 @@ pub struct GuiMainNotebook { pub check_button_image_ignore_same_size: CheckButton, pub check_button_video_ignore_same_size: CheckButton, - pub check_button_image_fast_compare: CheckButton, - pub label_image_similarity: Label, pub label_image_similarity_max: Label, @@ -223,8 +221,6 @@ impl GuiMainNotebook { let scale_similarity_similar_images: Scale = builder.object("scale_similarity_similar_images").unwrap(); let scale_similarity_similar_videos: Scale = builder.object("scale_similarity_similar_videos").unwrap(); - let check_button_image_fast_compare: CheckButton = builder.object("check_button_image_fast_compare").unwrap(); - let combo_box_image_resize_algorithm: ComboBoxText = builder.object("combo_box_image_resize_algorithm").unwrap(); let combo_box_image_hash_algorithm: ComboBoxText = builder.object("combo_box_image_hash_algorithm").unwrap(); let combo_box_image_hash_size: ComboBoxText = builder.object("combo_box_image_hash_size").unwrap(); @@ -319,7 +315,6 @@ impl GuiMainNotebook { combo_box_duplicate_hash_type, combo_box_image_hash_size, check_button_video_ignore_same_size, - check_button_image_fast_compare, check_button_duplicate_case_sensitive_name, evk_tree_view_bad_extensions, gc_tree_view_duplicate_finder, @@ -407,10 +402,6 @@ impl GuiMainNotebook { self.check_button_image_ignore_same_size.set_label(Some(&flg!("check_button_general_same_size"))); self.check_button_video_ignore_same_size.set_label(Some(&flg!("check_button_general_same_size"))); - self.check_button_image_fast_compare.set_label(Some(&flg!("main_notebook_image_fast_compare"))); - self.check_button_image_fast_compare - .set_tooltip_text(Some(&flg!("main_notebook_image_fast_compare_tooltip"))); - self.check_button_broken_files_audio.set_label(Some(&flg!("main_check_box_broken_files_audio"))); self.check_button_broken_files_archive.set_label(Some(&flg!("main_check_box_broken_files_archive"))); self.check_button_broken_files_image.set_label(Some(&flg!("main_check_box_broken_files_image"))); @@ -421,20 +412,19 @@ impl GuiMainNotebook { let hash_size = IMAGES_HASH_SIZE_COMBO_BOX[hash_size_index]; match hash_size { 8 => { - self.label_similar_images_minimal_similarity - .set_text(&get_string_from_similarity(&Similarity::Similar(SIMILAR_VALUES[0][5]), 8)); + self.label_similar_images_minimal_similarity.set_text(&get_string_from_similarity(&SIMILAR_VALUES[0][5], 8)); } 16 => { self.label_similar_images_minimal_similarity - .set_text(&get_string_from_similarity(&Similarity::Similar(SIMILAR_VALUES[1][5]), 16)); + .set_text(&get_string_from_similarity(&SIMILAR_VALUES[1][5], 16)); } 32 => { self.label_similar_images_minimal_similarity - .set_text(&get_string_from_similarity(&Similarity::Similar(SIMILAR_VALUES[2][5]), 32)); + .set_text(&get_string_from_similarity(&SIMILAR_VALUES[2][5], 32)); } 64 => { self.label_similar_images_minimal_similarity - .set_text(&get_string_from_similarity(&Similarity::Similar(SIMILAR_VALUES[3][5]), 64)); + .set_text(&get_string_from_similarity(&SIMILAR_VALUES[3][5], 64)); } _ => panic!(), } diff --git a/czkawka_gui/src/saving_loading.rs b/czkawka_gui/src/saving_loading.rs index 9b4efaf..8fe555b 100644 --- a/czkawka_gui/src/saving_loading.rs +++ b/czkawka_gui/src/saving_loading.rs @@ -48,10 +48,9 @@ const DEFAULT_BROKEN_FILES_ARCHIVE: bool = true; const DEFAULT_BROKEN_FILES_IMAGE: bool = true; const DEFAULT_NUMBER_OF_BIGGEST_FILES: &str = "50"; -const DEFAULT_SIMILAR_IMAGES_SIMILARITY: i32 = 0; +const DEFAULT_SIMILAR_IMAGES_SIMILARITY: f32 = 0.0; const DEFAULT_SIMILAR_IMAGES_IGNORE_SAME_SIZE: bool = false; -const DEFAULT_SIMILAR_IMAGES_FAST_COMPARE: bool = false; -const DEFAULT_SIMILAR_VIDEOS_SIMILARITY: i32 = 15; +const DEFAULT_SIMILAR_VIDEOS_SIMILARITY: f32 = 15.0; const DEFAULT_SIMILAR_VIDEOS_IGNORE_SAME_SIZE: bool = false; pub const DEFAULT_MINIMAL_FILE_SIZE: &str = "16384"; @@ -126,7 +125,7 @@ impl LoadSaveStruct { default_value } - pub fn get_integer(&self, key: String, default_value: T) -> T { + pub fn get_object(&self, key: String, default_value: T) -> T { if self.loaded_items.contains_key(&key) { let item = self.loaded_items.get(&key).unwrap().clone().into_iter().filter(|e| !e.is_empty()).collect::>(); @@ -428,7 +427,6 @@ enum LoadText { NumberOfBiggestFiles, SimilarImagesSimilarity, SimilarImagesIgnoreSameSize, - SimilarImagesFastCompare, SimilarVideosSimilarity, SimilarVideosIgnoreSameSize, MusicApproximateComparison, @@ -473,7 +471,6 @@ fn create_hash_map() -> (HashMap, HashMap) { (LoadText::NumberOfBiggestFiles, "number_of_biggest_files"), (LoadText::SimilarImagesSimilarity, "similar_images_similarity"), (LoadText::SimilarImagesIgnoreSameSize, "similar_images_ignore_same_size"), - (LoadText::SimilarImagesFastCompare, "similar_images_fast_compare"), (LoadText::SimilarVideosSimilarity, "similar_videos_similarity"), (LoadText::SimilarVideosIgnoreSameSize, "similar_videos_ignore_same_size"), (LoadText::MusicApproximateComparison, "music_approximate_comparison"), @@ -671,10 +668,6 @@ pub fn save_configuration(manual_execution: bool, upper_notebook: &GuiUpperNoteb hashmap_ls.get(&LoadText::SimilarImagesIgnoreSameSize).unwrap().to_string(), main_notebook.check_button_image_ignore_same_size.is_active(), ); - saving_struct.save_var( - hashmap_ls.get(&LoadText::SimilarImagesFastCompare).unwrap().to_string(), - main_notebook.check_button_image_fast_compare.is_active(), - ); saving_struct.save_var( hashmap_ls.get(&LoadText::SimilarVideosSimilarity).unwrap().to_string(), main_notebook.scale_similarity_similar_videos.value(), @@ -760,37 +753,36 @@ pub fn load_configuration( let cache_minimal_size: String = loaded_entries.get_integer_string(hashmap_ls.get(&LoadText::MinimalCacheSize).unwrap().clone(), DEFAULT_MINIMAL_CACHE_SIZE.to_string()); let short_language = loaded_entries.get_string(hashmap_ls.get(&LoadText::Language).unwrap().clone(), short_language); - let combo_box_duplicate_hash_type = loaded_entries.get_integer(hashmap_ls.get(&LoadText::ComboBoxDuplicateHashType).unwrap().clone(), 0); - let combo_box_duplicate_checking_method = loaded_entries.get_integer(hashmap_ls.get(&LoadText::ComboBoxDuplicateCheckMethod).unwrap().clone(), 0); - let combo_box_image_hash_size = loaded_entries.get_integer(hashmap_ls.get(&LoadText::ComboBoxImageHashSize).unwrap().clone(), 0); - let combo_box_image_hash_algorithm = loaded_entries.get_integer(hashmap_ls.get(&LoadText::ComboBoxImageHashType).unwrap().clone(), 0); - let combo_box_image_resize_algorithm = loaded_entries.get_integer(hashmap_ls.get(&LoadText::ComboBoxImageResizeAlgorithm).unwrap().clone(), 0); - let combo_box_big_files_mode = loaded_entries.get_integer(hashmap_ls.get(&LoadText::ComboBoxBigFiles).unwrap().clone(), 0); + let combo_box_duplicate_hash_type = loaded_entries.get_object(hashmap_ls.get(&LoadText::ComboBoxDuplicateHashType).unwrap().clone(), 0); + let combo_box_duplicate_checking_method = loaded_entries.get_object(hashmap_ls.get(&LoadText::ComboBoxDuplicateCheckMethod).unwrap().clone(), 0); + let combo_box_image_hash_size = loaded_entries.get_object(hashmap_ls.get(&LoadText::ComboBoxImageHashSize).unwrap().clone(), 0); + let combo_box_image_hash_algorithm = loaded_entries.get_object(hashmap_ls.get(&LoadText::ComboBoxImageHashType).unwrap().clone(), 0); + let combo_box_image_resize_algorithm = loaded_entries.get_object(hashmap_ls.get(&LoadText::ComboBoxImageResizeAlgorithm).unwrap().clone(), 0); + let combo_box_big_files_mode = loaded_entries.get_object(hashmap_ls.get(&LoadText::ComboBoxBigFiles).unwrap().clone(), 0); let number_of_biggest_files = loaded_entries.get_integer_string( hashmap_ls.get(&LoadText::NumberOfBiggestFiles).unwrap().clone(), DEFAULT_NUMBER_OF_BIGGEST_FILES.to_string(), ); - let similar_images_similarity = loaded_entries.get_integer(hashmap_ls.get(&LoadText::SimilarImagesSimilarity).unwrap().clone(), DEFAULT_SIMILAR_IMAGES_SIMILARITY); + let similar_images_similarity = loaded_entries.get_object(hashmap_ls.get(&LoadText::SimilarImagesSimilarity).unwrap().clone(), DEFAULT_SIMILAR_IMAGES_SIMILARITY); let similar_images_ignore_same_size = loaded_entries.get_bool( hashmap_ls.get(&LoadText::SimilarImagesIgnoreSameSize).unwrap().clone(), DEFAULT_SIMILAR_IMAGES_IGNORE_SAME_SIZE, ); - let similar_images_fast_compare = loaded_entries.get_bool(hashmap_ls.get(&LoadText::SimilarImagesFastCompare).unwrap().clone(), DEFAULT_SIMILAR_IMAGES_FAST_COMPARE); - let similar_videos_similarity = loaded_entries.get_integer(hashmap_ls.get(&LoadText::SimilarVideosSimilarity).unwrap().clone(), DEFAULT_SIMILAR_VIDEOS_SIMILARITY); + let similar_videos_similarity = loaded_entries.get_object(hashmap_ls.get(&LoadText::SimilarVideosSimilarity).unwrap().clone(), DEFAULT_SIMILAR_VIDEOS_SIMILARITY); let similar_videos_ignore_same_size = loaded_entries.get_bool( hashmap_ls.get(&LoadText::SimilarVideosIgnoreSameSize).unwrap().clone(), DEFAULT_SIMILAR_VIDEOS_IGNORE_SAME_SIZE, ); - let check_button_case_sensitive_name = loaded_entries.get_integer( + let check_button_case_sensitive_name = loaded_entries.get_object( hashmap_ls.get(&LoadText::DuplicateNameCaseSensitive).unwrap().clone(), DEFAULT_DUPLICATE_CASE_SENSITIVE_NAME_CHECKING, ); - let check_button_broken_files_archive = loaded_entries.get_integer(hashmap_ls.get(&LoadText::BrokenFilesArchive).unwrap().clone(), DEFAULT_BROKEN_FILES_ARCHIVE); - let check_button_broken_files_pdf = loaded_entries.get_integer(hashmap_ls.get(&LoadText::BrokenFilesPdf).unwrap().clone(), DEFAULT_BROKEN_FILES_ARCHIVE); - let check_button_broken_files_image = loaded_entries.get_integer(hashmap_ls.get(&LoadText::BrokenFilesImage).unwrap().clone(), DEFAULT_BROKEN_FILES_ARCHIVE); - let check_button_broken_files_audio = loaded_entries.get_integer(hashmap_ls.get(&LoadText::BrokenFilesAudio).unwrap().clone(), DEFAULT_BROKEN_FILES_ARCHIVE); + let check_button_broken_files_archive = loaded_entries.get_object(hashmap_ls.get(&LoadText::BrokenFilesArchive).unwrap().clone(), DEFAULT_BROKEN_FILES_ARCHIVE); + let check_button_broken_files_pdf = loaded_entries.get_object(hashmap_ls.get(&LoadText::BrokenFilesPdf).unwrap().clone(), DEFAULT_BROKEN_FILES_ARCHIVE); + let check_button_broken_files_image = loaded_entries.get_object(hashmap_ls.get(&LoadText::BrokenFilesImage).unwrap().clone(), DEFAULT_BROKEN_FILES_ARCHIVE); + let check_button_broken_files_audio = loaded_entries.get_object(hashmap_ls.get(&LoadText::BrokenFilesAudio).unwrap().clone(), DEFAULT_BROKEN_FILES_ARCHIVE); // Setting data if manual_execution || loading_at_start { @@ -912,7 +904,6 @@ pub fn load_configuration( main_notebook.check_button_duplicate_case_sensitive_name.set_active(check_button_case_sensitive_name); main_notebook.entry_big_files_number.set_text(&number_of_biggest_files); main_notebook.check_button_image_ignore_same_size.set_active(similar_images_ignore_same_size); - main_notebook.check_button_image_fast_compare.set_active(similar_images_fast_compare); main_notebook.check_button_video_ignore_same_size.set_active(similar_videos_ignore_same_size); main_notebook.scale_similarity_similar_videos.set_value(similar_videos_similarity as f64); @@ -1056,7 +1047,6 @@ pub fn reset_configuration(manual_clearing: bool, upper_notebook: &GuiUpperNoteb main_notebook.entry_big_files_number.set_text(DEFAULT_NUMBER_OF_BIGGEST_FILES); main_notebook.scale_similarity_similar_images.set_value(DEFAULT_SIMILAR_IMAGES_SIMILARITY as f64); main_notebook.check_button_image_ignore_same_size.set_active(DEFAULT_SIMILAR_IMAGES_IGNORE_SAME_SIZE); - main_notebook.check_button_image_fast_compare.set_active(DEFAULT_SIMILAR_IMAGES_FAST_COMPARE); main_notebook.check_button_video_ignore_same_size.set_active(DEFAULT_SIMILAR_VIDEOS_IGNORE_SAME_SIZE); main_notebook.scale_similarity_similar_videos.set_value(DEFAULT_SIMILAR_VIDEOS_SIMILARITY as f64); } diff --git a/czkawka_gui/ui/about_dialog.ui b/czkawka_gui/ui/about_dialog.ui index 9f64420..8d76729 100644 --- a/czkawka_gui/ui/about_dialog.ui +++ b/czkawka_gui/ui/about_dialog.ui @@ -1,5 +1,5 @@ - + diff --git a/czkawka_gui/ui/compare_images.ui b/czkawka_gui/ui/compare_images.ui index 735140b..93fa8be 100644 --- a/czkawka_gui/ui/compare_images.ui +++ b/czkawka_gui/ui/compare_images.ui @@ -1,5 +1,5 @@ - + diff --git a/czkawka_gui/ui/czkawka.cmb b/czkawka_gui/ui/czkawka.cmb index 30fb4c0..fdff19c 100755 --- a/czkawka_gui/ui/czkawka.cmb +++ b/czkawka_gui/ui/czkawka.cmb @@ -1,6 +1,6 @@ - + (3,None,"about_dialog.ui","about_dialog.ui",None,None,None,None,None,None), (4,None,"compare_images.ui","compare_images.ui",None,None,None,None,None,None), @@ -136,7 +136,6 @@ (5,98,"GtkScale","scale_similarity_similar_images",95,None,None,None,2), (5,99,"GtkLabel","label_similar_images_minimal_similarity",95,None,None,None,3), (5,100,"GtkCheckButton","check_button_image_ignore_same_size",95,None,None,None,4), - (5,101,"GtkCheckButton","check_button_image_fast_compare",95,None,None,None,5), (5,102,"GtkScrolledWindow","scrolled_window_similar_images_finder",87,None,None,None,2), (5,103,"GtkImage","image_preview_similar_images",86,None,None,None,1), (5,104,"GtkLabel",None,85,None,None,None,None), @@ -558,9 +557,6 @@ (5,100,"GtkCheckButton","label","Ignore same size",1,None,None,None,None), (5,100,"GtkWidget","focusable","1",None,None,None,None,None), (5,100,"GtkWidget","margin-start","7",None,None,None,None,None), - (5,101,"GtkCheckButton","label","Fast compare",1,None,None,None,None), - (5,101,"GtkWidget","focusable","1",None,None,None,None,None), - (5,101,"GtkWidget","margin-start","7",None,None,None,None,None), (5,102,"GtkWidget","focusable","1",None,None,None,None,None), (5,102,"GtkWidget","margin-end","5",None,None,None,None,None), (5,102,"GtkWidget","vexpand","1",None,None,None,None,None), diff --git a/czkawka_gui/ui/main_window.ui b/czkawka_gui/ui/main_window.ui index 9097abc..45dc966 100644 --- a/czkawka_gui/ui/main_window.ui +++ b/czkawka_gui/ui/main_window.ui @@ -1,5 +1,5 @@ - + @@ -578,13 +578,6 @@ 7 - - - 1 - Fast compare - 7 - - diff --git a/czkawka_gui/ui/popover_right_click.ui b/czkawka_gui/ui/popover_right_click.ui index 693263a..8f3bb95 100644 --- a/czkawka_gui/ui/popover_right_click.ui +++ b/czkawka_gui/ui/popover_right_click.ui @@ -1,5 +1,5 @@ - + diff --git a/czkawka_gui/ui/popover_select.ui b/czkawka_gui/ui/popover_select.ui index 110463a..a94e706 100644 --- a/czkawka_gui/ui/popover_select.ui +++ b/czkawka_gui/ui/popover_select.ui @@ -1,5 +1,5 @@ - + diff --git a/czkawka_gui/ui/progress.ui b/czkawka_gui/ui/progress.ui index 79ca55d..ffd8b15 100644 --- a/czkawka_gui/ui/progress.ui +++ b/czkawka_gui/ui/progress.ui @@ -1,5 +1,5 @@ - + diff --git a/czkawka_gui/ui/settings.ui b/czkawka_gui/ui/settings.ui index 5333862..afe7b70 100644 --- a/czkawka_gui/ui/settings.ui +++ b/czkawka_gui/ui/settings.ui @@ -1,5 +1,5 @@ - +