1
0
Fork 0
mirror of synced 2024-04-26 16:52:19 +12:00

Change image hash compare algorithm and add multithreading (#762)

* New image compare algorithm

* Par iter

* Ending words
This commit is contained in:
Rafał Mikrut 2022-07-02 21:30:59 +02:00 committed by GitHub
parent 4765bee87f
commit d1c66fda1b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 415 additions and 363 deletions

115
Cargo.lock generated
View file

@ -68,9 +68,9 @@ dependencies = [
[[package]]
name = "anyhow"
version = "1.0.57"
version = "1.0.58"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc"
checksum = "bb07d2053ccdbe10e2af2995a2f116c1330396493dc1269f6a91d0ae82e19704"
[[package]]
name = "arrayref"
@ -254,9 +254,9 @@ dependencies = [
[[package]]
name = "cairo-rs"
version = "0.15.11"
version = "0.15.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62be3562254e90c1c6050a72aa638f6315593e98c5cdaba9017cedbabf0a5dee"
checksum = "c76ee391b03d35510d9fa917357c7f1855bd9a6659c95a1b392e33f49b3369bc"
dependencies = [
"bitflags",
"cairo-sys-rs",
@ -284,9 +284,9 @@ checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
[[package]]
name = "cfb"
version = "0.7.0"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25f6cc832d96f9961bfe67da666f1fb96387305d385db5222cbb7bce21121016"
checksum = "19ff6ea9647f5b4fb422a553d5b6fe1b398986a6e4f458d1219eb77e0e6e0606"
dependencies = [
"byteorder",
"fnv",
@ -341,9 +341,9 @@ dependencies = [
[[package]]
name = "clap"
version = "3.2.5"
version = "3.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d53da17d37dba964b9b3ecb5c5a1f193a2762c700e6829201e645b9381c99dc7"
checksum = "190814073e85d238f31ff738fcb0bf6910cedeb73376c87cd69291028966fd83"
dependencies = [
"atty",
"bitflags",
@ -358,9 +358,9 @@ dependencies = [
[[package]]
name = "clap_derive"
version = "3.2.5"
version = "3.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c11d40217d16aee8508cc8e5fde8b4ff24639758608e5374e731b53f85749fb9"
checksum = "759bf187376e1afa7b85b959e6a664a3e7a95203415dba952ad19139e798f902"
dependencies = [
"heck",
"proc-macro-error",
@ -371,9 +371,9 @@ dependencies = [
[[package]]
name = "clap_lex"
version = "0.2.2"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5538cd660450ebeb4234cfecf8f2284b844ffc4c50531e66d584ad5b91293613"
checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5"
dependencies = [
"os_str_bytes",
]
@ -445,9 +445,9 @@ dependencies = [
[[package]]
name = "crossbeam-utils"
version = "0.8.9"
version = "0.8.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ff1f980957787286a554052d03c7aee98d99cc32e09f6d45f0a814133c87978"
checksum = "7d82ee10ce34d7bc12c2122495e7593a9c41347ecdd64185af4ecf72cb1a7f83"
dependencies = [
"cfg-if",
"once_cell",
@ -498,6 +498,7 @@ dependencies = [
"libheif-rs",
"lofty",
"mime_guess",
"num_cpus",
"once_cell",
"pdf",
"rawloader",
@ -626,9 +627,9 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
[[package]]
name = "either"
version = "1.6.1"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
checksum = "3f107b87b6afc2a64fd13cac55fe06d6c8859f12d4b14cbcdd2c67d0976781be"
[[package]]
name = "encoding_rs"
@ -990,9 +991,9 @@ dependencies = [
[[package]]
name = "gif"
version = "0.11.3"
version = "0.11.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3a7187e78088aead22ceedeee99779455b23fc231fe13ec443f99bb71694e5b"
checksum = "3edd93c6756b4dfaf2709eafcc345ba2636565295c198a9cfbf75fa5e3e00b06"
dependencies = [
"color_quant",
"weezl",
@ -1000,9 +1001,9 @@ dependencies = [
[[package]]
name = "gio"
version = "0.15.11"
version = "0.15.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f132be35e05d9662b9fa0fee3f349c6621f7782e0105917f4cc73c1bf47eceb"
checksum = "68fdbc90312d462781a395f7a16d96a2b379bb6ef8cd6310a2df272771c4283b"
dependencies = [
"bitflags",
"futures-channel",
@ -1030,9 +1031,9 @@ dependencies = [
[[package]]
name = "glib"
version = "0.15.11"
version = "0.15.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd124026a2fa8c33a3d17a3fe59c103f2d9fa5bd92c19e029e037736729abeab"
checksum = "edb0306fbad0ab5428b0ca674a23893db909a98582969c9b537be4ced78c505d"
dependencies = [
"bitflags",
"futures-channel",
@ -1362,7 +1363,7 @@ dependencies = [
"gif",
"jpeg-decoder 0.2.6",
"num-iter",
"num-rational 0.4.0",
"num-rational 0.4.1",
"num-traits",
"png 0.17.5",
"scoped_threadpool",
@ -1404,9 +1405,9 @@ dependencies = [
[[package]]
name = "indexmap"
version = "1.9.0"
version = "1.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c6392766afd7964e2531940894cffe4bd8d7d17dbc3c1c4857040fd4b33bdb3"
checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e"
dependencies = [
"autocfg",
"hashbrown",
@ -1540,9 +1541,9 @@ dependencies = [
[[package]]
name = "linked-hash-map"
version = "0.5.4"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3"
checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
[[package]]
name = "locale_config"
@ -1569,9 +1570,9 @@ dependencies = [
[[package]]
name = "lofty"
version = "0.6.3"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e09702a8eff21fa1cf105d189d8eea3609d7074ab1503d7c49d5b37b7403a65"
checksum = "3f4e4309226629ef3486925c6a1b346241abeb30758e307b3ca9cfd66587bf4f"
dependencies = [
"base64 0.13.0",
"byteorder",
@ -1732,9 +1733,9 @@ dependencies = [
[[package]]
name = "num-rational"
version = "0.4.0"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d41702bd167c2df5520b384281bc111a4b5efcf7fbc4c9c222c815b07e0a6a6a"
checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0"
dependencies = [
"autocfg",
"num-integer",
@ -1980,18 +1981,18 @@ dependencies = [
[[package]]
name = "pin-project"
version = "1.0.10"
version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e"
checksum = "78203e83c48cffbe01e4a2d35d566ca4de445d79a85372fc64e378bfc812a260"
dependencies = [
"pin-project-internal",
]
[[package]]
name = "pin-project-internal"
version = "1.0.10"
version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb"
checksum = "710faf75e1b33345361201d36d04e98ac1ed8909151a017ed384700836104c74"
dependencies = [
"proc-macro2",
"quote",
@ -2091,9 +2092,9 @@ dependencies = [
[[package]]
name = "proc-macro2"
version = "1.0.39"
version = "1.0.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f"
checksum = "dd96a1e8ed2596c337f8eae5f24924ec83f5ad5ab21ea8e455d3566c69fbcaf7"
dependencies = [
"unicode-ident",
]
@ -2109,9 +2110,9 @@ dependencies = [
[[package]]
name = "quote"
version = "1.0.18"
version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1"
checksum = "3bcdf212e9776fbcb2d23ab029360416bb1706b1aea2d1a5ba002727cbcab804"
dependencies = [
"proc-macro2",
]
@ -2286,7 +2287,7 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
dependencies = [
"semver 1.0.10",
"semver 1.0.12",
]
[[package]]
@ -2378,9 +2379,9 @@ dependencies = [
[[package]]
name = "semver"
version = "1.0.10"
version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a41d061efea015927ac527063765e73601444cdc344ba855bc7bd44578b25e1c"
checksum = "a2333e6df6d6598f2b1974829f853c2b4c5f4a6e503c10af918081aa6f8564e1"
[[package]]
name = "semver-parser"
@ -2393,18 +2394,18 @@ dependencies = [
[[package]]
name = "serde"
version = "1.0.137"
version = "1.0.138"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1"
checksum = "1578c6245786b9d168c5447eeacfb96856573ca56c9d68fdcf394be134882a47"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.137"
version = "1.0.138"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be"
checksum = "023e9b1467aef8a10fb88f25611870ada9800ef7e22afce356bb0d2387b6f27c"
dependencies = [
"proc-macro2",
"quote",
@ -2413,9 +2414,9 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.81"
version = "1.0.82"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c"
checksum = "82c2c1fdcd807d1098552c5b9a36e425e42e9fbd7c6a37a8425f390f781f7fa7"
dependencies = [
"itoa",
"ryu",
@ -2477,9 +2478,9 @@ checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32"
[[package]]
name = "smallvec"
version = "1.8.0"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83"
checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1"
[[package]]
name = "snafu"
@ -2713,9 +2714,9 @@ dependencies = [
[[package]]
name = "syn"
version = "1.0.96"
version = "1.0.98"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0748dd251e24453cb8717f0354206b91557e4ec8703673a4b30208f2abaf1ebf"
checksum = "c50aef8a904de4c23c788f104b7dddc7d6f79c647c7c8ce4cc8f73eb0ca773dd"
dependencies = [
"proc-macro2",
"quote",
@ -2828,9 +2829,9 @@ dependencies = [
[[package]]
name = "time"
version = "0.3.9"
version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd"
checksum = "72c91f41dcb2f096c05f0873d667dceec1087ce5bcf984ec8ffb19acddbb3217"
dependencies = [
"itoa",
"libc",
@ -2969,9 +2970,9 @@ checksum = "5bd2fe26506023ed7b5e1e315add59d6f584c621d037f9368fea9cfb988f368c"
[[package]]
name = "unicode-normalization"
version = "0.1.19"
version = "0.1.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9"
checksum = "854cbdc4f7bc6ae19c820d44abdc3277ac3e1b2b93db20a636825d9322fb60e6"
dependencies = [
"tinyvec",
]
@ -3254,5 +3255,5 @@ dependencies = [
"hmac",
"pbkdf2",
"sha1",
"time 0.3.9",
"time 0.3.11",
]

View file

@ -25,7 +25,7 @@ hamming = "0.1.3"
# Needed by same music
bitflags = "1.3.2"
lofty="0.6.2"
lofty="0.7.0"
# Futures - needed by async progress sender
futures = "0.3.21"
@ -65,6 +65,8 @@ imagepipe = "0.5.0"
mime_guess = "2.0.4"
infer = "0.8.0"
num_cpus = "1.13.1"
libheif-rs = { version = "0.15.0", optional = true }
anyhow = { version = "1.0.57", optional = true }

View file

@ -45,11 +45,6 @@ pub struct ProgressData {
pub images_to_check: usize,
}
#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Debug, Serialize, Deserialize)]
pub enum Similarity {
Similar(u32),
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct FileEntry {
pub path: PathBuf,
@ -57,7 +52,7 @@ pub struct FileEntry {
pub dimensions: String,
pub modified_date: u64,
pub hash: Vec<u8>,
pub similarity: Similarity,
pub similarity: u32,
}
/// Used by CLI tool when we cannot use directly values
@ -100,7 +95,7 @@ pub struct SimilarImages {
maximal_file_size: u64,
image_hashes: HashMap<Vec<u8>, Vec<FileEntry>>, // Hashmap with image hashes and Vector with names of files
stopped_search: bool,
similarity: Similarity,
similarity: u32,
images_to_check: HashMap<String, FileEntry>,
hash_size: u8,
hash_alg: HashAlg,
@ -109,7 +104,6 @@ pub struct SimilarImages {
delete_outdated_cache: bool,
exclude_images_with_same_size: bool,
use_reference_folders: bool,
fast_comparing: bool,
save_also_as_json: bool,
}
@ -144,7 +138,7 @@ impl SimilarImages {
maximal_file_size: u64::MAX,
image_hashes: Default::default(),
stopped_search: false,
similarity: Similarity::Similar(1),
similarity: 0,
images_to_check: Default::default(),
hash_size: 8,
hash_alg: HashAlg::Gradient,
@ -153,7 +147,6 @@ impl SimilarImages {
delete_outdated_cache: true,
exclude_images_with_same_size: false,
use_reference_folders: false,
fast_comparing: false,
save_also_as_json: false,
}
}
@ -183,9 +176,6 @@ impl SimilarImages {
self.image_filter = image_filter;
}
pub fn set_fast_comparing(&mut self, fast_comparing: bool) {
self.fast_comparing = fast_comparing;
}
pub fn set_save_also_as_json(&mut self, save_also_as_json: bool) {
self.save_also_as_json = save_also_as_json;
}
@ -243,7 +233,7 @@ impl SimilarImages {
t => t,
};
}
pub fn set_similarity(&mut self, similarity: Similarity) {
pub fn set_similarity(&mut self, similarity: u32) {
self.similarity = similarity;
}
@ -310,7 +300,7 @@ impl SimilarImages {
progress_send
.unbounded_send(ProgressData {
current_stage: 0,
max_stage: 2,
max_stage: 3,
images_checked: atomic_file_counter.load(Ordering::Relaxed) as usize,
images_to_check: 0,
})
@ -448,7 +438,7 @@ impl SimilarImages {
},
hash: Vec::new(),
similarity: Similarity::Similar(0),
similarity: 0,
};
fe_result.push((current_file_name.to_string_lossy().to_string(), fe));
@ -540,7 +530,7 @@ impl SimilarImages {
progress_send
.unbounded_send(ProgressData {
current_stage: 1,
max_stage: 2,
max_stage: 3,
images_checked: atomic_file_counter.load(Ordering::Relaxed) as usize,
images_to_check,
})
@ -684,195 +674,268 @@ impl SimilarImages {
fn find_similar_hashes(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender<ProgressData>>) -> bool {
let hash_map_modification = SystemTime::now();
let Similarity::Similar(similarity) = self.similarity;
let tolerance = self.similarity;
// Results
let mut collected_similar_images: HashMap<Vec<u8>, Vec<FileEntry>> = Default::default();
let mut temp_hashes = Default::default();
mem::swap(&mut temp_hashes, &mut self.image_hashes);
let mut all_hashed_images = Default::default();
mem::swap(&mut all_hashed_images, &mut self.image_hashes);
let mut this_time_check_hashes; // Temporary variable which
let mut master_of_group: HashSet<Vec<u8>> = Default::default(); // Hashes which are "master of groups",
let all_hashes: Vec<_> = all_hashed_images.keys().collect();
let mut all_hashes_to_check: HashMap<Vec<u8>, Vec<FileEntry>> = temp_hashes.clone(); // List of all hashes, which are or can be master of group
let mut available_hashes: HashMap<Vec<u8>, Vec<FileEntry>> = Default::default(); // List of hashes which can be used as similar images
for (hash, vec_file_entry) in temp_hashes {
// There exists 2 or more images with same hash
if vec_file_entry.len() >= 2 {
master_of_group.insert(hash.clone());
collected_similar_images.insert(hash, vec_file_entry);
} else {
self.bktree.add(hash.clone());
available_hashes.insert(hash, vec_file_entry);
// Checking entries with tolerance 0 is really easy and fast, because only entries with same hashes needs to be checked
if tolerance == 0 {
for (hash, vec_file_entry) in all_hashed_images.clone() {
if vec_file_entry.len() >= 2 {
collected_similar_images.insert(hash, vec_file_entry);
}
}
}
//// PROGRESS THREAD START
let progress_thread_run = Arc::new(AtomicBool::new(true));
let atomic_mode_counter = Arc::new(AtomicUsize::new(0));
let progress_thread_handle = if let Some(progress_sender) = progress_sender {
let progress_send = progress_sender.clone();
let progress_thread_run = progress_thread_run.clone();
let atomic_mode_counter = atomic_mode_counter.clone();
let all_images = match self.fast_comparing {
false => similarity as usize * available_hashes.len(),
true => available_hashes.len(),
};
thread::spawn(move || loop {
progress_send
.unbounded_send(ProgressData {
current_stage: 2,
max_stage: 2,
images_checked: atomic_mode_counter.load(Ordering::Relaxed) as usize,
images_to_check: all_images,
})
.unwrap();
if !progress_thread_run.load(Ordering::Relaxed) {
break;
}
sleep(Duration::from_millis(LOOP_DURATION as u64));
})
} else {
thread::spawn(|| {})
};
//// PROGRESS THREAD END
if similarity >= 1 {
if self.fast_comparing {
this_time_check_hashes = all_hashes_to_check.clone();
//// PROGRESS THREAD START
let check_was_stopped = AtomicBool::new(false); // Used for breaking from GUI and ending check thread
let progress_thread_run = Arc::new(AtomicBool::new(true));
let atomic_mode_counter = Arc::new(AtomicUsize::new(0));
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
return false;
}
for (hash, mut vec_file_entry) in this_time_check_hashes.into_iter() {
atomic_mode_counter.fetch_add(1, Ordering::Relaxed);
// It is not available, because in same iteration, was already taken out
if !all_hashes_to_check.contains_key(&hash) {
continue;
let progress_thread_handle = if let Some(progress_sender) = progress_sender {
let progress_send = progress_sender.clone();
let progress_thread_run = progress_thread_run.clone();
let atomic_mode_counter = atomic_mode_counter.clone();
let all_combinations_to_check = all_hashes.len();
thread::spawn(move || loop {
progress_send
.unbounded_send(ProgressData {
current_stage: 2,
max_stage: 2,
images_checked: atomic_mode_counter.load(Ordering::Relaxed) as usize,
images_to_check: all_combinations_to_check,
})
.unwrap();
if !progress_thread_run.load(Ordering::Relaxed) {
break;
}
// Finds hashes with specific distance to original one
let vector_with_found_similar_hashes = self
.bktree
.find(&hash, similarity)
.filter(|(similarity, hash)| *similarity != 0 && available_hashes.contains_key(*hash))
.collect::<Vec<_>>();
// Not found any hash with specific distance
if vector_with_found_similar_hashes.is_empty() {
continue;
}
// Current checked hash isn't in any group of similarity, so we create one, because found similar images
if !master_of_group.contains(&hash) {
master_of_group.insert(hash.clone());
collected_similar_images.insert(hash.clone(), Vec::new());
let _ = available_hashes.remove(&hash); // Cannot be used anymore as non master
collected_similar_images.get_mut(&hash).unwrap().append(&mut vec_file_entry);
// This shouldn't be executed too much times, so it should be quite fast to check this
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
return false;
}
}
vector_with_found_similar_hashes.iter().for_each(|(similarity, other_hash)| {
let _ = all_hashes_to_check.remove(*other_hash); // Cannot be used anymore as master record
let mut vec_fe = available_hashes.remove(*other_hash).unwrap();
for fe in &mut vec_fe {
fe.similarity = Similarity::Similar(*similarity)
}
collected_similar_images.get_mut(&hash).unwrap().append(&mut vec_fe);
});
}
sleep(Duration::from_millis(LOOP_DURATION as u64));
})
} else {
for current_similarity in 1..=similarity {
this_time_check_hashes = all_hashes_to_check.clone();
thread::spawn(|| {})
};
//// PROGRESS THREAD END
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
return false;
}
for hash in &all_hashes {
self.bktree.add(hash.to_vec());
}
for (hash, mut vec_file_entry) in this_time_check_hashes.into_iter() {
atomic_mode_counter.fetch_add(1, Ordering::Relaxed);
let number_of_processors = num_cpus::get();
let chunks: Vec<_> = all_hashes.chunks(all_hashes.len() / number_of_processors).collect();
// It is not available, because in same iteration, was already taken out
if !all_hashes_to_check.contains_key(&hash) {
continue;
let parts: Vec<_> = chunks
.into_par_iter()
.map(|hashes_to_check| {
let mut hashes_parents: HashMap<&Vec<u8>, u32> = Default::default(); // Hash used as parent, childrens
let mut hashes_similarity: HashMap<&Vec<u8>, (&Vec<u8>, u32)> = Default::default(); // Hash used as child, (parent_hash,similarity)
// Sprawdź czy hash nie jest użyty jako master gdzie indziej
// Jeśli tak to przejdź do sprawdzania kolejnego elementu
// Zweryfikuj czy sprawdzany element ma rodzica
// Jeśli ma to sprawdź czy similarity nowego rodzica jest mniejsze niż starego
// // Jeśli tak to zmniejsz ilość dzieci starego rodzica, dodaj ilość dzieci w nowym rodzicu i podmień rekord hashes_similarity
// // Jeśli nie to dodaj nowy rekord w hashes_similarity jak i hashes_parents z liczbą dzieci równą 1
for (index, hash_to_check) in hashes_to_check.iter().enumerate() {
// Don't check for user stop too often
// Also don't add too ofter data to variables
const CYCLES_COUNTER: usize = 50;
if index % CYCLES_COUNTER == 0 && index != 0 {
atomic_mode_counter.fetch_add(CYCLES_COUNTER, Ordering::Relaxed);
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
check_was_stopped.store(true, Ordering::Relaxed);
return None;
}
}
// Finds hashes with specific distance to original one
let vector_with_found_similar_hashes = self
let mut found_items = self
.bktree
.find(&hash, similarity)
.filter(|(similarity, hash)| (*similarity == current_similarity) && available_hashes.contains_key(*hash))
.find(hash_to_check, tolerance)
.filter(|(similarity, _hash)| *similarity != 0)
.collect::<Vec<_>>();
// Not found any hash with specific distance
if vector_with_found_similar_hashes.is_empty() {
continue;
}
found_items.sort_unstable_by_key(|f| f.0);
// Current checked hash isn't in any group of similarity, so we create one, because found similar images
if !master_of_group.contains(&hash) {
master_of_group.insert(hash.clone());
collected_similar_images.insert(hash.clone(), Vec::new());
let _ = available_hashes.remove(&hash); // Cannot be used anymore as non master
collected_similar_images.get_mut(&hash).unwrap().append(&mut vec_file_entry);
// This shouldn't be executed too much times, so it should be quite fast to check this
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
return false;
}
}
vector_with_found_similar_hashes.iter().for_each(|(similarity, other_hash)| {
let _ = all_hashes_to_check.remove(*other_hash); // Cannot be used anymore as master record
let mut vec_fe = available_hashes.remove(*other_hash).unwrap();
for fe in &mut vec_fe {
fe.similarity = Similarity::Similar(*similarity)
for (similarity, other_hash) in found_items {
// SSSTART
// Cannot use hash if already is used as master record(have more than 0 children)
if let Some(children_number) = hashes_parents.get(other_hash) {
if *children_number > 0 {
continue;
}
}
collected_similar_images.get_mut(&hash).unwrap().append(&mut vec_fe);
});
// If there is already record, with smaller sensitivity, then replace it
let mut need_to_add = false;
let mut need_to_check = false;
// TODO replace variables from above with closures
// If current checked hash, have parent, first we must check if similarity between them is lower than checked item
if let Some((current_parent_hash, current_similarity_with_parent)) = hashes_similarity.get(hash_to_check) {
if *current_similarity_with_parent > similarity {
need_to_check = true;
*hashes_parents.get_mut(current_parent_hash).unwrap() -= 1;
hashes_similarity.remove(hash_to_check).unwrap();
}
} else {
need_to_check = true;
}
if need_to_check {
if let Some((other_parent_hash, other_similarity)) = hashes_similarity.get(other_hash) {
if *other_similarity > similarity {
need_to_add = true;
*hashes_parents.get_mut(other_parent_hash).unwrap() -= 1;
}
}
// But when there is no record, just add it
else {
need_to_add = true
}
}
if need_to_add {
hashes_similarity.insert(other_hash, (hash_to_check, similarity));
if let Some(number_of_children) = hashes_parents.get_mut(hash_to_check) {
*number_of_children += 1;
} else {
hashes_parents.insert(hash_to_check, 1);
}
}
// ENND
}
}
#[cfg(debug_assertions)]
debug_check_for_duplicated_things(hashes_parents.clone(), hashes_similarity.clone(), all_hashed_images.clone(), "BEFORE");
Some((hashes_parents, hashes_similarity))
})
.while_some()
.collect();
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
if check_was_stopped.load(Ordering::Relaxed) {
return false;
}
{
let mut hashes_parents: HashMap<&Vec<u8>, u32> = Default::default();
let mut hashes_similarity: HashMap<&Vec<u8>, (&Vec<u8>, u32)> = Default::default();
let mut iter = parts.into_iter();
// At start fill arrays with first item
// Normal algorithm would do exactly same thing, but slower, one record after one
if let Some((first_hashes_parents, first_hashes_similarity)) = iter.next() {
hashes_parents = first_hashes_parents;
hashes_similarity = first_hashes_similarity;
}
for (_partial_hashes_with_parents, partial_hashes_with_similarity) in iter {
for (hash_to_check, (other_hash, similarity)) in partial_hashes_with_similarity {
// SSSTART
// Cannot use hash if already is used as master record(have more than 0 children)
if let Some(children_number) = hashes_parents.get(other_hash) {
if *children_number > 0 {
continue;
}
}
// If there is already record, with smaller sensitivity, then replace it
let mut need_to_add = false;
let mut need_to_check = false;
// TODO replace variables from above with closures
// If current checked hash, have parent, first we must check if similarity between them is lower than checked item
if let Some((current_parent_hash, current_similarity_with_parent)) = hashes_similarity.get(hash_to_check) {
if *current_similarity_with_parent > similarity {
need_to_check = true;
*hashes_parents.get_mut(current_parent_hash).unwrap() -= 1;
hashes_similarity.remove(hash_to_check).unwrap();
}
} else {
need_to_check = true;
}
if need_to_check {
if let Some((other_parent_hash, other_similarity)) = hashes_similarity.get(other_hash) {
if *other_similarity > similarity {
need_to_add = true;
*hashes_parents.get_mut(other_parent_hash).unwrap() -= 1;
}
}
// But when there is no record, just add it
else {
need_to_add = true
}
}
if need_to_add {
hashes_similarity.insert(other_hash, (hash_to_check, similarity));
if let Some(number_of_children) = hashes_parents.get_mut(hash_to_check) {
*number_of_children += 1;
} else {
hashes_parents.insert(hash_to_check, 1);
}
}
// ENND
}
}
#[cfg(debug_assertions)]
debug_check_for_duplicated_things(hashes_parents.clone(), hashes_similarity.clone(), all_hashed_images.clone(), "LATTER");
// Collecting results
for (parent_hash, child_number) in hashes_parents {
if child_number > 0 {
let vec_fe = all_hashed_images.get(parent_hash).unwrap().clone();
collected_similar_images.insert(parent_hash.clone(), vec_fe);
}
}
for (child_hash, (parent_hash, similarity)) in hashes_similarity {
let mut vec_fe = all_hashed_images.get(child_hash).unwrap().clone();
for mut fe in &mut vec_fe {
fe.similarity = similarity;
}
collected_similar_images.get_mut(parent_hash).unwrap().append(&mut vec_fe);
}
}
}
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
// Validating if group contains duplicated results
#[cfg(debug_assertions)]
{
let mut result_hashset: HashSet<String> = Default::default();
let mut found = false;
for (_hash, vec_file_entry) in collected_similar_images.iter() {
if vec_file_entry.is_empty() {
println!("Empty Element {:?}", vec_file_entry);
found = true;
continue;
}
if vec_file_entry.len() == 1 {
println!("Single Element {:?}", vec_file_entry);
found = true;
continue;
}
for file_entry in vec_file_entry {
let st = file_entry.path.to_string_lossy().to_string();
if result_hashset.contains(&st) {
found = true;
println!("Invalid Element {}", st);
println!("Duplicated Element {}", st);
} else {
result_hashset.insert(st);
}
@ -1148,14 +1211,14 @@ pub fn load_hashes_from_file(
fn get_cache_file(hash_size: &u8, hash_alg: &HashAlg, image_filter: &FilterType) -> String {
format!(
"cache_similar_images_{}_{}_{}.bin",
"cache_similar_images_{}_{}_{}_50.bin",
hash_size,
convert_algorithm_to_string(hash_alg),
convert_filters_to_string(image_filter),
)
}
pub fn get_string_from_similarity(similarity: &Similarity, hash_size: u8) -> String {
pub fn get_string_from_similarity(similarity: &u32, hash_size: u8) -> String {
let index_preset = match hash_size {
8 => 0,
16 => 1,
@ -1164,52 +1227,44 @@ pub fn get_string_from_similarity(similarity: &Similarity, hash_size: u8) -> Str
_ => panic!(),
};
match similarity {
// Similarity::None => {
// panic!()
// }
Similarity::Similar(h) => {
// #[cfg(debug_assertions)]
// {
// if *h <= SIMILAR_VALUES[index_preset][0] {
// format!("{} {}", flc!("core_similarity_very_high"), *h)
// } else if *h <= SIMILAR_VALUES[index_preset][1] {
// format!("{} {}", flc!("core_similarity_high"), *h)
// } else if *h <= SIMILAR_VALUES[index_preset][2] {
// format!("{} {}", flc!("core_similarity_medium"), *h)
// } else if *h <= SIMILAR_VALUES[index_preset][3] {
// format!("{} {}", flc!("core_similarity_small"), *h)
// } else if *h <= SIMILAR_VALUES[index_preset][4] {
// format!("{} {}", flc!("core_similarity_very_small"), *h)
// } else if *h <= SIMILAR_VALUES[index_preset][5] {
// format!("{} {}", flc!("core_similarity_minimal"), *h)
// } else {
// panic!();
// }
// }
// #[cfg(not(debug_assertions))]
{
if *h <= SIMILAR_VALUES[index_preset][0] {
flc!("core_similarity_very_high")
} else if *h <= SIMILAR_VALUES[index_preset][1] {
flc!("core_similarity_high")
} else if *h <= SIMILAR_VALUES[index_preset][2] {
flc!("core_similarity_medium")
} else if *h <= SIMILAR_VALUES[index_preset][3] {
flc!("core_similarity_small")
} else if *h <= SIMILAR_VALUES[index_preset][4] {
flc!("core_similarity_very_small")
} else if *h <= SIMILAR_VALUES[index_preset][5] {
flc!("core_similarity_minimal")
} else {
panic!();
}
}
}
// #[cfg(debug_assertions)]
// {
// if *similarity <= SIMILAR_VALUES[index_preset][0] {
// format!("{} {}", flc!("core_similarity_very_high"), *similarity)
// } else if *similarity <= SIMILAR_VALUES[index_preset][1] {
// format!("{} {}", flc!("core_similarity_high"), *similarity)
// } else if *similarity <= SIMILAR_VALUES[index_preset][2] {
// format!("{} {}", flc!("core_similarity_medium"), *similarity)
// } else if *similarity <= SIMILAR_VALUES[index_preset][3] {
// format!("{} {}", flc!("core_similarity_small"), *similarity)
// } else if *similarity <= SIMILAR_VALUES[index_preset][4] {
// format!("{} {}", flc!("core_similarity_very_small"), *similarity)
// } else if *similarity <= SIMILAR_VALUES[index_preset][5] {
// format!("{} {}", flc!("core_similarity_minimal"), *similarity)
// } else {
// panic!();
// }
// }
// #[cfg(not(debug_assertions))]
if *similarity <= SIMILAR_VALUES[index_preset][0] {
flc!("core_similarity_very_high")
} else if *similarity <= SIMILAR_VALUES[index_preset][1] {
flc!("core_similarity_high")
} else if *similarity <= SIMILAR_VALUES[index_preset][2] {
flc!("core_similarity_medium")
} else if *similarity <= SIMILAR_VALUES[index_preset][3] {
flc!("core_similarity_small")
} else if *similarity <= SIMILAR_VALUES[index_preset][4] {
flc!("core_similarity_very_small")
} else if *similarity <= SIMILAR_VALUES[index_preset][5] {
flc!("core_similarity_minimal")
} else {
panic!();
}
}
pub fn return_similarity_from_similarity_preset(similarity_preset: &SimilarityPreset, hash_size: u8) -> Similarity {
pub fn return_similarity_from_similarity_preset(similarity_preset: &SimilarityPreset, hash_size: u8) -> u32 {
let index_preset = match hash_size {
8 => 0,
16 => 1,
@ -1218,12 +1273,12 @@ pub fn return_similarity_from_similarity_preset(similarity_preset: &SimilarityPr
_ => panic!(),
};
match similarity_preset {
SimilarityPreset::VeryHigh => Similarity::Similar(SIMILAR_VALUES[index_preset][0]),
SimilarityPreset::High => Similarity::Similar(SIMILAR_VALUES[index_preset][1]),
SimilarityPreset::Medium => Similarity::Similar(SIMILAR_VALUES[index_preset][2]),
SimilarityPreset::Small => Similarity::Similar(SIMILAR_VALUES[index_preset][3]),
SimilarityPreset::VerySmall => Similarity::Similar(SIMILAR_VALUES[index_preset][4]),
SimilarityPreset::Minimal => Similarity::Similar(SIMILAR_VALUES[index_preset][5]),
SimilarityPreset::VeryHigh => SIMILAR_VALUES[index_preset][0],
SimilarityPreset::High => SIMILAR_VALUES[index_preset][1],
SimilarityPreset::Medium => SIMILAR_VALUES[index_preset][2],
SimilarityPreset::Small => SIMILAR_VALUES[index_preset][3],
SimilarityPreset::VerySmall => SIMILAR_VALUES[index_preset][4],
SimilarityPreset::Minimal => SIMILAR_VALUES[index_preset][5],
SimilarityPreset::None => panic!(""),
}
}
@ -1290,3 +1345,44 @@ pub fn test_image_conversion_speed() {
}
}
}
#[allow(dead_code)]
fn debug_check_for_duplicated_things(
hashes_parents: HashMap<&Vec<u8>, u32>,
hashes_similarity: HashMap<&Vec<u8>, (&Vec<u8>, u32)>,
all_hashed_images: HashMap<Vec<u8>, Vec<FileEntry>>,
numm: &str,
) {
let mut hashmap_hashes: HashSet<_> = Default::default();
let mut hashmap_names: HashSet<_> = Default::default();
for (hash, number_of_children) in &hashes_parents {
if *number_of_children > 0 {
if hashmap_hashes.contains(*hash) {
println!("------1--HASH--{} {:?}", numm, all_hashed_images.get(*hash).unwrap());
}
hashmap_hashes.insert(hash.to_vec());
for i in all_hashed_images.get(*hash).unwrap() {
let name = i.path.to_string_lossy().to_string();
if hashmap_names.contains(&name) {
println!("------1--NAME--{} {:?}", numm, name);
}
hashmap_names.insert(name);
}
}
}
for hash in hashes_similarity.keys() {
if hashmap_hashes.contains(*hash) {
println!("------2--HASH--{} {:?}", numm, all_hashed_images.get(*hash).unwrap());
}
hashmap_hashes.insert(hash.to_vec());
for i in all_hashed_images.get(*hash).unwrap() {
let name = i.path.to_string_lossy().to_string();
if hashmap_names.contains(&name) {
println!("------2--NAME--{} {:?}", numm, name);
}
hashmap_names.insert(name);
}
}
}

View file

@ -803,7 +803,7 @@ pub fn load_hashes_from_file(text_messages: &mut Messages, delete_outdated_cache
}
fn get_cache_file() -> String {
"cache_similar_videos.bin".to_string()
"cache_similar_videos_50.bin".to_string()
}
pub fn check_if_ffmpeg_is_installed() -> bool {

View file

@ -66,14 +66,6 @@ big_files_mode_label = Checked files
big_files_mode_smallest_combo_box = The Smallest
big_files_mode_biggest_combo_box = The Biggest
main_notebook_image_fast_compare = Fast compare
main_notebook_image_fast_compare_tooltip =
Speedup searching and comparing hashes.
As opposed to normal mode - where each hash is compared to each other x times (where x is the similarity the user chose) - in this mode, exactly one comparison will be used.
This option is recommended when comparing >10000 images with non 0 (Very High) similarity.
main_notebook_duplicates = Duplicate Files
main_notebook_empty_directories = Empty Directories
main_notebook_big_files = Big Files

View file

@ -818,10 +818,8 @@ pub fn connect_compute_results(gui_data: &GuiData, glib_stop_receiver: Receiver<
// Sort
let vec_file_entry = if vec_file_entry.len() >= 2 {
let mut vec_file_entry = vec_file_entry.clone();
vec_file_entry.sort_by_key(|e| {
let t = split_path(e.path.as_path());
(t.0, t.1)
});
// Use comparison by similarity, because it is more important that path here
vec_file_entry.sort_unstable_by_key(|e| e.similarity);
vec_file_entry
} else {
vec_file_entry.clone()
@ -882,10 +880,8 @@ pub fn connect_compute_results(gui_data: &GuiData, glib_stop_receiver: Receiver<
// Sort
let vec_file_entry = if vec_file_entry.len() >= 2 {
let mut vec_file_entry = vec_file_entry.clone();
vec_file_entry.sort_by_key(|e| {
let t = split_path(e.path.as_path());
(t.0, t.1)
});
// Use comparsion by similarity, because it is more important that path here
vec_file_entry.sort_unstable_by_key(|e| e.similarity);
vec_file_entry
} else {
vec_file_entry.clone()

View file

@ -113,7 +113,6 @@ pub fn connect_button_search(
let button_settings = gui_data.header.button_settings.clone();
let button_app_info = gui_data.header.button_app_info.clone();
let check_button_music_approximate_comparison = gui_data.main_notebook.check_button_music_approximate_comparison.clone();
let check_button_image_fast_compare = gui_data.main_notebook.check_button_image_fast_compare.clone();
let check_button_settings_save_also_json = gui_data.settings.check_button_settings_save_also_json.clone();
buttons_search_clone.connect_clicked(move |_| {
@ -326,12 +325,10 @@ pub fn connect_button_search(
let ignore_same_size = check_button_image_ignore_same_size.is_active();
let similarity = similar_images::Similarity::Similar(scale_similarity_similar_images.value() as u32);
let similarity = scale_similarity_similar_images.value() as u32;
let delete_outdated_cache = check_button_settings_similar_images_delete_outdated_cache.is_active();
let fast_compare = check_button_image_fast_compare.is_active();
let futures_sender_similar_images = futures_sender_similar_images.clone();
// Find similar images
thread::spawn(move || {
@ -352,7 +349,6 @@ pub fn connect_button_search(
sf.set_allowed_extensions(allowed_extensions);
sf.set_delete_outdated_cache(delete_outdated_cache);
sf.set_exclude_images_with_same_size(ignore_same_size);
sf.set_fast_comparing(fast_compare);
sf.set_save_also_as_json(save_also_as_json);
sf.find_similar_images(Some(&stop_receiver), Some(&futures_sender_similar_images));
let _ = glib_stop_sender.send(Message::SimilarImages(sf));

View file

@ -1,13 +1,13 @@
use gtk4::prelude::*;
use czkawka_core::similar_images::{get_string_from_similarity, Similarity, SIMILAR_VALUES};
use czkawka_core::similar_images::{get_string_from_similarity, SIMILAR_VALUES};
use crate::gui_structs::gui_data::GuiData;
use crate::help_combo_box::IMAGES_HASH_SIZE_COMBO_BOX;
pub fn connect_similar_image_size_change(gui_data: &GuiData) {
let label_similar_images_minimal_similarity = gui_data.main_notebook.label_similar_images_minimal_similarity.clone();
label_similar_images_minimal_similarity.set_text(&get_string_from_similarity(&Similarity::Similar(SIMILAR_VALUES[0][5]), 8));
label_similar_images_minimal_similarity.set_text(&get_string_from_similarity(&SIMILAR_VALUES[0][5], 8));
let combo_box_image_hash_size = gui_data.main_notebook.combo_box_image_hash_size.clone();
let label_similar_images_minimal_similarity = gui_data.main_notebook.label_similar_images_minimal_similarity.clone();
@ -26,6 +26,6 @@ pub fn connect_similar_image_size_change(gui_data: &GuiData) {
scale_similarity_similar_images.set_range(0_f64, SIMILAR_VALUES[index][5] as f64);
scale_similarity_similar_images.set_fill_level(SIMILAR_VALUES[index][5] as f64);
label_similar_images_minimal_similarity.set_text(&get_string_from_similarity(&Similarity::Similar(SIMILAR_VALUES[index][5]), hash_size as u8));
label_similar_images_minimal_similarity.set_text(&get_string_from_similarity(&SIMILAR_VALUES[index][5], hash_size as u8));
});
}

View file

@ -4,7 +4,7 @@ use gtk4::{Builder, CheckButton, ComboBoxText, Entry, EventControllerKey, Gestur
use czkawka_core::common_dir_traversal::CheckingMethod;
use czkawka_core::localizer_core::{fnc_get_similarity_minimal, fnc_get_similarity_very_high};
use czkawka_core::similar_images::{get_string_from_similarity, Similarity, SIMILAR_VALUES};
use czkawka_core::similar_images::{get_string_from_similarity, SIMILAR_VALUES};
use crate::flg;
use crate::help_combo_box::{BIG_FILES_CHECK_METHOD_COMBO_BOX, DUPLICATES_CHECK_METHOD_COMBO_BOX, IMAGES_HASH_SIZE_COMBO_BOX};
@ -94,8 +94,6 @@ pub struct GuiMainNotebook {
pub check_button_image_ignore_same_size: CheckButton,
pub check_button_video_ignore_same_size: CheckButton,
pub check_button_image_fast_compare: CheckButton,
pub label_image_similarity: Label,
pub label_image_similarity_max: Label,
@ -223,8 +221,6 @@ impl GuiMainNotebook {
let scale_similarity_similar_images: Scale = builder.object("scale_similarity_similar_images").unwrap();
let scale_similarity_similar_videos: Scale = builder.object("scale_similarity_similar_videos").unwrap();
let check_button_image_fast_compare: CheckButton = builder.object("check_button_image_fast_compare").unwrap();
let combo_box_image_resize_algorithm: ComboBoxText = builder.object("combo_box_image_resize_algorithm").unwrap();
let combo_box_image_hash_algorithm: ComboBoxText = builder.object("combo_box_image_hash_algorithm").unwrap();
let combo_box_image_hash_size: ComboBoxText = builder.object("combo_box_image_hash_size").unwrap();
@ -319,7 +315,6 @@ impl GuiMainNotebook {
combo_box_duplicate_hash_type,
combo_box_image_hash_size,
check_button_video_ignore_same_size,
check_button_image_fast_compare,
check_button_duplicate_case_sensitive_name,
evk_tree_view_bad_extensions,
gc_tree_view_duplicate_finder,
@ -407,10 +402,6 @@ impl GuiMainNotebook {
self.check_button_image_ignore_same_size.set_label(Some(&flg!("check_button_general_same_size")));
self.check_button_video_ignore_same_size.set_label(Some(&flg!("check_button_general_same_size")));
self.check_button_image_fast_compare.set_label(Some(&flg!("main_notebook_image_fast_compare")));
self.check_button_image_fast_compare
.set_tooltip_text(Some(&flg!("main_notebook_image_fast_compare_tooltip")));
self.check_button_broken_files_audio.set_label(Some(&flg!("main_check_box_broken_files_audio")));
self.check_button_broken_files_archive.set_label(Some(&flg!("main_check_box_broken_files_archive")));
self.check_button_broken_files_image.set_label(Some(&flg!("main_check_box_broken_files_image")));
@ -421,20 +412,19 @@ impl GuiMainNotebook {
let hash_size = IMAGES_HASH_SIZE_COMBO_BOX[hash_size_index];
match hash_size {
8 => {
self.label_similar_images_minimal_similarity
.set_text(&get_string_from_similarity(&Similarity::Similar(SIMILAR_VALUES[0][5]), 8));
self.label_similar_images_minimal_similarity.set_text(&get_string_from_similarity(&SIMILAR_VALUES[0][5], 8));
}
16 => {
self.label_similar_images_minimal_similarity
.set_text(&get_string_from_similarity(&Similarity::Similar(SIMILAR_VALUES[1][5]), 16));
.set_text(&get_string_from_similarity(&SIMILAR_VALUES[1][5], 16));
}
32 => {
self.label_similar_images_minimal_similarity
.set_text(&get_string_from_similarity(&Similarity::Similar(SIMILAR_VALUES[2][5]), 32));
.set_text(&get_string_from_similarity(&SIMILAR_VALUES[2][5], 32));
}
64 => {
self.label_similar_images_minimal_similarity
.set_text(&get_string_from_similarity(&Similarity::Similar(SIMILAR_VALUES[3][5]), 64));
.set_text(&get_string_from_similarity(&SIMILAR_VALUES[3][5], 64));
}
_ => panic!(),
}

View file

@ -48,10 +48,9 @@ const DEFAULT_BROKEN_FILES_ARCHIVE: bool = true;
const DEFAULT_BROKEN_FILES_IMAGE: bool = true;
const DEFAULT_NUMBER_OF_BIGGEST_FILES: &str = "50";
const DEFAULT_SIMILAR_IMAGES_SIMILARITY: i32 = 0;
const DEFAULT_SIMILAR_IMAGES_SIMILARITY: f32 = 0.0;
const DEFAULT_SIMILAR_IMAGES_IGNORE_SAME_SIZE: bool = false;
const DEFAULT_SIMILAR_IMAGES_FAST_COMPARE: bool = false;
const DEFAULT_SIMILAR_VIDEOS_SIMILARITY: i32 = 15;
const DEFAULT_SIMILAR_VIDEOS_SIMILARITY: f32 = 15.0;
const DEFAULT_SIMILAR_VIDEOS_IGNORE_SAME_SIZE: bool = false;
pub const DEFAULT_MINIMAL_FILE_SIZE: &str = "16384";
@ -126,7 +125,7 @@ impl LoadSaveStruct {
default_value
}
pub fn get_integer<T: std::str::FromStr>(&self, key: String, default_value: T) -> T {
pub fn get_object<T: std::str::FromStr>(&self, key: String, default_value: T) -> T {
if self.loaded_items.contains_key(&key) {
let item = self.loaded_items.get(&key).unwrap().clone().into_iter().filter(|e| !e.is_empty()).collect::<Vec<String>>();
@ -428,7 +427,6 @@ enum LoadText {
NumberOfBiggestFiles,
SimilarImagesSimilarity,
SimilarImagesIgnoreSameSize,
SimilarImagesFastCompare,
SimilarVideosSimilarity,
SimilarVideosIgnoreSameSize,
MusicApproximateComparison,
@ -473,7 +471,6 @@ fn create_hash_map() -> (HashMap<LoadText, String>, HashMap<String, LoadText>) {
(LoadText::NumberOfBiggestFiles, "number_of_biggest_files"),
(LoadText::SimilarImagesSimilarity, "similar_images_similarity"),
(LoadText::SimilarImagesIgnoreSameSize, "similar_images_ignore_same_size"),
(LoadText::SimilarImagesFastCompare, "similar_images_fast_compare"),
(LoadText::SimilarVideosSimilarity, "similar_videos_similarity"),
(LoadText::SimilarVideosIgnoreSameSize, "similar_videos_ignore_same_size"),
(LoadText::MusicApproximateComparison, "music_approximate_comparison"),
@ -671,10 +668,6 @@ pub fn save_configuration(manual_execution: bool, upper_notebook: &GuiUpperNoteb
hashmap_ls.get(&LoadText::SimilarImagesIgnoreSameSize).unwrap().to_string(),
main_notebook.check_button_image_ignore_same_size.is_active(),
);
saving_struct.save_var(
hashmap_ls.get(&LoadText::SimilarImagesFastCompare).unwrap().to_string(),
main_notebook.check_button_image_fast_compare.is_active(),
);
saving_struct.save_var(
hashmap_ls.get(&LoadText::SimilarVideosSimilarity).unwrap().to_string(),
main_notebook.scale_similarity_similar_videos.value(),
@ -760,37 +753,36 @@ pub fn load_configuration(
let cache_minimal_size: String = loaded_entries.get_integer_string(hashmap_ls.get(&LoadText::MinimalCacheSize).unwrap().clone(), DEFAULT_MINIMAL_CACHE_SIZE.to_string());
let short_language = loaded_entries.get_string(hashmap_ls.get(&LoadText::Language).unwrap().clone(), short_language);
let combo_box_duplicate_hash_type = loaded_entries.get_integer(hashmap_ls.get(&LoadText::ComboBoxDuplicateHashType).unwrap().clone(), 0);
let combo_box_duplicate_checking_method = loaded_entries.get_integer(hashmap_ls.get(&LoadText::ComboBoxDuplicateCheckMethod).unwrap().clone(), 0);
let combo_box_image_hash_size = loaded_entries.get_integer(hashmap_ls.get(&LoadText::ComboBoxImageHashSize).unwrap().clone(), 0);
let combo_box_image_hash_algorithm = loaded_entries.get_integer(hashmap_ls.get(&LoadText::ComboBoxImageHashType).unwrap().clone(), 0);
let combo_box_image_resize_algorithm = loaded_entries.get_integer(hashmap_ls.get(&LoadText::ComboBoxImageResizeAlgorithm).unwrap().clone(), 0);
let combo_box_big_files_mode = loaded_entries.get_integer(hashmap_ls.get(&LoadText::ComboBoxBigFiles).unwrap().clone(), 0);
let combo_box_duplicate_hash_type = loaded_entries.get_object(hashmap_ls.get(&LoadText::ComboBoxDuplicateHashType).unwrap().clone(), 0);
let combo_box_duplicate_checking_method = loaded_entries.get_object(hashmap_ls.get(&LoadText::ComboBoxDuplicateCheckMethod).unwrap().clone(), 0);
let combo_box_image_hash_size = loaded_entries.get_object(hashmap_ls.get(&LoadText::ComboBoxImageHashSize).unwrap().clone(), 0);
let combo_box_image_hash_algorithm = loaded_entries.get_object(hashmap_ls.get(&LoadText::ComboBoxImageHashType).unwrap().clone(), 0);
let combo_box_image_resize_algorithm = loaded_entries.get_object(hashmap_ls.get(&LoadText::ComboBoxImageResizeAlgorithm).unwrap().clone(), 0);
let combo_box_big_files_mode = loaded_entries.get_object(hashmap_ls.get(&LoadText::ComboBoxBigFiles).unwrap().clone(), 0);
let number_of_biggest_files = loaded_entries.get_integer_string(
hashmap_ls.get(&LoadText::NumberOfBiggestFiles).unwrap().clone(),
DEFAULT_NUMBER_OF_BIGGEST_FILES.to_string(),
);
let similar_images_similarity = loaded_entries.get_integer(hashmap_ls.get(&LoadText::SimilarImagesSimilarity).unwrap().clone(), DEFAULT_SIMILAR_IMAGES_SIMILARITY);
let similar_images_similarity = loaded_entries.get_object(hashmap_ls.get(&LoadText::SimilarImagesSimilarity).unwrap().clone(), DEFAULT_SIMILAR_IMAGES_SIMILARITY);
let similar_images_ignore_same_size = loaded_entries.get_bool(
hashmap_ls.get(&LoadText::SimilarImagesIgnoreSameSize).unwrap().clone(),
DEFAULT_SIMILAR_IMAGES_IGNORE_SAME_SIZE,
);
let similar_images_fast_compare = loaded_entries.get_bool(hashmap_ls.get(&LoadText::SimilarImagesFastCompare).unwrap().clone(), DEFAULT_SIMILAR_IMAGES_FAST_COMPARE);
let similar_videos_similarity = loaded_entries.get_integer(hashmap_ls.get(&LoadText::SimilarVideosSimilarity).unwrap().clone(), DEFAULT_SIMILAR_VIDEOS_SIMILARITY);
let similar_videos_similarity = loaded_entries.get_object(hashmap_ls.get(&LoadText::SimilarVideosSimilarity).unwrap().clone(), DEFAULT_SIMILAR_VIDEOS_SIMILARITY);
let similar_videos_ignore_same_size = loaded_entries.get_bool(
hashmap_ls.get(&LoadText::SimilarVideosIgnoreSameSize).unwrap().clone(),
DEFAULT_SIMILAR_VIDEOS_IGNORE_SAME_SIZE,
);
let check_button_case_sensitive_name = loaded_entries.get_integer(
let check_button_case_sensitive_name = loaded_entries.get_object(
hashmap_ls.get(&LoadText::DuplicateNameCaseSensitive).unwrap().clone(),
DEFAULT_DUPLICATE_CASE_SENSITIVE_NAME_CHECKING,
);
let check_button_broken_files_archive = loaded_entries.get_integer(hashmap_ls.get(&LoadText::BrokenFilesArchive).unwrap().clone(), DEFAULT_BROKEN_FILES_ARCHIVE);
let check_button_broken_files_pdf = loaded_entries.get_integer(hashmap_ls.get(&LoadText::BrokenFilesPdf).unwrap().clone(), DEFAULT_BROKEN_FILES_ARCHIVE);
let check_button_broken_files_image = loaded_entries.get_integer(hashmap_ls.get(&LoadText::BrokenFilesImage).unwrap().clone(), DEFAULT_BROKEN_FILES_ARCHIVE);
let check_button_broken_files_audio = loaded_entries.get_integer(hashmap_ls.get(&LoadText::BrokenFilesAudio).unwrap().clone(), DEFAULT_BROKEN_FILES_ARCHIVE);
let check_button_broken_files_archive = loaded_entries.get_object(hashmap_ls.get(&LoadText::BrokenFilesArchive).unwrap().clone(), DEFAULT_BROKEN_FILES_ARCHIVE);
let check_button_broken_files_pdf = loaded_entries.get_object(hashmap_ls.get(&LoadText::BrokenFilesPdf).unwrap().clone(), DEFAULT_BROKEN_FILES_ARCHIVE);
let check_button_broken_files_image = loaded_entries.get_object(hashmap_ls.get(&LoadText::BrokenFilesImage).unwrap().clone(), DEFAULT_BROKEN_FILES_ARCHIVE);
let check_button_broken_files_audio = loaded_entries.get_object(hashmap_ls.get(&LoadText::BrokenFilesAudio).unwrap().clone(), DEFAULT_BROKEN_FILES_ARCHIVE);
// Setting data
if manual_execution || loading_at_start {
@ -912,7 +904,6 @@ pub fn load_configuration(
main_notebook.check_button_duplicate_case_sensitive_name.set_active(check_button_case_sensitive_name);
main_notebook.entry_big_files_number.set_text(&number_of_biggest_files);
main_notebook.check_button_image_ignore_same_size.set_active(similar_images_ignore_same_size);
main_notebook.check_button_image_fast_compare.set_active(similar_images_fast_compare);
main_notebook.check_button_video_ignore_same_size.set_active(similar_videos_ignore_same_size);
main_notebook.scale_similarity_similar_videos.set_value(similar_videos_similarity as f64);
@ -1056,7 +1047,6 @@ pub fn reset_configuration(manual_clearing: bool, upper_notebook: &GuiUpperNoteb
main_notebook.entry_big_files_number.set_text(DEFAULT_NUMBER_OF_BIGGEST_FILES);
main_notebook.scale_similarity_similar_images.set_value(DEFAULT_SIMILAR_IMAGES_SIMILARITY as f64);
main_notebook.check_button_image_ignore_same_size.set_active(DEFAULT_SIMILAR_IMAGES_IGNORE_SAME_SIZE);
main_notebook.check_button_image_fast_compare.set_active(DEFAULT_SIMILAR_IMAGES_FAST_COMPARE);
main_notebook.check_button_video_ignore_same_size.set_active(DEFAULT_SIMILAR_VIDEOS_IGNORE_SAME_SIZE);
main_notebook.scale_similarity_similar_videos.set_value(DEFAULT_SIMILAR_VIDEOS_SIMILARITY as f64);
}

View file

@ -1,5 +1,5 @@
<?xml version='1.0' encoding='UTF-8'?>
<!-- Created with Cambalache 0.9.1 -->
<!-- Created with Cambalache 0.10.2 -->
<interface>
<!-- interface-name about_dialog.ui -->
<requires lib="gtk" version="4.0"/>

View file

@ -1,5 +1,5 @@
<?xml version='1.0' encoding='UTF-8'?>
<!-- Created with Cambalache 0.9.1 -->
<!-- Created with Cambalache 0.10.2 -->
<interface>
<!-- interface-name compare_images.ui -->
<requires lib="gtk" version="4.0"/>

View file

@ -1,6 +1,6 @@
<?xml version='1.0' encoding='UTF-8' standalone='no'?>
<!DOCTYPE cambalache-project SYSTEM "cambalache-project.dtd">
<cambalache-project version="0.9.1" target_tk="gtk-4.0">
<cambalache-project version="0.10.2" target_tk="gtk-4.0">
<ui>
(3,None,"about_dialog.ui","about_dialog.ui",None,None,None,None,None,None),
(4,None,"compare_images.ui","compare_images.ui",None,None,None,None,None,None),
@ -136,7 +136,6 @@
(5,98,"GtkScale","scale_similarity_similar_images",95,None,None,None,2),
(5,99,"GtkLabel","label_similar_images_minimal_similarity",95,None,None,None,3),
(5,100,"GtkCheckButton","check_button_image_ignore_same_size",95,None,None,None,4),
(5,101,"GtkCheckButton","check_button_image_fast_compare",95,None,None,None,5),
(5,102,"GtkScrolledWindow","scrolled_window_similar_images_finder",87,None,None,None,2),
(5,103,"GtkImage","image_preview_similar_images",86,None,None,None,1),
(5,104,"GtkLabel",None,85,None,None,None,None),
@ -558,9 +557,6 @@
(5,100,"GtkCheckButton","label","Ignore same size",1,None,None,None,None),
(5,100,"GtkWidget","focusable","1",None,None,None,None,None),
(5,100,"GtkWidget","margin-start","7",None,None,None,None,None),
(5,101,"GtkCheckButton","label","Fast compare",1,None,None,None,None),
(5,101,"GtkWidget","focusable","1",None,None,None,None,None),
(5,101,"GtkWidget","margin-start","7",None,None,None,None,None),
(5,102,"GtkWidget","focusable","1",None,None,None,None,None),
(5,102,"GtkWidget","margin-end","5",None,None,None,None,None),
(5,102,"GtkWidget","vexpand","1",None,None,None,None,None),

View file

@ -1,5 +1,5 @@
<?xml version='1.0' encoding='UTF-8'?>
<!-- Created with Cambalache 0.9.1 -->
<!-- Created with Cambalache 0.10.2 -->
<interface>
<!-- interface-name main_window.ui -->
<requires lib="gtk" version="4.0"/>
@ -578,13 +578,6 @@
<property name="margin-start">7</property>
</object>
</child>
<child>
<object class="GtkCheckButton" id="check_button_image_fast_compare">
<property name="focusable">1</property>
<property name="label" translatable="yes">Fast compare</property>
<property name="margin-start">7</property>
</object>
</child>
</object>
</child>
<child>

View file

@ -1,5 +1,5 @@
<?xml version='1.0' encoding='UTF-8'?>
<!-- Created with Cambalache 0.9.1 -->
<!-- Created with Cambalache 0.10.2 -->
<interface>
<!-- interface-name popover_right_click.ui -->
<requires lib="gtk" version="4.0"/>

View file

@ -1,5 +1,5 @@
<?xml version='1.0' encoding='UTF-8'?>
<!-- Created with Cambalache 0.9.1 -->
<!-- Created with Cambalache 0.10.2 -->
<interface>
<!-- interface-name popover_select.ui -->
<requires lib="gtk" version="4.0"/>

View file

@ -1,5 +1,5 @@
<?xml version='1.0' encoding='UTF-8'?>
<!-- Created with Cambalache 0.9.1 -->
<!-- Created with Cambalache 0.10.2 -->
<interface>
<!-- interface-name progress.ui -->
<requires lib="gtk" version="4.0"/>

View file

@ -1,5 +1,5 @@
<?xml version='1.0' encoding='UTF-8'?>
<!-- Created with Cambalache 0.9.1 -->
<!-- Created with Cambalache 0.10.2 -->
<interface>
<!-- interface-name settings.ui -->
<requires lib="gtk" version="4.0"/>