CodeMirror/czkawka
1
0
Fork 0
mirror of synced 2024-05-14 17:33:48 +12:00
Code Issues Projects Releases Wiki Activity

Add new step to check progress of comparing images. (#525)

Browse source 
Fix empty loaded items in e.g. Excluded Directories.
Fix bug with non-loading proper image results
This commit is contained in:
Rafał Mikrut 2021-12-25 22:23:18 +01:00 committed by GitHub
parent e2494d240f
commit 1f5fca3f46
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 133 additions and 131 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
czkawka_core/src/broken_files.rs
View file

@ -748,7 +748,7 @@ fn check_extension_avaibility(file_name_lowercase: &str) -> TypeOfFile {
// Checking allowed image extensions
let allowed_image_extensions = [
".jpg", ".jpeg", ".png", /*, ".bmp"*/
".tiff", ".tif", ".tga", ".ff", /*, ".gif"*/
/*".tiff", ".tif",*/ ".tga", ".ff", /*, ".gif"*/
// Gif will be reenabled in image-rs 0.24
".jif", ".jfi", /*, ".ico"*/
// Ico and bmp crashes are not fixed yet

228
czkawka_core/src/similar_images.rs
View file

@ -17,6 +17,7 @@ use humansize::{file_size_opts as options, FileSize};
use image::GenericImageView;
use img_hash::{FilterType, HashAlg, HasherConfig};
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use crate::common::Common;
use crate::common_directory::Directories;
@ -42,14 +43,15 @@ pub struct ProgressData {
pub images_checked: usize,
pub images_to_check: usize,
}
const LOOP_DURATION: u32 = 200; //ms
#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Debug)]
#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Debug, Serialize, Deserialize)]
pub enum Similarity {
None,
Similar(u32),
}
#[derive(Clone, Debug)]
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct FileEntry {
pub path: PathBuf,
pub size: u64,
@ -259,7 +261,7 @@ impl SimilarImages {
self.allowed_extensions.extend_allowed_extensions(&[
".jpg", ".jpeg", ".png", /*, ".bmp"*/
".tiff", ".tif", ".tga", ".ff", /*, ".gif"*/
/*".tiff", ".tif",*/ ".tga", ".ff", /*, ".gif"*/
".jif", ".jfi", /*, ".webp"*/
]); // webp cannot be seen in preview, gif needs to be enabled after releasing image crate 0.24.0, bmp needs to be fixed in image crate
@ -269,7 +271,6 @@ impl SimilarImages {
}
//// PROGRESS THREAD START
const LOOP_DURATION: u32 = 200; //in ms
let progress_thread_run = Arc::new(AtomicBool::new(true));
let atomic_file_counter = Arc::new(AtomicUsize::new(0));
@ -282,7 +283,7 @@ impl SimilarImages {
progress_send
.unbounded_send(ProgressData {
current_stage: 0,
max_stage: 1,
max_stage: 2,
images_checked: atomic_file_counter.load(Ordering::Relaxed) as usize,
images_to_check: 0,
})
@ -489,7 +490,6 @@ impl SimilarImages {
let hash_map_modification = SystemTime::now();
//// PROGRESS THREAD START
const LOOP_DURATION: u32 = 200; //in ms
let progress_thread_run = Arc::new(AtomicBool::new(true));
let atomic_file_counter = Arc::new(AtomicUsize::new(0));
@ -503,7 +503,7 @@ impl SimilarImages {
progress_send
.unbounded_send(ProgressData {
current_stage: 1,
max_stage: 1,
max_stage: 2,
images_checked: atomic_file_counter.load(Ordering::Relaxed) as usize,
images_to_check,
})
@ -602,33 +602,54 @@ impl SimilarImages {
_ => panic!(),
};
// TODO
// Maybe also add here progress report
let mut collected_similar_images: BTreeMap<Vec<u8>, Vec<FileEntry>> = Default::default();
let mut available_hashes = self.image_hashes.clone();
let mut this_time_check_hashes;
let mut master_of_group: BTreeSet<Vec<u8>> = Default::default(); // Lista wszystkich głównych hashy, które odpowiadają za porównywanie
// TODO optimize this for big temp_max_similarity values
// TODO maybe Simialar(u32) is enough instead SIMILAR_VALUES value?
let temp_max_similarity = match self.hash_size {
8 => SIMILAR_VALUES[0][5],
16 => SIMILAR_VALUES[1][4],
32 => SIMILAR_VALUES[2][3],
64 => SIMILAR_VALUES[3][2],
_ => panic!(),
};
//// PROGRESS THREAD START
let progress_thread_run = Arc::new(AtomicBool::new(true));
for current_similarity in 0..=temp_max_similarity {
let atomic_mode_counter = Arc::new(AtomicUsize::new(0));
let progress_thread_handle = if let Some(progress_sender) = progress_sender {
let progress_send = progress_sender.clone();
let progress_thread_run = progress_thread_run.clone();
let atomic_mode_counter = atomic_mode_counter.clone();
let all_images = similarity as usize * available_hashes.len();
thread::spawn(move || loop {
progress_send
.unbounded_send(ProgressData {
current_stage: 2,
max_stage: 2,
images_checked: atomic_mode_counter.load(Ordering::Relaxed) as usize,
images_to_check: all_images,
})
.unwrap();
if !progress_thread_run.load(Ordering::Relaxed) {
break;
}
sleep(Duration::from_millis(LOOP_DURATION as u64));
})
} else {
thread::spawn(|| {})
};
//// PROGRESS THREAD END
for current_similarity in 0..=similarity {
this_time_check_hashes = available_hashes.clone();
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
return false;
}
for (hash, vec_file_entry) in this_time_check_hashes.iter() {
for (hash, vec_file_entry) in &this_time_check_hashes {
atomic_mode_counter.fetch_add(1, Ordering::Relaxed);
let vector_with_found_similar_hashes = self
.bktree
.find(hash, similarity)
@ -687,6 +708,9 @@ impl SimilarImages {
}
}
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
self.similar_vectors = collected_similar_images.values().cloned().collect();
if self.exclude_images_with_same_size {
@ -880,7 +904,8 @@ pub fn save_hashes_to_file(hashmap: &BTreeMap<String, FileEntry>, text_messages:
text_messages.messages.push(format!("Cannot create config dir {}, reason {}", cache_dir.display(), e));
return;
}
let cache_file = cache_dir.join(get_cache_file(&hash_size, &hash_alg, &image_filter));
let cache_file = cache_dir.join(cache_dir.join(get_cache_file(&hash_size, &hash_alg, &image_filter)));
let file_handler = match OpenOptions::new().truncate(true).write(true).create(true).open(&cache_file) {
Ok(t) => t,
Err(e) => {
@ -890,29 +915,24 @@ pub fn save_hashes_to_file(hashmap: &BTreeMap<String, FileEntry>, text_messages:
return;
}
};
let mut writer = BufWriter::new(file_handler);
for file_entry in hashmap.values() {
let mut string: String = format!(
"{}//{}//{}//{}",
file_entry.path.display(),
file_entry.size,
file_entry.dimensions,
file_entry.modified_date
);
for hash in &file_entry.hash {
string.push_str("//");
string.push_str(hash.to_string().as_str());
}
if let Err(e) = writeln!(writer, "{}", string) {
text_messages
.messages
.push(format!("Failed to save some data to cache file {}, reason {}", cache_file.display(), e));
return;
};
let writer = BufWriter::new(file_handler);
#[cfg(not(debug_assertions))]
if let Err(e) = bincode::serialize_into(writer, hashmap) {
text_messages
.messages
.push(format!("Cannot write data to cache file {}, reason {}", cache_file.display(), e));
return;
}
#[cfg(debug_assertions)]
if let Err(e) = serde_json::to_writer(writer, hashmap) {
text_messages
.messages
.push(format!("Cannot write data to cache file {}, reason {}", cache_file.display(), e));
return;
}
text_messages.messages.push(format!("Properly saved to file {} cache entries.", hashmap.len()));
}
}
@ -929,97 +949,40 @@ pub fn load_hashes_from_file(
let file_handler = match OpenOptions::new().read(true).open(&cache_file) {
Ok(t) => t,
Err(_inspected) => {
// text_messages.messages.push(format!("Cannot find or open cache file {}", cache_file.display())); // This shouldn't be write to output
// text_messages.messages.push(format!("Cannot find or open cache file {}", cache_file.display())); // No error warning
return None;
}
};
let reader = BufReader::new(file_handler);
let mut hashmap_loaded_entries: BTreeMap<String, FileEntry> = Default::default();
let number_of_results: usize = hash_size as usize * hash_size as usize / 8;
// Read the file line by line using the lines() iterator from std::io::BufRead.
for (index, line) in reader.lines().enumerate() {
let line = match line {
Ok(t) => t,
Err(e) => {
text_messages
.warnings
.push(format!("Failed to load line number {} from cache file {}, reason {}", index + 1, cache_file.display(), e));
return None;
}
};
let uuu = line.split("//").collect::<Vec<&str>>();
if uuu.len() != (number_of_results + 4) {
text_messages.warnings.push(format!(
"Found invalid data in line {} - ({}) in cache file {}, expected {} values, found {}",
index + 1,
line,
cache_file.display(),
number_of_results + 4,
uuu.len()
));
continue;
#[cfg(debug_assertions)]
let mut hashmap_loaded_entries: BTreeMap<String, FileEntry> = match serde_json::from_reader(reader) {
Ok(t) => t,
Err(e) => {
text_messages
.warnings
.push(format!("Failed to load data from cache file {}, reason {}", cache_file.display(), e));
return None;
}
// Don't load cache data if destination file not exists
if !delete_outdated_cache || Path::new(uuu[0]).exists() {
let mut hash: Vec<u8> = Vec::new();
for i in 0..number_of_results {
hash.push(match uuu[4 + i as usize].parse::<u8>() {
Ok(t) => t,
Err(e) => {
text_messages.warnings.push(format!(
"Found invalid hash value in line {} - ({}) in cache file {}, reason {}",
index + 1,
line,
cache_file.display(),
e
));
continue;
}
});
}
hashmap_loaded_entries.insert(
uuu[0].to_string(),
FileEntry {
path: PathBuf::from(uuu[0]),
size: match uuu[1].parse::<u64>() {
Ok(t) => t,
Err(e) => {
text_messages.warnings.push(format!(
"Found invalid size value in line {} - ({}) in cache file {}, reason {}",
index + 1,
line,
cache_file.display(),
e
));
continue;
}
},
dimensions: uuu[2].to_string(),
modified_date: match uuu[3].parse::<u64>() {
Ok(t) => t,
Err(e) => {
text_messages.warnings.push(format!(
"Found invalid modified date value in line {} - ({}) in cache file {}, reason {}",
index + 1,
line,
cache_file.display(),
e
));
continue;
}
},
hash,
similarity: Similarity::None,
},
);
};
#[cfg(not(debug_assertions))]
let mut hashmap_loaded_entries: BTreeMap<String, FileEntry> = match bincode::deserialize_from(reader) {
Ok(t) => t,
Err(e) => {
text_messages
.warnings
.push(format!("Failed to load data from cache file {}, reason {}", cache_file.display(), e));
return None;
}
};
// Don't load cache data if destination file not exists
if delete_outdated_cache {
hashmap_loaded_entries.retain(|src_path, _file_entry| Path::new(src_path).exists());
}
text_messages.messages.push(format!("Properly loaded {} cache entries.", hashmap_loaded_entries.len()));
return Some(hashmap_loaded_entries);
}
@ -1028,11 +991,22 @@ pub fn load_hashes_from_file(
}
fn get_cache_file(hash_size: &u8, hash_alg: &HashAlg, image_filter: &FilterType) -> String {
let extension;
#[cfg(debug_assertions)]
{
extension = "json";
}
#[cfg(not(debug_assertions))]
{
extension = "bin";
}
format!(
"cache_similar_images_{}_{}_{}.txt",
"cache_similar_images_{}_{}_{}.{}",
hash_size,
convert_algorithm_to_string(hash_alg),
convert_filters_to_string(image_filter)
convert_filters_to_string(image_filter),
extension
)
}

21
czkawka_gui/src/connect_progress_window.rs
View file

@ -256,7 +256,7 @@ pub fn connect_progress_window(
item.images_to_check as u64 * (item.max_stage + 1) as u64,
);
} else {
progress_bar_all_stages.set_fraction((1f64) / (item.max_stage + 1) as f64);
progress_bar_all_stages.set_fraction((item.current_stage as f64) / (item.max_stage + 1) as f64);
progress_bar_current_stage.set_fraction(0f64);
taskbar_state.borrow().set_progress_value(1, (item.max_stage + 1) as u64);
}
@ -265,6 +265,25 @@ pub fn connect_progress_window(
generate_translation_hashmap(vec![("file_checked", item.images_checked.to_string()), ("all_files", item.images_to_check.to_string())])
));
}
2 => {
progress_bar_current_stage.show();
if item.images_to_check != 0 {
progress_bar_all_stages.set_fraction((2f64 + (item.images_checked) as f64 / item.images_to_check as f64) / (item.max_stage + 1) as f64);
progress_bar_current_stage.set_fraction((item.images_checked) as f64 / item.images_to_check as f64);
taskbar_state.borrow().set_progress_value(
(item.images_to_check + item.images_checked) as u64,
item.images_to_check as u64 * (item.max_stage + 1) as u64,
);
} else {
progress_bar_all_stages.set_fraction((item.current_stage as f64) / (item.max_stage + 1) as f64);
progress_bar_current_stage.set_fraction(0f64);
taskbar_state.borrow().set_progress_value(2, (item.max_stage + 1) as u64);
}
label_stage.set_text(&fl!(
"progress_comparing_image_hashes",
generate_translation_hashmap(vec![("file_checked", item.images_checked.to_string()), ("all_files", item.images_to_check.to_string())])
));
}
_ => {
panic!();
}

4
czkawka_gui/src/initialize_gui.rs
View file

@ -719,7 +719,9 @@ fn show_preview(
let file_name = file_name.as_str();
if let Some(extension) = Path::new(file_name).extension() {
if !["jpg", "jpeg", "png", "bmp", "tiff", "tif", "tga", "ff", "gif", "jif", "jfi"].contains(&extension.to_string_lossy().to_string().to_lowercase().as_str()) {
if !["jpg", "jpeg", "png", "bmp", /*"tiff", "tif",*/ "tga", "ff", "gif", "jif", "jfi"]
.contains(&extension.to_string_lossy().to_string().to_lowercase().as_str())
{
break 'dir;
}

8
czkawka_gui/src/saving_loading.rs
View file

@ -47,7 +47,13 @@ impl LoadSaveStruct {
pub fn get_vector_string(&self, key: String, default_value: Vec<String>) -> Vec<String> {
if self.loaded_items.contains_key(&key) {
return self.loaded_items.get(&key).unwrap().clone();
let mut new_vector = Vec::new();
for i in self.loaded_items.get(&key).unwrap() {
if !i.trim().is_empty() {
new_vector.push(i.trim().to_string());
}
}
return new_vector;
}
default_value

1
i18n/en/czkawka_gui.ftl
View file

@ -383,6 +383,7 @@ progress_scanning_general_file = Scanning {$file_number} file
progress_scanning_broken_files = Checking {$file_checked}/{$all_files} file
progress_scanning_video = Hashing of {$file_checked}/{$all_files} video
progress_scanning_image = Hashing of {$file_checked}/{$all_files} image
progress_comparing_image_hashes = Comparing {$file_checked}/{$all_files} image hash
progress_scanning_music_tags_end = Comparing tags of {$file_checked}/{$all_files} music file
progress_scanning_music_tags = Reading tags of {$file_checked}/{$all_files} music file
progress_scanning_empty_folders = Scanning {$folder_number} folder