1
0
Fork 0
mirror of synced 2024-05-18 03:13:36 +12:00
This commit is contained in:
Rafał Mikrut 2023-10-14 14:53:02 +02:00
parent e92a8a65de
commit acca1ec71a
6 changed files with 146 additions and 73 deletions

96
Cargo.lock generated
View file

@ -27,9 +27,9 @@ dependencies = [
[[package]]
name = "aho-corasick"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea5d730647d4fadd988536d06fecce94b7b4f2a7efdae548f1cf4b63205518ab"
checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0"
dependencies = [
"memchr",
]
@ -758,9 +758,12 @@ dependencies = [
[[package]]
name = "deranged"
version = "0.3.8"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946"
checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3"
dependencies = [
"powerfmt",
]
[[package]]
name = "digest"
@ -945,9 +948,9 @@ checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6"
[[package]]
name = "flate2"
version = "1.0.27"
version = "1.0.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010"
checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e"
dependencies = [
"crc32fast",
"miniz_oxide",
@ -1035,9 +1038,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fun_time"
version = "0.3.1"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9af29f347b6ae4821c45565e1238231caf5b57a951bd011222752ba0f5a47eae"
checksum = "783fdd8402ab5955698d66b0ba307bb2d3b3bfb911a4dfe7bd6bbc7f140be6a5"
dependencies = [
"fun_time_derive",
"log",
@ -1045,11 +1048,12 @@ dependencies = [
[[package]]
name = "fun_time_derive"
version = "0.3.1"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e992455767376a16164ee4cc69bd799530c1c41c701bdd8c8a541ec6095c078e"
checksum = "a3d89b974360ce9cd00ef9dd26353bc6f1ec9ff12d95e741ad83b36385f3a736"
dependencies = [
"darling",
"log",
"proc-macro2",
"quote",
"syn 1.0.109",
@ -1814,9 +1818,9 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
[[package]]
name = "jobserver"
version = "0.1.26"
version = "0.1.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2"
checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d"
dependencies = [
"libc",
]
@ -2158,9 +2162,9 @@ dependencies = [
[[package]]
name = "num-derive"
version = "0.4.0"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e6a0fd4f737c707bd9086cc16c925f294943eb62eb71499e9fd4cf71f8b9f4e"
checksum = "cfb77679af88f8b125209d354a202862602672222e7f2313fdd6dc349bad4712"
dependencies = [
"proc-macro2",
"quote",
@ -2426,6 +2430,12 @@ dependencies = [
"miniz_oxide",
]
[[package]]
name = "powerfmt"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
[[package]]
name = "ppv-lite86"
version = "0.2.17"
@ -2666,14 +2676,14 @@ dependencies = [
[[package]]
name = "regex"
version = "1.9.6"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff"
checksum = "d119d7c7ca818f8a53c300863d4f87566aac09943aef5b355bb83969dae75d87"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata 0.3.9",
"regex-syntax 0.7.5",
"regex-automata 0.4.1",
"regex-syntax 0.8.1",
]
[[package]]
@ -2687,13 +2697,13 @@ dependencies = [
[[package]]
name = "regex-automata"
version = "0.3.9"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9"
checksum = "465c6fc0621e4abc4187a2bda0937bfd4f722c2730b29562e19689ea796c9a4b"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax 0.7.5",
"regex-syntax 0.8.1",
]
[[package]]
@ -2704,9 +2714,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
[[package]]
name = "regex-syntax"
version = "0.7.5"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
checksum = "56d84fdd47036b038fc80dd333d10b6aab10d5d31f4a366e20014def75328d33"
[[package]]
name = "rgb"
@ -2849,9 +2859,9 @@ dependencies = [
[[package]]
name = "rustix"
version = "0.38.17"
version = "0.38.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f25469e9ae0f3d0047ca8b93fc56843f38e6774f0914a107ff8b41be8be8e0b7"
checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed"
dependencies = [
"bitflags 2.4.0",
"errno",
@ -2911,27 +2921,27 @@ checksum = "1ef965a420fe14fdac7dd018862966a4c14094f900e1650bbc71ddd7d580c8af"
[[package]]
name = "semver"
version = "1.0.19"
version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad977052201c6de01a8ef2aa3378c4bd23217a056337d1d6da40468d267a4fb0"
checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090"
dependencies = [
"serde",
]
[[package]]
name = "serde"
version = "1.0.188"
version = "1.0.189"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e"
checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.188"
version = "1.0.189"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2"
checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5"
dependencies = [
"proc-macro2",
"quote",
@ -3389,14 +3399,15 @@ dependencies = [
[[package]]
name = "time"
version = "0.3.29"
version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe"
checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5"
dependencies = [
"deranged",
"itoa",
"libc",
"num_threads",
"powerfmt",
"serde",
"time-core",
"time-macros",
@ -3497,11 +3508,10 @@ dependencies = [
[[package]]
name = "tracing"
version = "0.1.37"
version = "0.1.39"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
checksum = "ee2ef2af84856a50c1d430afce2fdded0a4ec7eda868db86409b4543df0797f9"
dependencies = [
"cfg-if",
"pin-project-lite",
"tracing-attributes",
"tracing-core",
@ -3509,9 +3519,9 @@ dependencies = [
[[package]]
name = "tracing-attributes"
version = "0.1.26"
version = "0.1.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
dependencies = [
"proc-macro2",
"quote",
@ -3520,9 +3530,9 @@ dependencies = [
[[package]]
name = "tracing-core"
version = "0.1.31"
version = "0.1.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a"
checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"
dependencies = [
"once_cell",
"valuable",
@ -3693,7 +3703,7 @@ checksum = "85db69f33d00031c1b07f7292e56317d5aa9475bdbd3d27ef18f3633438a697e"
dependencies = [
"cfg-if",
"noop_proc_macro",
"num-derive 0.4.0",
"num-derive 0.4.1",
"num-traits",
"rust_hawktracer",
]
@ -3983,9 +3993,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
[[package]]
name = "winnow"
version = "0.5.16"
version = "0.5.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "037711d82167854aff2018dfd193aa0fef5370f456732f0d5a0c59b0f1b4b907"
checksum = "a3b801d0e0a6726477cc207f60162da452f3a95adb368399bef20a946e06f65c"
dependencies = [
"memchr",
]

View file

@ -78,6 +78,7 @@ where
}
text_messages.messages.push(format!("Properly saved to file {} cache entries.", hashmap.len()));
debug!("Properly saved to file {} cache entries.", hashmap.len());
} else {
debug!("Failed to save cache to file {cache_file_name} because not exists");
}

View file

@ -34,6 +34,7 @@ pub enum HashType {
Xxh3,
}
const MAX_STAGE: u8 = 5;
impl HashType {
fn hasher(self: &HashType) -> Box<dyn MyHasher> {
match self {
@ -305,7 +306,7 @@ impl DuplicateFinder {
fn check_files_size(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
let max_stage = match self.check_method {
CheckingMethod::Size => 0,
CheckingMethod::Hash => 2,
CheckingMethod::Hash => MAX_STAGE,
_ => panic!(),
};
let result = DirTraversalBuilder::new()
@ -494,17 +495,24 @@ impl DuplicateFinder {
pre_checked_map: &mut BTreeMap<u64, Vec<FileEntry>>,
) -> Option<()> {
let check_type = self.hash_type;
let (progress_thread_handle, progress_thread_run, _atomic_counter, check_was_stopped) =
prepare_thread_handler_common(progress_sender, 1, MAX_STAGE, 0, self.check_method, self.common_data.tool_type);
let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.prehash_load_cache_at_start();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
if check_was_stopped.load(Ordering::Relaxed) {
return None;
}
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) = prepare_thread_handler_common(
progress_sender,
1,
2,
self.files_with_identical_size.values().map(Vec::len).sum(),
MAX_STAGE,
non_cached_files_to_check.values().map(Vec::len).sum(),
self.check_method,
self.common_data.tool_type,
);
let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.prehash_load_cache_at_start();
debug!("Starting calculating prehash");
#[allow(clippy::type_complexity)]
let pre_hash_results: Vec<(u64, BTreeMap<String, Vec<FileEntry>>, Vec<String>)> = non_cached_files_to_check
@ -536,10 +544,11 @@ impl DuplicateFinder {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
// Check if user aborted search(only from GUI)
if check_was_stopped.load(Ordering::Relaxed) {
return None;
}
let stopped_search = check_was_stopped.load(Ordering::Relaxed);
// Saving into cache
let (progress_thread_handle, progress_thread_run, _atomic_counter, check_was_stopped) =
prepare_thread_handler_common(progress_sender, 3, MAX_STAGE, 0, self.check_method, self.common_data.tool_type);
// Add data from cache
for (size, vec_file_entry) in &records_already_cached {
@ -560,6 +569,11 @@ impl DuplicateFinder {
self.prehash_save_cache_at_exit(loaded_hash_map, &pre_hash_results);
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
if check_was_stopped.load(Ordering::Relaxed) || stopped_search {
return None;
}
Some(())
}
@ -666,28 +680,33 @@ impl DuplicateFinder {
self.get_text_messages_mut().extend_with_another_messages(messages);
}
#[fun_time(message = "full_hashing")]
// #[fun_time(message = "full_hashing")]
fn full_hashing(
&mut self,
stop_receiver: Option<&Receiver<()>>,
progress_sender: Option<&UnboundedSender<ProgressData>>,
pre_checked_map: BTreeMap<u64, Vec<FileEntry>>,
) -> Option<()> {
let check_type = self.hash_type;
let (progress_thread_handle, progress_thread_run, _atomic_counter, check_was_stopped) =
prepare_thread_handler_common(progress_sender, 4, MAX_STAGE, 0, self.check_method, self.common_data.tool_type);
let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.full_hashing_load_cache_at_start(pre_checked_map);
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
if check_was_stopped.load(Ordering::Relaxed) {
return None;
}
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) = prepare_thread_handler_common(
progress_sender,
2,
2,
pre_checked_map.values().map(Vec::len).sum(),
5,
MAX_STAGE,
non_cached_files_to_check.values().map(Vec::len).sum(),
self.check_method,
self.common_data.tool_type,
);
///////////////////////////////////////////////////////////////////////////// HASHING START
let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.full_hashing_load_cache_at_start(pre_checked_map);
let check_type = self.hash_type;
debug!("Starting full hashing of {} files", non_cached_files_to_check.values().map(Vec::len).sum::<usize>());
let mut full_hash_results: Vec<(u64, BTreeMap<String, Vec<FileEntry>>, Vec<String>)> = non_cached_files_to_check
.into_par_iter()
@ -717,15 +736,15 @@ impl DuplicateFinder {
.collect();
debug!("Finished full hashing");
// Even if clicked stop, save items to cache and show results
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
let (progress_thread_handle, progress_thread_run, _atomic_counter, _check_was_stopped) =
prepare_thread_handler_common(progress_sender, 6, MAX_STAGE, 0, self.check_method, self.common_data.tool_type);
self.full_hashing_save_cache_at_exit(records_already_cached, &mut full_hash_results, loaded_hash_map);
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
// Break if stop was clicked after saving to cache
if check_was_stopped.load(Ordering::Relaxed) {
return None;
}
for (size, hash_map, mut errors) in full_hash_results {
self.common_data.text_messages.warnings.append(&mut errors);
for (_hash, vec_file_entry) in hash_map {

View file

@ -471,6 +471,12 @@ progress_scanning_size_name = Scanning name and size of {$file_number} file
progress_scanning_name = Scanning name of {$file_number} file
progress_analyzed_partial_hash = Analyzed partial hash of {$file_checked}/{$all_files} files
progress_analyzed_full_hash = Analyzed full hash of {$file_checked}/{$all_files} files
progress_prehash_cache_loading = Loading prehash cache
progress_prehash_cache_saving = Saving prehash cache
progress_hash_cache_loading = Loading hash cache
progress_hash_cache_saving = Saving hash cache
progress_cache_loading = Loading cache
progress_cache_saving = Saving cache
progress_current_stage = Current Stage:{" "}
progress_all_stages = All Stages:{" "}

View file

@ -4,6 +4,7 @@ use std::path::PathBuf;
use std::rc::Rc;
use chrono::NaiveDateTime;
use fun_time::fun_time;
use glib::Receiver;
use gtk4::prelude::*;
use gtk4::{Entry, ListStore, TextView, TreeView, Widget};
@ -229,6 +230,7 @@ pub fn connect_compute_results(gui_data: &GuiData, glib_stop_receiver: Receiver<
});
}
#[fun_time(message = "computer_bad_extensions")]
fn computer_bad_extensions(
be: BadExtensions,
entry_info: &Entry,
@ -302,6 +304,7 @@ fn computer_bad_extensions(
}
}
#[fun_time(message = "computer_broken_files")]
fn computer_broken_files(
br: BrokenFiles,
entry_info: &Entry,
@ -375,6 +378,7 @@ fn computer_broken_files(
}
}
#[fun_time(message = "computer_invalid_symlinks")]
fn computer_invalid_symlinks(
ifs: InvalidSymlinks,
entry_info: &Entry,
@ -446,6 +450,7 @@ fn computer_invalid_symlinks(
}
}
#[fun_time(message = "computer_same_music")]
fn computer_same_music(
mf: SameMusic,
entry_info: &Entry,
@ -620,6 +625,7 @@ fn computer_same_music(
}
}
#[fun_time(message = "computer_similar_videos")]
fn computer_similar_videos(
ff: SimilarVideos,
entry_info: &Entry,
@ -721,6 +727,7 @@ fn computer_similar_videos(
}
}
#[fun_time(message = "computer_similar_images")]
fn computer_similar_images(
sf: SimilarImages,
entry_info: &Entry,
@ -853,6 +860,7 @@ fn computer_similar_images(
}
}
#[fun_time(message = "computer_temporary_files")]
fn computer_temporary_files(
tf: Temporary,
entry_info: &Entry,
@ -924,6 +932,7 @@ fn computer_temporary_files(
}
}
#[fun_time(message = "computer_big_files")]
fn computer_big_files(
bf: BigFile,
entry_info: &Entry,
@ -991,6 +1000,7 @@ fn computer_big_files(
}
}
#[fun_time(message = "computer_empty_files")]
fn computer_empty_files(
vf: EmptyFiles,
entry_info: &Entry,
@ -1057,6 +1067,7 @@ fn computer_empty_files(
}
}
#[fun_time(message = "computer_empty_folders")]
fn computer_empty_folders(
ef: EmptyFolder,
entry_info: &Entry,
@ -1128,6 +1139,7 @@ fn computer_empty_folders(
}
}
#[fun_time(message = "computer_duplicate_finder")]
fn computer_duplicate_finder(
df: DuplicateFinder,
entry_info: &Entry,

View file

@ -235,21 +235,42 @@ fn process_bar_duplicates(gui_data: &GuiData, item: &ProgressData) {
// Checking Size
0 => {
progress_bar_current_stage.hide();
// progress_bar_all_stages.hide();
progress_bar_all_stages.set_fraction(0 as f64);
progress_bar_all_stages.set_fraction(0f64);
label_stage.set_text(&flg!("progress_scanning_size", file_number_tm(item)));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
// Loading cache
1 | 4 => {
progress_bar_current_stage.hide();
common_set_data(item, &progress_bar_all_stages, &progress_bar_current_stage, &taskbar_state);
if item.current_stage == 1 {
label_stage.set_text(&flg!("progress_prehash_cache_loading"));
} else {
label_stage.set_text(&flg!("progress_hash_cache_loading"));
}
}
// Saving cache
3 | 6 => {
progress_bar_current_stage.hide();
common_set_data(item, &progress_bar_all_stages, &progress_bar_current_stage, &taskbar_state);
if item.current_stage == 3 {
label_stage.set_text(&flg!("progress_prehash_cache_saving"));
} else {
label_stage.set_text(&flg!("progress_hash_cache_saving"));
}
}
// Hash - first 1KB file
1 => {
2 => {
progress_bar_current_stage.show();
// progress_bar_all_stages.show();
common_set_data(item, &progress_bar_all_stages, &progress_bar_current_stage, &taskbar_state);
label_stage.set_text(&flg!("progress_analyzed_partial_hash", progress_ratio_tm(item)));
}
// Hash - normal hash
2 => {
5 => {
progress_bar_current_stage.show();
common_set_data(item, &progress_bar_all_stages, &progress_bar_current_stage, &taskbar_state);
label_stage.set_text(&flg!("progress_analyzed_full_hash", progress_ratio_tm(item)));
}
@ -285,14 +306,18 @@ fn process_bar_duplicates(gui_data: &GuiData, item: &ProgressData) {
fn common_set_data(item: &ProgressData, progress_bar_all_stages: &ProgressBar, progress_bar_current_stage: &ProgressBar, taskbar_state: &Rc<RefCell<TaskbarProgress>>) {
if item.entries_to_check != 0 {
progress_bar_all_stages.set_fraction((item.current_stage as f64 + (item.entries_checked) as f64 / item.entries_to_check as f64) / (item.max_stage + 1) as f64);
let all_stages = (item.current_stage as f64 + (item.entries_checked) as f64 / item.entries_to_check as f64) / (item.max_stage + 1) as f64;
let all_stages = if all_stages > 0.99 { 0.99 } else { all_stages };
progress_bar_all_stages.set_fraction(all_stages);
progress_bar_current_stage.set_fraction((item.entries_checked) as f64 / item.entries_to_check as f64);
taskbar_state.borrow().set_progress_value(
((item.current_stage as usize) * item.entries_to_check + item.entries_checked) as u64,
item.entries_to_check as u64 * (item.max_stage + 1) as u64,
);
} else {
progress_bar_all_stages.set_fraction((item.current_stage as f64) / (item.max_stage + 1) as f64);
let all_stages = (item.current_stage as f64) / (item.max_stage + 1) as f64;
let all_stages = if all_stages > 0.99 { 0.99 } else { all_stages };
progress_bar_all_stages.set_fraction(all_stages);
progress_bar_current_stage.set_fraction(0f64);
taskbar_state.borrow().set_progress_value(item.current_stage as u64, 1 + item.max_stage as u64);
}