CodeMirror/czkawka
1
0
Fork 0
mirror of synced 2024-05-17 19:03:08 +12:00
Code Issues Projects Releases Wiki Activity

Also

Browse source
This commit is contained in:
Rafał Mikrut 2023-05-02 09:17:39 +02:00
parent e59d209c57
commit 45da05c173
10 changed files with 90 additions and 136 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
czkawka_core/src/bad_extensions.rs
View file

@ -13,7 +13,7 @@ use crossbeam_channel::Receiver;
use mime_guess::get_mime_extensions;
use rayon::prelude::*;
use crate::common::{prepare_thread_handler_common, Common};
use crate::common::{prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, Common};
use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData};
use crate::common_directory::Directories;
use crate::common_extensions::Extensions;
@ -25,7 +25,7 @@ static DISABLED_EXTENSIONS: &[&str] = &["file", "cache", "bak", "data"]; // Such
// This adds several workarounds for bugs/invalid recognizing types by external libraries
// ("real_content_extension", "current_file_extension")
static WORKAROUNDS: &[(&str, &str)] = &[
const WORKAROUNDS: &[(&str, &str)] = &[
// Wine/Windows
("der", "cat"),
("exe", "acm"),
@ -322,8 +322,6 @@ impl BadExtensions {
let check_was_stopped = AtomicBool::new(false); // Used for breaking from GUI and ending check thread
//// PROGRESS THREAD START
let progress_thread_run = Arc::new(AtomicBool::new(true));
let atomic_counter = Arc::new(AtomicUsize::new(0));
@ -339,7 +337,6 @@ impl BadExtensions {
let mut files_to_check = Default::default();
mem::swap(&mut files_to_check, &mut self.files_to_check);
//// PROGRESS THREAD END
let mut hashmap_workarounds: HashMap<&str, Vec<&str>> = Default::default();
for (proper, found) in WORKAROUNDS {
@ -352,9 +349,7 @@ impl BadExtensions {
self.bad_extensions_files = self.verify_extensions(files_to_check, &atomic_counter, stop_receiver, &check_was_stopped, &hashmap_workarounds);
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
// Break if stop was clicked
if check_was_stopped.load(Ordering::Relaxed) {

10
czkawka_core/src/big_file.rs
View file

@ -15,7 +15,7 @@ use humansize::format_size;
use humansize::BINARY;
use rayon::prelude::*;
use crate::common::{check_folder_children, split_path};
use crate::common::{check_folder_children, send_info_and_wait_for_ending_all_threads, split_path};
use crate::common::{Common, LOOP_DURATION};
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time};
use crate::common_directory::Directories;
@ -189,9 +189,7 @@ impl BigFile {
while !folders_to_check.is_empty() {
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
}
@ -243,9 +241,7 @@ impl BigFile {
}
}
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
self.extract_n_biggest_files(old_map);

97
czkawka_core/src/broken_files.rs
View file

@ -1,5 +1,5 @@
use std::collections::BTreeMap;
use std::fs::File;
use std::fs::{DirEntry, File, Metadata};
use std::io::prelude::*;
use std::io::{BufReader, BufWriter};
use std::path::{Path, PathBuf};
@ -17,7 +17,7 @@ use pdf::PdfError::Try;
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use crate::common::{check_folder_children, create_crash_message, open_cache_folder, Common, LOOP_DURATION, PDF_FILES_EXTENSIONS};
use crate::common::{check_folder_children, create_crash_message, open_cache_folder, send_info_and_wait_for_ending_all_threads, Common, LOOP_DURATION, PDF_FILES_EXTENSIONS};
use crate::common::{AUDIO_FILES_EXTENSIONS, IMAGE_RS_BROKEN_FILES_EXTENSIONS, ZIP_FILES_EXTENSIONS};
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time};
use crate::common_directory::Directories;
@ -238,18 +238,13 @@ impl BrokenFiles {
folders_to_check.push(id.clone());
}
//// PROGRESS THREAD START
let progress_thread_run = Arc::new(AtomicBool::new(true));
let atomic_counter = Arc::new(AtomicUsize::new(0));
let progress_thread_handle = self.prepare_thread_handler_broken_files(progress_sender, &progress_thread_run, &atomic_counter, 0, 1, 0);
//// PROGRESS THREAD END
while !folders_to_check.is_empty() {
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
}
@ -265,7 +260,7 @@ impl BrokenFiles {
};
// Check every sub folder/file/link etc.
'dir: for entry in read_dir {
for entry in read_dir {
let Some((entry_data,metadata)) = common_get_entry_data_metadata(&entry, &mut warnings, current_folder) else {
continue;
};
@ -281,39 +276,9 @@ impl BrokenFiles {
&self.excluded_items,
);
} else if metadata.is_file() {
atomic_counter.fetch_add(1, Ordering::Relaxed);
let Some(file_name_lowercase) = get_lowercase_name(entry_data, &mut warnings) else {
continue 'dir;
};
if !self.allowed_extensions.matches_filename(&file_name_lowercase) {
continue 'dir;
if let Some(file_entry) = self.get_file_entry(&metadata, &atomic_counter, entry_data, &mut warnings, current_folder) {
fe_result.push((file_entry.path.to_string_lossy().to_string(), file_entry));
}
let type_of_file = check_extension_availability(&file_name_lowercase);
if type_of_file == TypeOfFile::Unknown {
continue 'dir;
}
if !check_extension_allowed(&type_of_file, &self.checked_types) {
continue 'dir;
}
let current_file_name = current_folder.join(entry_data.file_name());
if self.excluded_items.is_excluded(&current_file_name) {
continue 'dir;
}
let fe: FileEntry = FileEntry {
path: current_file_name.clone(),
modified_date: get_modified_time(&metadata, &mut warnings, &current_file_name, false),
size: metadata.len(),
type_of_file,
error_string: String::new(),
};
fe_result.push((current_file_name.to_string_lossy().to_string(), fe));
}
}
(dir_result, warnings, fe_result)
@ -333,13 +298,53 @@ impl BrokenFiles {
}
}
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
Common::print_time(start_time, SystemTime::now(), "check_files");
true
}
fn get_file_entry(
&self,
metadata: &Metadata,
atomic_counter: &Arc<AtomicUsize>,
entry_data: &DirEntry,
warnings: &mut Vec<String>,
current_folder: &Path,
) -> Option<FileEntry> {
atomic_counter.fetch_add(1, Ordering::Relaxed);
let Some(file_name_lowercase) = get_lowercase_name(entry_data, warnings) else {
return None;
};
if !self.allowed_extensions.matches_filename(&file_name_lowercase) {
return None;
}
let type_of_file = check_extension_availability(&file_name_lowercase);
if type_of_file == TypeOfFile::Unknown {
return None;
}
if !check_extension_allowed(&type_of_file, &self.checked_types) {
return None;
}
let current_file_name = current_folder.join(entry_data.file_name());
if self.excluded_items.is_excluded(&current_file_name) {
return None;
}
let fe: FileEntry = FileEntry {
path: current_file_name.clone(),
modified_date: get_modified_time(metadata, warnings, &current_file_name, false),
size: metadata.len(),
type_of_file,
error_string: String::new(),
};
Some(fe)
}
fn look_for_broken_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender<ProgressData>>) -> bool {
let system_time = SystemTime::now();
@ -493,9 +498,7 @@ impl BrokenFiles {
.map(Option::unwrap)
.collect::<Vec<FileEntry>>();
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
// Just connect loaded results with already calculated
for (_name, file_entry) in records_already_cached {

5
czkawka_core/src/common.rs
View file

@ -392,6 +392,11 @@ pub fn prepare_thread_handler_common(
}
}
pub fn send_info_and_wait_for_ending_all_threads(progress_thread_run: &Arc<AtomicBool>, progress_thread_handle: JoinHandle<()>) {
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
}
#[cfg(test)]
mod test {
use std::path::PathBuf;

10
czkawka_core/src/common_dir_traversal.rs
View file

@ -9,7 +9,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
use crossbeam_channel::Receiver;
use rayon::prelude::*;
use crate::common::prepare_thread_handler_common;
use crate::common::{prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads};
use crate::common_directory::Directories;
use crate::common_extensions::Extensions;
use crate::common_items::ExcludedItems;
@ -358,9 +358,7 @@ where
while !folders_to_check.is_empty() {
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return DirTraversalResult::Stopped;
}
@ -590,9 +588,7 @@ where
}
}
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
match collect {
Collect::Files | Collect::InvalidSymlinks => DirTraversalResult::SuccessFiles {

14
czkawka_core/src/duplicate.rs
View file

@ -20,7 +20,7 @@ use humansize::format_size;
use humansize::BINARY;
use rayon::prelude::*;
use crate::common::{open_cache_folder, prepare_thread_handler_common, Common};
use crate::common::{open_cache_folder, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, Common};
use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData};
use crate::common_directory::Directories;
use crate::common_extensions::Extensions;
@ -734,9 +734,7 @@ impl DuplicateFinder {
.while_some()
.collect();
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
// Check if user aborted search(only from GUI)
if check_was_stopped.load(Ordering::Relaxed) {
@ -798,7 +796,7 @@ impl DuplicateFinder {
let check_type = self.hash_type;
let start_time: SystemTime = SystemTime::now();
//// PROGRESS THREAD START
let progress_thread_run = Arc::new(AtomicBool::new(true));
let atomic_counter = Arc::new(AtomicUsize::new(0));
@ -812,8 +810,6 @@ impl DuplicateFinder {
self.check_method,
);
//// PROGRESS THREAD END
///////////////////////////////////////////////////////////////////////////// HASHING START
{
#[allow(clippy::type_complexity)]
@ -922,9 +918,7 @@ impl DuplicateFinder {
save_hashes_to_file(&all_results, &mut self.text_messages, &self.hash_type, false, self.minimal_cache_file_size);
}
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
// Break if stop was clicked after saving to cache
if check_was_stopped.load(Ordering::Relaxed) {

34
czkawka_core/src/same_music.rs
View file

@ -15,7 +15,7 @@ use lofty::{read_from, AudioFile, ItemKey};
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use crate::common::{create_crash_message, prepare_thread_handler_common, AUDIO_FILES_EXTENSIONS};
use crate::common::{create_crash_message, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, AUDIO_FILES_EXTENSIONS};
use crate::common::{open_cache_folder, Common};
use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData};
use crate::common_directory::Directories;
@ -475,9 +475,7 @@ impl SameMusic {
.map(Option::unwrap)
.collect::<Vec<_>>();
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
// Just connect loaded results with already calculated
for (_name, file_entry) in records_already_cached {
@ -528,9 +526,7 @@ impl SameMusic {
for vec_file_entry in old_duplicates {
atomic_counter.fetch_add(1, Ordering::Relaxed);
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
}
let mut hash_map: BTreeMap<String, Vec<MusicEntry>> = Default::default();
@ -556,9 +552,7 @@ impl SameMusic {
for vec_file_entry in old_duplicates {
atomic_counter.fetch_add(1, Ordering::Relaxed);
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
}
let mut hash_map: BTreeMap<String, Vec<MusicEntry>> = Default::default();
@ -584,9 +578,7 @@ impl SameMusic {
for vec_file_entry in old_duplicates {
atomic_counter.fetch_add(1, Ordering::Relaxed);
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
}
let mut hash_map: BTreeMap<String, Vec<MusicEntry>> = Default::default();
@ -609,9 +601,7 @@ impl SameMusic {
for vec_file_entry in old_duplicates {
atomic_counter.fetch_add(1, Ordering::Relaxed);
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
}
let mut hash_map: BTreeMap<String, Vec<MusicEntry>> = Default::default();
@ -634,9 +624,7 @@ impl SameMusic {
for vec_file_entry in old_duplicates {
atomic_counter.fetch_add(1, Ordering::Relaxed);
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
}
let mut hash_map: BTreeMap<String, Vec<MusicEntry>> = Default::default();
@ -659,9 +647,7 @@ impl SameMusic {
for vec_file_entry in old_duplicates {
atomic_counter.fetch_add(1, Ordering::Relaxed);
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
}
let mut hash_map: BTreeMap<String, Vec<MusicEntry>> = Default::default();
@ -683,9 +669,7 @@ impl SameMusic {
// new_duplicates = Vec::new();
}
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
self.duplicated_music_entries = old_duplicates;

20
czkawka_core/src/similar_images.rs
View file

@ -22,8 +22,8 @@ use serde::{Deserialize, Serialize};
#[cfg(feature = "heif")]
use crate::common::get_dynamic_image_from_heic;
use crate::common::{
check_folder_children, create_crash_message, get_dynamic_image_from_raw_image, get_number_of_threads, open_cache_folder, Common, HEIC_EXTENSIONS,
IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS, LOOP_DURATION, RAW_IMAGE_EXTENSIONS,
check_folder_children, create_crash_message, get_dynamic_image_from_raw_image, get_number_of_threads, open_cache_folder, send_info_and_wait_for_ending_all_threads, Common,
HEIC_EXTENSIONS, IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS, LOOP_DURATION, RAW_IMAGE_EXTENSIONS,
};
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time};
use crate::common_directory::Directories;
@ -339,9 +339,7 @@ impl SimilarImages {
while !folders_to_check.is_empty() {
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
}
@ -420,9 +418,7 @@ impl SimilarImages {
}
}
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
Common::print_time(start_time, SystemTime::now(), "check_for_similar_images");
true
}
@ -556,9 +552,7 @@ impl SimilarImages {
.map(Option::unwrap)
.collect::<Vec<(FileEntry, Vec<u8>)>>();
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
Common::print_time(hash_map_modification, SystemTime::now(), "sort_images - reading data from files in parallel");
let hash_map_modification = SystemTime::now();
@ -748,9 +742,7 @@ impl SimilarImages {
.while_some()
.collect();
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
if check_was_stopped.load(Ordering::Relaxed) {
return false;

15
czkawka_core/src/similar_videos.rs
View file

@ -18,7 +18,7 @@ use serde::{Deserialize, Serialize};
use vid_dup_finder_lib::HashCreationErrorKind::DetermineVideo;
use vid_dup_finder_lib::{NormalizedTolerance, VideoHash};
use crate::common::{check_folder_children, VIDEO_FILES_EXTENSIONS};
use crate::common::{check_folder_children, send_info_and_wait_for_ending_all_threads, VIDEO_FILES_EXTENSIONS};
use crate::common::{open_cache_folder, Common, LOOP_DURATION};
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time};
use crate::common_directory::Directories;
@ -305,9 +305,7 @@ impl SimilarVideos {
while !folders_to_check.is_empty() {
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
}
@ -386,9 +384,7 @@ impl SimilarVideos {
}
}
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
Common::print_time(start_time, SystemTime::now(), "check_for_similar_videos");
true
}
@ -428,7 +424,6 @@ impl SimilarVideos {
Common::print_time(hash_map_modification, SystemTime::now(), "sort_videos - reading data from cache and preparing them");
let hash_map_modification = SystemTime::now();
//// PROGRESS THREAD START
let check_was_stopped = AtomicBool::new(false); // Used for breaking from GUI and ending check thread
let progress_thread_run = Arc::new(AtomicBool::new(true));
@ -462,9 +457,7 @@ impl SimilarVideos {
.while_some()
.collect::<Vec<FileEntry>>();
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
Common::print_time(hash_map_modification, SystemTime::now(), "sort_videos - reading data from files in parallel");
let hash_map_modification = SystemTime::now();

10
czkawka_core/src/temporary.rs
View file

@ -11,7 +11,7 @@ use std::{fs, thread};
use crossbeam_channel::Receiver;
use rayon::prelude::*;
use crate::common::{check_folder_children, Common, LOOP_DURATION};
use crate::common::{check_folder_children, send_info_and_wait_for_ending_all_threads, Common, LOOP_DURATION};
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time};
use crate::common_directory::Directories;
use crate::common_items::ExcludedItems;
@ -192,9 +192,7 @@ impl Temporary {
while !folders_to_check.is_empty() {
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
}
@ -248,9 +246,7 @@ impl Temporary {
}
}
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
self.information.number_of_temporary_files = self.temporary_files.len();
Common::print_time(start_time, SystemTime::now(), "check_files_size");