diff --git a/czkawka_core/src/bad_extensions.rs b/czkawka_core/src/bad_extensions.rs index 420d35d..34a9eb1 100644 --- a/czkawka_core/src/bad_extensions.rs +++ b/czkawka_core/src/bad_extensions.rs @@ -5,15 +5,15 @@ use std::io::BufWriter; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::Arc; -use std::thread::{sleep, JoinHandle}; -use std::time::{Duration, SystemTime}; -use std::{mem, thread}; + +use std::mem; +use std::time::SystemTime; use crossbeam_channel::Receiver; use mime_guess::get_mime_extensions; use rayon::prelude::*; -use crate::common::{Common, LOOP_DURATION}; +use crate::common::{prepare_thread_handler_common, Common}; use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData}; use crate::common_directory::Directories; use crate::common_extensions::Extensions; @@ -317,39 +317,6 @@ impl BadExtensions { } } - fn prepare_bad_extension_thread_handler( - &self, - progress_sender: Option<&futures::channel::mpsc::UnboundedSender>, - progress_thread_run: Arc, - atomic_counter: Arc, - current_stage: u8, - max_stage: u8, - max_value: usize, - ) -> JoinHandle<()> { - if let Some(progress_sender) = progress_sender { - let progress_send = progress_sender.clone(); - let progress_thread_run = progress_thread_run; - let atomic_counter = atomic_counter; - thread::spawn(move || loop { - progress_send - .unbounded_send(ProgressData { - checking_method: CheckingMethod::None, - current_stage, - max_stage, - entries_checked: atomic_counter.load(Ordering::Relaxed), - entries_to_check: max_value, - }) - .unwrap(); - if !progress_thread_run.load(Ordering::Relaxed) { - break; - } - sleep(Duration::from_millis(LOOP_DURATION as u64)); - }) - } else { - thread::spawn(|| {}) - } - } - fn look_for_bad_extensions_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { let system_time = SystemTime::now(); @@ -360,8 +327,15 @@ impl BadExtensions { let progress_thread_run = Arc::new(AtomicBool::new(true)); let atomic_file_counter = Arc::new(AtomicUsize::new(0)); - let progress_thread_handle = - self.prepare_bad_extension_thread_handler(progress_sender, progress_thread_run.clone(), atomic_file_counter.clone(), 1, 1, self.files_to_check.len()); + let progress_thread_handle = prepare_thread_handler_common( + progress_sender, + &progress_thread_run, + &atomic_file_counter, + 1, + 1, + self.files_to_check.len(), + CheckingMethod::None, + ); let mut files_to_check = Default::default(); mem::swap(&mut files_to_check, &mut self.files_to_check); diff --git a/czkawka_core/src/big_file.rs b/czkawka_core/src/big_file.rs index 0e7acec..8b5f2c6 100644 --- a/czkawka_core/src/big_file.rs +++ b/czkawka_core/src/big_file.rs @@ -5,7 +5,7 @@ use std::path::{Path, PathBuf}; use std::sync::atomic::Ordering; use std::sync::atomic::{AtomicBool, AtomicU64}; use std::sync::Arc; -use std::thread::sleep; +use std::thread::{sleep, JoinHandle}; use std::time::Duration; use std::time::{SystemTime, UNIX_EPOCH}; use std::{fs, thread}; @@ -148,22 +148,13 @@ impl BigFile { self.allowed_extensions.set_allowed_extensions(allowed_extensions, &mut self.text_messages); } - fn look_for_big_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { - let start_time: SystemTime = SystemTime::now(); - let mut folders_to_check: Vec = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector - let mut old_map: BTreeMap> = Default::default(); - - // Add root folders for finding - for id in &self.directories.included_directories { - folders_to_check.push(id.clone()); - } - - //// PROGRESS THREAD START - let progress_thread_run = Arc::new(AtomicBool::new(true)); - - let atomic_file_counter = Arc::new(AtomicU64::new(0)); - - let progress_thread_handle = if let Some(progress_sender) = progress_sender { + pub fn prepare_thread_handler( + &self, + progress_sender: Option<&futures::channel::mpsc::UnboundedSender>, + progress_thread_run: &Arc, + atomic_file_counter: &Arc, + ) -> JoinHandle<()> { + if let Some(progress_sender) = progress_sender { let progress_send = progress_sender.clone(); let progress_thread_run = progress_thread_run.clone(); let atomic_file_counter = atomic_file_counter.clone(); @@ -180,9 +171,23 @@ impl BigFile { }) } else { thread::spawn(|| {}) - }; + } + } + + fn look_for_big_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { + let start_time: SystemTime = SystemTime::now(); + let mut folders_to_check: Vec = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector + let mut old_map: BTreeMap> = Default::default(); + + // Add root folders for finding + for id in &self.directories.included_directories { + folders_to_check.push(id.clone()); + } + + let progress_thread_run = Arc::new(AtomicBool::new(true)); + let atomic_file_counter = Arc::new(AtomicU64::new(0)); + let progress_thread_handle = self.prepare_thread_handler(progress_sender, &progress_thread_run, &atomic_file_counter); - //// PROGRESS THREAD END while !folders_to_check.is_empty() { if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { // End thread which send info to gui diff --git a/czkawka_core/src/broken_files.rs b/czkawka_core/src/broken_files.rs index 1750291..72de6f8 100644 --- a/czkawka_core/src/broken_files.rs +++ b/czkawka_core/src/broken_files.rs @@ -5,7 +5,7 @@ use std::io::{BufReader, BufWriter}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::Arc; -use std::thread::sleep; +use std::thread::{sleep, JoinHandle}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use std::{fs, mem, panic, thread}; @@ -198,6 +198,38 @@ impl BrokenFiles { self.excluded_items.set_excluded_items(excluded_items, &mut self.text_messages); } + pub fn prepare_thread_handler_broken_files( + &self, + progress_sender: Option<&futures::channel::mpsc::UnboundedSender>, + progress_thread_run: &Arc, + atomic_counter: &Arc, + current_stage: u8, + max_stage: u8, + max_value: usize, + ) -> JoinHandle<()> { + if let Some(progress_sender) = progress_sender { + let progress_send = progress_sender.clone(); + let progress_thread_run = progress_thread_run.clone(); + let atomic_counter = atomic_counter.clone(); + thread::spawn(move || loop { + progress_send + .unbounded_send(ProgressData { + current_stage, + max_stage, + files_checked: atomic_counter.load(Ordering::Relaxed), + files_to_check: max_value, + }) + .unwrap(); + if !progress_thread_run.load(Ordering::Relaxed) { + break; + } + sleep(Duration::from_millis(LOOP_DURATION as u64)); + }) + } else { + thread::spawn(|| {}) + } + } + fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { let start_time: SystemTime = SystemTime::now(); let mut folders_to_check: Vec = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector @@ -209,30 +241,9 @@ impl BrokenFiles { //// PROGRESS THREAD START let progress_thread_run = Arc::new(AtomicBool::new(true)); - let atomic_file_counter = Arc::new(AtomicUsize::new(0)); + let progress_thread_handle = self.prepare_thread_handler_broken_files(progress_sender, &progress_thread_run, &atomic_file_counter, 0, 1, 0); - let progress_thread_handle = if let Some(progress_sender) = progress_sender { - let progress_send = progress_sender.clone(); - let progress_thread_run = progress_thread_run.clone(); - let atomic_file_counter = atomic_file_counter.clone(); - thread::spawn(move || loop { - progress_send - .unbounded_send(ProgressData { - current_stage: 0, - max_stage: 1, - files_checked: atomic_file_counter.load(Ordering::Relaxed), - files_to_check: 0, - }) - .unwrap(); - if !progress_thread_run.load(Ordering::Relaxed) { - break; - } - sleep(Duration::from_millis(LOOP_DURATION as u64)); - }) - } else { - thread::spawn(|| {}) - }; //// PROGRESS THREAD END while !folders_to_check.is_empty() { @@ -431,33 +442,10 @@ impl BrokenFiles { non_cached_files_to_check = files_to_check; } - //// PROGRESS THREAD START let progress_thread_run = Arc::new(AtomicBool::new(true)); let atomic_file_counter = Arc::new(AtomicUsize::new(0)); + let progress_thread_handle = self.prepare_thread_handler_broken_files(progress_sender, &progress_thread_run, &atomic_file_counter, 1, 1, non_cached_files_to_check.len()); - let progress_thread_handle = if let Some(progress_sender) = progress_sender { - let progress_send = progress_sender.clone(); - let progress_thread_run = progress_thread_run.clone(); - let atomic_file_counter = atomic_file_counter.clone(); - let files_to_check = non_cached_files_to_check.len(); - thread::spawn(move || loop { - progress_send - .unbounded_send(ProgressData { - current_stage: 1, - max_stage: 1, - files_checked: atomic_file_counter.load(Ordering::Relaxed), - files_to_check, - }) - .unwrap(); - if !progress_thread_run.load(Ordering::Relaxed) { - break; - } - sleep(Duration::from_millis(LOOP_DURATION as u64)); - }) - } else { - thread::spawn(|| {}) - }; - //// PROGRESS THREAD END let mut vec_file_entry: Vec = non_cached_files_to_check .into_par_iter() .map(|(_, mut file_entry)| { diff --git a/czkawka_core/src/common.rs b/czkawka_core/src/common.rs index bafd5c1..528be1f 100644 --- a/czkawka_core/src/common.rs +++ b/czkawka_core/src/common.rs @@ -1,9 +1,12 @@ use std::ffi::OsString; -use std::fs; use std::fs::{File, OpenOptions}; use std::io::BufReader; use std::path::{Path, PathBuf}; -use std::time::SystemTime; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Arc; +use std::thread::{sleep, JoinHandle}; +use std::time::{Duration, SystemTime}; +use std::{fs, thread}; #[cfg(feature = "heif")] use anyhow::Result; @@ -12,6 +15,7 @@ use image::{DynamicImage, ImageBuffer, Rgb}; use imagepipe::{ImageSource, Pipeline}; // #[cfg(feature = "heif")] // use libheif_rs::LibHeif; +use crate::common_dir_traversal::{CheckingMethod, ProgressData}; #[cfg(feature = "heif")] use libheif_rs::{ColorSpace, HeifContext, RgbChroma}; @@ -318,6 +322,38 @@ impl Common { } } } +pub fn prepare_thread_handler_common( + progress_sender: Option<&futures::channel::mpsc::UnboundedSender>, + progress_thread_run: &Arc, + atomic_counter: &Arc, + current_stage: u8, + max_stage: u8, + max_value: usize, + checking_method: CheckingMethod, +) -> JoinHandle<()> { + if let Some(progress_sender) = progress_sender { + let progress_send = progress_sender.clone(); + let progress_thread_run = progress_thread_run.clone(); + let atomic_counter = atomic_counter.clone(); + thread::spawn(move || loop { + progress_send + .unbounded_send(ProgressData { + checking_method, + current_stage, + max_stage, + entries_checked: atomic_counter.load(Ordering::Relaxed), + entries_to_check: max_value, + }) + .unwrap(); + if !progress_thread_run.load(Ordering::Relaxed) { + break; + } + sleep(Duration::from_millis(LOOP_DURATION as u64)); + }) + } else { + thread::spawn(|| {}) + } +} #[cfg(test)] mod test { diff --git a/czkawka_core/src/duplicate.rs b/czkawka_core/src/duplicate.rs index 77edae1..0bdbb32 100644 --- a/czkawka_core/src/duplicate.rs +++ b/czkawka_core/src/duplicate.rs @@ -11,16 +11,16 @@ use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::Arc; -use std::thread::{sleep, JoinHandle}; -use std::time::{Duration, SystemTime}; -use std::{fs, mem, thread}; + +use std::time::SystemTime; +use std::{fs, mem}; use crossbeam_channel::Receiver; use humansize::format_size; use humansize::BINARY; use rayon::prelude::*; -use crate::common::{open_cache_folder, Common, LOOP_DURATION}; +use crate::common::{open_cache_folder, prepare_thread_handler_common, Common}; use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData}; use crate::common_directory::Directories; use crate::common_extensions::Extensions; @@ -644,41 +644,6 @@ impl DuplicateFinder { } } - // TODO Generalize this if possible with different tools - fn prepare_hash_thread_handler( - &self, - progress_sender: Option<&futures::channel::mpsc::UnboundedSender>, - progress_thread_run: Arc, - atomic_counter: Arc, - current_stage: u8, - max_stage: u8, - max_value: usize, - ) -> JoinHandle<()> { - if let Some(progress_sender) = progress_sender { - let progress_send = progress_sender.clone(); - let progress_thread_run = progress_thread_run; - let atomic_counter = atomic_counter; - let checking_method = self.check_method; - thread::spawn(move || loop { - progress_send - .unbounded_send(ProgressData { - checking_method, - current_stage, - max_stage, - entries_checked: atomic_counter.load(Ordering::Relaxed), - entries_to_check: max_value, - }) - .unwrap(); - if !progress_thread_run.load(Ordering::Relaxed) { - break; - } - sleep(Duration::from_millis(LOOP_DURATION as u64)); - }) - } else { - thread::spawn(|| {}) - } - } - fn prehashing( &mut self, stop_receiver: Option<&Receiver<()>>, @@ -691,13 +656,14 @@ impl DuplicateFinder { let progress_thread_run = Arc::new(AtomicBool::new(true)); let atomic_file_counter = Arc::new(AtomicUsize::new(0)); - let progress_thread_handle = self.prepare_hash_thread_handler( + let progress_thread_handle = prepare_thread_handler_common( progress_sender, - progress_thread_run.clone(), - atomic_file_counter.clone(), + &progress_thread_run, + &atomic_file_counter, 1, 2, self.files_with_identical_size.values().map(Vec::len).sum(), + self.check_method, ); let loaded_hash_map; @@ -836,13 +802,14 @@ impl DuplicateFinder { let progress_thread_run = Arc::new(AtomicBool::new(true)); let atomic_file_counter = Arc::new(AtomicUsize::new(0)); - let progress_thread_handle = self.prepare_hash_thread_handler( + let progress_thread_handle = prepare_thread_handler_common( progress_sender, - progress_thread_run.clone(), - atomic_file_counter.clone(), + &progress_thread_run, + &atomic_file_counter, 2, 2, pre_checked_map.values().map(Vec::len).sum(), + self.check_method, ); //// PROGRESS THREAD END