Parallel BFS with Rayon (#502)
* Tester * Parallel BFS with Rayon * Rustfmt * Refactor code * Use `extend` instead of `append` * Remove no longer needed derives
This commit is contained in:
parent
c041ab416e
commit
859e01b30e
|
@ -185,7 +185,7 @@ impl DuplicateFinder {
|
|||
|
||||
match self.check_method {
|
||||
CheckingMethod::Name => {
|
||||
if !self.check_files_name(stop_receiver, progress_sender) {
|
||||
if !self.check_files_name_threaded(stop_receiver, progress_sender) {
|
||||
self.stopped_search = true;
|
||||
return;
|
||||
}
|
||||
|
@ -314,7 +314,7 @@ impl DuplicateFinder {
|
|||
self.excluded_items.set_excluded_items(excluded_items, &mut self.text_messages);
|
||||
}
|
||||
|
||||
fn check_files_name(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender<ProgressData>>) -> bool {
|
||||
fn check_files_name_threaded(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender<ProgressData>>) -> bool {
|
||||
let start_time: SystemTime = SystemTime::now();
|
||||
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
|
||||
|
||||
|
@ -363,14 +363,18 @@ impl DuplicateFinder {
|
|||
return false;
|
||||
}
|
||||
|
||||
let current_folder = folders_to_check.pop().unwrap();
|
||||
|
||||
let segments: Vec<_> = folders_to_check
|
||||
.par_iter()
|
||||
.map(|current_folder| {
|
||||
let mut dir_result = vec![];
|
||||
let mut warnings = vec![];
|
||||
let mut fe_result = vec![];
|
||||
// Read current dir, if permission are denied just go to next
|
||||
let read_dir = match fs::read_dir(¤t_folder) {
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
self.text_messages.warnings.push(format!("Cannot open dir {}, reason {}", current_folder.display(), e));
|
||||
continue;
|
||||
warnings.push(format!("Cannot open dir {}, reason {}", current_folder.display(), e));
|
||||
return (dir_result, warnings, fe_result);
|
||||
} // Permissions denied
|
||||
};
|
||||
|
||||
|
@ -379,14 +383,14 @@ impl DuplicateFinder {
|
|||
let entry_data = match entry {
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
self.text_messages.warnings.push(format!("Cannot read entry in dir {}, reason {}", current_folder.display(), e));
|
||||
warnings.push(format!("Cannot read entry in dir {}, reason {}", current_folder.display(), e));
|
||||
continue 'dir;
|
||||
} //Permissions denied
|
||||
};
|
||||
let metadata: Metadata = match entry_data.metadata() {
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
self.text_messages.warnings.push(format!("Cannot read metadata in dir {}, reason {}", current_folder.display(), e));
|
||||
warnings.push(format!("Cannot read metadata in dir {}, reason {}", current_folder.display(), e));
|
||||
continue 'dir;
|
||||
} //Permissions denied
|
||||
};
|
||||
|
@ -404,7 +408,7 @@ impl DuplicateFinder {
|
|||
continue 'dir;
|
||||
}
|
||||
|
||||
folders_to_check.push(next_folder);
|
||||
dir_result.push(next_folder);
|
||||
} else if metadata.is_file() {
|
||||
atomic_file_counter.fetch_add(1, Ordering::Relaxed);
|
||||
// let mut have_valid_extension: bool;
|
||||
|
@ -440,12 +444,12 @@ impl DuplicateFinder {
|
|||
Ok(t) => match t.duration_since(UNIX_EPOCH) {
|
||||
Ok(d) => d.as_secs(),
|
||||
Err(_inspected) => {
|
||||
self.text_messages.warnings.push(format!("File {} seems to be modified before Unix Epoch.", current_file_name.display()));
|
||||
warnings.push(format!("File {} seems to be modified before Unix Epoch.", current_file_name.display()));
|
||||
0
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
self.text_messages.warnings.push(format!("Unable to get modification date from file {}, reason {}", current_file_name.display(), e));
|
||||
warnings.push(format!("Unable to get modification date from file {}, reason {}", current_file_name.display(), e));
|
||||
0
|
||||
} // Permissions Denied
|
||||
},
|
||||
|
@ -453,11 +457,26 @@ impl DuplicateFinder {
|
|||
};
|
||||
|
||||
// Adding files to BTreeMap
|
||||
self.files_with_identical_names.entry(entry_data.file_name().to_string_lossy().to_string()).or_insert_with(Vec::new);
|
||||
self.files_with_identical_names.get_mut(&entry_data.file_name().to_string_lossy().to_string()).unwrap().push(fe);
|
||||
fe_result.push((entry_data.file_name().to_string_lossy().to_string(), fe));
|
||||
}
|
||||
}
|
||||
}
|
||||
(dir_result, warnings, fe_result)
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Advance the frontier
|
||||
folders_to_check.clear();
|
||||
|
||||
// Process collected data
|
||||
for (segment, warnings, fe_result) in segments {
|
||||
folders_to_check.extend(segment);
|
||||
self.text_messages.warnings.extend(warnings);
|
||||
for (name, fe) in fe_result {
|
||||
self.files_with_identical_names.entry(name.clone()).or_insert_with(Vec::new);
|
||||
self.files_with_identical_names.get_mut(&name).unwrap().push(fe);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// End thread which send info to gui
|
||||
|
|
Loading…
Reference in a new issue