CodeMirror/czkawka
1
0
Fork 0
mirror of synced 2024-05-08 06:23:00 +12:00
Code Issues Projects Releases Wiki Activity

Parallel BFS with Rayon (#502)

Browse source 
* Tester

* Parallel BFS with Rayon

* Rustfmt

* Refactor code

* Use `extend` instead of `append`

* Remove no longer needed derives
This commit is contained in:
Peter Blackson 2021-12-16 20:49:59 +01:00 committed by GitHub
parent c041ab416e
commit 859e01b30e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 109 additions and 90 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
czkawka_core/src/duplicate.rs
View file

@ -185,7 +185,7 @@ impl DuplicateFinder {
match self.check_method {
CheckingMethod::Name => {
if !self.check_files_name(stop_receiver, progress_sender) {
if !self.check_files_name_threaded(stop_receiver, progress_sender) {
self.stopped_search = true;
return;
}
@ -314,7 +314,7 @@ impl DuplicateFinder {
self.excluded_items.set_excluded_items(excluded_items, &mut self.text_messages);
}
fn check_files_name(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender<ProgressData>>) -> bool {
fn check_files_name_threaded(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender<ProgressData>>) -> bool {
let start_time: SystemTime = SystemTime::now();
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
@ -363,14 +363,18 @@ impl DuplicateFinder {
return false;
}
let current_folder = folders_to_check.pop().unwrap();
let segments: Vec<_> = folders_to_check
.par_iter()
.map(|current_folder| {
let mut dir_result = vec![];
let mut warnings = vec![];
let mut fe_result = vec![];
// Read current dir, if permission are denied just go to next
let read_dir = match fs::read_dir(&current_folder) {
Ok(t) => t,
Err(e) => {
self.text_messages.warnings.push(format!("Cannot open dir {}, reason {}", current_folder.display(), e));
continue;
warnings.push(format!("Cannot open dir {}, reason {}", current_folder.display(), e));
return (dir_result, warnings, fe_result);
} // Permissions denied
};
@ -379,14 +383,14 @@ impl DuplicateFinder {
let entry_data = match entry {
Ok(t) => t,
Err(e) => {
self.text_messages.warnings.push(format!("Cannot read entry in dir {}, reason {}", current_folder.display(), e));
warnings.push(format!("Cannot read entry in dir {}, reason {}", current_folder.display(), e));
continue 'dir;
} //Permissions denied
};
let metadata: Metadata = match entry_data.metadata() {
Ok(t) => t,
Err(e) => {
self.text_messages.warnings.push(format!("Cannot read metadata in dir {}, reason {}", current_folder.display(), e));
warnings.push(format!("Cannot read metadata in dir {}, reason {}", current_folder.display(), e));
continue 'dir;
} //Permissions denied
};
@ -404,7 +408,7 @@ impl DuplicateFinder {
continue 'dir;
}
folders_to_check.push(next_folder);
dir_result.push(next_folder);
} else if metadata.is_file() {
atomic_file_counter.fetch_add(1, Ordering::Relaxed);
// let mut have_valid_extension: bool;
@ -440,12 +444,12 @@ impl DuplicateFinder {
Ok(t) => match t.duration_since(UNIX_EPOCH) {
Ok(d) => d.as_secs(),
Err(_inspected) => {
self.text_messages.warnings.push(format!("File {} seems to be modified before Unix Epoch.", current_file_name.display()));
warnings.push(format!("File {} seems to be modified before Unix Epoch.", current_file_name.display()));
0
}
},
Err(e) => {
self.text_messages.warnings.push(format!("Unable to get modification date from file {}, reason {}", current_file_name.display(), e));
warnings.push(format!("Unable to get modification date from file {}, reason {}", current_file_name.display(), e));
0
} // Permissions Denied
},
@ -453,11 +457,26 @@ impl DuplicateFinder {
};
// Adding files to BTreeMap
self.files_with_identical_names.entry(entry_data.file_name().to_string_lossy().to_string()).or_insert_with(Vec::new);
self.files_with_identical_names.get_mut(&entry_data.file_name().to_string_lossy().to_string()).unwrap().push(fe);
fe_result.push((entry_data.file_name().to_string_lossy().to_string(), fe));
}
}
}
(dir_result, warnings, fe_result)
})
.collect();
// Advance the frontier
folders_to_check.clear();
// Process collected data
for (segment, warnings, fe_result) in segments {
folders_to_check.extend(segment);
self.text_messages.warnings.extend(warnings);
for (name, fe) in fe_result {
self.files_with_identical_names.entry(name.clone()).or_insert_with(Vec::new);
self.files_with_identical_names.get_mut(&name).unwrap().push(fe);
}
}
}
// End thread which send info to gui