Add buffered write/read (#186)

This commit is contained in:
Rafał Mikrut 2021-01-09 11:52:43 +01:00 committed by GitHub
parent 3adbfdc5b6
commit 9975605f75
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 87 additions and 67 deletions

View File

@ -9,7 +9,7 @@ use humansize::{file_size_opts as options, FileSize};
use std::collections::BTreeMap;
use std::ffi::OsStr;
use std::fs::{File, Metadata};
use std::io::Write;
use std::io::{BufWriter, Write};
use std::path::PathBuf;
use std::sync::atomic::Ordering;
use std::sync::atomic::{AtomicBool, AtomicU64};
@ -361,16 +361,17 @@ impl SaveResults for BigFile {
k => k.to_string(),
};
let mut file = match File::create(&file_name) {
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(_) => {
self.text_messages.errors.push("Failed to create file ".to_string() + file_name.as_str());
return false;
}
};
let mut writer = BufWriter::new(file_handler);
if writeln!(
file,
writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.directories.included_directories, self.directories.excluded_directories, self.excluded_items.items
)
@ -381,15 +382,15 @@ impl SaveResults for BigFile {
}
if self.information.number_of_real_files != 0 {
write!(file, "{} the biggest files.\n\n", self.information.number_of_real_files).unwrap();
write!(writer, "{} the biggest files.\n\n", self.information.number_of_real_files).unwrap();
for (size, files) in self.big_files.iter().rev() {
for file_entry in files {
writeln!(file, "{} ({}) - {}", size.file_size(options::BINARY).unwrap(), size, file_entry.path.display()).unwrap();
writeln!(writer, "{} ({}) - {}", size.file_size(options::BINARY).unwrap(), size, file_entry.path.display()).unwrap();
}
}
} else {
write!(file, "Not found any files.").unwrap();
write!(writer, "Not found any files.").unwrap();
}
Common::print_time(start_time, SystemTime::now(), "save_results_to_file".to_string());
true

View File

@ -14,6 +14,7 @@ use crate::common_items::ExcludedItems;
use crate::common_messages::Messages;
use crate::common_traits::*;
use rayon::prelude::*;
use std::io::BufWriter;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
use std::thread::sleep;
@ -904,16 +905,17 @@ impl SaveResults for DuplicateFinder {
k => k.to_string(),
};
let mut file = match File::create(&file_name) {
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(_) => {
self.text_messages.errors.push(format!("Failed to create file {}", file_name));
return false;
}
};
let mut writer = BufWriter::new(file_handler);
if writeln!(
file,
writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.directories.included_directories, self.directories.excluded_directories, self.excluded_items.items
)
@ -925,29 +927,29 @@ impl SaveResults for DuplicateFinder {
match self.check_method {
CheckingMethod::Name => {
if !self.files_with_identical_size.is_empty() {
writeln!(file, "-------------------------------------------------Files with same names-------------------------------------------------").unwrap();
writeln!(writer, "-------------------------------------------------Files with same names-------------------------------------------------").unwrap();
writeln!(
file,
writer,
"Found {} files in {} groups with same name(may have different content)",
self.information.number_of_duplicated_files_by_name, self.information.number_of_groups_by_name,
)
.unwrap();
for (name, vector) in self.files_with_identical_names.iter().rev() {
writeln!(file, "Name - {} - {} files ", name, vector.len()).unwrap();
writeln!(writer, "Name - {} - {} files ", name, vector.len()).unwrap();
for j in vector {
writeln!(file, "{}", j.path.display()).unwrap();
writeln!(writer, "{}", j.path.display()).unwrap();
}
writeln!(file).unwrap();
writeln!(writer).unwrap();
}
} else {
write!(file, "Not found any files with same names.").unwrap();
write!(writer, "Not found any files with same names.").unwrap();
}
}
CheckingMethod::Size => {
if !self.files_with_identical_size.is_empty() {
writeln!(file, "-------------------------------------------------Files with same size-------------------------------------------------").unwrap();
writeln!(writer, "-------------------------------------------------Files with same size-------------------------------------------------").unwrap();
writeln!(
file,
writer,
"Found {} duplicated files which in {} groups which takes {}.",
self.information.number_of_duplicated_files_by_size,
self.information.number_of_groups_by_size,
@ -955,20 +957,20 @@ impl SaveResults for DuplicateFinder {
)
.unwrap();
for (size, vector) in self.files_with_identical_size.iter().rev() {
write!(file, "\n---- Size {} ({}) - {} files \n", size.file_size(options::BINARY).unwrap(), size, vector.len()).unwrap();
write!(writer, "\n---- Size {} ({}) - {} files \n", size.file_size(options::BINARY).unwrap(), size, vector.len()).unwrap();
for file_entry in vector {
writeln!(file, "{}", file_entry.path.display()).unwrap();
writeln!(writer, "{}", file_entry.path.display()).unwrap();
}
}
} else {
write!(file, "Not found any duplicates.").unwrap();
write!(writer, "Not found any duplicates.").unwrap();
}
}
CheckingMethod::Hash | CheckingMethod::HashMB => {
if !self.files_with_identical_hashes.is_empty() {
writeln!(file, "-------------------------------------------------Files with same hashes-------------------------------------------------").unwrap();
writeln!(writer, "-------------------------------------------------Files with same hashes-------------------------------------------------").unwrap();
writeln!(
file,
writer,
"Found {} duplicated files which in {} groups which takes {}.",
self.information.number_of_duplicated_files_by_hash,
self.information.number_of_groups_by_hash,
@ -977,14 +979,14 @@ impl SaveResults for DuplicateFinder {
.unwrap();
for (size, vectors_vector) in self.files_with_identical_hashes.iter().rev() {
for vector in vectors_vector {
writeln!(file, "\n---- Size {} ({}) - {} files", size.file_size(options::BINARY).unwrap(), size, vector.len()).unwrap();
writeln!(writer, "\n---- Size {} ({}) - {} files", size.file_size(options::BINARY).unwrap(), size, vector.len()).unwrap();
for file_entry in vector {
writeln!(file, "{}", file_entry.path.display()).unwrap();
writeln!(writer, "{}", file_entry.path.display()).unwrap();
}
}
}
} else {
write!(file, "Not found any duplicates.").unwrap();
write!(writer, "Not found any duplicates.").unwrap();
}
}
CheckingMethod::None => {

View File

@ -11,6 +11,7 @@ use crate::common_items::ExcludedItems;
use crate::common_messages::Messages;
use crate::common_traits::*;
use crossbeam_channel::Receiver;
use std::io::BufWriter;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
use std::thread::sleep;
@ -329,16 +330,17 @@ impl SaveResults for EmptyFiles {
k => k.to_string(),
};
let mut file = match File::create(&file_name) {
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(_) => {
self.text_messages.errors.push(format!("Failed to create file {}", file_name));
return false;
}
};
let mut writer = BufWriter::new(file_handler);
if writeln!(
file,
writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.directories.included_directories, self.directories.excluded_directories, self.excluded_items.items
)
@ -349,12 +351,12 @@ impl SaveResults for EmptyFiles {
}
if !self.empty_files.is_empty() {
writeln!(file, "Found {} empty files.", self.information.number_of_empty_files).unwrap();
writeln!(writer, "Found {} empty files.", self.information.number_of_empty_files).unwrap();
for file_entry in self.empty_files.iter() {
writeln!(file, "{}", file_entry.path.display()).unwrap();
writeln!(writer, "{}", file_entry.path.display()).unwrap();
}
} else {
write!(file, "Not found any empty files.").unwrap();
write!(writer, "Not found any empty files.").unwrap();
}
Common::print_time(start_time, SystemTime::now(), "save_results_to_file".to_string());
true

View File

@ -6,7 +6,7 @@ use crate::common_traits::{DebugPrint, PrintResults, SaveResults};
use crossbeam_channel::Receiver;
use std::collections::BTreeMap;
use std::fs::{File, Metadata};
use std::io::Write;
use std::io::{BufWriter, Write};
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
@ -333,27 +333,28 @@ impl SaveResults for EmptyFolder {
k => k.to_string(),
};
let mut file = match File::create(&file_name) {
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(_) => {
self.text_messages.errors.push("Failed to create file ".to_string() + file_name.as_str());
return false;
}
};
let mut writer = BufWriter::new(file_handler);
if writeln!(file, "Results of searching {:?} with excluded directories {:?}", self.directories.included_directories, self.directories.excluded_directories).is_err() {
if writeln!(writer, "Results of searching {:?} with excluded directories {:?}", self.directories.included_directories, self.directories.excluded_directories).is_err() {
self.text_messages.errors.push("Failed to save results to file ".to_string() + file_name.as_str());
return false;
}
if !self.empty_folder_list.is_empty() {
writeln!(file, "-------------------------------------------------Empty folder list-------------------------------------------------").unwrap();
writeln!(file, "Found {} empty folders", self.information.number_of_empty_folders).unwrap();
writeln!(writer, "-------------------------------------------------Empty folder list-------------------------------------------------").unwrap();
writeln!(writer, "Found {} empty folders", self.information.number_of_empty_folders).unwrap();
for name in self.empty_folder_list.keys() {
writeln!(file, "{}", name.display()).unwrap();
writeln!(writer, "{}", name.display()).unwrap();
}
} else {
write!(file, "Not found any empty folders.").unwrap();
write!(writer, "Not found any empty folders.").unwrap();
}
Common::print_time(start_time, SystemTime::now(), "save_results_to_file".to_string());
true

View File

@ -11,6 +11,7 @@ use crate::common_items::ExcludedItems;
use crate::common_messages::Messages;
use crate::common_traits::*;
use crossbeam_channel::Receiver;
use std::io::BufWriter;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
use std::thread::sleep;
@ -376,16 +377,17 @@ impl SaveResults for InvalidSymlinks {
k => k.to_string(),
};
let mut file = match File::create(&file_name) {
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(_) => {
self.text_messages.errors.push(format!("Failed to create file {}", file_name));
return false;
}
};
let mut writer = BufWriter::new(file_handler);
if writeln!(
file,
writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.directories.included_directories, self.directories.excluded_directories, self.excluded_items.items
)
@ -396,10 +398,10 @@ impl SaveResults for InvalidSymlinks {
}
if !self.invalid_symlinks.is_empty() {
writeln!(file, "Found {} invalid symlinks.", self.information.number_of_invalid_symlinks).unwrap();
writeln!(writer, "Found {} invalid symlinks.", self.information.number_of_invalid_symlinks).unwrap();
for file_entry in self.invalid_symlinks.iter() {
writeln!(
file,
writer,
"{}\t\t{}\t\t{}",
file_entry.symlink_path.display(),
file_entry.destination_path.display(),
@ -411,7 +413,7 @@ impl SaveResults for InvalidSymlinks {
.unwrap();
}
} else {
write!(file, "Not found any invalid symlinks.").unwrap();
write!(writer, "Not found any invalid symlinks.").unwrap();
}
Common::print_time(start_time, SystemTime::now(), "save_results_to_file".to_string());
true

View File

@ -13,6 +13,7 @@ use audiotags::Tag;
use crossbeam_channel::Receiver;
use rayon::prelude::*;
use std::collections::HashMap;
use std::io::BufWriter;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
use std::thread::sleep;
@ -664,16 +665,17 @@ impl SaveResults for SameMusic {
k => k.to_string(),
};
let mut file = match File::create(&file_name) {
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(_) => {
self.text_messages.errors.push(format!("Failed to create file {}", file_name));
return false;
}
};
let mut writer = BufWriter::new(file_handler);
if writeln!(
file,
writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.directories.included_directories, self.directories.excluded_directories, self.excluded_items.items
)
@ -684,12 +686,12 @@ impl SaveResults for SameMusic {
}
if !self.music_entries.is_empty() {
writeln!(file, "Found {} same music files.", self.information.number_of_music_entries).unwrap();
writeln!(writer, "Found {} same music files.", self.information.number_of_music_entries).unwrap();
for file_entry in self.music_entries.iter() {
writeln!(file, "{}", file_entry.path.display()).unwrap();
writeln!(writer, "{}", file_entry.path.display()).unwrap();
}
} else {
write!(file, "Not found any empty files.").unwrap();
write!(writer, "Not found any empty files.").unwrap();
}
Common::print_time(start_time, SystemTime::now(), "save_results_to_file".to_string());
true

View File

@ -571,16 +571,17 @@ impl SaveResults for SimilarImages {
k => k.to_string(),
};
let mut file = match File::create(&file_name) {
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(_) => {
self.text_messages.errors.push("Failed to create file ".to_string() + file_name.as_str());
return false;
}
};
let mut writer = BufWriter::new(file_handler);
if writeln!(
file,
writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.directories.included_directories, self.directories.excluded_directories, self.excluded_items.items
)
@ -591,19 +592,18 @@ impl SaveResults for SimilarImages {
}
if !self.similar_vectors.is_empty() {
write!(file, "{} images which have similar friends\n\n", self.similar_vectors.len()).unwrap();
write!(writer, "{} images which have similar friends\n\n", self.similar_vectors.len()).unwrap();
// for struct_similar in self.similar_vectors.iter() {
// writeln!(file, "Image {:?} have {} similar images", struct_similar.base_image.path, struct_similar.similar_images.len()).unwrap();
// writeln!(writer, "Image {:?} have {} similar images", struct_similar.base_image.path, struct_similar.similar_images.len()).unwrap();
// for similar_picture in struct_similar.similar_images.iter() {
// writeln!(file, "{:?} - Similarity Level: {}", similar_picture.path, get_string_from_similarity(&similar_picture.similarity)).unwrap();
// writeln!(writer, "{:?} - Similarity Level: {}", similar_picture.path, get_string_from_similarity(&similar_picture.similarity)).unwrap();
// }
// writeln!(file).unwrap();
// writeln!(writer).unwrap();
// }
} else {
write!(file, "Not found any similar images.").unwrap();
write!(writer, "Not found any similar images.").unwrap();
}
let _ = file.flush();
Common::print_time(start_time, SystemTime::now(), "save_results_to_file".to_string());
true
@ -660,13 +660,14 @@ fn save_hashes_to_file(hashmap: &HashMap<String, FileEntry>, text_messages: &mut
return;
}
let config_file = config_dir.join(CACHE_FILE_NAME);
let mut file_handler = match OpenOptions::new().truncate(true).write(true).create(true).open(&config_file) {
let file_handler = match OpenOptions::new().truncate(true).write(true).create(true).open(&config_file) {
Ok(t) => t,
Err(_) => {
text_messages.messages.push(format!("Cannot create or open cache file {}", config_file.display()));
return;
}
};
let mut writer = BufWriter::new(file_handler);
for file_entry in hashmap.values() {
let mut string: String = "".to_string();
@ -678,7 +679,7 @@ fn save_hashes_to_file(hashmap: &HashMap<String, FileEntry>, text_messages: &mut
}
string += file_entry.hash[file_entry.hash.len() - 1].to_string().as_str();
if writeln!(file_handler, "{}", string).is_err() {
if writeln!(writer, "{}", string).is_err() {
text_messages.messages.push(format!("Failed to save some data to cache file {}", config_file.display()));
return;
};
@ -730,7 +731,12 @@ fn load_hashes_from_file(text_messages: &mut Messages) -> Option<HashMap<String,
have_at_least += 1;
}
}
assert!(have_at_least != hash.len() as u8);
if have_at_least == hash.len() as u8 {
println!("ERROR START - {}", line);
println!("have_at_least == hash.len() as u8");
println!("ERROR END hash.len() - {} == have_at_least - {}", hash.len(), have_at_least);
continue; // Just skip this entry, it is very very unlikelly that something have this hash, but if it has, then just ignore it
}
}
hashmap_loaded_entries.insert(

View File

@ -10,6 +10,7 @@ use crate::common_items::ExcludedItems;
use crate::common_messages::Messages;
use crate::common_traits::*;
use crossbeam_channel::Receiver;
use std::io::BufWriter;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
use std::thread::sleep;
@ -314,16 +315,17 @@ impl SaveResults for Temporary {
k => k.to_string(),
};
let mut file = match File::create(&file_name) {
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(_) => {
self.text_messages.errors.push(format!("Failed to create file {}", file_name));
return false;
}
};
let mut writer = BufWriter::new(file_handler);
if writeln!(
file,
writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.directories.included_directories, self.directories.excluded_directories, self.excluded_items.items
)
@ -334,12 +336,12 @@ impl SaveResults for Temporary {
}
if !self.temporary_files.is_empty() {
writeln!(file, "Found {} temporary files.", self.information.number_of_temporary_files).unwrap();
writeln!(writer, "Found {} temporary files.", self.information.number_of_temporary_files).unwrap();
for file_entry in self.temporary_files.iter() {
writeln!(file, "{}", file_entry.path.display()).unwrap();
writeln!(writer, "{}", file_entry.path.display()).unwrap();
}
} else {
write!(file, "Not found any temporary files.").unwrap();
write!(writer, "Not found any temporary files.").unwrap();
}
Common::print_time(start_time, SystemTime::now(), "save_results_to_file".to_string());
true

View File

@ -12,6 +12,7 @@ use crate::common_messages::Messages;
use crate::common_traits::*;
use crossbeam_channel::Receiver;
use rayon::prelude::*;
use std::io::BufWriter;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
use std::thread::sleep;
@ -452,16 +453,17 @@ impl SaveResults for ZeroedFiles {
k => k.to_string(),
};
let mut file = match File::create(&file_name) {
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(_) => {
self.text_messages.errors.push(format!("Failed to create file {}", file_name));
return false;
}
};
let mut writer = BufWriter::new(file_handler);
if writeln!(
file,
writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.directories.included_directories, self.directories.excluded_directories, self.excluded_items.items
)
@ -472,12 +474,12 @@ impl SaveResults for ZeroedFiles {
}
if !self.zeroed_files.is_empty() {
writeln!(file, "Found {} zeroed files.", self.information.number_of_zeroed_files).unwrap();
writeln!(writer, "Found {} zeroed files.", self.information.number_of_zeroed_files).unwrap();
for file_entry in self.zeroed_files.iter() {
writeln!(file, "{}", file_entry.path.display()).unwrap();
writeln!(writer, "{}", file_entry.path.display()).unwrap();
}
} else {
write!(file, "Not found any zeroed files.").unwrap();
write!(writer, "Not found any zeroed files.").unwrap();
}
Common::print_time(start_time, SystemTime::now(), "save_results_to_file".to_string());
true