Loading cache
This commit is contained in:
parent
edfc8e7b5f
commit
8053ce81d2
9 changed files with 129 additions and 176 deletions
|
@ -1,7 +1,7 @@
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
use std::fs::{DirEntry, File, Metadata};
|
use std::fs::{DirEntry, File, Metadata};
|
||||||
use std::io::prelude::*;
|
use std::io::prelude::*;
|
||||||
use std::io::{BufReader, BufWriter};
|
use std::io::BufWriter;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
@ -18,8 +18,8 @@ use rayon::prelude::*;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use crate::common::{
|
use crate::common::{
|
||||||
check_folder_children, create_crash_message, open_cache_folder, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, AUDIO_FILES_EXTENSIONS,
|
check_folder_children, create_crash_message, load_cache_from_file_generalized, open_cache_folder, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads,
|
||||||
IMAGE_RS_BROKEN_FILES_EXTENSIONS, PDF_FILES_EXTENSIONS, ZIP_FILES_EXTENSIONS,
|
AUDIO_FILES_EXTENSIONS, IMAGE_RS_BROKEN_FILES_EXTENSIONS, PDF_FILES_EXTENSIONS, ZIP_FILES_EXTENSIONS,
|
||||||
};
|
};
|
||||||
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
|
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
|
||||||
use crate::common_messages::Messages;
|
use crate::common_messages::Messages;
|
||||||
|
@ -349,10 +349,9 @@ impl BrokenFiles {
|
||||||
let files_to_check = mem::take(&mut self.files_to_check);
|
let files_to_check = mem::take(&mut self.files_to_check);
|
||||||
|
|
||||||
if self.common_data.use_cache {
|
if self.common_data.use_cache {
|
||||||
loaded_hash_map = match load_cache_from_file(&mut self.common_data.text_messages, self.common_data.delete_outdated_cache) {
|
let (messages, loaded_items) = load_cache_from_file_generalized::<FileEntry>(&get_cache_file(), self.get_delete_outdated_cache());
|
||||||
Some(t) => t,
|
self.get_text_messages_mut().extend_with_another_messages(messages);
|
||||||
None => Default::default(),
|
loaded_hash_map = loaded_items.unwrap_or_default();
|
||||||
};
|
|
||||||
|
|
||||||
for (name, file_entry) in files_to_check {
|
for (name, file_entry) in files_to_check {
|
||||||
let checked_extension = check_extension_allowed(&file_entry.type_of_file, &self.checked_types); // Only broken
|
let checked_extension = check_extension_allowed(&file_entry.type_of_file, &self.checked_types); // Only broken
|
||||||
|
@ -440,7 +439,8 @@ impl BrokenFiles {
|
||||||
for (_name, file_entry) in loaded_hash_map {
|
for (_name, file_entry) in loaded_hash_map {
|
||||||
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry);
|
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry);
|
||||||
}
|
}
|
||||||
save_cache_to_file(&all_results, &mut self.common_data.text_messages, self.common_data.save_also_as_json);
|
let save_as_json = self.get_save_also_as_json();
|
||||||
|
save_cache_to_file(&all_results, &mut self.common_data.text_messages, save_as_json);
|
||||||
}
|
}
|
||||||
debug!("save_to_cache - end");
|
debug!("save_to_cache - end");
|
||||||
}
|
}
|
||||||
|
@ -571,45 +571,6 @@ fn save_cache_to_file(old_hashmap: &BTreeMap<String, FileEntry>, text_messages:
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn load_cache_from_file(text_messages: &mut Messages, delete_outdated_cache: bool) -> Option<BTreeMap<String, FileEntry>> {
|
|
||||||
if let Some(((file_handler, cache_file), (file_handler_json, cache_file_json))) = open_cache_folder(&get_cache_file(), false, true, &mut text_messages.warnings) {
|
|
||||||
let mut hashmap_loaded_entries: BTreeMap<String, FileEntry>;
|
|
||||||
if let Some(file_handler) = file_handler {
|
|
||||||
let reader = BufReader::new(file_handler);
|
|
||||||
hashmap_loaded_entries = match bincode::deserialize_from(reader) {
|
|
||||||
Ok(t) => t,
|
|
||||||
Err(e) => {
|
|
||||||
text_messages
|
|
||||||
.warnings
|
|
||||||
.push(format!("Failed to load data from cache file {}, reason {}", cache_file.display(), e));
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
} else {
|
|
||||||
let reader = BufReader::new(file_handler_json.unwrap()); // Unwrap cannot fail, because at least one file must be valid
|
|
||||||
hashmap_loaded_entries = match serde_json::from_reader(reader) {
|
|
||||||
Ok(t) => t,
|
|
||||||
Err(e) => {
|
|
||||||
text_messages
|
|
||||||
.warnings
|
|
||||||
.push(format!("Failed to load data from cache file {}, reason {}", cache_file_json.display(), e));
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
// Don't load cache data if destination file not exists
|
|
||||||
if delete_outdated_cache {
|
|
||||||
hashmap_loaded_entries.retain(|src_path, _file_entry| Path::new(src_path).exists());
|
|
||||||
}
|
|
||||||
|
|
||||||
text_messages.messages.push(format!("Properly loaded {} cache entries.", hashmap_loaded_entries.len()));
|
|
||||||
|
|
||||||
return Some(hashmap_loaded_entries);
|
|
||||||
}
|
|
||||||
None
|
|
||||||
}
|
|
||||||
|
|
||||||
fn get_cache_file() -> String {
|
fn get_cache_file() -> String {
|
||||||
"cache_broken_files.bin".to_string()
|
"cache_broken_files.bin".to_string()
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
use std::collections::BTreeMap;
|
||||||
use std::ffi::OsString;
|
use std::ffi::OsString;
|
||||||
use std::fs::{DirEntry, File, OpenOptions};
|
use std::fs::{DirEntry, File, OpenOptions};
|
||||||
use std::io::BufReader;
|
use std::io::BufReader;
|
||||||
|
@ -18,12 +19,14 @@ use imagepipe::{ImageSource, Pipeline};
|
||||||
#[cfg(feature = "heif")]
|
#[cfg(feature = "heif")]
|
||||||
use libheif_rs::{ColorSpace, HeifContext, RgbChroma};
|
use libheif_rs::{ColorSpace, HeifContext, RgbChroma};
|
||||||
use log::{debug, LevelFilter, Record};
|
use log::{debug, LevelFilter, Record};
|
||||||
|
use serde::Deserialize;
|
||||||
|
|
||||||
// #[cfg(feature = "heif")]
|
// #[cfg(feature = "heif")]
|
||||||
// use libheif_rs::LibHeif;
|
// use libheif_rs::LibHeif;
|
||||||
use crate::common_dir_traversal::{CheckingMethod, ProgressData, ToolType};
|
use crate::common_dir_traversal::{CheckingMethod, ProgressData, ToolType};
|
||||||
use crate::common_directory::Directories;
|
use crate::common_directory::Directories;
|
||||||
use crate::common_items::ExcludedItems;
|
use crate::common_items::ExcludedItems;
|
||||||
|
use crate::common_messages::Messages;
|
||||||
use crate::common_traits::ResultEntry;
|
use crate::common_traits::ResultEntry;
|
||||||
|
|
||||||
static NUMBER_OF_THREADS: state::InitCell<usize> = state::InitCell::new();
|
static NUMBER_OF_THREADS: state::InitCell<usize> = state::InitCell::new();
|
||||||
|
@ -155,6 +158,56 @@ pub fn open_cache_folder(cache_file_name: &str, save_to_cache: bool, use_json: b
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn load_cache_from_file_generalized<T>(cache_file_name: &str, delete_outdated_cache: bool) -> (Messages, Option<BTreeMap<String, T>>)
|
||||||
|
where
|
||||||
|
for<'a> T: Deserialize<'a>,
|
||||||
|
{
|
||||||
|
debug!("Loading cache from file {} (or json alternative)", cache_file_name);
|
||||||
|
let mut text_messages = Messages::new();
|
||||||
|
|
||||||
|
if let Some(((file_handler, cache_file), (file_handler_json, cache_file_json))) = open_cache_folder(cache_file_name, false, true, &mut text_messages.warnings) {
|
||||||
|
let mut hashmap_loaded_entries: BTreeMap<String, T>;
|
||||||
|
if let Some(file_handler) = file_handler {
|
||||||
|
let reader = BufReader::new(file_handler);
|
||||||
|
hashmap_loaded_entries = match bincode::deserialize_from(reader) {
|
||||||
|
Ok(t) => t,
|
||||||
|
Err(e) => {
|
||||||
|
text_messages
|
||||||
|
.warnings
|
||||||
|
.push(format!("Failed to load data from cache file {}, reason {}", cache_file.display(), e));
|
||||||
|
debug!("Failed to load cache from file {:?}", cache_file);
|
||||||
|
return (text_messages, None);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
} else {
|
||||||
|
let reader = BufReader::new(file_handler_json.unwrap()); // Unwrap cannot fail, because at least one file must be valid
|
||||||
|
hashmap_loaded_entries = match serde_json::from_reader(reader) {
|
||||||
|
Ok(t) => t,
|
||||||
|
Err(e) => {
|
||||||
|
text_messages
|
||||||
|
.warnings
|
||||||
|
.push(format!("Failed to load data from cache file {}, reason {}", cache_file_json.display(), e));
|
||||||
|
debug!("Failed to load cache from file {:?}", cache_file);
|
||||||
|
return (text_messages, None);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Don't load cache data if destination file not exists
|
||||||
|
if delete_outdated_cache {
|
||||||
|
debug!("Starting to removing outdated cache entries");
|
||||||
|
hashmap_loaded_entries.retain(|src_path, _file_entry| Path::new(src_path).exists());
|
||||||
|
debug!("Completed removing outdated cache entries");
|
||||||
|
}
|
||||||
|
|
||||||
|
text_messages.messages.push(format!("Properly loaded {} cache entries.", hashmap_loaded_entries.len()));
|
||||||
|
|
||||||
|
debug!("Loaded cache from file {} (or json alternative)", cache_file_name);
|
||||||
|
return (text_messages, Some(hashmap_loaded_entries));
|
||||||
|
}
|
||||||
|
(text_messages, None)
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(feature = "heif")]
|
#[cfg(feature = "heif")]
|
||||||
pub fn get_dynamic_image_from_heic(path: &str) -> Result<DynamicImage> {
|
pub fn get_dynamic_image_from_heic(path: &str) -> Result<DynamicImage> {
|
||||||
// let libheif = LibHeif::new();
|
// let libheif = LibHeif::new();
|
||||||
|
|
|
@ -8,6 +8,7 @@ use std::time::UNIX_EPOCH;
|
||||||
use crossbeam_channel::Receiver;
|
use crossbeam_channel::Receiver;
|
||||||
use futures::channel::mpsc::UnboundedSender;
|
use futures::channel::mpsc::UnboundedSender;
|
||||||
use rayon::prelude::*;
|
use rayon::prelude::*;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use crate::common::{prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads};
|
use crate::common::{prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads};
|
||||||
use crate::common_directory::Directories;
|
use crate::common_directory::Directories;
|
||||||
|
@ -44,7 +45,7 @@ pub enum ToolType {
|
||||||
None,
|
None,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(PartialEq, Eq, Clone, Debug, Copy, Default)]
|
#[derive(PartialEq, Eq, Clone, Debug, Copy, Default, Deserialize, Serialize)]
|
||||||
pub enum CheckingMethod {
|
pub enum CheckingMethod {
|
||||||
#[default]
|
#[default]
|
||||||
None,
|
None,
|
||||||
|
@ -56,7 +57,7 @@ pub enum CheckingMethod {
|
||||||
AudioContent,
|
AudioContent,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct FileEntry {
|
pub struct FileEntry {
|
||||||
pub path: PathBuf,
|
pub path: PathBuf,
|
||||||
pub size: u64,
|
pub size: u64,
|
||||||
|
@ -75,13 +76,13 @@ impl ResultEntry for FileEntry {
|
||||||
|
|
||||||
const MAX_NUMBER_OF_SYMLINK_JUMPS: i32 = 20;
|
const MAX_NUMBER_OF_SYMLINK_JUMPS: i32 = 20;
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
|
||||||
pub struct SymlinkInfo {
|
pub struct SymlinkInfo {
|
||||||
pub destination_path: PathBuf,
|
pub destination_path: PathBuf,
|
||||||
pub type_of_error: ErrorType,
|
pub type_of_error: ErrorType,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Copy)]
|
#[derive(Clone, Debug, PartialEq, Eq, Copy, Deserialize, Serialize)]
|
||||||
pub enum ErrorType {
|
pub enum ErrorType {
|
||||||
InfiniteRecursion,
|
InfiniteRecursion,
|
||||||
NonExistentFile,
|
NonExistentFile,
|
||||||
|
|
|
@ -53,4 +53,11 @@ impl Messages {
|
||||||
self.warnings.extend(warnings);
|
self.warnings.extend(warnings);
|
||||||
self.errors.extend(errors);
|
self.errors.extend(errors);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn extend_with_another_messages(&mut self, messages: Messages) {
|
||||||
|
let (messages, warnings, errors) = (messages.messages, messages.warnings, messages.errors);
|
||||||
|
self.messages.extend(messages);
|
||||||
|
self.warnings.extend(warnings);
|
||||||
|
self.errors.extend(errors);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -103,6 +103,9 @@ pub trait CommonData {
|
||||||
fn get_text_messages(&self) -> &Messages {
|
fn get_text_messages(&self) -> &Messages {
|
||||||
&self.get_cd().text_messages
|
&self.get_cd().text_messages
|
||||||
}
|
}
|
||||||
|
fn get_text_messages_mut(&mut self) -> &mut Messages {
|
||||||
|
&mut self.get_cd_mut().text_messages
|
||||||
|
}
|
||||||
|
|
||||||
fn set_save_also_as_json(&mut self, save_also_as_json: bool) {
|
fn set_save_also_as_json(&mut self, save_also_as_json: bool) {
|
||||||
self.get_cd_mut().save_also_as_json = save_also_as_json;
|
self.get_cd_mut().save_also_as_json = save_also_as_json;
|
||||||
|
|
|
@ -1,8 +1,8 @@
|
||||||
use std::cmp::max;
|
use std::cmp::max;
|
||||||
use std::collections::{BTreeMap, HashMap, HashSet};
|
use std::collections::{BTreeMap, HashSet};
|
||||||
use std::fs::File;
|
use std::fs::File;
|
||||||
use std::io::prelude::*;
|
use std::io::prelude::*;
|
||||||
use std::io::{BufReader, BufWriter};
|
use std::io::BufWriter;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
@ -24,7 +24,8 @@ use symphonia::core::meta::MetadataOptions;
|
||||||
use symphonia::core::probe::Hint;
|
use symphonia::core::probe::Hint;
|
||||||
|
|
||||||
use crate::common::{
|
use crate::common::{
|
||||||
create_crash_message, filter_reference_folders_generic, open_cache_folder, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, AUDIO_FILES_EXTENSIONS,
|
create_crash_message, filter_reference_folders_generic, load_cache_from_file_generalized, open_cache_folder, prepare_thread_handler_common,
|
||||||
|
send_info_and_wait_for_ending_all_threads, AUDIO_FILES_EXTENSIONS,
|
||||||
};
|
};
|
||||||
use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData, ToolType};
|
use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData, ToolType};
|
||||||
use crate::common_messages::Messages;
|
use crate::common_messages::Messages;
|
||||||
|
@ -102,7 +103,7 @@ pub struct Info {
|
||||||
pub struct SameMusic {
|
pub struct SameMusic {
|
||||||
common_data: CommonToolData,
|
common_data: CommonToolData,
|
||||||
information: Info,
|
information: Info,
|
||||||
music_to_check: HashMap<String, MusicEntry>,
|
music_to_check: BTreeMap<String, MusicEntry>,
|
||||||
music_entries: Vec<MusicEntry>,
|
music_entries: Vec<MusicEntry>,
|
||||||
duplicated_music_entries: Vec<Vec<MusicEntry>>,
|
duplicated_music_entries: Vec<Vec<MusicEntry>>,
|
||||||
duplicated_music_entries_referenced: Vec<(MusicEntry, Vec<MusicEntry>)>,
|
duplicated_music_entries_referenced: Vec<(MusicEntry, Vec<MusicEntry>)>,
|
||||||
|
@ -221,18 +222,17 @@ impl SameMusic {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn load_cache(&mut self, checking_tags: bool) -> (HashMap<String, MusicEntry>, HashMap<String, MusicEntry>, HashMap<String, MusicEntry>) {
|
fn load_cache(&mut self, checking_tags: bool) -> (BTreeMap<String, MusicEntry>, BTreeMap<String, MusicEntry>, BTreeMap<String, MusicEntry>) {
|
||||||
debug!("load_cache - start, using cache {}", self.common_data.use_cache);
|
debug!("load_cache - start, using cache {}", self.common_data.use_cache);
|
||||||
let loaded_hash_map;
|
let loaded_hash_map;
|
||||||
|
|
||||||
let mut records_already_cached: HashMap<String, MusicEntry> = Default::default();
|
let mut records_already_cached: BTreeMap<String, MusicEntry> = Default::default();
|
||||||
let mut non_cached_files_to_check: HashMap<String, MusicEntry> = Default::default();
|
let mut non_cached_files_to_check: BTreeMap<String, MusicEntry> = Default::default();
|
||||||
|
|
||||||
if self.common_data.use_cache {
|
if self.common_data.use_cache {
|
||||||
loaded_hash_map = match load_cache_from_file(&mut self.common_data.text_messages, self.common_data.delete_outdated_cache, checking_tags) {
|
let (messages, loaded_items) = load_cache_from_file_generalized::<MusicEntry>(get_cache_file(checking_tags), self.get_delete_outdated_cache());
|
||||||
Some(t) => t,
|
self.get_text_messages_mut().extend_with_another_messages(messages);
|
||||||
None => Default::default(),
|
loaded_hash_map = loaded_items.unwrap_or_default();
|
||||||
};
|
|
||||||
|
|
||||||
for (name, file_entry) in &self.music_to_check {
|
for (name, file_entry) in &self.music_to_check {
|
||||||
if !loaded_hash_map.contains_key(name) {
|
if !loaded_hash_map.contains_key(name) {
|
||||||
|
@ -257,18 +257,19 @@ impl SameMusic {
|
||||||
(loaded_hash_map, records_already_cached, non_cached_files_to_check)
|
(loaded_hash_map, records_already_cached, non_cached_files_to_check)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn save_cache(&mut self, vec_file_entry: Vec<MusicEntry>, loaded_hash_map: HashMap<String, MusicEntry>, checking_tags: bool) {
|
fn save_cache(&mut self, vec_file_entry: Vec<MusicEntry>, loaded_hash_map: BTreeMap<String, MusicEntry>, checking_tags: bool) {
|
||||||
debug!("save_cache - start, using cache {}", self.common_data.use_cache);
|
debug!("save_cache - start, using cache {}", self.common_data.use_cache);
|
||||||
if !self.common_data.use_cache {
|
if !self.common_data.use_cache {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
// Must save all results to file, old loaded from file with all currently counted results
|
// Must save all results to file, old loaded from file with all currently counted results
|
||||||
let mut all_results: HashMap<String, MusicEntry> = loaded_hash_map;
|
let mut all_results: BTreeMap<String, MusicEntry> = loaded_hash_map;
|
||||||
|
|
||||||
for file_entry in vec_file_entry {
|
for file_entry in vec_file_entry {
|
||||||
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry);
|
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry);
|
||||||
}
|
}
|
||||||
save_cache_to_file(&all_results, &mut self.common_data.text_messages, self.common_data.save_also_as_json, checking_tags);
|
let save_also_as_json = self.get_save_also_as_json();
|
||||||
|
save_cache_to_file(&all_results, &mut self.common_data.text_messages, save_also_as_json, checking_tags);
|
||||||
debug!("save_cache - end");
|
debug!("save_cache - end");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -745,7 +746,7 @@ impl SameMusic {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn save_cache_to_file(hashmap: &HashMap<String, MusicEntry>, text_messages: &mut Messages, save_also_as_json: bool, checking_tags: bool) {
|
fn save_cache_to_file(hashmap: &BTreeMap<String, MusicEntry>, text_messages: &mut Messages, save_also_as_json: bool, checking_tags: bool) {
|
||||||
if let Some(((file_handler, cache_file), (file_handler_json, cache_file_json))) =
|
if let Some(((file_handler, cache_file), (file_handler_json, cache_file_json))) =
|
||||||
open_cache_folder(get_cache_file(checking_tags), true, save_also_as_json, &mut text_messages.warnings)
|
open_cache_folder(get_cache_file(checking_tags), true, save_also_as_json, &mut text_messages.warnings)
|
||||||
{
|
{
|
||||||
|
@ -774,45 +775,6 @@ fn save_cache_to_file(hashmap: &HashMap<String, MusicEntry>, text_messages: &mut
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn load_cache_from_file(text_messages: &mut Messages, delete_outdated_cache: bool, checking_tags: bool) -> Option<HashMap<String, MusicEntry>> {
|
|
||||||
if let Some(((file_handler, cache_file), (file_handler_json, cache_file_json))) = open_cache_folder(get_cache_file(checking_tags), false, true, &mut text_messages.warnings) {
|
|
||||||
let mut hashmap_loaded_entries: HashMap<String, MusicEntry>;
|
|
||||||
if let Some(file_handler) = file_handler {
|
|
||||||
let reader = BufReader::new(file_handler);
|
|
||||||
hashmap_loaded_entries = match bincode::deserialize_from(reader) {
|
|
||||||
Ok(t) => t,
|
|
||||||
Err(e) => {
|
|
||||||
text_messages
|
|
||||||
.warnings
|
|
||||||
.push(format!("Failed to load data from cache file {}, reason {}", cache_file.display(), e));
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
} else {
|
|
||||||
let reader = BufReader::new(file_handler_json.unwrap()); // Unwrap cannot fail, because at least one file must be valid
|
|
||||||
hashmap_loaded_entries = match serde_json::from_reader(reader) {
|
|
||||||
Ok(t) => t,
|
|
||||||
Err(e) => {
|
|
||||||
text_messages
|
|
||||||
.warnings
|
|
||||||
.push(format!("Failed to load data from cache file {}, reason {}", cache_file_json.display(), e));
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
// Don't load cache data if destination file not exists
|
|
||||||
if delete_outdated_cache {
|
|
||||||
hashmap_loaded_entries.retain(|src_path, _file_entry| Path::new(src_path).exists());
|
|
||||||
}
|
|
||||||
|
|
||||||
text_messages.messages.push(format!("Properly loaded {} cache entries.", hashmap_loaded_entries.len()));
|
|
||||||
|
|
||||||
return Some(hashmap_loaded_entries);
|
|
||||||
}
|
|
||||||
None
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO this should be taken from rusty-chromaprint repo, not reimplemented here
|
// TODO this should be taken from rusty-chromaprint repo, not reimplemented here
|
||||||
fn calc_fingerprint_helper(path: impl AsRef<Path>, config: &Configuration) -> anyhow::Result<Vec<u32>> {
|
fn calc_fingerprint_helper(path: impl AsRef<Path>, config: &Configuration) -> anyhow::Result<Vec<u32>> {
|
||||||
let path = path.as_ref();
|
let path = path.as_ref();
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
use std::collections::{BTreeSet, HashMap, HashSet};
|
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
|
||||||
use std::fs::{DirEntry, File, Metadata};
|
use std::fs::{DirEntry, File, Metadata};
|
||||||
use std::io::{Write, *};
|
use std::io::{Write, *};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
@ -19,8 +19,8 @@ use serde::{Deserialize, Serialize};
|
||||||
#[cfg(feature = "heif")]
|
#[cfg(feature = "heif")]
|
||||||
use crate::common::get_dynamic_image_from_heic;
|
use crate::common::get_dynamic_image_from_heic;
|
||||||
use crate::common::{
|
use crate::common::{
|
||||||
check_folder_children, create_crash_message, get_dynamic_image_from_raw_image, open_cache_folder, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads,
|
check_folder_children, create_crash_message, get_dynamic_image_from_raw_image, load_cache_from_file_generalized, open_cache_folder, prepare_thread_handler_common,
|
||||||
HEIC_EXTENSIONS, IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS, RAW_IMAGE_EXTENSIONS,
|
send_info_and_wait_for_ending_all_threads, HEIC_EXTENSIONS, IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS, RAW_IMAGE_EXTENSIONS,
|
||||||
};
|
};
|
||||||
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
|
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
|
||||||
use crate::common_messages::Messages;
|
use crate::common_messages::Messages;
|
||||||
|
@ -89,7 +89,7 @@ pub struct SimilarImages {
|
||||||
image_hashes: HashMap<ImHash, Vec<FileEntry>>,
|
image_hashes: HashMap<ImHash, Vec<FileEntry>>,
|
||||||
// Hashmap with image hashes and Vector with names of files
|
// Hashmap with image hashes and Vector with names of files
|
||||||
similarity: u32,
|
similarity: u32,
|
||||||
images_to_check: HashMap<String, FileEntry>,
|
images_to_check: BTreeMap<String, FileEntry>,
|
||||||
hash_size: u8,
|
hash_size: u8,
|
||||||
hash_alg: HashAlg,
|
hash_alg: HashAlg,
|
||||||
image_filter: FilterType,
|
image_filter: FilterType,
|
||||||
|
@ -273,24 +273,18 @@ impl SimilarImages {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn hash_images_load_cache(&mut self) -> (HashMap<String, FileEntry>, HashMap<String, FileEntry>, HashMap<String, FileEntry>) {
|
fn hash_images_load_cache(&mut self) -> (BTreeMap<String, FileEntry>, BTreeMap<String, FileEntry>, BTreeMap<String, FileEntry>) {
|
||||||
debug!("hash_images_load_cache - start, use cache: {}", self.common_data.use_cache);
|
debug!("hash_images_load_cache - start, use cache: {}", self.common_data.use_cache);
|
||||||
let loaded_hash_map;
|
let loaded_hash_map;
|
||||||
|
|
||||||
let mut records_already_cached: HashMap<String, FileEntry> = Default::default();
|
let mut records_already_cached: BTreeMap<String, FileEntry> = Default::default();
|
||||||
let mut non_cached_files_to_check: HashMap<String, FileEntry> = Default::default();
|
let mut non_cached_files_to_check: BTreeMap<String, FileEntry> = Default::default();
|
||||||
|
|
||||||
if self.common_data.use_cache {
|
if self.common_data.use_cache {
|
||||||
loaded_hash_map = match load_hashes_from_file(
|
let (messages, loaded_items) =
|
||||||
&mut self.common_data.text_messages,
|
load_cache_from_file_generalized::<FileEntry>(&get_cache_file(&self.hash_size, &self.hash_alg, &self.image_filter), self.get_delete_outdated_cache());
|
||||||
self.common_data.delete_outdated_cache,
|
self.get_text_messages_mut().extend_with_another_messages(messages);
|
||||||
self.hash_size,
|
loaded_hash_map = loaded_items.unwrap_or_default();
|
||||||
self.hash_alg,
|
|
||||||
self.image_filter,
|
|
||||||
) {
|
|
||||||
Some(t) => t,
|
|
||||||
None => Default::default(),
|
|
||||||
};
|
|
||||||
|
|
||||||
for (name, file_entry) in &self.images_to_check {
|
for (name, file_entry) in &self.images_to_check {
|
||||||
if !loaded_hash_map.contains_key(name) {
|
if !loaded_hash_map.contains_key(name) {
|
||||||
|
@ -373,18 +367,19 @@ impl SimilarImages {
|
||||||
true
|
true
|
||||||
}
|
}
|
||||||
|
|
||||||
fn save_to_cache(&mut self, vec_file_entry: Vec<(FileEntry, ImHash)>, loaded_hash_map: HashMap<String, FileEntry>) {
|
fn save_to_cache(&mut self, vec_file_entry: Vec<(FileEntry, ImHash)>, loaded_hash_map: BTreeMap<String, FileEntry>) {
|
||||||
debug!("save_to_cache - start, using cache: {}", self.common_data.use_cache);
|
debug!("save_to_cache - start, using cache: {}", self.common_data.use_cache);
|
||||||
if self.common_data.use_cache {
|
if self.common_data.use_cache {
|
||||||
// Must save all results to file, old loaded from file with all currently counted results
|
// Must save all results to file, old loaded from file with all currently counted results
|
||||||
let mut all_results: HashMap<String, FileEntry> = loaded_hash_map;
|
let mut all_results: BTreeMap<String, FileEntry> = loaded_hash_map;
|
||||||
for (file_entry, _hash) in vec_file_entry {
|
for (file_entry, _hash) in vec_file_entry {
|
||||||
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry);
|
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry);
|
||||||
}
|
}
|
||||||
|
let save_also_as_json = self.get_save_also_as_json();
|
||||||
save_hashes_to_file(
|
save_hashes_to_file(
|
||||||
&all_results,
|
&all_results,
|
||||||
&mut self.common_data.text_messages,
|
&mut self.common_data.text_messages,
|
||||||
self.common_data.save_also_as_json,
|
save_also_as_json,
|
||||||
self.hash_size,
|
self.hash_size,
|
||||||
self.hash_alg,
|
self.hash_alg,
|
||||||
self.image_filter,
|
self.image_filter,
|
||||||
|
@ -946,7 +941,7 @@ impl PrintResults for SimilarImages {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn save_hashes_to_file(
|
pub fn save_hashes_to_file(
|
||||||
hashmap: &HashMap<String, FileEntry>,
|
hashmap: &BTreeMap<String, FileEntry>,
|
||||||
text_messages: &mut Messages,
|
text_messages: &mut Messages,
|
||||||
save_also_as_json: bool,
|
save_also_as_json: bool,
|
||||||
hash_size: u8,
|
hash_size: u8,
|
||||||
|
@ -1028,7 +1023,7 @@ pub fn load_hashes_from_file(
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_cache_file(hash_size: &u8, hash_alg: &HashAlg, image_filter: &FilterType) -> String {
|
pub fn get_cache_file(hash_size: &u8, hash_alg: &HashAlg, image_filter: &FilterType) -> String {
|
||||||
format!(
|
format!(
|
||||||
"cache_similar_images_{}_{}_{}_50.bin",
|
"cache_similar_images_{}_{}_{}_50.bin",
|
||||||
hash_size,
|
hash_size,
|
||||||
|
|
|
@ -15,7 +15,9 @@ use serde::{Deserialize, Serialize};
|
||||||
use vid_dup_finder_lib::HashCreationErrorKind::DetermineVideo;
|
use vid_dup_finder_lib::HashCreationErrorKind::DetermineVideo;
|
||||||
use vid_dup_finder_lib::{NormalizedTolerance, VideoHash};
|
use vid_dup_finder_lib::{NormalizedTolerance, VideoHash};
|
||||||
|
|
||||||
use crate::common::{check_folder_children, open_cache_folder, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, VIDEO_FILES_EXTENSIONS};
|
use crate::common::{
|
||||||
|
check_folder_children, load_cache_from_file_generalized, open_cache_folder, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, VIDEO_FILES_EXTENSIONS,
|
||||||
|
};
|
||||||
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
|
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
|
||||||
use crate::common_messages::Messages;
|
use crate::common_messages::Messages;
|
||||||
use crate::common_tool::{CommonData, CommonToolData};
|
use crate::common_tool::{CommonData, CommonToolData};
|
||||||
|
@ -259,10 +261,9 @@ impl SimilarVideos {
|
||||||
let mut non_cached_files_to_check: BTreeMap<String, FileEntry> = Default::default();
|
let mut non_cached_files_to_check: BTreeMap<String, FileEntry> = Default::default();
|
||||||
|
|
||||||
if self.common_data.use_cache {
|
if self.common_data.use_cache {
|
||||||
loaded_hash_map = match load_hashes_from_file(&mut self.common_data.text_messages, self.common_data.delete_outdated_cache) {
|
let (messages, loaded_items) = load_cache_from_file_generalized::<FileEntry>(&get_cache_file(), self.get_delete_outdated_cache());
|
||||||
Some(t) => t,
|
self.get_text_messages_mut().extend_with_another_messages(messages);
|
||||||
None => Default::default(),
|
loaded_hash_map = loaded_items.unwrap_or_default();
|
||||||
};
|
|
||||||
|
|
||||||
for (name, file_entry) in &self.videos_to_check {
|
for (name, file_entry) in &self.videos_to_check {
|
||||||
if !loaded_hash_map.contains_key(name) {
|
if !loaded_hash_map.contains_key(name) {
|
||||||
|
@ -375,7 +376,8 @@ impl SimilarVideos {
|
||||||
for file_entry in vec_file_entry {
|
for file_entry in vec_file_entry {
|
||||||
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry);
|
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry);
|
||||||
}
|
}
|
||||||
save_hashes_to_file(&all_results, &mut self.common_data.text_messages, self.common_data.save_also_as_json);
|
let save_also_as_json = self.get_save_also_as_json();
|
||||||
|
save_hashes_to_file(&all_results, &mut self.common_data.text_messages, save_also_as_json);
|
||||||
}
|
}
|
||||||
debug!("save_cache - end");
|
debug!("save_cache - end");
|
||||||
}
|
}
|
||||||
|
@ -539,46 +541,7 @@ pub fn save_hashes_to_file(hashmap: &BTreeMap<String, FileEntry>, text_messages:
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn load_hashes_from_file(text_messages: &mut Messages, delete_outdated_cache: bool) -> Option<BTreeMap<String, FileEntry>> {
|
pub fn get_cache_file() -> String {
|
||||||
if let Some(((file_handler, cache_file), (file_handler_json, cache_file_json))) = open_cache_folder(&get_cache_file(), false, true, &mut text_messages.warnings) {
|
|
||||||
let mut hashmap_loaded_entries: BTreeMap<String, FileEntry>;
|
|
||||||
if let Some(file_handler) = file_handler {
|
|
||||||
let reader = BufReader::new(file_handler);
|
|
||||||
hashmap_loaded_entries = match bincode::deserialize_from(reader) {
|
|
||||||
Ok(t) => t,
|
|
||||||
Err(e) => {
|
|
||||||
text_messages
|
|
||||||
.warnings
|
|
||||||
.push(format!("Failed to load data from cache file {}, reason {}", cache_file.display(), e));
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
} else {
|
|
||||||
let reader = BufReader::new(file_handler_json.unwrap()); // Unwrap cannot fail, because at least one file must be valid
|
|
||||||
hashmap_loaded_entries = match serde_json::from_reader(reader) {
|
|
||||||
Ok(t) => t,
|
|
||||||
Err(e) => {
|
|
||||||
text_messages
|
|
||||||
.warnings
|
|
||||||
.push(format!("Failed to load data from cache file {}, reason {}", cache_file_json.display(), e));
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
// Don't load cache data if destination file not exists
|
|
||||||
if delete_outdated_cache {
|
|
||||||
hashmap_loaded_entries.retain(|src_path, _file_entry| Path::new(src_path).exists());
|
|
||||||
}
|
|
||||||
|
|
||||||
text_messages.messages.push(format!("Properly loaded {} cache entries.", hashmap_loaded_entries.len()));
|
|
||||||
|
|
||||||
return Some(hashmap_loaded_entries);
|
|
||||||
}
|
|
||||||
None
|
|
||||||
}
|
|
||||||
|
|
||||||
fn get_cache_file() -> String {
|
|
||||||
"cache_similar_videos.bin".to_string()
|
"cache_similar_videos.bin".to_string()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
use std::default::Default;
|
use std::default::Default;
|
||||||
|
|
||||||
|
use czkawka_core::common::load_cache_from_file_generalized;
|
||||||
use directories_next::ProjectDirs;
|
use directories_next::ProjectDirs;
|
||||||
use gtk4::prelude::*;
|
use gtk4::prelude::*;
|
||||||
use gtk4::{Label, ResponseType, Window};
|
use gtk4::{Label, ResponseType, Window};
|
||||||
|
@ -169,7 +170,12 @@ pub fn connect_settings(gui_data: &GuiData) {
|
||||||
FilterType::Triangle,
|
FilterType::Triangle,
|
||||||
] {
|
] {
|
||||||
for hash_alg in &[HashAlg::Blockhash, HashAlg::Gradient, HashAlg::DoubleGradient, HashAlg::VertGradient, HashAlg::Mean] {
|
for hash_alg in &[HashAlg::Blockhash, HashAlg::Gradient, HashAlg::DoubleGradient, HashAlg::VertGradient, HashAlg::Mean] {
|
||||||
if let Some(cache_entries) = czkawka_core::similar_images::load_hashes_from_file(&mut messages, true, *hash_size, *hash_alg, *image_filter) {
|
let (mut messages, loaded_items) = load_cache_from_file_generalized::<czkawka_core::similar_images::FileEntry>(
|
||||||
|
&czkawka_core::similar_images::get_cache_file(hash_size, hash_alg, image_filter),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
|
||||||
|
if let Some(cache_entries) = loaded_items {
|
||||||
czkawka_core::similar_images::save_hashes_to_file(&cache_entries, &mut messages, false, *hash_size, *hash_alg, *image_filter);
|
czkawka_core::similar_images::save_hashes_to_file(&cache_entries, &mut messages, false, *hash_size, *hash_alg, *image_filter);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -196,8 +202,10 @@ pub fn connect_settings(gui_data: &GuiData) {
|
||||||
|
|
||||||
dialog.connect_response(move |dialog, response_type| {
|
dialog.connect_response(move |dialog, response_type| {
|
||||||
if response_type == ResponseType::Ok {
|
if response_type == ResponseType::Ok {
|
||||||
let mut messages: Messages = Messages::new();
|
let (mut messages, loaded_items) =
|
||||||
if let Some(cache_entries) = czkawka_core::similar_videos::load_hashes_from_file(&mut messages, true) {
|
load_cache_from_file_generalized::<czkawka_core::similar_videos::FileEntry>(&czkawka_core::similar_videos::get_cache_file(), true);
|
||||||
|
|
||||||
|
if let Some(cache_entries) = loaded_items {
|
||||||
czkawka_core::similar_videos::save_hashes_to_file(&cache_entries, &mut messages, false);
|
czkawka_core::similar_videos::save_hashes_to_file(&cache_entries, &mut messages, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue