Fix cache and improve performance of validating items
This commit is contained in:
parent
d892ec1935
commit
244c6086d1
|
@ -23,8 +23,8 @@ use symphonia::core::io::MediaSourceStream;
|
||||||
use symphonia::core::meta::MetadataOptions;
|
use symphonia::core::meta::MetadataOptions;
|
||||||
use symphonia::core::probe::Hint;
|
use symphonia::core::probe::Hint;
|
||||||
|
|
||||||
|
use crate::common::open_cache_folder;
|
||||||
use crate::common::{create_crash_message, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, AUDIO_FILES_EXTENSIONS};
|
use crate::common::{create_crash_message, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, AUDIO_FILES_EXTENSIONS};
|
||||||
use crate::common::{open_cache_folder};
|
|
||||||
use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData};
|
use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData};
|
||||||
use crate::common_directory::Directories;
|
use crate::common_directory::Directories;
|
||||||
use crate::common_extensions::Extensions;
|
use crate::common_extensions::Extensions;
|
||||||
|
@ -160,7 +160,7 @@ impl SameMusic {
|
||||||
use_reference_folders: false,
|
use_reference_folders: false,
|
||||||
duplicated_music_entries_referenced: vec![],
|
duplicated_music_entries_referenced: vec![],
|
||||||
save_also_as_json: false,
|
save_also_as_json: false,
|
||||||
check_type: AudioCheckMethod::Tags,
|
check_type: AudioCheckMethod::Content,
|
||||||
hash_preset_config: Configuration::preset_test1(), // TODO allow to change this
|
hash_preset_config: Configuration::preset_test1(), // TODO allow to change this
|
||||||
minimal_segment_duration: 10.0,
|
minimal_segment_duration: 10.0,
|
||||||
minimum_similarity_score: 2.0,
|
minimum_similarity_score: 2.0,
|
||||||
|
@ -355,21 +355,20 @@ impl SameMusic {
|
||||||
for (name, file_entry) in &self.music_to_check {
|
for (name, file_entry) in &self.music_to_check {
|
||||||
#[allow(clippy::if_same_then_else)]
|
#[allow(clippy::if_same_then_else)]
|
||||||
if !loaded_hash_map.contains_key(name) {
|
if !loaded_hash_map.contains_key(name) {
|
||||||
println!("Checking completelly not cached item");
|
|
||||||
// If loaded data doesn't contains current image info
|
// If loaded data doesn't contains current image info
|
||||||
non_cached_files_to_check.insert(name.clone(), file_entry.clone());
|
non_cached_files_to_check.insert(name.clone(), file_entry.clone());
|
||||||
} else if (checking_tags && ![1, 3].contains(&file_entry.cache_type)) || (!checking_tags && ![2, 3].contains(&file_entry.cache_type)) {
|
|
||||||
println!("File was not checked with current mode");
|
|
||||||
// File was not cheched with current mode
|
|
||||||
non_cached_files_to_check.insert(name.clone(), file_entry.clone());
|
|
||||||
} else if file_entry.size != loaded_hash_map.get(name).unwrap().size || file_entry.modified_date != loaded_hash_map.get(name).unwrap().modified_date {
|
|
||||||
println!("File have different size or modified date");
|
|
||||||
// When size or modification date of image changed, then it is clear that is different image
|
|
||||||
non_cached_files_to_check.insert(name.clone(), file_entry.clone());
|
|
||||||
} else {
|
} else {
|
||||||
println!("File was cached");
|
let loaded_item = loaded_hash_map.get(name).unwrap();
|
||||||
// Checking may be omitted when already there is entry with same size and modification date
|
if (checking_tags && [0, 2].contains(&loaded_item.cache_type)) || (!checking_tags && [0, 1].contains(&loaded_item.cache_type)) {
|
||||||
records_already_cached.insert(name.clone(), loaded_hash_map.get(name).unwrap().clone());
|
// File was not checked with current mode
|
||||||
|
non_cached_files_to_check.insert(name.clone(), file_entry.clone());
|
||||||
|
} else if file_entry.size != loaded_item.size || file_entry.modified_date != loaded_item.modified_date {
|
||||||
|
// When size or modification date of image changed, then it is clear that is different image
|
||||||
|
non_cached_files_to_check.insert(name.clone(), file_entry.clone());
|
||||||
|
} else {
|
||||||
|
// Checking may be omitted when already there is entry with same size and modification date
|
||||||
|
records_already_cached.insert(name.clone(), loaded_item.clone());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -400,16 +400,18 @@ impl SimilarImages {
|
||||||
};
|
};
|
||||||
|
|
||||||
for (name, file_entry) in &self.images_to_check {
|
for (name, file_entry) in &self.images_to_check {
|
||||||
#[allow(clippy::if_same_then_else)]
|
|
||||||
if !loaded_hash_map.contains_key(name) {
|
if !loaded_hash_map.contains_key(name) {
|
||||||
// If loaded data doesn't contains current image info
|
// If loaded data doesn't contains current image info
|
||||||
non_cached_files_to_check.insert(name.clone(), file_entry.clone());
|
non_cached_files_to_check.insert(name.clone(), file_entry.clone());
|
||||||
} else if file_entry.size != loaded_hash_map.get(name).unwrap().size || file_entry.modified_date != loaded_hash_map.get(name).unwrap().modified_date {
|
|
||||||
// When size or modification date of image changed, then it is clear that is different image
|
|
||||||
non_cached_files_to_check.insert(name.clone(), file_entry.clone());
|
|
||||||
} else {
|
} else {
|
||||||
// Checking may be omitted when already there is entry with same size and modification date
|
let loaded_item = loaded_hash_map.get(name).unwrap();
|
||||||
records_already_cached.insert(name.clone(), loaded_hash_map.get(name).unwrap().clone());
|
if file_entry.size != loaded_item.size || file_entry.modified_date != loaded_item.modified_date {
|
||||||
|
// When size or modification date of image changed, then it is clear that is different image
|
||||||
|
non_cached_files_to_check.insert(name.clone(), file_entry.clone());
|
||||||
|
} else {
|
||||||
|
// Checking may be omitted when already there is entry with same size and modification date
|
||||||
|
records_already_cached.insert(name.clone(), loaded_item.clone());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -7,7 +7,6 @@ use std::path::{Path, PathBuf};
|
||||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
|
||||||
use crossbeam_channel::Receiver;
|
use crossbeam_channel::Receiver;
|
||||||
use ffmpeg_cmdline_utils::FfmpegErrorKind::FfmpegNotFound;
|
use ffmpeg_cmdline_utils::FfmpegErrorKind::FfmpegNotFound;
|
||||||
use futures::channel::mpsc::UnboundedSender;
|
use futures::channel::mpsc::UnboundedSender;
|
||||||
|
@ -18,8 +17,8 @@ use serde::{Deserialize, Serialize};
|
||||||
use vid_dup_finder_lib::HashCreationErrorKind::DetermineVideo;
|
use vid_dup_finder_lib::HashCreationErrorKind::DetermineVideo;
|
||||||
use vid_dup_finder_lib::{NormalizedTolerance, VideoHash};
|
use vid_dup_finder_lib::{NormalizedTolerance, VideoHash};
|
||||||
|
|
||||||
|
use crate::common::open_cache_folder;
|
||||||
use crate::common::{check_folder_children, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, VIDEO_FILES_EXTENSIONS};
|
use crate::common::{check_folder_children, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, VIDEO_FILES_EXTENSIONS};
|
||||||
use crate::common::{open_cache_folder};
|
|
||||||
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData};
|
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData};
|
||||||
use crate::common_directory::Directories;
|
use crate::common_directory::Directories;
|
||||||
use crate::common_extensions::Extensions;
|
use crate::common_extensions::Extensions;
|
||||||
|
@ -364,16 +363,18 @@ impl SimilarVideos {
|
||||||
};
|
};
|
||||||
|
|
||||||
for (name, file_entry) in &self.videos_to_check {
|
for (name, file_entry) in &self.videos_to_check {
|
||||||
#[allow(clippy::if_same_then_else)]
|
|
||||||
if !loaded_hash_map.contains_key(name) {
|
if !loaded_hash_map.contains_key(name) {
|
||||||
// If loaded data doesn't contains current videos info
|
// If loaded data doesn't contains current videos info
|
||||||
non_cached_files_to_check.insert(name.clone(), file_entry.clone());
|
non_cached_files_to_check.insert(name.clone(), file_entry.clone());
|
||||||
} else if file_entry.size != loaded_hash_map.get(name).unwrap().size || file_entry.modified_date != loaded_hash_map.get(name).unwrap().modified_date {
|
|
||||||
// When size or modification date of video changed, then it is clear that is different video
|
|
||||||
non_cached_files_to_check.insert(name.clone(), file_entry.clone());
|
|
||||||
} else {
|
} else {
|
||||||
// Checking may be omitted when already there is entry with same size and modification date
|
let loaded_item = loaded_hash_map.get(name).unwrap();
|
||||||
records_already_cached.insert(name.clone(), loaded_hash_map.get(name).unwrap().clone());
|
if file_entry.size != loaded_item.size || file_entry.modified_date != loaded_item.modified_date {
|
||||||
|
// When size or modification date of video changed, then it is clear that is different video
|
||||||
|
non_cached_files_to_check.insert(name.clone(), file_entry.clone());
|
||||||
|
} else {
|
||||||
|
// Checking may be omitted when already there is entry with same size and modification date
|
||||||
|
records_already_cached.insert(name.clone(), loaded_item.clone());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|
Loading…
Reference in a new issue