diff --git a/Cargo.lock b/Cargo.lock index 12d9a44..3d58dfe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -138,6 +138,15 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bindgen" version = "0.56.0" @@ -479,6 +488,7 @@ name = "czkawka_core" version = "3.3.1" dependencies = [ "audiotags", + "bincode", "bitflags", "bk-tree", "blake3", @@ -493,6 +503,7 @@ dependencies = [ "img_hash", "rayon", "rodio", + "serde", "tempfile", "vid_dup_finder_lib", "xxhash-rust", @@ -2270,7 +2281,8 @@ checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" [[package]] name = "vid_dup_finder_lib" version = "0.1.0" -source = "git+https://github.com/qarmin/vid_dup_finder_lib#a4809772aea8f73c9a22da6fb43df50bfdd1b31d" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64bf85b5022ad9df9ec55750118ec38f33406165fcf9edf19c71eb2bd6823232" dependencies = [ "ffmpeg_cmdline_utils", "image", diff --git a/czkawka_core/Cargo.toml b/czkawka_core/Cargo.toml index a9b3d14..e1411de 100644 --- a/czkawka_core/Cargo.toml +++ b/czkawka_core/Cargo.toml @@ -42,8 +42,11 @@ xxhash-rust = { version = "0.8.2", features = ["xxh3"] } tempfile = "3.2.0" # Video Duplactes -vid_dup_finder_lib = { git = "https://github.com/qarmin/vid_dup_finder_lib" } +vid_dup_finder_lib = "0.1.0" ffmpeg_cmdline_utils = "0.1.0" +serde = "1.0.130" +bincode = "1.3.3" + [features] default = [] diff --git a/czkawka_core/src/similar_videos.rs b/czkawka_core/src/similar_videos.rs index 8916b61..fc91580 100644 --- a/czkawka_core/src/similar_videos.rs +++ b/czkawka_core/src/similar_videos.rs @@ -15,6 +15,7 @@ use directories_next::ProjectDirs; use ffmpeg_cmdline_utils::FfmpegErrorKind::FfmpegNotFound; use humansize::{file_size_opts as options, FileSize}; use rayon::prelude::*; +use serde::{Deserialize, Serialize}; use vid_dup_finder_lib::HashCreationErrorKind::DetermineVideo; use vid_dup_finder_lib::{NormalizedTolerance, VideoHash}; @@ -27,8 +28,6 @@ use crate::common_traits::{DebugPrint, PrintResults, SaveResults}; pub const MAX_TOLERANCE: i32 = 20; -const HASH_SIZE: usize = 19; - #[derive(Debug)] pub struct ProgressData { pub current_stage: u8, @@ -37,7 +36,7 @@ pub struct ProgressData { pub videos_to_check: usize, } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct FileEntry { pub path: PathBuf, pub size: u64, @@ -611,7 +610,7 @@ pub fn save_hashes_to_file(hashmap: &BTreeMap, text_messages: text_messages.messages.push(format!("Cannot create config dir {}, reason {}", cache_dir.display(), e)); return; } - let cache_file = cache_dir.join("cache_similar_videos.txt"); + let cache_file = cache_dir.join("cache_similar_videos.bin"); let file_handler = match OpenOptions::new().truncate(true).write(true).create(true).open(&cache_file) { Ok(t) => t, Err(e) => { @@ -619,31 +618,10 @@ pub fn save_hashes_to_file(hashmap: &BTreeMap, text_messages: return; } }; - let mut writer = BufWriter::new(file_handler); - for file_entry in hashmap.values() { - let mut string: String = String::with_capacity(256); - - string += format!( - "{}//{}//{}//{}//{}//{}", - file_entry.path.display(), - file_entry.size, - file_entry.modified_date, - file_entry.vhash.num_frames(), - file_entry.vhash.duration(), - file_entry.error - ) - .as_str(); - - for i in file_entry.vhash.hash() { - string.push_str("//"); - string.push_str(i.to_string().as_str()); - } - - if let Err(e) = writeln!(writer, "{}", string) { - text_messages.messages.push(format!("Failed to save some data to cache file {}, reason {}", cache_file.display(), e)); - return; - }; + let writer = BufWriter::new(file_handler); + if let Err(e) = bincode::serialize_into(writer, hashmap) { + text_messages.messages.push(format!("cannot write data to cache file {}, reason {}", cache_file.display(), e)); } } } @@ -651,7 +629,7 @@ pub fn save_hashes_to_file(hashmap: &BTreeMap, text_messages: pub fn load_hashes_from_file(text_messages: &mut Messages, delete_outdated_cache: bool) -> Option> { if let Some(proj_dirs) = ProjectDirs::from("pl", "Qarmin", "Czkawka") { let cache_dir = PathBuf::from(proj_dirs.cache_dir()); - let cache_file = cache_dir.join("cache_similar_videos.txt"); + let cache_file = cache_dir.join("cache_similar_videos.bin"); let file_handler = match OpenOptions::new().read(true).open(&cache_file) { Ok(t) => t, Err(_inspected) => { @@ -661,74 +639,16 @@ pub fn load_hashes_from_file(text_messages: &mut Messages, delete_outdated_cache }; let reader = BufReader::new(file_handler); - - let mut hashmap_loaded_entries: BTreeMap = Default::default(); - - // Read the file line by line using the lines() iterator from std::io::BufRead. - for (index, line) in reader.lines().enumerate() { - let line = match line { - Ok(t) => t, - Err(e) => { - text_messages.warnings.push(format!("Failed to load line number {} from cache file {}, reason {}", index + 1, cache_file.display(), e)); - return None; - } - }; - let uuu = line.split("//").collect::>(); - // Hash size + other things - if uuu.len() != (HASH_SIZE + 6) { - text_messages.warnings.push(format!( - "Found invalid data in line {} - ({}) in cache file {}, expected {} values, found {}", - index + 1, - line, - cache_file.display(), - HASH_SIZE + 6, - uuu.len(), - )); - continue; - }; - // Don't load cache data if destination file not exists - if !delete_outdated_cache || Path::new(uuu[0]).exists() { - let mut hash: [u64; 19] = [0; 19]; - for i in 0..HASH_SIZE { - hash[i] = match uuu[6 + i as usize].parse::() { - Ok(t) => t, - Err(e) => { - text_messages - .warnings - .push(format!("Found invalid hash value in line {} - ({}) in cache file {}, reason {}", index + 1, line, cache_file.display(), e)); - continue; - } - }; - } - - hashmap_loaded_entries.insert( - uuu[0].to_string(), - FileEntry { - path: PathBuf::from(uuu[0]), - size: match uuu[1].parse::() { - Ok(t) => t, - Err(e) => { - text_messages - .warnings - .push(format!("Found invalid size value in line {} - ({}) in cache file {}, reason {}", index + 1, line, cache_file.display(), e)); - continue; - } - }, - modified_date: match uuu[2].parse::() { - Ok(t) => t, - Err(e) => { - text_messages - .warnings - .push(format!("Found invalid modified date value in line {} - ({}) in cache file {}, reason {}", index + 1, line, cache_file.display(), e)); - continue; - } - }, - vhash: VideoHash::with_start_data(uuu[4].parse::().unwrap_or(0), uuu[0], hash, uuu[3].parse::().unwrap_or(10)), - error: uuu[5].to_string(), - }, - ); + let mut hashmap_loaded_entries: BTreeMap = match bincode::deserialize_from(reader) { + Ok(t) => t, + Err(e) => { + text_messages.warnings.push(format!("Failed to load data from cache file {}, reason {}", cache_file.display(), e)); + return None; } - } + }; + + // Don't load cache data if destination file not exists + hashmap_loaded_entries.retain(|src_path, _file_entry| Path::new(src_path).exists() && !delete_outdated_cache); return Some(hashmap_loaded_entries); }