CodeMirror/czkawka
1
0
Fork 0
mirror of synced 2024-04-28 09:33:30 +12:00
Code Issues Projects Releases Wiki Activity

Add finding similar audio by content (#970)

Browse source 
* In prehashing check for user clicks, less often

* Similar audio

* Remove ugly time checking

* Fix using cache

* Fix cache and improve performance of validating items

* Remove cache type - cache should be saved to two different cache files(because )

* Working

* Simple multithreading

* Basic Generalization

* Reference folder and swap cleaning

* Split into multiple files

* Commons, improved GUI message

* Simplifying thread run

* Check was stopped

* Fix checking same files

* Make read single file tag more general

* Remove unnnecessary clone

* Reading tags

* Base

* Search

* Gui Fix

* Gui Fix

* Tooltip
This commit is contained in:
Rafał Mikrut 2023-05-07 20:54:05 +02:00 committed by GitHub
parent 14ccb49e45
commit 78d00eeb99
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
32 changed files with 1900 additions and 1612 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

52
Cargo.lock generated
View file

@ -389,9 +389,9 @@ dependencies = [
[[package]]
name = "clap"
version = "4.2.6"
version = "4.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d70680e56dc65cb226c361aaa4e4a16d1f7e082bfed9ffceaee39c2012384ec"
checksum = "34d21f9bf1b425d2968943631ec91202fe5e837264063503708b83013f8fc938"
dependencies = [
"clap_builder",
"clap_derive",
@ -400,9 +400,9 @@ dependencies = [
[[package]]
name = "clap_builder"
version = "4.2.6"
version = "4.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fad499d5e07338414687350c5fdb82b1ab0001e9b26aa6275deccb684b14164"
checksum = "914c8c79fb560f238ef6429439a30023c862f7a28e688c58f7203f12b29970bd"
dependencies = [
"anstream",
"anstyle",
@ -744,13 +744,13 @@ dependencies = [
[[package]]
name = "displaydoc"
version = "0.2.3"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3bf95dc3f046b9da4f2d51833c0d3547d8564ef6910f5c1ed130306a75b92886"
checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d"
dependencies = [
"proc-macro2",
"quote",
"syn 1.0.109",
"syn 2.0.15",
]
[[package]]
@ -1780,9 +1780,9 @@ checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8"
[[package]]
name = "libc"
version = "0.2.142"
version = "0.2.143"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317"
checksum = "edc207893e85c5d6be840e969b496b53d94cec8be2d501b214f50daa97fa8024"
[[package]]
name = "libheif-rs"
@ -1827,9 +1827,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
[[package]]
name = "linux-raw-sys"
version = "0.3.6"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b64f40e5e03e0d54f03845c8197d0291253cdbedfb1cb46b13c2c117554a9f4c"
checksum = "ece97ea872ece730aed82664c424eb4c8291e1ff2480247ccf7409044bc6479f"
[[package]]
name = "locale_config"
@ -2293,9 +2293,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "pkg-config"
version = "0.3.26"
version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
[[package]]
name = "png"
@ -2633,9 +2633,9 @@ dependencies = [
[[package]]
name = "rustix"
version = "0.37.18"
version = "0.37.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8bbfc1d1c7c40c01715f47d71444744a81669ca84e8b63e25a55e169b1f86433"
checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d"
dependencies = [
"bitflags 1.3.2",
"errno",
@ -2708,18 +2708,18 @@ checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed"
[[package]]
name = "serde"
version = "1.0.160"
version = "1.0.162"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c"
checksum = "71b2f6e1ab5c2b98c05f0f35b236b22e8df7ead6ffbf51d7808da7f8817e7ab6"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.160"
version = "1.0.162"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df"
checksum = "a2a0814352fd64b58489904a44ea8d90cb1a91dcb6b4f5ebabc32c8318e93cb6"
dependencies = [
"proc-macro2",
"quote",
@ -3178,9 +3178,9 @@ dependencies = [
[[package]]
name = "time"
version = "0.3.20"
version = "0.3.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890"
checksum = "8f3403384eaacbca9923fa06940178ac13e4edb725486d70e8e15881d0c836cc"
dependencies = [
"serde",
"time-core",
@ -3188,9 +3188,9 @@ dependencies = [
[[package]]
name = "time-core"
version = "0.1.0"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd"
checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb"
[[package]]
name = "tinystr"
@ -3770,9 +3770,9 @@ dependencies = [
[[package]]
name = "zip"
version = "0.6.4"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0445d0fbc924bb93539b4316c11afb121ea39296f99a3c4c9edad09e3658cdef"
checksum = "7e92305c174683d78035cbf1b70e18db6329cc0f1b9cae0a52ca90bf5bfe7125"
dependencies = [
"aes 0.7.5",
"byteorder",
@ -3784,7 +3784,7 @@ dependencies = [
"hmac",
"pbkdf2",
"sha1",
"time 0.3.20",
"time 0.3.21",
]
[[package]]

6
czkawka_core/Cargo.toml
View file

@ -38,7 +38,7 @@ pdf = "0.8"
# Needed by audio similarity feature
rusty-chromaprint = "0.1"
symphonia = { version = "0.5", features = ["mp3", "aac", "alac", "flac", "isomp4", "mkv", "ogg", "pcm", "vorbis", "wav"] }
symphonia = { version = "0.5", features = ["all"] }
# Hashes for duplicate files
blake3 = "1.3"
@ -74,10 +74,10 @@ num_cpus = "1.15"
# Heif/Heic
libheif-rs = { version = "0.18.0", optional = true } # Do not upgrade now, since Ubuntu 22.04 not works with newer version
anyhow = { version = "1.0", optional = true }
anyhow = { version = "1.0" }
state = "0.5"
[features]
default = []
heif = ["dep:libheif-rs", "dep:anyhow"]
heif = ["dep:libheif-rs"]

39
czkawka_core/src/bad_extensions.rs
View file

@ -6,14 +6,13 @@ use std::mem;
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::SystemTime;
use crossbeam_channel::Receiver;
use futures::channel::mpsc::UnboundedSender;
use mime_guess::get_mime_extensions;
use rayon::prelude::*;
use crate::common::{prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, Common};
use crate::common::{prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads};
use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData};
use crate::common_directory::Directories;
use crate::common_extensions::Extensions;
@ -298,16 +297,12 @@ impl BadExtensions {
.build()
.run();
match result {
DirTraversalResult::SuccessFiles {
start_time,
grouped_file_entries,
warnings,
} => {
DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => {
if let Some(files_to_check) = grouped_file_entries.get(&()) {
self.files_to_check = files_to_check.clone();
}
self.text_messages.warnings.extend(warnings);
Common::print_time(start_time, SystemTime::now(), "check_files");
true
}
DirTraversalResult::SuccessFolders { .. } => {
@ -318,24 +313,10 @@ impl BadExtensions {
}
fn look_for_bad_extensions_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
let system_time = SystemTime::now();
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
prepare_thread_handler_common(progress_sender, 1, 1, self.files_to_check.len(), CheckingMethod::None);
let check_was_stopped = AtomicBool::new(false); // Used for breaking from GUI and ending check thread
let progress_thread_run = Arc::new(AtomicBool::new(true));
let atomic_counter = Arc::new(AtomicUsize::new(0));
let progress_thread_handle = prepare_thread_handler_common(
progress_sender,
&progress_thread_run,
&atomic_counter,
1,
1,
self.files_to_check.len(),
CheckingMethod::None,
);
let mut files_to_check = Default::default();
mem::swap(&mut files_to_check, &mut self.files_to_check);
let files_to_check = mem::take(&mut self.files_to_check);
let mut hashmap_workarounds: HashMap<&str, Vec<&str>> = Default::default();
for (proper, found) in WORKAROUNDS {
@ -357,8 +338,6 @@ impl BadExtensions {
self.information.number_of_files_with_bad_extension = self.bad_extensions_files.len();
Common::print_time(system_time, SystemTime::now(), "bad extension finding");
// Clean unused data
self.files_to_check = Default::default();
@ -525,7 +504,6 @@ impl DebugPrint for BadExtensions {
impl SaveResults for BadExtensions {
fn save_results_to_file(&mut self, file_name: &str) -> bool {
let start_time: SystemTime = SystemTime::now();
let file_name: String = match file_name {
"" => "results.txt".to_string(),
k => k.to_string(),
@ -557,7 +535,7 @@ impl SaveResults for BadExtensions {
} else {
write!(writer, "Not found any files with invalid extension.").unwrap();
}
Common::print_time(start_time, SystemTime::now(), "save_results_to_file");
true
}
}
@ -566,12 +544,9 @@ impl PrintResults for BadExtensions {
/// Print information's about duplicated entries
/// Only needed for CLI
fn print_results(&self) {
let start_time: SystemTime = SystemTime::now();
println!("Found {} files with invalid extension.\n", self.information.number_of_files_with_bad_extension);
for file_entry in &self.bad_extensions_files {
println!("{} ----- {}", file_entry.path.display(), file_entry.proper_extensions);
}
Common::print_time(start_time, SystemTime::now(), "print_entries");
}
}

19
czkawka_core/src/big_file.rs
View file

@ -3,10 +3,9 @@ use std::fs;
use std::fs::{DirEntry, File, Metadata};
use std::io::{BufWriter, Write};
use std::path::{Path, PathBuf};
use std::sync::atomic::AtomicBool;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::SystemTime;
use crossbeam_channel::Receiver;
use futures::channel::mpsc::UnboundedSender;
@ -14,7 +13,6 @@ use humansize::format_size;
use humansize::BINARY;
use rayon::prelude::*;
use crate::common::Common;
use crate::common::{check_folder_children, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, split_path};
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData};
use crate::common_directory::Directories;
@ -142,7 +140,6 @@ impl BigFile {
}
fn look_for_big_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
let start_time: SystemTime = SystemTime::now();
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
let mut old_map: BTreeMap<u64, Vec<FileEntry>> = Default::default();
@ -151,9 +148,7 @@ impl BigFile {
folders_to_check.push(id.clone());
}
let progress_thread_run = Arc::new(AtomicBool::new(true));
let atomic_counter = Arc::new(AtomicUsize::new(0));
let progress_thread_handle = prepare_thread_handler_common(progress_sender, &progress_thread_run, &atomic_counter, 0, 0, 0, CheckingMethod::None);
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) = prepare_thread_handler_common(progress_sender, 0, 0, 0, CheckingMethod::None);
while !folders_to_check.is_empty() {
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
@ -213,7 +208,6 @@ impl BigFile {
self.extract_n_biggest_files(old_map);
Common::print_time(start_time, SystemTime::now(), "look_for_big_files");
true
}
@ -308,8 +302,6 @@ impl BigFile {
/// Function to delete files, from filed Vector
fn delete_files(&mut self) {
let start_time: SystemTime = SystemTime::now();
match self.delete_method {
DeleteMethod::Delete => {
for (_, file_entry) in &self.big_files {
@ -322,8 +314,6 @@ impl BigFile {
//Just do nothing
}
}
Common::print_time(start_time, SystemTime::now(), "delete_files");
}
}
@ -365,7 +355,6 @@ impl DebugPrint for BigFile {
impl SaveResults for BigFile {
/// Saving results to provided file
fn save_results_to_file(&mut self, file_name: &str) -> bool {
let start_time: SystemTime = SystemTime::now();
let file_name: String = match file_name {
"" => "results.txt".to_string(),
k => k.to_string(),
@ -401,14 +390,13 @@ impl SaveResults for BigFile {
} else {
write!(writer, "Not found any files.").unwrap();
}
Common::print_time(start_time, SystemTime::now(), "save_results_to_file");
true
}
}
impl PrintResults for BigFile {
fn print_results(&self) {
let start_time: SystemTime = SystemTime::now();
if self.search_mode == SearchMode::BiggestFiles {
println!("{} the biggest files.\n\n", self.information.number_of_real_files);
} else {
@ -417,6 +405,5 @@ impl PrintResults for BigFile {
for (size, file_entry) in &self.big_files {
println!("{} ({}) - {}", format_size(*size, BINARY), size, file_entry.path.display());
}
Common::print_time(start_time, SystemTime::now(), "print_entries");
}
}

46
czkawka_core/src/broken_files.rs
View file

@ -3,9 +3,9 @@ use std::fs::{DirEntry, File, Metadata};
use std::io::prelude::*;
use std::io::{BufReader, BufWriter};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::SystemTime;
use std::{fs, mem, panic};
use crossbeam_channel::Receiver;
@ -18,7 +18,7 @@ use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use crate::common::{
check_folder_children, create_crash_message, open_cache_folder, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, Common, PDF_FILES_EXTENSIONS,
check_folder_children, create_crash_message, open_cache_folder, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, PDF_FILES_EXTENSIONS,
};
use crate::common::{AUDIO_FILES_EXTENSIONS, IMAGE_RS_BROKEN_FILES_EXTENSIONS, ZIP_FILES_EXTENSIONS};
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData};
@ -193,7 +193,6 @@ impl BrokenFiles {
}
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
let start_time: SystemTime = SystemTime::now();
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
// Add root folders for finding
@ -201,9 +200,7 @@ impl BrokenFiles {
folders_to_check.push(id.clone());
}
let progress_thread_run = Arc::new(AtomicBool::new(true));
let atomic_counter = Arc::new(AtomicUsize::new(0));
let progress_thread_handle = prepare_thread_handler_common(progress_sender, &progress_thread_run, &atomic_counter, 0, 1, 0, CheckingMethod::None);
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) = prepare_thread_handler_common(progress_sender, 0, 1, 0, CheckingMethod::None);
while !folders_to_check.is_empty() {
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
@ -263,7 +260,6 @@ impl BrokenFiles {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
Common::print_time(start_time, SystemTime::now(), "check_files");
true
}
fn get_file_entry(
@ -405,14 +401,11 @@ impl BrokenFiles {
}
fn look_for_broken_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
let system_time = SystemTime::now();
let loaded_hash_map;
let mut records_already_cached: BTreeMap<String, FileEntry> = Default::default();
let mut non_cached_files_to_check: BTreeMap<String, FileEntry> = Default::default();
let mut files_to_check = Default::default();
mem::swap(&mut self.files_to_check, &mut files_to_check);
let files_to_check = mem::take(&mut self.files_to_check);
if self.use_cache {
loaded_hash_map = match load_cache_from_file(&mut self.text_messages, self.delete_outdated_cache) {
@ -442,17 +435,8 @@ impl BrokenFiles {
non_cached_files_to_check = files_to_check;
}
let progress_thread_run = Arc::new(AtomicBool::new(true));
let atomic_counter = Arc::new(AtomicUsize::new(0));
let progress_thread_handle = prepare_thread_handler_common(
progress_sender,
&progress_thread_run,
&atomic_counter,
1,
1,
non_cached_files_to_check.len(),
CheckingMethod::None,
);
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) =
prepare_thread_handler_common(progress_sender, 1, 1, non_cached_files_to_check.len(), CheckingMethod::None);
let mut vec_file_entry: Vec<FileEntry> = non_cached_files_to_check
.into_par_iter()
@ -479,9 +463,7 @@ impl BrokenFiles {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
// Just connect loaded results with already calculated
for (_name, file_entry) in records_already_cached {
vec_file_entry.push(file_entry.clone());
}
vec_file_entry.extend(records_already_cached.into_values());
if self.use_cache {
// Must save all results to file, old loaded from file with all currently counted results
@ -503,8 +485,6 @@ impl BrokenFiles {
self.information.number_of_broken_files = self.broken_files.len();
Common::print_time(system_time, SystemTime::now(), "sort_images - reading data from files in parallel");
// Clean unused data
self.files_to_check = Default::default();
@ -513,8 +493,6 @@ impl BrokenFiles {
/// Function to delete files, from filed Vector
fn delete_files(&mut self) {
let start_time: SystemTime = SystemTime::now();
match self.delete_method {
DeleteMethod::Delete => {
for file_entry in &self.broken_files {
@ -527,8 +505,6 @@ impl BrokenFiles {
//Just do nothing
}
}
Common::print_time(start_time, SystemTime::now(), "delete_files");
}
}
@ -569,7 +545,6 @@ impl DebugPrint for BrokenFiles {
impl SaveResults for BrokenFiles {
fn save_results_to_file(&mut self, file_name: &str) -> bool {
let start_time: SystemTime = SystemTime::now();
let file_name: String = match file_name {
"" => "results.txt".to_string(),
k => k.to_string(),
@ -601,7 +576,7 @@ impl SaveResults for BrokenFiles {
} else {
write!(writer, "Not found any broken files.").unwrap();
}
Common::print_time(start_time, SystemTime::now(), "save_results_to_file");
true
}
}
@ -610,13 +585,10 @@ impl PrintResults for BrokenFiles {
/// Print information's about duplicated entries
/// Only needed for CLI
fn print_results(&self) {
let start_time: SystemTime = SystemTime::now();
println!("Found {} broken files.\n", self.information.number_of_broken_files);
for file_entry in &self.broken_files {
println!("{} - {}", file_entry.path.display(), file_entry.error_string);
}
Common::print_time(start_time, SystemTime::now(), "print_entries");
}
}

34
czkawka_core/src/common.rs
View file

@ -22,6 +22,7 @@ use libheif_rs::{ColorSpace, HeifContext, RgbChroma};
use crate::common_dir_traversal::{CheckingMethod, ProgressData};
use crate::common_directory::Directories;
use crate::common_items::ExcludedItems;
use crate::common_traits::ResultEntry;
static NUMBER_OF_THREADS: state::Storage<usize> = state::Storage::new();
@ -364,16 +365,38 @@ pub fn check_folder_children(
dir_result.push(next_folder);
}
#[must_use]
pub fn filter_reference_folders_generic<T>(entries_to_check: Vec<Vec<T>>, directories: &Directories) -> Vec<(T, Vec<T>)>
where
T: ResultEntry,
{
entries_to_check
.into_iter()
.filter_map(|vec_file_entry| {
let (mut files_from_referenced_folders, normal_files): (Vec<_>, Vec<_>) =
vec_file_entry.into_iter().partition(|e| directories.is_in_referenced_directory(e.get_path()));
if files_from_referenced_folders.is_empty() || normal_files.is_empty() {
None
} else {
Some((files_from_referenced_folders.pop().unwrap(), normal_files))
}
})
.collect::<Vec<(T, Vec<T>)>>()
}
#[must_use]
pub fn prepare_thread_handler_common(
progress_sender: Option<&UnboundedSender<ProgressData>>,
progress_thread_run: &Arc<AtomicBool>,
atomic_counter: &Arc<AtomicUsize>,
current_stage: u8,
max_stage: u8,
max_value: usize,
checking_method: CheckingMethod,
) -> JoinHandle<()> {
if let Some(progress_sender) = progress_sender {
) -> (JoinHandle<()>, Arc<AtomicBool>, Arc<AtomicUsize>, AtomicBool) {
let progress_thread_run = Arc::new(AtomicBool::new(true));
let atomic_counter = Arc::new(AtomicUsize::new(0));
let check_was_stopped = AtomicBool::new(false);
let progress_thread_sender = if let Some(progress_sender) = progress_sender {
let progress_send = progress_sender.clone();
let progress_thread_run = progress_thread_run.clone();
let atomic_counter = atomic_counter.clone();
@ -394,7 +417,8 @@ pub fn prepare_thread_handler_common(
})
} else {
thread::spawn(|| {})
}
};
(progress_thread_sender, progress_thread_run, atomic_counter, check_was_stopped)
}
pub fn send_info_and_wait_for_ending_all_threads(progress_thread_run: &Arc<AtomicBool>, progress_thread_handle: JoinHandle<()>) {

24
czkawka_core/src/common_dir_traversal.rs
View file

@ -2,9 +2,9 @@ use std::collections::BTreeMap;
use std::fs;
use std::fs::{DirEntry, Metadata, ReadDir};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use std::sync::atomic::Ordering;
use std::time::UNIX_EPOCH;
use crossbeam_channel::Receiver;
use futures::channel::mpsc::UnboundedSender;
@ -14,6 +14,7 @@ use crate::common::{prepare_thread_handler_common, send_info_and_wait_for_ending
use crate::common_directory::Directories;
use crate::common_extensions::Extensions;
use crate::common_items::ExcludedItems;
use crate::common_traits::ResultEntry;
use crate::flc;
use crate::localizer_core::generate_translation_hashmap;
@ -33,6 +34,8 @@ pub enum CheckingMethod {
SizeName,
Size,
Hash,
AudioTags,
AudioContent,
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
@ -43,6 +46,11 @@ pub struct FileEntry {
pub hash: String,
pub symlink_info: Option<SymlinkInfo>,
}
impl ResultEntry for FileEntry {
fn get_path(&self) -> &Path {
&self.path
}
}
// Symlinks
@ -280,12 +288,10 @@ impl<'a, 'b, F> DirTraversalBuilder<'a, 'b, F> {
pub enum DirTraversalResult<T: Ord + PartialOrd> {
SuccessFiles {
start_time: SystemTime,
warnings: Vec<String>,
grouped_file_entries: BTreeMap<T, Vec<FileEntry>>,
},
SuccessFolders {
start_time: SystemTime,
warnings: Vec<String>,
folder_entries: BTreeMap<PathBuf, FolderEntry>, // Path, FolderEntry
},
@ -314,7 +320,6 @@ where
let mut all_warnings = vec![];
let mut grouped_file_entries: BTreeMap<T, Vec<FileEntry>> = BTreeMap::new();
let mut folder_entries: BTreeMap<PathBuf, FolderEntry> = BTreeMap::new();
let start_time: SystemTime = SystemTime::now();
// Add root folders into result (only for empty folder collection)
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
@ -333,9 +338,8 @@ where
// Add root folders for finding
folders_to_check.extend(self.root_dirs);
let progress_thread_run = Arc::new(AtomicBool::new(true));
let atomic_counter = Arc::new(AtomicUsize::new(0));
let progress_thread_handle = prepare_thread_handler_common(self.progress_sender, &progress_thread_run, &atomic_counter, 0, self.max_stage, 0, self.checking_method);
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) =
prepare_thread_handler_common(self.progress_sender, 0, self.max_stage, 0, self.checking_method);
let DirTraversal {
collect,
@ -468,12 +472,10 @@ where
match collect {
Collect::Files | Collect::InvalidSymlinks => DirTraversalResult::SuccessFiles {
start_time,
grouped_file_entries,
warnings: all_warnings,
},
Collect::EmptyFolders => DirTraversalResult::SuccessFolders {
start_time,
folder_entries,
warnings: all_warnings,
},

16
czkawka_core/src/common_directory.rs
View file

@ -1,5 +1,5 @@
use std::path::{Path, PathBuf};
use std::time::SystemTime;
#[cfg(target_family = "unix")]
use std::{fs, os::unix::fs::MetadataExt};
@ -30,8 +30,6 @@ impl Directories {
/// Setting included directories, at least one must be provided or scan won't start
pub fn set_included_directory(&mut self, included_directory: Vec<PathBuf>, text_messages: &mut Messages) -> bool {
let start_time: SystemTime = SystemTime::now();
if included_directory.is_empty() {
text_messages.errors.push(flc!("core_missing_no_chosen_included_directory"));
return false;
@ -90,13 +88,11 @@ impl Directories {
self.included_directories = checked_directories;
Common::print_time(start_time, SystemTime::now(), "set_included_directory");
true
}
/// Setting absolute path to exclude from search
pub fn set_excluded_directory(&mut self, excluded_directory: Vec<PathBuf>, text_messages: &mut Messages) {
let start_time: SystemTime = SystemTime::now();
if excluded_directory.is_empty() {
return;
}
@ -148,8 +144,6 @@ impl Directories {
checked_directories.push(directory);
}
self.excluded_directories = checked_directories;
Common::print_time(start_time, SystemTime::now(), "set_excluded_directory");
}
#[cfg(target_family = "unix")]
@ -159,8 +153,6 @@ impl Directories {
/// Remove unused entries when included or excluded overlaps with each other or are duplicated etc.
pub fn optimize_directories(&mut self, recursive_search: bool, text_messages: &mut Messages) -> bool {
let start_time: SystemTime = SystemTime::now();
let mut optimized_included: Vec<PathBuf> = Vec::new();
let mut optimized_excluded: Vec<PathBuf> = Vec::new();
@ -295,7 +287,6 @@ impl Directories {
// Not needed, but better is to have sorted everything
self.excluded_directories.sort_unstable();
self.included_directories.sort_unstable();
Common::print_time(start_time, SystemTime::now(), "optimize_directories");
// Get device IDs for included directories
#[cfg(target_family = "unix")]
@ -314,6 +305,11 @@ impl Directories {
true
}
#[must_use]
pub fn is_in_referenced_directory(&self, path: &Path) -> bool {
self.reference_directories.iter().any(|e| path.starts_with(e))
}
/// Checks whether a specified directory is excluded from searching
pub fn is_excluded(&self, path: impl AsRef<Path>) -> bool {
let path = path.as_ref();

5
czkawka_core/src/common_extensions.rs
View file

@ -1,6 +1,3 @@
use std::time::SystemTime;
use crate::common::Common;
use crate::common_messages::Messages;
#[derive(Clone, Default)]
@ -16,7 +13,6 @@ impl Extensions {
/// List of allowed extensions, only files with this extensions will be checking if are duplicates
/// After, extensions cannot contains any dot, commas etc.
pub fn set_allowed_extensions(&mut self, mut allowed_extensions: String, text_messages: &mut Messages) {
let start_time: SystemTime = SystemTime::now();
if allowed_extensions.trim().is_empty() {
return;
}
@ -57,7 +53,6 @@ impl Extensions {
.messages
.push("No valid extensions were provided, so allowing all extensions by default.".to_string());
}
Common::print_time(start_time, SystemTime::now(), "set_allowed_extensions");
}
#[must_use]

4
czkawka_core/src/common_items.rs
View file

@ -1,5 +1,4 @@
use std::path::Path;
use std::time::SystemTime;
use crate::common::Common;
use crate::common_messages::Messages;
@ -17,8 +16,6 @@ impl ExcludedItems {
/// Setting excluded items which needs to contains * wildcard
/// Are a lot of slower than absolute path, so it should be used to heavy
pub fn set_excluded_items(&mut self, excluded_items: Vec<String>, text_messages: &mut Messages) {
let start_time: SystemTime = SystemTime::now();
if excluded_items.is_empty() {
return;
}
@ -55,7 +52,6 @@ impl ExcludedItems {
checked_expressions.push(expression);
}
self.items = checked_expressions;
Common::print_time(start_time, SystemTime::now(), "set_excluded_items");
}
/// Checks whether a specified path is excluded from searching

6
czkawka_core/src/common_traits.rs
View file

@ -1,3 +1,5 @@
use std::path::Path;
pub trait DebugPrint {
fn debug_print(&self);
}
@ -9,3 +11,7 @@ pub trait SaveResults {
pub trait PrintResults {
fn print_results(&self);
}
pub trait ResultEntry {
fn get_path(&self) -> &Path;
}

167
czkawka_core/src/duplicate.rs
View file

@ -9,9 +9,8 @@ use std::io::{BufReader, BufWriter};
#[cfg(target_family = "unix")]
use std::os::unix::fs::MetadataExt;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::SystemTime;
use std::sync::atomic::Ordering;
use std::{fs, mem};
use crossbeam_channel::Receiver;
@ -21,7 +20,7 @@ use humansize::BINARY;
use rayon::prelude::*;
use xxhash_rust::xxh3::Xxh3;
use crate::common::{open_cache_folder, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, Common};
use crate::common::{open_cache_folder, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads};
use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData};
use crate::common_directory::Directories;
use crate::common_extensions::Extensions;
@ -182,9 +181,7 @@ impl DuplicateFinder {
return;
}
}
CheckingMethod::None => {
panic!();
}
_ => panic!(),
}
self.delete_files();
self.debug_print();
@ -364,11 +361,7 @@ impl DuplicateFinder {
.build()
.run();
match result {
DirTraversalResult::SuccessFiles {
start_time,
grouped_file_entries,
warnings,
} => {
DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => {
self.files_with_identical_names = grouped_file_entries;
self.text_messages.warnings.extend(warnings);
@ -384,21 +377,11 @@ impl DuplicateFinder {
// Reference - only use in size, because later hash will be counted differently
if self.use_reference_folders {
let mut btree_map = Default::default();
mem::swap(&mut self.files_with_identical_names, &mut btree_map);
let reference_directories = self.directories.reference_directories.clone();
let vec = btree_map
let vec = mem::take(&mut self.files_with_identical_names)
.into_iter()
.filter_map(|(_size, vec_file_entry)| {
let mut files_from_referenced_folders = Vec::new();
let mut normal_files = Vec::new();
for file_entry in vec_file_entry {
if reference_directories.iter().any(|e| file_entry.path.starts_with(e)) {
files_from_referenced_folders.push(file_entry);
} else {
normal_files.push(file_entry);
}
}
.filter_map(|(_name, vec_file_entry)| {
let (mut files_from_referenced_folders, normal_files): (Vec<_>, Vec<_>) =
vec_file_entry.into_iter().partition(|e| self.directories.is_in_referenced_directory(e.get_path()));
if files_from_referenced_folders.is_empty() || normal_files.is_empty() {
None
@ -413,7 +396,6 @@ impl DuplicateFinder {
}
self.calculate_name_stats();
Common::print_time(start_time, SystemTime::now(), "check_files_name");
true
}
DirTraversalResult::SuccessFolders { .. } => {
@ -459,11 +441,7 @@ impl DuplicateFinder {
.build()
.run();
match result {
DirTraversalResult::SuccessFiles {
start_time,
grouped_file_entries,
warnings,
} => {
DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => {
self.files_with_identical_size_names = grouped_file_entries;
self.text_messages.warnings.extend(warnings);
@ -479,21 +457,11 @@ impl DuplicateFinder {
// Reference - only use in size, because later hash will be counted differently
if self.use_reference_folders {
let mut btree_map = Default::default();
mem::swap(&mut self.files_with_identical_size_names, &mut btree_map);
let reference_directories = self.directories.reference_directories.clone();
let vec = btree_map
let vec = mem::take(&mut self.files_with_identical_size_names)
.into_iter()
.filter_map(|(_size, vec_file_entry)| {
let mut files_from_referenced_folders = Vec::new();
let mut normal_files = Vec::new();
for file_entry in vec_file_entry {
if reference_directories.iter().any(|e| file_entry.path.starts_with(e)) {
files_from_referenced_folders.push(file_entry);
} else {
normal_files.push(file_entry);
}
}
let (mut files_from_referenced_folders, normal_files): (Vec<_>, Vec<_>) =
vec_file_entry.into_iter().partition(|e| self.directories.is_in_referenced_directory(e.get_path()));
if files_from_referenced_folders.is_empty() || normal_files.is_empty() {
None
@ -509,7 +477,6 @@ impl DuplicateFinder {
}
self.calculate_size_name_stats();
Common::print_time(start_time, SystemTime::now(), "check_files_size_name");
true
}
DirTraversalResult::SuccessFolders { .. } => {
@ -559,17 +526,12 @@ impl DuplicateFinder {
.build()
.run();
match result {
DirTraversalResult::SuccessFiles {
start_time,
grouped_file_entries,
warnings,
} => {
DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => {
self.files_with_identical_size = grouped_file_entries;
self.text_messages.warnings.extend(warnings);
// Create new BTreeMap without single size entries(files have not duplicates)
let mut old_map: BTreeMap<u64, Vec<FileEntry>> = Default::default();
mem::swap(&mut old_map, &mut self.files_with_identical_size);
let old_map: BTreeMap<u64, Vec<FileEntry>> = mem::take(&mut self.files_with_identical_size);
for (size, vec) in old_map {
if vec.len() <= 1 {
@ -586,7 +548,6 @@ impl DuplicateFinder {
self.filter_reference_folders_by_size();
self.calculate_size_stats();
Common::print_time(start_time, SystemTime::now(), "check_files_size");
true
}
DirTraversalResult::SuccessFolders { .. } => {
@ -616,21 +577,11 @@ impl DuplicateFinder {
/// This is needed, because later reference folders looks for hashes, not size
fn filter_reference_folders_by_size(&mut self) {
if self.use_reference_folders && self.check_method == CheckingMethod::Size {
let mut btree_map = Default::default();
mem::swap(&mut self.files_with_identical_size, &mut btree_map);
let reference_directories = self.directories.reference_directories.clone();
let vec = btree_map
let vec = mem::take(&mut self.files_with_identical_size)
.into_iter()
.filter_map(|(_size, vec_file_entry)| {
let mut files_from_referenced_folders = Vec::new();
let mut normal_files = Vec::new();
for file_entry in vec_file_entry {
if reference_directories.iter().any(|e| file_entry.path.starts_with(e)) {
files_from_referenced_folders.push(file_entry);
} else {
normal_files.push(file_entry);
}
}
let (mut files_from_referenced_folders, normal_files): (Vec<_>, Vec<_>) =
vec_file_entry.into_iter().partition(|e| self.directories.is_in_referenced_directory(e.get_path()));
if files_from_referenced_folders.is_empty() || normal_files.is_empty() {
None
@ -723,21 +674,9 @@ impl DuplicateFinder {
progress_sender: Option<&UnboundedSender<ProgressData>>,
pre_checked_map: &mut BTreeMap<u64, Vec<FileEntry>>,
) -> Option<()> {
let start_time: SystemTime = SystemTime::now();
let check_type = self.hash_type;
let check_was_stopped = AtomicBool::new(false); // Used for breaking from GUI and ending check thread
let progress_thread_run = Arc::new(AtomicBool::new(true));
let atomic_counter = Arc::new(AtomicUsize::new(0));
let progress_thread_handle = prepare_thread_handler_common(
progress_sender,
&progress_thread_run,
&atomic_counter,
1,
2,
self.files_with_identical_size.values().map(Vec::len).sum(),
self.check_method,
);
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
prepare_thread_handler_common(progress_sender, 1, 2, self.files_with_identical_size.values().map(Vec::len).sum(), self.check_method);
let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.prehash_load_cache_at_start();
@ -750,11 +689,11 @@ impl DuplicateFinder {
let mut buffer = [0u8; 1024 * 2];
atomic_counter.fetch_add(vec_file_entry.len(), Ordering::Relaxed);
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
check_was_stopped.store(true, Ordering::Relaxed);
return None;
}
for file_entry in vec_file_entry {
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
check_was_stopped.store(true, Ordering::Relaxed);
return None;
}
match hash_calculation(&mut buffer, file_entry, &check_type, 0) {
Ok(hash_string) => {
hashmap_with_hash.entry(hash_string.clone()).or_insert_with(Vec::new).push(file_entry.clone());
@ -791,8 +730,6 @@ impl DuplicateFinder {
self.prehash_save_cache_at_exit(loaded_hash_map, &pre_hash_results);
Common::print_time(start_time, SystemTime::now(), "check_files_hash - prehash");
Some(())
}
@ -891,23 +828,10 @@ impl DuplicateFinder {
progress_sender: Option<&UnboundedSender<ProgressData>>,
pre_checked_map: BTreeMap<u64, Vec<FileEntry>>,
) -> Option<()> {
let check_was_stopped = AtomicBool::new(false); // Used for breaking from GUI and ending check thread
let check_type = self.hash_type;
let start_time: SystemTime = SystemTime::now();
let progress_thread_run = Arc::new(AtomicBool::new(true));
let atomic_counter = Arc::new(AtomicUsize::new(0));
let progress_thread_handle = prepare_thread_handler_common(
progress_sender,
&progress_thread_run,
&atomic_counter,
2,
2,
pre_checked_map.values().map(Vec::len).sum(),
self.check_method,
);
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
prepare_thread_handler_common(progress_sender, 2, 2, pre_checked_map.values().map(Vec::len).sum(), self.check_method);
///////////////////////////////////////////////////////////////////////////// HASHING START
{
@ -958,30 +882,20 @@ impl DuplicateFinder {
}
}
}
Common::print_time(start_time, SystemTime::now(), "delete_files");
Some(())
}
fn hash_reference_folders(&mut self) {
// Reference - only use in size, because later hash will be counted differently
if self.use_reference_folders {
let mut btree_map = Default::default();
mem::swap(&mut self.files_with_identical_hashes, &mut btree_map);
let reference_directories = self.directories.reference_directories.clone();
let vec = btree_map
let vec = mem::take(&mut self.files_with_identical_hashes)
.into_iter()
.filter_map(|(_size, vec_vec_file_entry)| {
let mut all_results_with_same_size = Vec::new();
for vec_file_entry in vec_vec_file_entry {
let mut files_from_referenced_folders = Vec::new();
let mut normal_files = Vec::new();
for file_entry in vec_file_entry {
if reference_directories.iter().any(|e| file_entry.path.starts_with(e)) {
files_from_referenced_folders.push(file_entry);
} else {
normal_files.push(file_entry);
}
}
let (mut files_from_referenced_folders, normal_files): (Vec<_>, Vec<_>) =
vec_file_entry.into_iter().partition(|e| self.directories.is_in_referenced_directory(e.get_path()));
if files_from_referenced_folders.is_empty() || normal_files.is_empty() {
continue;
@ -1045,7 +959,6 @@ impl DuplicateFinder {
/// Function to delete files, from filed before `BTreeMap`
/// Using another function to delete files to avoid duplicates data
fn delete_files(&mut self) {
let start_time: SystemTime = SystemTime::now();
if self.delete_method == DeleteMethod::None {
return;
}
@ -1073,13 +986,8 @@ impl DuplicateFinder {
let _tuple: (u64, usize, usize) = delete_files(vector, &self.delete_method, &mut self.text_messages, self.dryrun);
}
}
CheckingMethod::None => {
//Just do nothing
panic!("Checking method should never be none.");
}
_ => panic!(),
}
Common::print_time(start_time, SystemTime::now(), "delete_files");
}
}
@ -1146,7 +1054,6 @@ impl DebugPrint for DuplicateFinder {
impl SaveResults for DuplicateFinder {
fn save_results_to_file(&mut self, file_name: &str) -> bool {
let start_time: SystemTime = SystemTime::now();
let file_name: String = match file_name {
"" => "results.txt".to_string(),
k => k.to_string(),
@ -1270,11 +1177,9 @@ impl SaveResults for DuplicateFinder {
write!(writer, "Not found any duplicates.").unwrap();
}
}
CheckingMethod::None => {
panic!();
}
_ => panic!(),
}
Common::print_time(start_time, SystemTime::now(), "save_results_to_file");
true
}
}
@ -1283,7 +1188,6 @@ impl PrintResults for DuplicateFinder {
/// Print information's about duplicated entries
/// Only needed for CLI
fn print_results(&self) {
let start_time: SystemTime = SystemTime::now();
let mut number_of_files: u64 = 0;
let mut number_of_groups: u64 = 0;
@ -1359,11 +1263,8 @@ impl PrintResults for DuplicateFinder {
println!();
}
}
CheckingMethod::None => {
panic!("Checking Method shouldn't be ever set to None");
}
_ => panic!(),
}
Common::print_time(start_time, SystemTime::now(), "print_entries");
}
}

20
czkawka_core/src/empty_files.rs
View file

@ -3,12 +3,10 @@ use std::fs::File;
use std::io::prelude::*;
use std::io::BufWriter;
use std::path::PathBuf;
use std::time::SystemTime;
use crossbeam_channel::Receiver;
use futures::channel::mpsc::UnboundedSender;
use crate::common::Common;
use crate::common_dir_traversal::{DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData};
use crate::common_directory::Directories;
use crate::common_extensions::Extensions;
@ -141,17 +139,13 @@ impl EmptyFiles {
.build()
.run();
match result {
DirTraversalResult::SuccessFiles {
start_time,
grouped_file_entries,
warnings,
} => {
DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => {
if let Some(empty_files) = grouped_file_entries.get(&()) {
self.empty_files = empty_files.clone();
}
self.information.number_of_empty_files = self.empty_files.len();
self.text_messages.warnings.extend(warnings);
Common::print_time(start_time, SystemTime::now(), "check_files_name");
true
}
DirTraversalResult::SuccessFolders { .. } => {
@ -163,8 +157,6 @@ impl EmptyFiles {
/// Function to delete files, from filed Vector
fn delete_files(&mut self) {
let start_time: SystemTime = SystemTime::now();
match self.delete_method {
DeleteMethod::Delete => {
for file_entry in &self.empty_files {
@ -177,8 +169,6 @@ impl EmptyFiles {
//Just do nothing
}
}
Common::print_time(start_time, SystemTime::now(), "delete_files");
}
}
@ -220,7 +210,6 @@ impl DebugPrint for EmptyFiles {
impl SaveResults for EmptyFiles {
fn save_results_to_file(&mut self, file_name: &str) -> bool {
let start_time: SystemTime = SystemTime::now();
let file_name: String = match file_name {
"" => "results.txt".to_string(),
k => k.to_string(),
@ -252,7 +241,7 @@ impl SaveResults for EmptyFiles {
} else {
write!(writer, "Not found any empty files.").unwrap();
}
Common::print_time(start_time, SystemTime::now(), "save_results_to_file");
true
}
}
@ -261,12 +250,9 @@ impl PrintResults for EmptyFiles {
/// Print information's about duplicated entries
/// Only needed for CLI
fn print_results(&self) {
let start_time: SystemTime = SystemTime::now();
println!("Found {} empty files.\n", self.information.number_of_empty_files);
for file_entry in &self.empty_files {
println!("{}", file_entry.path.display());
}
Common::print_time(start_time, SystemTime::now(), "print_entries");
}
}

15
czkawka_core/src/empty_folder.rs
View file

@ -3,12 +3,10 @@ use std::fs;
use std::fs::File;
use std::io::{BufWriter, Write};
use std::path::PathBuf;
use std::time::SystemTime;
use crossbeam_channel::Receiver;
use futures::channel::mpsc::UnboundedSender;
use crate::common::Common;
use crate::common_dir_traversal::{Collect, DirTraversalBuilder, DirTraversalResult, FolderEmptiness, FolderEntry, ProgressData};
use crate::common_directory::Directories;
use crate::common_items::ExcludedItems;
@ -145,11 +143,7 @@ impl EmptyFolder {
DirTraversalResult::SuccessFiles { .. } => {
unreachable!()
}
DirTraversalResult::SuccessFolders {
start_time,
folder_entries,
warnings,
} => {
DirTraversalResult::SuccessFolders { folder_entries, warnings } => {
// We need to set empty folder list
#[allow(unused_mut)] // Used is later by Windows build
for (mut name, folder_entry) in folder_entries {
@ -160,7 +154,6 @@ impl EmptyFolder {
self.text_messages.warnings.extend(warnings);
Common::print_time(start_time, SystemTime::now(), "check_for_empty_folder");
true
}
DirTraversalResult::Stopped => false,
@ -169,7 +162,6 @@ impl EmptyFolder {
/// Deletes earlier found empty folders
fn delete_empty_folders(&mut self) {
let start_time: SystemTime = SystemTime::now();
// Folders may be deleted or require too big privileges
for name in self.empty_folder_list.keys() {
match fs::remove_dir_all(name) {
@ -177,8 +169,6 @@ impl EmptyFolder {
Err(e) => self.text_messages.warnings.push(format!("Failed to remove folder {}, reason {}", name.display(), e)),
};
}
Common::print_time(start_time, SystemTime::now(), "delete_files");
}
/// Set included dir which needs to be relative, exists etc.
@ -211,7 +201,6 @@ impl DebugPrint for EmptyFolder {
impl SaveResults for EmptyFolder {
fn save_results_to_file(&mut self, file_name: &str) -> bool {
let start_time: SystemTime = SystemTime::now();
let file_name: String = match file_name {
"" => "results.txt".to_string(),
k => k.to_string(),
@ -248,7 +237,7 @@ impl SaveResults for EmptyFolder {
} else {
write!(writer, "Not found any empty folders.").unwrap();
}
Common::print_time(start_time, SystemTime::now(), "save_results_to_file");
true
}
}

18
czkawka_core/src/invalid_symlinks.rs
View file

@ -3,12 +3,10 @@ use std::fs::File;
use std::io::prelude::*;
use std::io::BufWriter;
use std::path::PathBuf;
use std::time::SystemTime;
use crossbeam_channel::Receiver;
use futures::channel::mpsc::UnboundedSender;
use crate::common::Common;
use crate::common_dir_traversal::{Collect, DirTraversalBuilder, DirTraversalResult, ErrorType, FileEntry, ProgressData};
use crate::common_directory::Directories;
use crate::common_extensions::Extensions;
@ -139,17 +137,12 @@ impl InvalidSymlinks {
.build()
.run();
match result {
DirTraversalResult::SuccessFiles {
start_time,
grouped_file_entries,
warnings,
} => {
DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => {
if let Some(((), invalid_symlinks)) = grouped_file_entries.into_iter().next() {
self.invalid_symlinks = invalid_symlinks;
}
self.information.number_of_invalid_symlinks = self.invalid_symlinks.len();
self.text_messages.warnings.extend(warnings);
Common::print_time(start_time, SystemTime::now(), "check_files_name");
true
}
DirTraversalResult::SuccessFolders { .. } => unreachable!(),
@ -159,8 +152,6 @@ impl InvalidSymlinks {
/// Function to delete files, from filed Vector
fn delete_files(&mut self) {
let start_time: SystemTime = SystemTime::now();
match self.delete_method {
DeleteMethod::Delete => {
for file_entry in &self.invalid_symlinks {
@ -173,8 +164,6 @@ impl InvalidSymlinks {
//Just do nothing
}
}
Common::print_time(start_time, SystemTime::now(), "delete_files");
}
}
@ -216,7 +205,6 @@ impl DebugPrint for InvalidSymlinks {
impl SaveResults for InvalidSymlinks {
fn save_results_to_file(&mut self, file_name: &str) -> bool {
let start_time: SystemTime = SystemTime::now();
let file_name: String = match file_name {
"" => "results.txt".to_string(),
k => k.to_string(),
@ -258,7 +246,6 @@ impl SaveResults for InvalidSymlinks {
} else {
write!(writer, "Not found any invalid symlinks.").unwrap();
}
Common::print_time(start_time, SystemTime::now(), "save_results_to_file");
true
}
}
@ -267,7 +254,6 @@ impl PrintResults for InvalidSymlinks {
/// Print information's about duplicated entries
/// Only needed for CLI
fn print_results(&self) {
let start_time: SystemTime = SystemTime::now();
println!("Found {} invalid symlinks.\n", self.information.number_of_invalid_symlinks);
for file_entry in &self.invalid_symlinks {
println!(
@ -280,7 +266,5 @@ impl PrintResults for InvalidSymlinks {
}
);
}
Common::print_time(start_time, SystemTime::now(), "print_entries");
}
}

683
czkawka_core/src/same_music.rs
View file

@ -1,22 +1,30 @@
use std::collections::{BTreeMap, HashMap};
use std::cmp::max;
use std::collections::{BTreeMap, HashMap, HashSet};
use std::fs::File;
use std::io::prelude::*;
use std::io::{BufReader, BufWriter};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::SystemTime;
use std::{mem, panic};
use anyhow::Context;
use crossbeam_channel::Receiver;
use futures::channel::mpsc::UnboundedSender;
use lofty::TaggedFileExt;
use lofty::{read_from, AudioFile, ItemKey};
use rayon::prelude::*;
use rusty_chromaprint::{match_fingerprints, Configuration, Fingerprinter};
use serde::{Deserialize, Serialize};
use symphonia::core::audio::SampleBuffer;
use symphonia::core::codecs::{DecoderOptions, CODEC_TYPE_NULL};
use symphonia::core::formats::FormatOptions;
use symphonia::core::io::MediaSourceStream;
use symphonia::core::meta::MetadataOptions;
use symphonia::core::probe::Hint;
use crate::common::{create_crash_message, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, AUDIO_FILES_EXTENSIONS};
use crate::common::{open_cache_folder, Common};
use crate::common::{filter_reference_folders_generic, open_cache_folder};
use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData};
use crate::common_directory::Directories;
use crate::common_extensions::Extensions;
@ -30,12 +38,6 @@ pub enum DeleteMethod {
Delete,
}
#[derive(Eq, PartialEq, Clone, Debug, Copy)]
pub enum AudioCheckMethod {
Tags,
Content,
}
bitflags! {
#[derive(PartialEq, Copy, Clone, Debug)]
pub struct MusicSimilarity : u32 {
@ -56,6 +58,7 @@ pub struct MusicEntry {
pub path: PathBuf,
pub modified_date: u64,
pub fingerprint: Vec<u32>,
pub track_title: String,
pub track_artist: String,
@ -65,6 +68,12 @@ pub struct MusicEntry {
pub bitrate: u32,
}
impl ResultEntry for MusicEntry {
fn get_path(&self) -> &Path {
&self.path
}
}
impl FileEntry {
fn to_music_entry(&self) -> MusicEntry {
MusicEntry {
@ -72,6 +81,7 @@ impl FileEntry {
path: self.path.clone(),
modified_date: self.modified_date,
fingerprint: vec![],
track_title: String::new(),
track_artist: String::new(),
year: String::new(),
@ -118,7 +128,10 @@ pub struct SameMusic {
delete_outdated_cache: bool, // TODO add this to GUI
use_reference_folders: bool,
save_also_as_json: bool,
check_type: AudioCheckMethod,
check_type: CheckingMethod,
hash_preset_config: Configuration,
minimum_segment_duration: f32,
maximum_difference: f64,
}
impl SameMusic {
@ -145,7 +158,10 @@ impl SameMusic {
use_reference_folders: false,
duplicated_music_entries_referenced: vec![],
save_also_as_json: false,
check_type: AudioCheckMethod::Tags,
check_type: CheckingMethod::AudioContent,
hash_preset_config: Configuration::preset_test1(), // TODO allow to change this
minimum_segment_duration: 10.0,
maximum_difference: 2.0,
}
}
@ -157,7 +173,7 @@ impl SameMusic {
return;
}
match self.check_type {
AudioCheckMethod::Tags => {
CheckingMethod::AudioTags => {
if !self.read_tags(stop_receiver, progress_sender) {
self.stopped_search = true;
return;
@ -167,9 +183,21 @@ impl SameMusic {
return;
}
}
AudioCheckMethod::Content => {
unimplemented!();
CheckingMethod::AudioContent => {
if !self.calculate_fingerprint(stop_receiver, progress_sender) {
self.stopped_search = true;
return;
}
if !self.check_for_duplicate_fingerprints(stop_receiver, progress_sender) {
self.stopped_search = true;
return;
}
if !self.read_tags_to_files_similar_by_content(stop_receiver, progress_sender) {
self.stopped_search = true;
return;
}
}
_ => panic!(),
}
self.delete_files();
self.debug_print();
@ -231,10 +259,27 @@ impl SameMusic {
self.directories.set_included_directory(included_directory, &mut self.text_messages);
}
pub fn set_maximum_difference(&mut self, maximum_difference: f64) {
self.maximum_difference = maximum_difference;
}
pub fn set_minimum_segment_duration(&mut self, minimum_segment_duration: f32) {
self.minimum_segment_duration = minimum_segment_duration;
}
pub fn set_reference_directory(&mut self, reference_directory: Vec<PathBuf>) {
self.directories.set_reference_directory(reference_directory);
}
pub fn set_check_type(&mut self, check_type: CheckingMethod) {
assert!([CheckingMethod::AudioTags, CheckingMethod::AudioContent].contains(&check_type));
self.check_type = check_type;
}
#[must_use]
pub fn get_check_type(&self) -> CheckingMethod {
self.check_type
}
pub fn set_excluded_directory(&mut self, excluded_directory: Vec<PathBuf>) {
self.directories.set_excluded_directory(excluded_directory, &mut self.text_messages);
}
@ -302,18 +347,14 @@ impl SameMusic {
.build()
.run();
match result {
DirTraversalResult::SuccessFiles {
start_time,
grouped_file_entries,
warnings,
} => {
DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => {
if let Some(music_to_check) = grouped_file_entries.get(&()) {
for fe in music_to_check {
self.music_to_check.insert(fe.path.to_string_lossy().to_string(), fe.to_music_entry());
}
}
self.text_messages.warnings.extend(warnings);
Common::print_time(start_time, SystemTime::now(), "check_files");
true
}
DirTraversalResult::SuccessFolders { .. } => {
@ -323,29 +364,31 @@ impl SameMusic {
}
}
fn read_tags_load_cache(&mut self) -> (HashMap<String, MusicEntry>, HashMap<String, MusicEntry>, HashMap<String, MusicEntry>) {
fn load_cache(&mut self, checking_tags: bool) -> (HashMap<String, MusicEntry>, HashMap<String, MusicEntry>, HashMap<String, MusicEntry>) {
let loaded_hash_map;
let mut records_already_cached: HashMap<String, MusicEntry> = Default::default();
let mut non_cached_files_to_check: HashMap<String, MusicEntry> = Default::default();
if self.use_cache {
loaded_hash_map = match load_cache_from_file(&mut self.text_messages, self.delete_outdated_cache) {
loaded_hash_map = match load_cache_from_file(&mut self.text_messages, self.delete_outdated_cache, checking_tags) {
Some(t) => t,
None => Default::default(),
};
for (name, file_entry) in &self.music_to_check {
#[allow(clippy::if_same_then_else)]
if !loaded_hash_map.contains_key(name) {
// If loaded data doesn't contains current image info
non_cached_files_to_check.insert(name.clone(), file_entry.clone());
} else if file_entry.size != loaded_hash_map.get(name).unwrap().size || file_entry.modified_date != loaded_hash_map.get(name).unwrap().modified_date {
// When size or modification date of image changed, then it is clear that is different image
non_cached_files_to_check.insert(name.clone(), file_entry.clone());
} else {
// Checking may be omitted when already there is entry with same size and modification date
records_already_cached.insert(name.clone(), loaded_hash_map.get(name).unwrap().clone());
let loaded_item = loaded_hash_map.get(name).unwrap();
if file_entry.size != loaded_item.size || file_entry.modified_date != loaded_item.modified_date {
// When size or modification date of image changed, then it is clear that is different image
non_cached_files_to_check.insert(name.clone(), file_entry.clone());
} else {
// Checking may be omitted when already there is entry with same size and modification date
records_already_cached.insert(name.clone(), loaded_item.clone());
}
}
}
} else {
@ -355,7 +398,7 @@ impl SameMusic {
(loaded_hash_map, records_already_cached, non_cached_files_to_check)
}
fn read_tags_save_cache(&mut self, vec_file_entry: Vec<MusicEntry>, loaded_hash_map: HashMap<String, MusicEntry>) {
fn save_cache(&mut self, vec_file_entry: Vec<MusicEntry>, loaded_hash_map: HashMap<String, MusicEntry>, checking_tags: bool) {
if !self.use_cache {
return;
}
@ -365,38 +408,32 @@ impl SameMusic {
for file_entry in vec_file_entry {
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry);
}
save_cache_to_file(&all_results, &mut self.text_messages, self.save_also_as_json);
save_cache_to_file(&all_results, &mut self.text_messages, self.save_also_as_json, checking_tags);
}
fn read_tags(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
let start_time: SystemTime = SystemTime::now();
fn calculate_fingerprint(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.load_cache(false);
let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.read_tags_load_cache();
let check_was_stopped = AtomicBool::new(false); // Used for breaking from GUI and ending check thread
let progress_thread_run = Arc::new(AtomicBool::new(true));
let atomic_counter = Arc::new(AtomicUsize::new(0));
let progress_thread_handle = prepare_thread_handler_common(
progress_sender,
&progress_thread_run,
&atomic_counter,
1,
2,
non_cached_files_to_check.len(),
CheckingMethod::None,
);
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
prepare_thread_handler_common(progress_sender, 1, 3, non_cached_files_to_check.len(), self.check_type);
let configuration = &self.hash_preset_config;
// Clean for duplicate files
let mut vec_file_entry = non_cached_files_to_check
.into_par_iter()
.map(|(path, music_entry)| {
.map(|(path, mut music_entry)| {
atomic_counter.fetch_add(1, Ordering::Relaxed);
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
check_was_stopped.store(true, Ordering::Relaxed);
return None;
}
Some(self.read_single_file_tag(&path, music_entry))
let Ok(fingerprint) = calc_fingerprint_helper(path, configuration) else {
return Some(None);
};
music_entry.fingerprint = fingerprint;
Some(Some(music_entry))
})
.while_some()
.filter(Option::is_some)
@ -406,131 +443,66 @@ impl SameMusic {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
// Just connect loaded results with already calculated
for (_name, file_entry) in records_already_cached {
vec_file_entry.push(file_entry.clone());
}
vec_file_entry.extend(records_already_cached.into_values());
self.music_entries = vec_file_entry.clone();
self.read_tags_save_cache(vec_file_entry, loaded_hash_map);
self.save_cache(vec_file_entry, loaded_hash_map, false);
// Break if stop was clicked after saving to cache
if check_was_stopped.load(Ordering::Relaxed) {
return false;
}
Common::print_time(start_time, SystemTime::now(), "read_tags");
true
}
fn read_tags(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.load_cache(true);
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
prepare_thread_handler_common(progress_sender, 1, 2, non_cached_files_to_check.len(), self.check_type);
// Clean for duplicate files
let mut vec_file_entry = non_cached_files_to_check
.into_par_iter()
.map(|(path, mut music_entry)| {
atomic_counter.fetch_add(1, Ordering::Relaxed);
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
check_was_stopped.store(true, Ordering::Relaxed);
return None;
}
if read_single_file_tag(&path, &mut music_entry) {
Some(Some(music_entry))
} else {
Some(None)
}
})
.while_some()
.filter(Option::is_some)
.map(Option::unwrap)
.collect::<Vec<_>>();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
// Just connect loaded results with already calculated
vec_file_entry.extend(records_already_cached.into_values());
self.music_entries = vec_file_entry.clone();
self.save_cache(vec_file_entry, loaded_hash_map, true);
// Break if stop was clicked after saving to cache
if check_was_stopped.load(Ordering::Relaxed) {
return false;
}
true
}
fn read_single_file_tag(&self, path: &str, mut music_entry: MusicEntry) -> Option<MusicEntry> {
let Ok(mut file) = File::open(path) else { return None; };
let result = panic::catch_unwind(move || {
match read_from(&mut file) {
Ok(t) => Some(t),
Err(_inspected) => {
// println!("Failed to open {}", path);
None
}
}
});
let tagged_file = if let Ok(t) = result {
match t {
Some(r) => r,
None => {
return Some(music_entry);
}
}
} else {
let message = create_crash_message("Lofty", path, "https://github.com/image-rs/image/issues");
println!("{message}");
return None;
};
let properties = tagged_file.properties();
let mut track_title = String::new();
let mut track_artist = String::new();
let mut year = String::new();
let mut genre = String::new();
let bitrate = properties.audio_bitrate().unwrap_or(0);
let mut length = properties.duration().as_millis().to_string();
if let Some(tag) = tagged_file.primary_tag() {
track_title = tag.get_string(&ItemKey::TrackTitle).unwrap_or("").to_string();
track_artist = tag.get_string(&ItemKey::TrackArtist).unwrap_or("").to_string();
year = tag.get_string(&ItemKey::Year).unwrap_or("").to_string();
genre = tag.get_string(&ItemKey::Genre).unwrap_or("").to_string();
}
for tag in tagged_file.tags() {
if track_title.is_empty() {
if let Some(tag_value) = tag.get_string(&ItemKey::TrackTitle) {
track_title = tag_value.to_string();
}
}
if track_artist.is_empty() {
if let Some(tag_value) = tag.get_string(&ItemKey::TrackArtist) {
track_artist = tag_value.to_string();
}
}
if year.is_empty() {
if let Some(tag_value) = tag.get_string(&ItemKey::Year) {
year = tag_value.to_string();
}
}
if genre.is_empty() {
if let Some(tag_value) = tag.get_string(&ItemKey::Genre) {
genre = tag_value.to_string();
}
}
// println!("{:?}", tag.items());
}
if let Ok(old_length_number) = length.parse::<u32>() {
let length_number = old_length_number / 60;
let minutes = length_number / 1000;
let seconds = (length_number % 1000) * 6 / 100;
if minutes != 0 || seconds != 0 {
length = format!("{minutes}:{seconds:02}");
} else if old_length_number > 0 {
// That means, that audio have length smaller that second, but length is properly read
length = "0:01".to_string();
} else {
length = String::new();
}
} else {
length = String::new();
}
music_entry.track_title = track_title;
music_entry.track_artist = track_artist;
music_entry.year = year;
music_entry.length = length;
music_entry.genre = genre;
music_entry.bitrate = bitrate;
Some(music_entry)
}
fn check_for_duplicate_tags(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
assert_ne!(MusicSimilarity::NONE, self.music_similarity, "This can't be none");
let start_time: SystemTime = SystemTime::now();
let progress_thread_run = Arc::new(AtomicBool::new(true));
let atomic_counter = Arc::new(AtomicUsize::new(0));
let progress_thread_handle = prepare_thread_handler_common(
progress_sender,
&progress_thread_run,
&atomic_counter,
2,
2,
self.music_to_check.len(),
CheckingMethod::None,
);
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) =
prepare_thread_handler_common(progress_sender, 2, 2, self.music_to_check.len(), self.check_type);
let mut old_duplicates: Vec<Vec<MusicEntry>> = vec![self.music_entries.clone()];
let mut new_duplicates: Vec<Vec<MusicEntry>> = Vec::new();
@ -580,9 +552,8 @@ impl SameMusic {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
}
let old_duplicates_len = old_duplicates.len();
for vec_file_entry in old_duplicates {
atomic_counter.fetch_add(1, Ordering::Relaxed);
let mut hash_map: BTreeMap<String, Vec<MusicEntry>> = Default::default();
for file_entry in vec_file_entry {
if file_entry.bitrate != 0 {
@ -598,6 +569,7 @@ impl SameMusic {
}
}
}
atomic_counter.fetch_add(old_duplicates_len, Ordering::Relaxed);
old_duplicates = new_duplicates;
}
@ -605,7 +577,9 @@ impl SameMusic {
self.duplicated_music_entries = old_duplicates;
self.filter_reference_folders();
if self.use_reference_folders {
self.duplicated_music_entries_referenced = filter_reference_folders_generic(mem::take(&mut self.duplicated_music_entries), &self.directories);
}
if self.use_reference_folders {
for (_fe, vector) in &self.duplicated_music_entries_referenced {
@ -619,7 +593,165 @@ impl SameMusic {
}
}
Common::print_time(start_time, SystemTime::now(), "check_for_duplicate_tags");
// Clear unused data
self.music_entries.clear();
true
}
fn read_tags_to_files_similar_by_content(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
let groups_to_check = max(self.duplicated_music_entries.len(), self.duplicated_music_entries_referenced.len());
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
prepare_thread_handler_common(progress_sender, 3, 3, groups_to_check, self.check_type);
// TODO is ther a way to just run iterator and not collect any info?
if !self.duplicated_music_entries.is_empty() {
let _: Vec<_> = self
.duplicated_music_entries
.par_iter_mut()
.map(|vec_me| {
atomic_counter.fetch_add(1, Ordering::Relaxed);
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
check_was_stopped.store(true, Ordering::Relaxed);
return None;
}
for me in vec_me {
let me_path = me.path.to_string_lossy().to_string();
read_single_file_tag(&me_path, me);
}
Some(())
})
.while_some()
.collect();
} else {
let _: Vec<_> = self
.duplicated_music_entries_referenced
.par_iter_mut()
.map(|(me_o, vec_me)| {
atomic_counter.fetch_add(1, Ordering::Relaxed);
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
check_was_stopped.store(true, Ordering::Relaxed);
return None;
}
let me_o_path = me_o.path.to_string_lossy().to_string();
read_single_file_tag(&me_o_path, me_o);
for me in vec_me {
let me_path = me.path.to_string_lossy().to_string();
read_single_file_tag(&me_path, me);
}
Some(())
})
.while_some()
.collect();
}
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
!check_was_stopped.load(Ordering::Relaxed)
}
fn split_fingerprints_to_check(&mut self) -> (Vec<MusicEntry>, Vec<MusicEntry>) {
let base_files: Vec<MusicEntry>;
let files_to_compare: Vec<MusicEntry>;
if self.use_reference_folders {
(base_files, files_to_compare) = mem::take(&mut self.music_entries)
.into_iter()
.partition(|f| self.directories.is_in_referenced_directory(f.get_path()));
} else {
base_files = self.music_entries.clone();
files_to_compare = mem::take(&mut self.music_entries);
}
(base_files, files_to_compare)
}
fn compare_fingerprints(
&mut self,
stop_receiver: Option<&Receiver<()>>,
atomic_counter: &Arc<AtomicUsize>,
base_files: Vec<MusicEntry>,
files_to_compare: &[MusicEntry],
) -> Option<Vec<Vec<MusicEntry>>> {
let mut used_paths: HashSet<String> = Default::default();
let configuration = &self.hash_preset_config;
let minimum_segment_duration = self.minimum_segment_duration;
let maximum_difference = self.maximum_difference;
let mut duplicated_music_entries = Vec::new();
for f_entry in base_files {
atomic_counter.fetch_add(1, Ordering::Relaxed);
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
return None;
}
let f_string = f_entry.path.to_string_lossy().to_string();
if used_paths.contains(&f_string) {
continue;
}
let mut collected_similar_items = files_to_compare
.par_iter()
.filter_map(|e_entry| {
let e_string = e_entry.path.to_string_lossy().to_string();
if used_paths.contains(&e_string) || e_string == f_string {
return None;
}
let mut segments = match_fingerprints(&f_entry.fingerprint, &e_entry.fingerprint, configuration).unwrap();
segments.retain(|s| s.duration(configuration) > minimum_segment_duration && s.score < maximum_difference);
if segments.is_empty() {
None
} else {
Some((e_string, e_entry))
}
})
.collect::<Vec<_>>();
collected_similar_items.retain(|(path, _entry)| !used_paths.contains(path));
if !collected_similar_items.is_empty() {
let mut music_entries = Vec::new();
for (path, entry) in collected_similar_items {
used_paths.insert(path);
music_entries.push(entry.clone());
}
used_paths.insert(f_string);
music_entries.push(f_entry.clone());
duplicated_music_entries.push(music_entries);
}
}
Some(duplicated_music_entries)
}
fn check_for_duplicate_fingerprints(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
let (base_files, files_to_compare) = self.split_fingerprints_to_check();
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) =
prepare_thread_handler_common(progress_sender, 2, 3, base_files.len(), self.check_type);
let Some(duplicated_music_entries) = self.compare_fingerprints(stop_receiver, &atomic_counter, base_files, &files_to_compare) else {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
};
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
self.duplicated_music_entries = duplicated_music_entries;
if self.use_reference_folders {
self.duplicated_music_entries_referenced = filter_reference_folders_generic(mem::take(&mut self.duplicated_music_entries), &self.directories);
}
if self.use_reference_folders {
for (_fe, vector) in &self.duplicated_music_entries_referenced {
self.information.number_of_duplicates += vector.len();
self.information.number_of_groups += 1;
}
} else {
for vector in &self.duplicated_music_entries {
self.information.number_of_duplicates += vector.len() - 1;
self.information.number_of_groups += 1;
}
}
// Clear unused data
self.music_entries.clear();
@ -635,8 +767,8 @@ impl SameMusic {
approximate_comparison: bool,
) -> Vec<Vec<MusicEntry>> {
let mut new_duplicates: Vec<_> = Default::default();
let old_duplicates_len = old_duplicates.len();
for vec_file_entry in old_duplicates {
atomic_counter.fetch_add(1, Ordering::Relaxed);
let mut hash_map: BTreeMap<String, Vec<MusicEntry>> = Default::default();
for file_entry in vec_file_entry {
let mut thing = get_item(&file_entry).trim().to_lowercase();
@ -653,40 +785,11 @@ impl SameMusic {
}
}
}
atomic_counter.fetch_add(old_duplicates_len, Ordering::Relaxed);
new_duplicates
}
fn filter_reference_folders(&mut self) {
if !self.use_reference_folders {
return;
}
let mut similar_vector = Default::default();
mem::swap(&mut self.duplicated_music_entries, &mut similar_vector);
let reference_directories = self.directories.reference_directories.clone();
self.duplicated_music_entries_referenced = similar_vector
.into_iter()
.filter_map(|vec_file_entry| {
let mut files_from_referenced_folders = Vec::new();
let mut normal_files = Vec::new();
for file_entry in vec_file_entry {
if reference_directories.iter().any(|e| file_entry.path.starts_with(e)) {
files_from_referenced_folders.push(file_entry);
} else {
normal_files.push(file_entry);
}
}
if files_from_referenced_folders.is_empty() || normal_files.is_empty() {
None
} else {
Some((files_from_referenced_folders.pop().unwrap(), normal_files))
}
})
.collect::<Vec<(MusicEntry, Vec<MusicEntry>)>>();
}
pub fn set_minimal_file_size(&mut self, minimal_file_size: u64) {
self.minimal_file_size = match minimal_file_size {
0 => 1,
@ -696,7 +799,7 @@ impl SameMusic {
/// Function to delete files, from filed Vector
fn delete_files(&mut self) {
let start_time: SystemTime = SystemTime::now();
// TODO
// match self.delete_method {
// DeleteMethod::Delete => {
@ -710,13 +813,13 @@ impl SameMusic {
// //Just do nothing
// }
// }
Common::print_time(start_time, SystemTime::now(), "delete_files");
}
}
fn save_cache_to_file(hashmap: &HashMap<String, MusicEntry>, text_messages: &mut Messages, save_also_as_json: bool) {
if let Some(((file_handler, cache_file), (file_handler_json, cache_file_json))) = open_cache_folder(&get_cache_file(), true, save_also_as_json, &mut text_messages.warnings) {
fn save_cache_to_file(hashmap: &HashMap<String, MusicEntry>, text_messages: &mut Messages, save_also_as_json: bool, checking_tags: bool) {
if let Some(((file_handler, cache_file), (file_handler_json, cache_file_json))) =
open_cache_folder(get_cache_file(checking_tags), true, save_also_as_json, &mut text_messages.warnings)
{
{
let writer = BufWriter::new(file_handler.unwrap()); // Unwrap because cannot fail here
if let Err(e) = bincode::serialize_into(writer, hashmap) {
@ -742,8 +845,8 @@ fn save_cache_to_file(hashmap: &HashMap<String, MusicEntry>, text_messages: &mut
}
}
fn load_cache_from_file(text_messages: &mut Messages, delete_outdated_cache: bool) -> Option<HashMap<String, MusicEntry>> {
if let Some(((file_handler, cache_file), (file_handler_json, cache_file_json))) = open_cache_folder(&get_cache_file(), false, true, &mut text_messages.warnings) {
fn load_cache_from_file(text_messages: &mut Messages, delete_outdated_cache: bool, checking_tags: bool) -> Option<HashMap<String, MusicEntry>> {
if let Some(((file_handler, cache_file), (file_handler_json, cache_file_json))) = open_cache_folder(get_cache_file(checking_tags), false, true, &mut text_messages.warnings) {
let mut hashmap_loaded_entries: HashMap<String, MusicEntry>;
if let Some(file_handler) = file_handler {
let reader = BufReader::new(file_handler);
@ -781,8 +884,172 @@ fn load_cache_from_file(text_messages: &mut Messages, delete_outdated_cache: boo
None
}
fn get_cache_file() -> String {
"cache_same_music.bin".to_string()
// TODO this should be taken from rusty-chromaprint repo, not reimplemented here
fn calc_fingerprint_helper(path: impl AsRef<Path>, config: &Configuration) -> anyhow::Result<Vec<u32>> {
let path = path.as_ref();
let src = File::open(path).context("failed to open file")?;
let mss = MediaSourceStream::new(Box::new(src), Default::default());
let mut hint = Hint::new();
if let Some(ext) = path.extension().and_then(std::ffi::OsStr::to_str) {
hint.with_extension(ext);
}
let meta_opts: MetadataOptions = Default::default();
let fmt_opts: FormatOptions = Default::default();
let probed = symphonia::default::get_probe().format(&hint, mss, &fmt_opts, &meta_opts).context("unsupported format")?;
let mut format = probed.format;
let track = format
.tracks()
.iter()
.find(|t| t.codec_params.codec != CODEC_TYPE_NULL)
.context("no supported audio tracks")?;
let dec_opts: DecoderOptions = Default::default();
let mut decoder = symphonia::default::get_codecs().make(&track.codec_params, &dec_opts).context("unsupported codec")?;
let track_id = track.id;
let mut printer = Fingerprinter::new(config);
let sample_rate = track.codec_params.sample_rate.context("missing sample rate")?;
let channels = track.codec_params.channels.context("missing audio channels")?.count() as u32;
printer.start(sample_rate, channels).context("initializing fingerprinter")?;
let mut sample_buf = None;
loop {
let Ok(packet) = format.next_packet() else { break };
if packet.track_id() != track_id {
continue;
}
match decoder.decode(&packet) {
Ok(audio_buf) => {
if sample_buf.is_none() {
let spec = *audio_buf.spec();
let duration = audio_buf.capacity() as u64;
sample_buf = Some(SampleBuffer::<i16>::new(duration, spec));
}
if let Some(buf) = &mut sample_buf {
buf.copy_interleaved_ref(audio_buf);
printer.consume(buf.samples());
}
}
Err(symphonia::core::errors::Error::DecodeError(_)) => (),
Err(_) => break,
}
}
printer.finish();
Ok(printer.fingerprint().to_vec())
}
fn read_single_file_tag(path: &str, music_entry: &mut MusicEntry) -> bool {
let Ok(mut file) = File::open(path) else { return false; };
let result = panic::catch_unwind(move || {
match read_from(&mut file) {
Ok(t) => Some(t),
Err(_inspected) => {
// println!("Failed to open {}", path);
None
}
}
});
let tagged_file = if let Ok(t) = result {
match t {
Some(r) => r,
None => {
return true;
}
}
} else {
let message = create_crash_message("Lofty", path, "https://github.com/image-rs/image/issues");
println!("{message}");
return false;
};
let properties = tagged_file.properties();
let mut track_title = String::new();
let mut track_artist = String::new();
let mut year = String::new();
let mut genre = String::new();
let bitrate = properties.audio_bitrate().unwrap_or(0);
let mut length = properties.duration().as_millis().to_string();
if let Some(tag) = tagged_file.primary_tag() {
track_title = tag.get_string(&ItemKey::TrackTitle).unwrap_or("").to_string();
track_artist = tag.get_string(&ItemKey::TrackArtist).unwrap_or("").to_string();
year = tag.get_string(&ItemKey::Year).unwrap_or("").to_string();
genre = tag.get_string(&ItemKey::Genre).unwrap_or("").to_string();
}
for tag in tagged_file.tags() {
if track_title.is_empty() {
if let Some(tag_value) = tag.get_string(&ItemKey::TrackTitle) {
track_title = tag_value.to_string();
}
}
if track_artist.is_empty() {
if let Some(tag_value) = tag.get_string(&ItemKey::TrackArtist) {
track_artist = tag_value.to_string();
}
}
if year.is_empty() {
if let Some(tag_value) = tag.get_string(&ItemKey::Year) {
year = tag_value.to_string();
}
}
if genre.is_empty() {
if let Some(tag_value) = tag.get_string(&ItemKey::Genre) {
genre = tag_value.to_string();
}
}
// println!("{:?}", tag.items());
}
if let Ok(old_length_number) = length.parse::<u32>() {
let length_number = old_length_number / 60;
let minutes = length_number / 1000;
let seconds = (length_number % 1000) * 6 / 100;
if minutes != 0 || seconds != 0 {
length = format!("{minutes}:{seconds:02}");
} else if old_length_number > 0 {
// That means, that audio have length smaller that second, but length is properly read
length = "0:01".to_string();
} else {
length = String::new();
}
} else {
length = String::new();
}
music_entry.track_title = track_title;
music_entry.track_artist = track_artist;
music_entry.year = year;
music_entry.length = length;
music_entry.genre = genre;
music_entry.bitrate = bitrate;
true
}
// Using different cache folders, because loading cache just for finding duplicated tags would be really slow
fn get_cache_file(checking_tags: bool) -> &'static str {
if checking_tags {
"cache_same_music_tags.bin"
} else {
"cache_same_music_fingerprints.bin"
}
}
impl Default for SameMusic {
@ -825,7 +1092,6 @@ impl DebugPrint for SameMusic {
impl SaveResults for SameMusic {
fn save_results_to_file(&mut self, file_name: &str) -> bool {
let start_time: SystemTime = SystemTime::now();
let file_name: String = match file_name {
"" => "results.txt".to_string(),
k => k.to_string(),
@ -857,7 +1123,7 @@ impl SaveResults for SameMusic {
} else {
write!(writer, "Not found any empty files.").unwrap();
}
Common::print_time(start_time, SystemTime::now(), "save_results_to_file");
true
}
}
@ -866,7 +1132,6 @@ impl PrintResults for SameMusic {
/// Print information's about duplicated entries
/// Only needed for CLI
fn print_results(&self) {
let start_time: SystemTime = SystemTime::now();
println!("Found {} similar music files.\n", self.duplicated_music_entries.len());
for vec_file_entry in &self.duplicated_music_entries {
for file_entry in vec_file_entry {
@ -883,8 +1148,6 @@ impl PrintResults for SameMusic {
}
println!();
}
Common::print_time(start_time, SystemTime::now(), "print_entries");
}
}

93
czkawka_core/src/similar_images.rs
View file

@ -23,14 +23,14 @@ use serde::{Deserialize, Serialize};
use crate::common::get_dynamic_image_from_heic;
use crate::common::{
check_folder_children, create_crash_message, get_dynamic_image_from_raw_image, get_number_of_threads, open_cache_folder, prepare_thread_handler_common,
send_info_and_wait_for_ending_all_threads, Common, HEIC_EXTENSIONS, IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS, RAW_IMAGE_EXTENSIONS,
send_info_and_wait_for_ending_all_threads, HEIC_EXTENSIONS, IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS, RAW_IMAGE_EXTENSIONS,
};
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData};
use crate::common_directory::Directories;
use crate::common_extensions::Extensions;
use crate::common_items::ExcludedItems;
use crate::common_messages::Messages;
use crate::common_traits::{DebugPrint, PrintResults, SaveResults};
use crate::common_traits::{DebugPrint, PrintResults, ResultEntry, SaveResults};
use crate::flc;
type ImHash = Vec<u8>;
@ -51,6 +51,11 @@ pub struct FileEntry {
pub hash: ImHash,
pub similarity: u32,
}
impl ResultEntry for FileEntry {
fn get_path(&self) -> &Path {
&self.path
}
}
/// Used by CLI tool when we cannot use directly values
#[derive(Clone, Debug, Copy)]
@ -275,7 +280,6 @@ impl SimilarImages {
/// Function to check if folder are empty.
/// Parameter `initial_checking` for second check before deleting to be sure that checked folder is still empty
fn check_for_similar_images(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
let start_time: SystemTime = SystemTime::now();
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
if !self.allowed_extensions.using_custom_extensions() {
@ -296,9 +300,7 @@ impl SimilarImages {
folders_to_check.push(id.clone());
}
let progress_thread_run = Arc::new(AtomicBool::new(true));
let atomic_counter = Arc::new(AtomicUsize::new(0));
let progress_thread_handle = prepare_thread_handler_common(progress_sender, &progress_thread_run, &atomic_counter, 0, 2, 0, CheckingMethod::None);
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) = prepare_thread_handler_common(progress_sender, 0, 2, 0, CheckingMethod::None);
while !folders_to_check.is_empty() {
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
@ -355,7 +357,7 @@ impl SimilarImages {
}
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
Common::print_time(start_time, SystemTime::now(), "check_for_similar_images");
true
}
@ -401,16 +403,18 @@ impl SimilarImages {
};
for (name, file_entry) in &self.images_to_check {
#[allow(clippy::if_same_then_else)]
if !loaded_hash_map.contains_key(name) {
// If loaded data doesn't contains current image info
non_cached_files_to_check.insert(name.clone(), file_entry.clone());
} else if file_entry.size != loaded_hash_map.get(name).unwrap().size || file_entry.modified_date != loaded_hash_map.get(name).unwrap().modified_date {
// When size or modification date of image changed, then it is clear that is different image
non_cached_files_to_check.insert(name.clone(), file_entry.clone());
} else {
// Checking may be omitted when already there is entry with same size and modification date
records_already_cached.insert(name.clone(), loaded_hash_map.get(name).unwrap().clone());
let loaded_item = loaded_hash_map.get(name).unwrap();
if file_entry.size != loaded_item.size || file_entry.modified_date != loaded_item.modified_date {
// When size or modification date of image changed, then it is clear that is different image
non_cached_files_to_check.insert(name.clone(), file_entry.clone());
} else {
// Checking may be omitted when already there is entry with same size and modification date
records_already_cached.insert(name.clone(), loaded_item.clone());
}
}
}
} else {
@ -422,31 +426,16 @@ impl SimilarImages {
// Cache algorithm:
// - Load data from file
// - Remove from data to search, already loaded entries from cache(size and modified datamust match)
// - Remove from data to search, already loaded entries from cache(size and modified date must match)
// - Check hash of files which doesn't have saved entry
// - Join already read hashes with hashes which were read from file
// - Join all hashes and save it to file
fn hash_images(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
let hash_map_modification = SystemTime::now();
let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.hash_images_load_cache();
Common::print_time(hash_map_modification, SystemTime::now(), "sort_images - reading data from cache and preparing them");
let hash_map_modification = SystemTime::now();
let check_was_stopped = AtomicBool::new(false); // Used for breaking from GUI and ending check thread
let progress_thread_run = Arc::new(AtomicBool::new(true));
let atomic_counter = Arc::new(AtomicUsize::new(0));
let progress_thread_handle = prepare_thread_handler_common(
progress_sender,
&progress_thread_run,
&atomic_counter,
1,
2,
non_cached_files_to_check.len(),
CheckingMethod::None,
);
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
prepare_thread_handler_common(progress_sender, 1, 2, non_cached_files_to_check.len(), CheckingMethod::None);
let mut vec_file_entry: Vec<(FileEntry, ImHash)> = non_cached_files_to_check
.into_par_iter()
@ -465,11 +454,8 @@ impl SimilarImages {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
Common::print_time(hash_map_modification, SystemTime::now(), "sort_images - reading data from files in parallel");
let hash_map_modification = SystemTime::now();
// Just connect loaded results with already calculated hashes
for (_name, file_entry) in records_already_cached {
for file_entry in records_already_cached.into_values() {
vec_file_entry.push((file_entry.clone(), file_entry.hash));
}
@ -502,7 +488,6 @@ impl SimilarImages {
return false;
}
Common::print_time(hash_map_modification, SystemTime::now(), "sort_images - saving data to files");
true
}
fn collect_image_file_entry(&self, mut file_entry: FileEntry) -> (FileEntry, ImHash) {
@ -813,14 +798,12 @@ impl SimilarImages {
return true;
}
let hash_map_modification = SystemTime::now();
let tolerance = self.similarity;
// Results
let mut collected_similar_images: HashMap<ImHash, Vec<FileEntry>> = Default::default();
let mut all_hashed_images = Default::default();
mem::swap(&mut all_hashed_images, &mut self.image_hashes);
let all_hashed_images = mem::take(&mut self.image_hashes);
let all_hashes: Vec<_> = all_hashed_images.clone().into_keys().collect();
@ -832,10 +815,8 @@ impl SimilarImages {
}
}
} else {
let check_was_stopped = AtomicBool::new(false); // Used for breaking from GUI and ending check thread
let progress_thread_run = Arc::new(AtomicBool::new(true));
let atomic_counter = Arc::new(AtomicUsize::new(0));
let progress_thread_handle = prepare_thread_handler_common(progress_sender, &progress_thread_run, &atomic_counter, 2, 2, all_hashes.len(), CheckingMethod::None);
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
prepare_thread_handler_common(progress_sender, 2, 2, all_hashes.len(), CheckingMethod::None);
// Don't use hashes with multiple images in bktree, because they will always be master of group and cannot be find by other hashes
@ -874,8 +855,6 @@ impl SimilarImages {
self.check_for_reference_folders();
Common::print_time(hash_map_modification, SystemTime::now(), "sort_images - selecting data from HashMap");
if self.use_reference_folders {
for (_fe, vector) in &self.similar_referenced_vectors {
self.information.number_of_duplicates += vector.len();
@ -888,7 +867,7 @@ impl SimilarImages {
}
}
// Clean unused data
// Clean unused data to save ram
self.image_hashes = Default::default();
self.images_to_check = Default::default();
self.bktree = BKTree::new(Hamming);
@ -898,9 +877,7 @@ impl SimilarImages {
fn exclude_items_with_same_size(&mut self) {
if self.exclude_images_with_same_size {
let mut new_vector = Default::default();
mem::swap(&mut self.similar_vectors, &mut new_vector);
for vec_file_entry in new_vector {
for vec_file_entry in mem::take(&mut self.similar_vectors) {
let mut bt_sizes: BTreeSet<u64> = Default::default();
let mut vec_values = Vec::new();
for file_entry in vec_file_entry {
@ -918,21 +895,11 @@ impl SimilarImages {
fn check_for_reference_folders(&mut self) {
if self.use_reference_folders {
let mut similar_vector = Default::default();
mem::swap(&mut self.similar_vectors, &mut similar_vector);
let reference_directories = self.directories.reference_directories.clone();
self.similar_referenced_vectors = similar_vector
self.similar_referenced_vectors = mem::take(&mut self.similar_vectors)
.into_iter()
.filter_map(|vec_file_entry| {
let mut files_from_referenced_folders = Vec::new();
let mut normal_files = Vec::new();
for file_entry in vec_file_entry {
if reference_directories.iter().any(|e| file_entry.path.starts_with(e)) {
files_from_referenced_folders.push(file_entry);
} else {
normal_files.push(file_entry);
}
}
let (mut files_from_referenced_folders, normal_files): (Vec<_>, Vec<_>) =
vec_file_entry.into_iter().partition(|e| self.directories.is_in_referenced_directory(e.get_path()));
if files_from_referenced_folders.is_empty() || normal_files.is_empty() {
None
@ -1077,7 +1044,6 @@ impl DebugPrint for SimilarImages {
impl SaveResults for SimilarImages {
fn save_results_to_file(&mut self, file_name: &str) -> bool {
let start_time: SystemTime = SystemTime::now();
let file_name: String = match file_name {
"" => "results.txt".to_string(),
k => k.to_string(),
@ -1123,7 +1089,6 @@ impl SaveResults for SimilarImages {
write!(writer, "Not found any similar images.").unwrap();
}
Common::print_time(start_time, SystemTime::now(), "save_results_to_file");
true
}
}

84
czkawka_core/src/similar_videos.rs
View file

@ -4,9 +4,7 @@ use std::io::Write;
use std::io::*;
use std::mem;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::SystemTime;
use std::sync::atomic::Ordering;
use crossbeam_channel::Receiver;
use ffmpeg_cmdline_utils::FfmpegErrorKind::FfmpegNotFound;
@ -18,14 +16,14 @@ use serde::{Deserialize, Serialize};
use vid_dup_finder_lib::HashCreationErrorKind::DetermineVideo;
use vid_dup_finder_lib::{NormalizedTolerance, VideoHash};
use crate::common::open_cache_folder;
use crate::common::{check_folder_children, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, VIDEO_FILES_EXTENSIONS};
use crate::common::{open_cache_folder, Common};
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData};
use crate::common_directory::Directories;
use crate::common_extensions::Extensions;
use crate::common_items::ExcludedItems;
use crate::common_messages::Messages;
use crate::common_traits::{DebugPrint, PrintResults, SaveResults};
use crate::common_traits::{DebugPrint, PrintResults, ResultEntry, SaveResults};
use crate::flc;
use crate::localizer_core::generate_translation_hashmap;
@ -39,6 +37,11 @@ pub struct FileEntry {
pub vhash: VideoHash,
pub error: String,
}
impl ResultEntry for FileEntry {
fn get_path(&self) -> &Path {
&self.path
}
}
/// Distance metric to use with the BK-tree.
struct Hamming;
@ -242,7 +245,6 @@ impl SimilarVideos {
/// Function to check if folder are empty.
/// Parameter `initial_checking` for second check before deleting to be sure that checked folder is still empty
fn check_for_similar_videos(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
let start_time: SystemTime = SystemTime::now();
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
if !self.allowed_extensions.using_custom_extensions() {
@ -259,9 +261,7 @@ impl SimilarVideos {
folders_to_check.push(id.clone());
}
let progress_thread_run = Arc::new(AtomicBool::new(true));
let atomic_counter = Arc::new(AtomicUsize::new(0));
let progress_thread_handle = prepare_thread_handler_common(progress_sender, &progress_thread_run, &atomic_counter, 0, 1, 0, CheckingMethod::None);
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) = prepare_thread_handler_common(progress_sender, 0, 1, 0, CheckingMethod::None);
while !folders_to_check.is_empty() {
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
@ -319,7 +319,7 @@ impl SimilarVideos {
}
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
Common::print_time(start_time, SystemTime::now(), "check_for_similar_videos");
true
}
@ -365,16 +365,18 @@ impl SimilarVideos {
};
for (name, file_entry) in &self.videos_to_check {
#[allow(clippy::if_same_then_else)]
if !loaded_hash_map.contains_key(name) {
// If loaded data doesn't contains current videos info
non_cached_files_to_check.insert(name.clone(), file_entry.clone());
} else if file_entry.size != loaded_hash_map.get(name).unwrap().size || file_entry.modified_date != loaded_hash_map.get(name).unwrap().modified_date {
// When size or modification date of video changed, then it is clear that is different video
non_cached_files_to_check.insert(name.clone(), file_entry.clone());
} else {
// Checking may be omitted when already there is entry with same size and modification date
records_already_cached.insert(name.clone(), loaded_hash_map.get(name).unwrap().clone());
let loaded_item = loaded_hash_map.get(name).unwrap();
if file_entry.size != loaded_item.size || file_entry.modified_date != loaded_item.modified_date {
// When size or modification date of video changed, then it is clear that is different video
non_cached_files_to_check.insert(name.clone(), file_entry.clone());
} else {
// Checking may be omitted when already there is entry with same size and modification date
records_already_cached.insert(name.clone(), loaded_item.clone());
}
}
}
} else {
@ -385,26 +387,10 @@ impl SimilarVideos {
}
fn sort_videos(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
let hash_map_modification = SystemTime::now();
let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.load_cache_at_start();
Common::print_time(hash_map_modification, SystemTime::now(), "sort_videos - reading data from cache and preparing them");
let hash_map_modification = SystemTime::now();
let check_was_stopped = AtomicBool::new(false); // Used for breaking from GUI and ending check thread
let progress_thread_run = Arc::new(AtomicBool::new(true));
let atomic_counter = Arc::new(AtomicUsize::new(0));
let progress_thread_handle = prepare_thread_handler_common(
progress_sender,
&progress_thread_run,
&atomic_counter,
1,
1,
non_cached_files_to_check.len(),
CheckingMethod::None,
);
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
prepare_thread_handler_common(progress_sender, 1, 1, non_cached_files_to_check.len(), CheckingMethod::None);
let mut vec_file_entry: Vec<FileEntry> = non_cached_files_to_check
.par_iter()
@ -435,13 +421,8 @@ impl SimilarVideos {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
Common::print_time(hash_map_modification, SystemTime::now(), "sort_videos - reading data from files in parallel");
let hash_map_modification = SystemTime::now();
// Just connect loaded results with already calculated hashes
for (_name, file_entry) in records_already_cached {
vec_file_entry.push(file_entry.clone());
}
vec_file_entry.extend(records_already_cached.into_values());
let mut hashmap_with_file_entries: HashMap<String, FileEntry> = Default::default();
let mut vector_of_hashes: Vec<VideoHash> = Vec::new();
@ -469,9 +450,6 @@ impl SimilarVideos {
return false;
}
Common::print_time(hash_map_modification, SystemTime::now(), "sort_videos - saving data to files");
let hash_map_modification = SystemTime::now();
self.match_groups_of_videos(vector_of_hashes, &hashmap_with_file_entries);
self.remove_from_reference_folders();
@ -487,8 +465,6 @@ impl SimilarVideos {
}
}
Common::print_time(hash_map_modification, SystemTime::now(), "sort_videos - selecting data from BtreeMap");
// Clean unused data
self.videos_hashes = Default::default();
self.videos_to_check = Default::default();
@ -523,21 +499,11 @@ impl SimilarVideos {
fn remove_from_reference_folders(&mut self) {
if self.use_reference_folders {
let mut similar_vector = Default::default();
mem::swap(&mut self.similar_vectors, &mut similar_vector);
let reference_directories = self.directories.reference_directories.clone();
self.similar_referenced_vectors = similar_vector
self.similar_referenced_vectors = mem::take(&mut self.similar_vectors)
.into_iter()
.filter_map(|vec_file_entry| {
let mut files_from_referenced_folders = Vec::new();
let mut normal_files = Vec::new();
for file_entry in vec_file_entry {
if reference_directories.iter().any(|e| file_entry.path.starts_with(e)) {
files_from_referenced_folders.push(file_entry);
} else {
normal_files.push(file_entry);
}
}
let (mut files_from_referenced_folders, normal_files): (Vec<_>, Vec<_>) =
vec_file_entry.into_iter().partition(|e| self.directories.is_in_referenced_directory(e.get_path()));
if files_from_referenced_folders.is_empty() || normal_files.is_empty() {
None
@ -590,7 +556,6 @@ impl DebugPrint for SimilarVideos {
impl SaveResults for SimilarVideos {
fn save_results_to_file(&mut self, file_name: &str) -> bool {
let start_time: SystemTime = SystemTime::now();
let file_name: String = match file_name {
"" => "results.txt".to_string(),
k => k.to_string(),
@ -628,7 +593,6 @@ impl SaveResults for SimilarVideos {
write!(writer, "Not found any similar videos.").unwrap();
}
Common::print_time(start_time, SystemTime::now(), "save_results_to_file");
true
}
}

21
czkawka_core/src/temporary.rs
View file

@ -3,15 +3,14 @@ use std::fs::{DirEntry, File, Metadata};
use std::io::prelude::*;
use std::io::BufWriter;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::SystemTime;
use crossbeam_channel::Receiver;
use futures::channel::mpsc::UnboundedSender;
use rayon::prelude::*;
use crate::common::{check_folder_children, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, Common};
use crate::common::{check_folder_children, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads};
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData};
use crate::common_directory::Directories;
use crate::common_items::ExcludedItems;
@ -143,7 +142,6 @@ impl Temporary {
}
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
let start_time: SystemTime = SystemTime::now();
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
// Add root folders for finding
@ -151,9 +149,7 @@ impl Temporary {
folders_to_check.push(id.clone());
}
let progress_thread_run = Arc::new(AtomicBool::new(true));
let atomic_counter = Arc::new(AtomicUsize::new(0));
let progress_thread_handle = prepare_thread_handler_common(progress_sender, &progress_thread_run, &atomic_counter, 0, 0, 0, CheckingMethod::None);
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) = prepare_thread_handler_common(progress_sender, 0, 0, 0, CheckingMethod::None);
while !folders_to_check.is_empty() {
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
@ -214,7 +210,6 @@ impl Temporary {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
self.information.number_of_temporary_files = self.temporary_files.len();
Common::print_time(start_time, SystemTime::now(), "check_files_size");
true
}
pub fn get_file_entry(
@ -248,8 +243,6 @@ impl Temporary {
/// Function to delete files, from filed Vector
fn delete_files(&mut self) {
let start_time: SystemTime = SystemTime::now();
match self.delete_method {
DeleteMethod::Delete => {
for file_entry in &self.temporary_files {
@ -262,8 +255,6 @@ impl Temporary {
//Just do nothing
}
}
Common::print_time(start_time, SystemTime::now(), "delete_files");
}
}
@ -304,7 +295,6 @@ impl DebugPrint for Temporary {
impl SaveResults for Temporary {
fn save_results_to_file(&mut self, file_name: &str) -> bool {
let start_time: SystemTime = SystemTime::now();
let file_name: String = match file_name {
"" => "results.txt".to_string(),
k => k.to_string(),
@ -336,19 +326,16 @@ impl SaveResults for Temporary {
} else {
write!(writer, "Not found any temporary files.").unwrap();
}
Common::print_time(start_time, SystemTime::now(), "save_results_to_file");
true
}
}
impl PrintResults for Temporary {
fn print_results(&self) {
let start_time: SystemTime = SystemTime::now();
println!("Found {} temporary files.\n", self.information.number_of_temporary_files);
for file_entry in &self.temporary_files {
println!("{}", file_entry.path.display());
}
Common::print_time(start_time, SystemTime::now(), "print_entries");
}
}

19
czkawka_gui/i18n/en/czkawka_gui.ftl
View file

@ -16,6 +16,23 @@ music_bitrate_checkbox = Bitrate
music_genre_checkbox = Genre
music_length_checkbox = Length
music_comparison_checkbox = Approximate Comparison
music_checking_by_tags = Tags
music_checking_by_content = Content
same_music_seconds_label = Minimal fragment second duration
same_music_similarity_label = Maximum difference
same_music_tooltip =
Searching for similar music files by its content can be configured by setting:
- The minimum fragment time after which music files can be identified as similar
- The maximum difference difference between two tested fragments
The key to good results is to find sensible combinations of these parameters, for provided.
Setting the minimum time to 5s and the maximum difference to 1.0, will look for almost identical fragments in the files.
A time of 20s and a maximum difference of 6.0, on the other hand, works well for finding remixes/live versions etc.
By default, each music file is compared to each other and this can take a lot of time when testing many files, so it is usually better to use reference folders and specifying which files are to be compared with each other(with same amount of files, comparing fingerprints will be faster at least 4x than without reference folders).
music_comparison_checkbox_tooltip =
It searches for similar music files using AI, which uses machine learning to remove parentheses from a phrase. For example, with this option enabled, the files in question will be considered duplicates:
@ -446,6 +463,8 @@ progress_scanning_image = Hashing of {$file_checked}/{$all_files} image
progress_comparing_image_hashes = Comparing {$file_checked}/{$all_files} image hash
progress_scanning_music_tags_end = Comparing tags of {$file_checked}/{$all_files} music file
progress_scanning_music_tags = Reading tags of {$file_checked}/{$all_files} music file
progress_scanning_music_content_end = Comparing fingerprint of {$file_checked}/{$all_files} music file
progress_scanning_music_content = Calculating fingerprint of {$file_checked}/{$all_files} music file
progress_scanning_empty_folders = Scanning {$folder_number} folder
progress_scanning_size = Scanning size of {$file_number} file
progress_scanning_size_name = Scanning name and size of {$file_number} file

14
czkawka_gui/src/compute_results.rs
View file

@ -550,7 +550,7 @@ fn computer_same_music(
} else {
let vector = mf.get_duplicated_music_entries();
let text: &str = "-----";
let text: &str = if mf.get_check_type() == CheckingMethod::AudioTags { "-----" } else { "" };
for vec_file_entry in vector {
// Sort
@ -1164,9 +1164,7 @@ fn computer_duplicate_finder(
duplicates_size = information.lost_space_by_size;
duplicates_group = information.number_of_groups_by_size_name;
}
CheckingMethod::None => {
panic!();
}
_ => panic!(),
}
if duplicates_size == 0 {
entry_info.set_text(
@ -1251,9 +1249,7 @@ fn computer_duplicate_finder(
}
}
}
CheckingMethod::None => {
panic!();
}
_ => panic!(),
}
} else {
match df.get_check_method() {
@ -1310,9 +1306,7 @@ fn computer_duplicate_finder(
}
}
}
CheckingMethod::None => {
panic!();
}
_ => panic!(),
}
}
print_text_messages_to_text_view(text_messages, text_view_errors);

1166
czkawka_gui/src/connect_things/connect_button_search.rs
View file

File diff suppressed because it is too large Load diff

746
czkawka_gui/src/connect_things/connect_progress_window.rs
View file

@ -1,7 +1,14 @@
use std::cell::RefCell;
use std::collections::HashMap;
use std::rc::Rc;
use futures::channel::mpsc::UnboundedReceiver;
use futures::StreamExt;
use glib::MainContext;
use gtk4::prelude::*;
use gtk4::ProgressBar;
use common_dir_traversal::CheckingMethod;
use czkawka_core::common_dir_traversal;
use czkawka_core::common_dir_traversal::ProgressData;
@ -9,464 +16,327 @@ use crate::flg;
use crate::gui_structs::gui_data::GuiData;
use crate::localizer_core::generate_translation_hashmap;
use crate::taskbar_progress::tbp_flags::TBPF_INDETERMINATE;
use crate::taskbar_progress::TaskbarProgress;
#[allow(clippy::too_many_arguments)]
pub fn connect_progress_window(
gui_data: &GuiData,
mut futures_receiver_duplicate_files: UnboundedReceiver<ProgressData>,
mut futures_receiver_empty_files: UnboundedReceiver<ProgressData>,
mut futures_receiver_empty_folder: UnboundedReceiver<ProgressData>,
mut futures_receiver_big_files: UnboundedReceiver<ProgressData>,
mut futures_receiver_same_music: UnboundedReceiver<ProgressData>,
mut futures_receiver_similar_images: UnboundedReceiver<ProgressData>,
mut futures_receiver_similar_videos: UnboundedReceiver<ProgressData>,
mut futures_receiver_temporary: UnboundedReceiver<ProgressData>,
mut futures_receiver_invalid_symlinks: UnboundedReceiver<ProgressData>,
mut futures_receiver_broken_files: UnboundedReceiver<ProgressData>,
mut futures_receiver_bad_extensions: UnboundedReceiver<ProgressData>,
futures_receiver_duplicate_files: UnboundedReceiver<ProgressData>,
futures_receiver_empty_files: UnboundedReceiver<ProgressData>,
futures_receiver_empty_folder: UnboundedReceiver<ProgressData>,
futures_receiver_big_files: UnboundedReceiver<ProgressData>,
futures_receiver_same_music: UnboundedReceiver<ProgressData>,
futures_receiver_similar_images: UnboundedReceiver<ProgressData>,
futures_receiver_similar_videos: UnboundedReceiver<ProgressData>,
futures_receiver_temporary: UnboundedReceiver<ProgressData>,
futures_receiver_invalid_symlinks: UnboundedReceiver<ProgressData>,
futures_receiver_broken_files: UnboundedReceiver<ProgressData>,
futures_receiver_bad_extensions: UnboundedReceiver<ProgressData>,
) {
let main_context = glib::MainContext::default();
let main_context = MainContext::default();
let _guard = main_context.acquire().unwrap();
{
// Duplicate Files
let label_stage = gui_data.progress_window.label_stage.clone();
let progress_bar_current_stage = gui_data.progress_window.progress_bar_current_stage.clone();
let progress_bar_all_stages = gui_data.progress_window.progress_bar_all_stages.clone();
let grid_progress_stages = gui_data.progress_window.grid_progress_stages.clone();
let taskbar_state = gui_data.taskbar_state.clone();
let future = async move {
while let Some(item) = futures_receiver_duplicate_files.next().await {
match item.checking_method {
common_dir_traversal::CheckingMethod::Hash => {
label_stage.show();
match item.current_stage {
// Checking Size
0 => {
progress_bar_current_stage.hide();
// progress_bar_all_stages.hide();
progress_bar_all_stages.set_fraction(0 as f64);
label_stage.set_text(&flg!(
"progress_scanning_size",
generate_translation_hashmap(vec![("file_number", item.entries_checked.to_string())])
));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
// Hash - first 1KB file
1 => {
progress_bar_current_stage.show();
// progress_bar_all_stages.show();
if item.entries_to_check != 0 {
progress_bar_all_stages.set_fraction((1f64 + (item.entries_checked) as f64 / item.entries_to_check as f64) / (item.max_stage + 1) as f64);
progress_bar_current_stage.set_fraction((item.entries_checked) as f64 / item.entries_to_check as f64);
taskbar_state.borrow().set_progress_value(
(item.entries_to_check + item.entries_checked) as u64,
item.entries_to_check as u64 * (item.max_stage + 1) as u64,
);
} else {
progress_bar_all_stages.set_fraction((1f64) / (item.max_stage + 1) as f64);
progress_bar_current_stage.set_fraction(0f64);
taskbar_state.borrow().set_progress_value(1, 1 + item.max_stage as u64);
}
process_bar_duplicates(gui_data, &main_context, futures_receiver_duplicate_files);
process_bar_empty_files(gui_data, &main_context, futures_receiver_empty_files);
process_bar_empty_folder(gui_data, &main_context, futures_receiver_empty_folder);
process_bar_big_files(gui_data, &main_context, futures_receiver_big_files);
process_bar_same_music(gui_data, &main_context, futures_receiver_same_music);
process_bar_similar_images(gui_data, &main_context, futures_receiver_similar_images);
process_bar_similar_videos(gui_data, &main_context, futures_receiver_similar_videos);
process_bar_temporary(gui_data, &main_context, futures_receiver_temporary);
process_bar_invalid_symlinks(gui_data, &main_context, futures_receiver_invalid_symlinks);
process_bar_broken_files(gui_data, &main_context, futures_receiver_broken_files);
process_bar_bad_extensions(gui_data, &main_context, futures_receiver_bad_extensions);
}
fn process_bar_empty_files(gui_data: &GuiData, main_context: &MainContext, mut futures_receiver_empty_files: UnboundedReceiver<ProgressData>) {
let label_stage = gui_data.progress_window.label_stage.clone();
let taskbar_state = gui_data.taskbar_state.clone();
let future = async move {
while let Some(item) = futures_receiver_empty_files.next().await {
label_stage.set_text(&flg!("progress_scanning_general_file", file_number_tm(&item)));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
};
main_context.spawn_local(future);
}
fn process_bar_empty_folder(gui_data: &GuiData, main_context: &MainContext, mut futures_receiver_empty_folder: UnboundedReceiver<ProgressData>) {
let label_stage = gui_data.progress_window.label_stage.clone();
let taskbar_state = gui_data.taskbar_state.clone();
let future = async move {
while let Some(item) = futures_receiver_empty_folder.next().await {
label_stage.set_text(&flg!(
"progress_scanning_empty_folders",
generate_translation_hashmap(vec![("folder_number", item.entries_checked.to_string())])
));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
};
main_context.spawn_local(future);
}
fn process_bar_big_files(gui_data: &GuiData, main_context: &MainContext, mut futures_receiver_big_files: UnboundedReceiver<ProgressData>) {
let label_stage = gui_data.progress_window.label_stage.clone();
let taskbar_state = gui_data.taskbar_state.clone();
let future = async move {
while let Some(item) = futures_receiver_big_files.next().await {
label_stage.set_text(&flg!("progress_scanning_general_file", file_number_tm(&item)));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
};
main_context.spawn_local(future);
}
fn process_bar_same_music(gui_data: &GuiData, main_context: &MainContext, mut futures_receiver_same_music: UnboundedReceiver<ProgressData>) {
let label_stage = gui_data.progress_window.label_stage.clone();
let progress_bar_current_stage = gui_data.progress_window.progress_bar_current_stage.clone();
let progress_bar_all_stages = gui_data.progress_window.progress_bar_all_stages.clone();
let taskbar_state = gui_data.taskbar_state.clone();
let future = async move {
while let Some(item) = futures_receiver_same_music.next().await {
match item.current_stage {
0 => {
progress_bar_current_stage.hide();
label_stage.set_text(&flg!("progress_scanning_general_file", file_number_tm(&item)));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
1 => {
progress_bar_current_stage.show();
common_set_data(&item, &progress_bar_all_stages, &progress_bar_current_stage, &taskbar_state);
label_stage.set_text(&flg!(
"progress_analyzed_partial_hash",
generate_translation_hashmap(vec![("file_checked", item.entries_checked.to_string()), ("all_files", item.entries_to_check.to_string())])
));
}
// Hash - normal hash
2 => {
if item.entries_to_check != 0 {
progress_bar_all_stages.set_fraction((2f64 + (item.entries_checked) as f64 / item.entries_to_check as f64) / (item.max_stage + 1) as f64);
progress_bar_current_stage.set_fraction((item.entries_checked) as f64 / item.entries_to_check as f64);
taskbar_state.borrow().set_progress_value(
(2 * item.entries_to_check + item.entries_checked) as u64,
item.entries_to_check as u64 * (item.max_stage + 1) as u64,
);
} else {
progress_bar_all_stages.set_fraction((2f64) / (item.max_stage + 1) as f64);
progress_bar_current_stage.set_fraction(0f64);
taskbar_state.borrow().set_progress_value(2, 1 + item.max_stage as u64);
}
label_stage.set_text(&flg!(
"progress_analyzed_full_hash",
generate_translation_hashmap(vec![("file_checked", item.entries_checked.to_string()), ("all_files", item.entries_to_check.to_string())])
));
}
_ => {
panic!("Not available current_stage");
}
}
}
common_dir_traversal::CheckingMethod::Name => {
label_stage.show();
grid_progress_stages.hide();
label_stage.set_text(&flg!(
"progress_scanning_name",
generate_translation_hashmap(vec![("file_number", item.entries_checked.to_string())])
));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
common_dir_traversal::CheckingMethod::SizeName => {
label_stage.show();
grid_progress_stages.hide();
label_stage.set_text(&flg!(
"progress_scanning_size_name",
generate_translation_hashmap(vec![("file_number", item.entries_checked.to_string())])
));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
common_dir_traversal::CheckingMethod::Size => {
label_stage.show();
grid_progress_stages.hide();
label_stage.set_text(&flg!(
"progress_scanning_size",
generate_translation_hashmap(vec![("file_number", item.entries_checked.to_string())])
));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
common_dir_traversal::CheckingMethod::None => {
panic!();
}
};
}
};
main_context.spawn_local(future);
}
{
// Empty Files
let label_stage = gui_data.progress_window.label_stage.clone();
let taskbar_state = gui_data.taskbar_state.clone();
let future = async move {
while let Some(item) = futures_receiver_empty_files.next().await {
label_stage.set_text(&flg!(
"progress_scanning_general_file",
generate_translation_hashmap(vec![("file_number", item.entries_checked.to_string())])
));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
};
main_context.spawn_local(future);
}
{
// Empty Folder
let label_stage = gui_data.progress_window.label_stage.clone();
let taskbar_state = gui_data.taskbar_state.clone();
let future = async move {
while let Some(item) = futures_receiver_empty_folder.next().await {
label_stage.set_text(&flg!(
"progress_scanning_empty_folders",
generate_translation_hashmap(vec![("folder_number", item.entries_checked.to_string())])
));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
};
main_context.spawn_local(future);
}
{
// Big Files
let label_stage = gui_data.progress_window.label_stage.clone();
let taskbar_state = gui_data.taskbar_state.clone();
let future = async move {
while let Some(item) = futures_receiver_big_files.next().await {
label_stage.set_text(&flg!(
"progress_scanning_general_file",
generate_translation_hashmap(vec![("file_number", item.entries_checked.to_string())])
));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
};
main_context.spawn_local(future);
}
{
// Same Music
let label_stage = gui_data.progress_window.label_stage.clone();
let progress_bar_current_stage = gui_data.progress_window.progress_bar_current_stage.clone();
let progress_bar_all_stages = gui_data.progress_window.progress_bar_all_stages.clone();
let taskbar_state = gui_data.taskbar_state.clone();
let future = async move {
while let Some(item) = futures_receiver_same_music.next().await {
match item.current_stage {
0 => {
progress_bar_current_stage.hide();
label_stage.set_text(&flg!(
"progress_scanning_general_file",
generate_translation_hashmap(vec![("file_number", item.entries_checked.to_string())])
));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
1 => {
progress_bar_current_stage.show();
if item.entries_to_check != 0 {
progress_bar_all_stages.set_fraction((1f64 + (item.entries_checked) as f64 / item.entries_to_check as f64) / (item.max_stage + 1) as f64);
progress_bar_current_stage.set_fraction((item.entries_checked) as f64 / item.entries_to_check as f64);
taskbar_state.borrow().set_progress_value(
(item.entries_to_check + item.entries_checked) as u64,
item.entries_to_check as u64 * (item.max_stage + 1) as u64,
);
} else {
progress_bar_all_stages.set_fraction((1f64) / (item.max_stage + 1) as f64);
progress_bar_current_stage.set_fraction(0f64);
taskbar_state.borrow().set_progress_value(1, (item.max_stage + 1) as u64);
}
label_stage.set_text(&flg!(
"progress_scanning_music_tags",
generate_translation_hashmap(vec![("file_checked", item.entries_checked.to_string()), ("all_files", item.entries_to_check.to_string())])
));
}
2 => {
if item.entries_to_check != 0 {
progress_bar_all_stages.set_fraction((2f64 + (item.entries_checked) as f64 / item.entries_to_check as f64) / (item.max_stage + 1) as f64);
progress_bar_current_stage.set_fraction((item.entries_checked) as f64 / item.entries_to_check as f64);
taskbar_state.borrow().set_progress_value(
(2 * item.entries_to_check + item.entries_checked) as u64,
item.entries_to_check as u64 * (item.max_stage + 1) as u64,
);
} else {
progress_bar_all_stages.set_fraction((2f64) / (item.max_stage + 1) as f64);
progress_bar_current_stage.set_fraction(0f64);
taskbar_state.borrow().set_progress_value(2, (item.max_stage + 1) as u64);
}
label_stage.set_text(&flg!(
"progress_scanning_music_tags_end",
generate_translation_hashmap(vec![("file_checked", item.entries_checked.to_string()), ("all_files", item.entries_to_check.to_string())])
));
}
_ => {
panic!();
match item.checking_method {
CheckingMethod::AudioTags => label_stage.set_text(&flg!("progress_scanning_music_tags", progress_ratio_tm(&item))),
CheckingMethod::AudioContent => label_stage.set_text(&flg!("progress_scanning_music_content", progress_ratio_tm(&item))),
_ => panic!(),
}
}
}
};
main_context.spawn_local(future);
}
{
// Similar Images
let label_stage = gui_data.progress_window.label_stage.clone();
let progress_bar_current_stage = gui_data.progress_window.progress_bar_current_stage.clone();
let progress_bar_all_stages = gui_data.progress_window.progress_bar_all_stages.clone();
let taskbar_state = gui_data.taskbar_state.clone();
let future = async move {
while let Some(item) = futures_receiver_similar_images.next().await {
match item.current_stage {
0 => {
progress_bar_current_stage.hide();
label_stage.set_text(&flg!(
"progress_scanning_general_file",
generate_translation_hashmap(vec![("file_number", item.entries_checked.to_string())])
));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
1 => {
progress_bar_current_stage.show();
if item.entries_to_check != 0 {
progress_bar_all_stages.set_fraction((1f64 + (item.entries_checked) as f64 / item.entries_to_check as f64) / (item.max_stage + 1) as f64);
progress_bar_current_stage.set_fraction((item.entries_checked) as f64 / item.entries_to_check as f64);
taskbar_state.borrow().set_progress_value(
(item.entries_to_check + item.entries_checked) as u64,
item.entries_to_check as u64 * (item.max_stage + 1) as u64,
);
} else {
progress_bar_all_stages.set_fraction((item.current_stage as f64) / (item.max_stage + 1) as f64);
progress_bar_current_stage.set_fraction(0f64);
taskbar_state.borrow().set_progress_value(1, (item.max_stage + 1) as u64);
}
label_stage.set_text(&flg!(
"progress_scanning_image",
generate_translation_hashmap(vec![("file_checked", item.entries_checked.to_string()), ("all_files", item.entries_to_check.to_string())])
));
}
2 => {
progress_bar_current_stage.show();
if item.entries_to_check != 0 {
progress_bar_all_stages.set_fraction((2f64 + (item.entries_checked) as f64 / item.entries_to_check as f64) / (item.max_stage + 1) as f64);
progress_bar_current_stage.set_fraction((item.entries_checked) as f64 / item.entries_to_check as f64);
taskbar_state.borrow().set_progress_value(
(item.entries_to_check + item.entries_checked) as u64,
item.entries_to_check as u64 * (item.max_stage + 1) as u64,
);
} else {
progress_bar_all_stages.set_fraction((item.current_stage as f64) / (item.max_stage + 1) as f64);
progress_bar_current_stage.set_fraction(0f64);
taskbar_state.borrow().set_progress_value(2, (item.max_stage + 1) as u64);
}
label_stage.set_text(&flg!(
"progress_comparing_image_hashes",
generate_translation_hashmap(vec![("file_checked", item.entries_checked.to_string()), ("all_files", item.entries_to_check.to_string())])
));
}
_ => {
panic!();
2 => {
common_set_data(&item, &progress_bar_all_stages, &progress_bar_current_stage, &taskbar_state);
match item.checking_method {
CheckingMethod::AudioTags => label_stage.set_text(&flg!("progress_scanning_music_tags_end", progress_ratio_tm(&item))),
CheckingMethod::AudioContent => label_stage.set_text(&flg!("progress_scanning_music_content_end", progress_ratio_tm(&item))),
_ => panic!(),
}
}
}
};
main_context.spawn_local(future);
}
{
// Similar Videos
let label_stage = gui_data.progress_window.label_stage.clone();
let progress_bar_current_stage = gui_data.progress_window.progress_bar_current_stage.clone();
let progress_bar_all_stages = gui_data.progress_window.progress_bar_all_stages.clone();
let taskbar_state = gui_data.taskbar_state.clone();
let future = async move {
while let Some(item) = futures_receiver_similar_videos.next().await {
match item.current_stage {
0 => {
progress_bar_current_stage.hide();
label_stage.set_text(&flg!(
"progress_scanning_general_file",
generate_translation_hashmap(vec![("file_number", item.entries_checked.to_string())])
));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
1 => {
progress_bar_current_stage.show();
if item.entries_to_check != 0 {
progress_bar_all_stages.set_fraction((1f64 + (item.entries_checked) as f64 / item.entries_to_check as f64) / (item.max_stage + 1) as f64);
progress_bar_current_stage.set_fraction((item.entries_checked) as f64 / item.entries_to_check as f64);
taskbar_state.borrow().set_progress_value(
(item.entries_to_check + item.entries_checked) as u64,
item.entries_to_check as u64 * (item.max_stage + 1) as u64,
);
} else {
progress_bar_all_stages.set_fraction((1f64) / (item.max_stage + 1) as f64);
progress_bar_current_stage.set_fraction(0f64);
taskbar_state.borrow().set_progress_value(1, (item.max_stage + 1) as u64);
}
label_stage.set_text(&flg!(
"progress_scanning_video",
generate_translation_hashmap(vec![("file_checked", item.entries_checked.to_string()), ("all_files", item.entries_to_check.to_string())])
));
}
_ => {
panic!();
3 => {
common_set_data(&item, &progress_bar_all_stages, &progress_bar_current_stage, &taskbar_state);
match item.checking_method {
CheckingMethod::AudioContent => label_stage.set_text(&flg!("progress_scanning_music_tags", progress_ratio_tm(&item))),
_ => panic!(),
}
}
_ => panic!(),
}
};
main_context.spawn_local(future);
}
{
// Temporary
let label_stage = gui_data.progress_window.label_stage.clone();
let taskbar_state = gui_data.taskbar_state.clone();
let future = async move {
while let Some(item) = futures_receiver_temporary.next().await {
label_stage.set_text(&flg!(
"progress_scanning_general_file",
generate_translation_hashmap(vec![("file_number", item.entries_checked.to_string())])
));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
};
main_context.spawn_local(future);
}
fn process_bar_similar_images(gui_data: &GuiData, main_context: &MainContext, mut futures_receiver_similar_images: UnboundedReceiver<ProgressData>) {
let label_stage = gui_data.progress_window.label_stage.clone();
let progress_bar_current_stage = gui_data.progress_window.progress_bar_current_stage.clone();
let progress_bar_all_stages = gui_data.progress_window.progress_bar_all_stages.clone();
let taskbar_state = gui_data.taskbar_state.clone();
let future = async move {
while let Some(item) = futures_receiver_similar_images.next().await {
match item.current_stage {
0 => {
progress_bar_current_stage.hide();
label_stage.set_text(&flg!("progress_scanning_general_file", file_number_tm(&item)));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
1 => {
progress_bar_current_stage.show();
common_set_data(&item, &progress_bar_all_stages, &progress_bar_current_stage, &taskbar_state);
label_stage.set_text(&flg!("progress_scanning_image", progress_ratio_tm(&item)));
}
2 => {
progress_bar_current_stage.show();
common_set_data(&item, &progress_bar_all_stages, &progress_bar_current_stage, &taskbar_state);
label_stage.set_text(&flg!("progress_comparing_image_hashes", progress_ratio_tm(&item)));
}
_ => panic!(),
}
};
main_context.spawn_local(future);
}
{
// Invalid Symlinks
let label_stage = gui_data.progress_window.label_stage.clone();
let taskbar_state = gui_data.taskbar_state.clone();
let future = async move {
while let Some(item) = futures_receiver_invalid_symlinks.next().await {
label_stage.set_text(&flg!(
"progress_scanning_general_file",
generate_translation_hashmap(vec![("file_number", item.entries_checked.to_string())])
));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
};
main_context.spawn_local(future);
}
fn process_bar_similar_videos(gui_data: &GuiData, main_context: &MainContext, mut futures_receiver_similar_videos: UnboundedReceiver<ProgressData>) {
let label_stage = gui_data.progress_window.label_stage.clone();
let progress_bar_current_stage = gui_data.progress_window.progress_bar_current_stage.clone();
let progress_bar_all_stages = gui_data.progress_window.progress_bar_all_stages.clone();
let taskbar_state = gui_data.taskbar_state.clone();
let future = async move {
while let Some(item) = futures_receiver_similar_videos.next().await {
match item.current_stage {
0 => {
progress_bar_current_stage.hide();
label_stage.set_text(&flg!("progress_scanning_general_file", file_number_tm(&item)));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
1 => {
progress_bar_current_stage.show();
common_set_data(&item, &progress_bar_all_stages, &progress_bar_current_stage, &taskbar_state);
label_stage.set_text(&flg!("progress_scanning_video", progress_ratio_tm(&item)));
}
_ => panic!(),
}
};
main_context.spawn_local(future);
}
{
// Broken Files
let label_stage = gui_data.progress_window.label_stage.clone();
let progress_bar_current_stage = gui_data.progress_window.progress_bar_current_stage.clone();
let progress_bar_all_stages = gui_data.progress_window.progress_bar_all_stages.clone();
let taskbar_state = gui_data.taskbar_state.clone();
let future = async move {
while let Some(item) = futures_receiver_broken_files.next().await {
match item.current_stage {
0 => {
progress_bar_current_stage.hide();
label_stage.set_text(&flg!(
"progress_scanning_general_file",
generate_translation_hashmap(vec![("file_number", item.entries_checked.to_string())])
));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
1 => {
progress_bar_current_stage.show();
if item.entries_to_check != 0 {
progress_bar_all_stages.set_fraction((1f64 + (item.entries_checked) as f64 / item.entries_to_check as f64) / (item.max_stage + 1) as f64);
progress_bar_current_stage.set_fraction((item.entries_checked) as f64 / item.entries_to_check as f64);
taskbar_state.borrow().set_progress_value(
(item.entries_to_check + item.entries_checked) as u64,
item.entries_to_check as u64 * (item.max_stage + 1) as u64,
);
} else {
progress_bar_all_stages.set_fraction((1f64) / (item.max_stage + 1) as f64);
progress_bar_current_stage.set_fraction(0f64);
taskbar_state.borrow().set_progress_value(1, (item.max_stage + 1) as u64);
}
};
main_context.spawn_local(future);
}
fn process_bar_temporary(gui_data: &GuiData, main_context: &MainContext, mut futures_receiver_temporary: UnboundedReceiver<ProgressData>) {
let label_stage = gui_data.progress_window.label_stage.clone();
let taskbar_state = gui_data.taskbar_state.clone();
let future = async move {
while let Some(item) = futures_receiver_temporary.next().await {
label_stage.set_text(&flg!("progress_scanning_general_file", file_number_tm(&item)));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
};
main_context.spawn_local(future);
}
fn process_bar_invalid_symlinks(gui_data: &GuiData, main_context: &MainContext, mut futures_receiver_invalid_symlinks: UnboundedReceiver<ProgressData>) {
let label_stage = gui_data.progress_window.label_stage.clone();
let taskbar_state = gui_data.taskbar_state.clone();
let future = async move {
while let Some(item) = futures_receiver_invalid_symlinks.next().await {
label_stage.set_text(&flg!("progress_scanning_general_file", file_number_tm(&item)));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
};
main_context.spawn_local(future);
}
fn process_bar_broken_files(gui_data: &GuiData, main_context: &MainContext, mut futures_receiver_broken_files: UnboundedReceiver<ProgressData>) {
let label_stage = gui_data.progress_window.label_stage.clone();
let progress_bar_current_stage = gui_data.progress_window.progress_bar_current_stage.clone();
let progress_bar_all_stages = gui_data.progress_window.progress_bar_all_stages.clone();
let taskbar_state = gui_data.taskbar_state.clone();
let future = async move {
while let Some(item) = futures_receiver_broken_files.next().await {
match item.current_stage {
0 => {
progress_bar_current_stage.hide();
label_stage.set_text(&flg!("progress_scanning_general_file", file_number_tm(&item)));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
1 => {
progress_bar_current_stage.show();
common_set_data(&item, &progress_bar_all_stages, &progress_bar_current_stage, &taskbar_state);
label_stage.set_text(&flg!("progress_scanning_broken_files", progress_ratio_tm(&item)));
}
_ => panic!(),
}
}
};
main_context.spawn_local(future);
}
fn process_bar_bad_extensions(gui_data: &GuiData, main_context: &MainContext, mut futures_receiver_bad_extensions: UnboundedReceiver<ProgressData>) {
let label_stage = gui_data.progress_window.label_stage.clone();
let progress_bar_current_stage = gui_data.progress_window.progress_bar_current_stage.clone();
let progress_bar_all_stages = gui_data.progress_window.progress_bar_all_stages.clone();
let taskbar_state = gui_data.taskbar_state.clone();
let future = async move {
while let Some(item) = futures_receiver_bad_extensions.next().await {
match item.current_stage {
0 => {
progress_bar_current_stage.hide();
label_stage.set_text(&flg!("progress_scanning_general_file", file_number_tm(&item)));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
1 => {
progress_bar_current_stage.show();
common_set_data(&item, &progress_bar_all_stages, &progress_bar_current_stage, &taskbar_state);
label_stage.set_text(&flg!("progress_scanning_extension_of_files", progress_ratio_tm(&item)));
}
_ => panic!(),
}
}
};
main_context.spawn_local(future);
}
fn process_bar_duplicates(gui_data: &GuiData, main_context: &MainContext, mut futures_receiver_duplicate_files: UnboundedReceiver<ProgressData>) {
let label_stage = gui_data.progress_window.label_stage.clone();
let progress_bar_current_stage = gui_data.progress_window.progress_bar_current_stage.clone();
let progress_bar_all_stages = gui_data.progress_window.progress_bar_all_stages.clone();
let grid_progress_stages = gui_data.progress_window.grid_progress_stages.clone();
let taskbar_state = gui_data.taskbar_state.clone();
let future = async move {
while let Some(item) = futures_receiver_duplicate_files.next().await {
match item.checking_method {
CheckingMethod::Hash => {
label_stage.show();
match item.current_stage {
// Checking Size
0 => {
progress_bar_current_stage.hide();
// progress_bar_all_stages.hide();
progress_bar_all_stages.set_fraction(0 as f64);
label_stage.set_text(&flg!("progress_scanning_size", file_number_tm(&item)));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
// Hash - first 1KB file
1 => {
progress_bar_current_stage.show();
// progress_bar_all_stages.show();
common_set_data(&item, &progress_bar_all_stages, &progress_bar_current_stage, &taskbar_state);
label_stage.set_text(&flg!("progress_analyzed_partial_hash", progress_ratio_tm(&item)));
}
// Hash - normal hash
2 => {
common_set_data(&item, &progress_bar_all_stages, &progress_bar_current_stage, &taskbar_state);
label_stage.set_text(&flg!("progress_analyzed_full_hash", progress_ratio_tm(&item)));
}
_ => {
panic!("Not available current_stage");
}
label_stage.set_text(&flg!(
"progress_scanning_broken_files",
generate_translation_hashmap(vec![("file_checked", item.entries_checked.to_string()), ("all_files", item.entries_to_check.to_string())])
));
}
_ => {
panic!();
}
}
}
};
main_context.spawn_local(future);
}
{
// Broken Files
let label_stage = gui_data.progress_window.label_stage.clone();
let progress_bar_current_stage = gui_data.progress_window.progress_bar_current_stage.clone();
let progress_bar_all_stages = gui_data.progress_window.progress_bar_all_stages.clone();
let taskbar_state = gui_data.taskbar_state.clone();
let future = async move {
while let Some(item) = futures_receiver_bad_extensions.next().await {
match item.current_stage {
0 => {
progress_bar_current_stage.hide();
label_stage.set_text(&flg!(
"progress_scanning_general_file",
generate_translation_hashmap(vec![("file_number", item.entries_checked.to_string())])
));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
1 => {
progress_bar_current_stage.show();
if item.entries_to_check != 0 {
progress_bar_all_stages.set_fraction((1f64 + (item.entries_checked) as f64 / item.entries_to_check as f64) / (item.max_stage + 1) as f64);
progress_bar_current_stage.set_fraction((item.entries_checked) as f64 / item.entries_to_check as f64);
taskbar_state.borrow().set_progress_value(
(item.entries_to_check + item.entries_checked) as u64,
item.entries_to_check as u64 * (item.max_stage + 1) as u64,
);
} else {
progress_bar_all_stages.set_fraction((1f64) / (item.max_stage + 1) as f64);
progress_bar_current_stage.set_fraction(0f64);
taskbar_state.borrow().set_progress_value(1, (item.max_stage + 1) as u64);
}
label_stage.set_text(&flg!(
"progress_scanning_extension_of_files",
generate_translation_hashmap(vec![("file_checked", item.entries_checked.to_string()), ("all_files", item.entries_to_check.to_string())])
));
}
_ => {
panic!();
}
CheckingMethod::Name => {
label_stage.show();
grid_progress_stages.hide();
label_stage.set_text(&flg!("progress_scanning_name", file_number_tm(&item)));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
}
};
main_context.spawn_local(future);
CheckingMethod::SizeName => {
label_stage.show();
grid_progress_stages.hide();
label_stage.set_text(&flg!("progress_scanning_size_name", file_number_tm(&item)));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
CheckingMethod::Size => {
label_stage.show();
grid_progress_stages.hide();
label_stage.set_text(&flg!("progress_scanning_size", file_number_tm(&item)));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
_ => panic!(),
};
}
};
main_context.spawn_local(future);
}
fn common_set_data(item: &ProgressData, progress_bar_all_stages: &ProgressBar, progress_bar_current_stage: &ProgressBar, taskbar_state: &Rc<RefCell<TaskbarProgress>>) {
if item.entries_to_check != 0 {
progress_bar_all_stages.set_fraction((item.current_stage as f64 + (item.entries_checked) as f64 / item.entries_to_check as f64) / (item.max_stage + 1) as f64);
progress_bar_current_stage.set_fraction((item.entries_checked) as f64 / item.entries_to_check as f64);
taskbar_state.borrow().set_progress_value(
((item.current_stage as usize) * item.entries_to_check + item.entries_checked) as u64,
item.entries_to_check as u64 * (item.max_stage + 1) as u64,
);
} else {
progress_bar_all_stages.set_fraction((item.current_stage as f64) / (item.max_stage + 1) as f64);
progress_bar_current_stage.set_fraction(0f64);
taskbar_state.borrow().set_progress_value(item.current_stage as u64, 1 + item.max_stage as u64);
}
}
fn file_number_tm(item: &ProgressData) -> HashMap<&'static str, String> {
generate_translation_hashmap(vec![("file_number", item.entries_checked.to_string())])
}
fn progress_ratio_tm(item: &ProgressData) -> HashMap<&'static str, String> {
generate_translation_hashmap(vec![("file_checked", item.entries_checked.to_string()), ("all_files", item.entries_to_check.to_string())])
}

77
czkawka_gui/src/connect_things/connect_same_music_mode_changed.rs Normal file
View file

@ -0,0 +1,77 @@
use gtk4::prelude::*;
use gtk4::{CheckButton, Widget};
use czkawka_core::common_dir_traversal::CheckingMethod;
use crate::gui_structs::gui_data::GuiData;
use crate::help_combo_box::AUDIO_TYPE_CHECK_METHOD_COMBO_BOX;
use crate::help_functions::scale_set_min_max_values;
const MINIMUM_SECONDS: f64 = 0.5;
const MAXIMUM_SECONDS: f64 = 180.0;
const DEFAULT_SECONDS: f64 = 15.0;
const MINIMUM_SIMILARITY: f64 = 0.0;
const MAXIMUM_SIMILARITY: f64 = 10.0;
const DEFAULT_SIMILARITY: f64 = 5.0;
pub fn connect_same_music_change_mode(gui_data: &GuiData) {
let check_button_music_title = gui_data.main_notebook.check_button_music_title.clone();
let check_button_music_approximate_comparison = gui_data.main_notebook.check_button_music_approximate_comparison.clone();
let check_button_music_bitrate = gui_data.main_notebook.check_button_music_bitrate.clone();
let check_button_music_artist = gui_data.main_notebook.check_button_music_artist.clone();
let check_button_music_genre = gui_data.main_notebook.check_button_music_genre.clone();
let check_button_music_length = gui_data.main_notebook.check_button_music_length.clone();
let check_button_music_year = gui_data.main_notebook.check_button_music_year.clone();
let buttons = [
check_button_music_title,
check_button_music_approximate_comparison,
check_button_music_bitrate,
check_button_music_artist,
check_button_music_genre,
check_button_music_year,
check_button_music_length,
];
let scale_seconds_same_music = gui_data.main_notebook.scale_seconds_same_music.clone();
let scale_similarity_same_music = gui_data.main_notebook.scale_similarity_same_music.clone();
let label_same_music_similarity = gui_data.main_notebook.label_same_music_similarity.clone();
let label_same_music_seconds = gui_data.main_notebook.label_same_music_seconds.clone();
scale_set_min_max_values(&scale_seconds_same_music, MINIMUM_SECONDS, MAXIMUM_SECONDS, DEFAULT_SECONDS, None);
scale_set_min_max_values(&scale_similarity_same_music, MINIMUM_SIMILARITY, MAXIMUM_SIMILARITY, DEFAULT_SIMILARITY, None);
let scales_and_labels = [
scale_seconds_same_music.into(),
scale_similarity_same_music.into(),
label_same_music_similarity.into(),
label_same_music_seconds.into(),
];
let combo_box_audio_check_type = gui_data.main_notebook.combo_box_audio_check_type.clone();
let check_method_index = combo_box_audio_check_type.active().unwrap() as usize;
let check_method = AUDIO_TYPE_CHECK_METHOD_COMBO_BOX[check_method_index].check_method;
disable_enable_buttons(&buttons, &scales_and_labels, check_method);
combo_box_audio_check_type.connect_changed(move |combo_box_text| {
if let Some(active) = combo_box_text.active() {
let check_method = AUDIO_TYPE_CHECK_METHOD_COMBO_BOX[active as usize].check_method;
disable_enable_buttons(&buttons, &scales_and_labels, check_method);
}
});
}
fn disable_enable_buttons(buttons: &[CheckButton; 7], scales: &[Widget; 4], current_mode: CheckingMethod) {
match current_mode {
CheckingMethod::AudioTags => {
buttons.iter().for_each(WidgetExt::show);
scales.iter().for_each(WidgetExt::hide);
}
CheckingMethod::AudioContent => {
buttons.iter().for_each(WidgetExt::hide);
scales.iter().for_each(WidgetExt::show);
}
_ => panic!(),
}
}

1
czkawka_gui/src/connect_things/mod.rs
View file

@ -15,6 +15,7 @@ pub mod connect_notebook_tabs;
pub mod connect_popovers_select;
pub mod connect_popovers_sort;
pub mod connect_progress_window;
pub mod connect_same_music_mode_changed;
pub mod connect_selection_of_directories;
pub mod connect_settings;
pub mod connect_show_hide_ui;

43
czkawka_gui/src/gui_structs/gui_main_notebook.rs
View file

@ -7,7 +7,7 @@ use czkawka_core::localizer_core::{fnc_get_similarity_minimal, fnc_get_similarit
use czkawka_core::similar_images::{get_string_from_similarity, SIMILAR_VALUES};
use crate::flg;
use crate::help_combo_box::{BIG_FILES_CHECK_METHOD_COMBO_BOX, DUPLICATES_CHECK_METHOD_COMBO_BOX, IMAGES_HASH_SIZE_COMBO_BOX};
use crate::help_combo_box::{AUDIO_TYPE_CHECK_METHOD_COMBO_BOX, BIG_FILES_CHECK_METHOD_COMBO_BOX, DUPLICATES_CHECK_METHOD_COMBO_BOX, IMAGES_HASH_SIZE_COMBO_BOX};
use crate::help_functions::get_all_direct_children;
use crate::notebook_enums::{NotebookMainEnum, NUMBER_OF_NOTEBOOK_MAIN_TABS};
@ -121,6 +121,12 @@ pub struct GuiMainNotebook {
pub check_button_music_genre: CheckButton,
pub check_button_music_length: CheckButton,
pub check_button_music_approximate_comparison: CheckButton,
pub label_audio_check_type: Label,
pub combo_box_audio_check_type: ComboBoxText,
pub label_same_music_seconds: Label,
pub label_same_music_similarity: Label,
pub scale_seconds_same_music: Scale,
pub scale_similarity_same_music: Scale,
}
impl GuiMainNotebook {
@ -247,6 +253,13 @@ impl GuiMainNotebook {
let image_preview_similar_images: Image = builder.object("image_preview_similar_images").unwrap();
let image_preview_duplicates: Image = builder.object("image_preview_duplicates").unwrap();
let label_audio_check_type: Label = builder.object("label_audio_check_type").unwrap();
let combo_box_audio_check_type: ComboBoxText = builder.object("combo_box_audio_check_type").unwrap();
let label_same_music_seconds: Label = builder.object("label_same_music_seconds").unwrap();
let label_same_music_similarity: Label = builder.object("label_same_music_similarity").unwrap();
let scale_seconds_same_music: Scale = builder.object("scale_seconds_same_music").unwrap();
let scale_similarity_same_music: Scale = builder.object("scale_similarity_same_music").unwrap();
Self {
notebook_main,
scrolled_window_duplicate_finder,
@ -289,6 +302,7 @@ impl GuiMainNotebook {
check_button_music_genre,
check_button_music_length,
check_button_music_approximate_comparison,
label_audio_check_type,
scale_similarity_similar_images,
scale_similarity_similar_videos,
check_button_broken_files_audio,
@ -331,6 +345,11 @@ impl GuiMainNotebook {
combo_box_big_files_mode,
label_big_files_mode,
check_button_broken_files_image,
combo_box_audio_check_type,
label_same_music_seconds,
label_same_music_similarity,
scale_seconds_same_music,
scale_similarity_same_music,
}
}
@ -407,6 +426,13 @@ impl GuiMainNotebook {
self.check_button_broken_files_image.set_label(Some(&flg!("main_check_box_broken_files_image")));
self.check_button_broken_files_pdf.set_label(Some(&flg!("main_check_box_broken_files_pdf")));
self.label_same_music_seconds.set_label(&flg!("same_music_seconds_label"));
self.label_same_music_similarity.set_label(&flg!("same_music_similarity_label"));
self.label_same_music_seconds.set_tooltip_text(Some(&flg!("same_music_tooltip")));
self.label_same_music_similarity.set_tooltip_text(Some(&flg!("same_music_tooltip")));
self.scale_seconds_same_music.set_tooltip_text(Some(&flg!("same_music_tooltip")));
self.scale_similarity_similar_videos.set_tooltip_text(Some(&flg!("same_music_tooltip")));
{
let hash_size_index = self.combo_box_image_hash_size.active().unwrap() as usize;
let hash_size = IMAGES_HASH_SIZE_COMBO_BOX[hash_size_index];
@ -541,6 +567,19 @@ impl GuiMainNotebook {
}
}
{
let active = self.combo_box_audio_check_type.active().unwrap_or(0);
self.combo_box_audio_check_type.remove_all();
for i in &AUDIO_TYPE_CHECK_METHOD_COMBO_BOX {
let text = match i.check_method {
CheckingMethod::AudioTags => flg!("music_checking_by_tags"),
CheckingMethod::AudioContent => flg!("music_checking_by_content"),
_ => panic!(),
};
self.combo_box_audio_check_type.append_text(&text);
}
self.combo_box_audio_check_type.set_active(Some(active));
}
{
let active = self.combo_box_duplicate_check_method.active().unwrap_or(0);
self.combo_box_duplicate_check_method.remove_all();
@ -550,7 +589,7 @@ impl GuiMainNotebook {
CheckingMethod::Size => flg!("duplicate_mode_size_combo_box"),
CheckingMethod::Name => flg!("duplicate_mode_name_combo_box"),
CheckingMethod::SizeName => flg!("duplicate_mode_size_name_combo_box"),
CheckingMethod::None => panic!(),
_ => panic!(),
};
self.combo_box_duplicate_check_method.append_text(&text);
}

17
czkawka_gui/src/help_combo_box.rs
View file

@ -48,6 +48,23 @@ pub const DUPLICATES_CHECK_METHOD_COMBO_BOX: [CheckMethodStruct; 4] = [
},
];
#[derive(Copy, Clone)]
pub struct AudioTypeStruct {
pub eng_name: &'static str,
pub check_method: CheckingMethod,
}
pub const AUDIO_TYPE_CHECK_METHOD_COMBO_BOX: [AudioTypeStruct; 2] = [
AudioTypeStruct {
eng_name: "Tags",
check_method: CheckingMethod::AudioTags,
},
AudioTypeStruct {
eng_name: "Content",
check_method: CheckingMethod::AudioContent,
},
];
#[derive(Copy, Clone)]
pub struct SearchModeStruct {
pub eng_name: &'static str,

13
czkawka_gui/src/help_functions.rs
View file

@ -7,7 +7,7 @@ use gdk4::gdk_pixbuf::{InterpType, Pixbuf};
use glib::signal::Inhibit;
use glib::Error;
use gtk4::prelude::*;
use gtk4::{ListStore, ScrollType, TextView, TreeView, Widget};
use gtk4::{ListStore, Scale, ScrollType, TextView, TreeView, Widget};
use image::codecs::jpeg::JpegEncoder;
use image::{DynamicImage, EncodableLayout};
use once_cell::sync::OnceCell;
@ -770,7 +770,16 @@ pub fn check_if_list_store_column_have_all_same_values(list_store: &ListStore, c
false
}
pub fn scale_step_function(scale: &gtk4::Scale, _scroll_type: ScrollType, value: f64) -> Inhibit {
pub fn scale_set_min_max_values(scale: &Scale, minimum: f64, maximum: f64, current_value: f64, step: Option<f64>) {
scale.set_range(minimum, maximum);
scale.set_fill_level(maximum);
scale.set_value(current_value);
if let Some(step) = step {
scale.adjustment().set_step_increment(step);
}
}
pub fn scale_step_function(scale: &Scale, _scroll_type: ScrollType, value: f64) -> Inhibit {
scale.set_increments(1_f64, 1_f64);
scale.set_round_digits(0);
scale.set_fill_level(value.round());

2
czkawka_gui/src/main.rs
View file

@ -42,6 +42,7 @@ use crate::compute_results::*;
use crate::connect_things::connect_button_sort::connect_button_sort;
use crate::connect_things::connect_popovers_select::connect_popover_select;
use crate::connect_things::connect_popovers_sort::connect_popover_sort;
use crate::connect_things::connect_same_music_mode_changed::connect_same_music_change_mode;
use crate::initialize_gui::*;
use crate::language_functions::LANGUAGES_ALL;
use crate::saving_loading::*;
@ -163,6 +164,7 @@ fn build_ui(application: &Application, arguments: &[OsString]) {
connect_button_about(&gui_data);
connect_about_buttons(&gui_data);
connect_similar_image_size_change(&gui_data);
connect_same_music_change_mode(&gui_data);
let window_main = gui_data.window_main.clone();
let taskbar_state = gui_data.taskbar_state.clone();

25
czkawka_gui/ui/czkawka.cmb
View file

@ -237,6 +237,10 @@
(5,232,"GtkLabel","label_buttons_sort",230,None,None,None,1),
(5,234,"GtkLabel","label_audio_check_type",124,None,None,None,None),
(5,235,"GtkComboBoxText","combo_box_audio_check_type",124,None,None,None,1),
(5,236,"GtkScale","scale_seconds_same_music",117,None,None,None,7),
(5,237,"GtkScale","scale_similarity_same_music",117,None,None,None,9),
(5,238,"GtkLabel","label_same_music_seconds",117,None,None,None,6),
(5,239,"GtkLabel","label_same_music_similarity",117,None,None,None,8),
(6,1,"GtkPopover","popover_right_click",None,None,None,None,None),
(6,2,"GtkBox",None,1,None,None,None,None),
(6,3,"GtkButton","buttons_popover_right_click_open_file",2,None,None,None,None),
@ -769,6 +773,26 @@
(5,232,"GtkLabel","label","SortMenu",None,None,None,None,None),
(5,234,"GtkLabel","label","Audio check type",None,None,None,None,None),
(5,234,"GtkWidget","margin-end","2",None,None,None,None,None),
(5,236,"GtkRange","fill-level","100",None,None,None,None,None),
(5,236,"GtkRange","round-digits","1",None,None,None,None,None),
(5,236,"GtkScale","digits","0",None,None,None,None,None),
(5,236,"GtkScale","draw-value","1",None,None,None,None,None),
(5,236,"GtkScale","value-pos","right",None,None,None,None,None),
(5,236,"GtkWidget","focusable","1",None,None,None,None,None),
(5,236,"GtkWidget","hexpand","1",None,None,None,None,None),
(5,237,"GtkRange","fill-level","100",None,None,None,None,None),
(5,237,"GtkRange","round-digits","1",None,None,None,None,None),
(5,237,"GtkScale","digits","0",None,None,None,None,None),
(5,237,"GtkScale","draw-value","1",None,None,None,None,None),
(5,237,"GtkScale","value-pos","right",None,None,None,None,None),
(5,237,"GtkWidget","focusable","1",None,None,None,None,None),
(5,237,"GtkWidget","hexpand","1",None,None,None,None,None),
(5,238,"GtkLabel","label","Minimal fragment second duration",None,None,None,None,None),
(5,238,"GtkWidget","margin-end","5",None,None,None,None,None),
(5,238,"GtkWidget","margin-start","5",None,None,None,None,None),
(5,239,"GtkLabel","label","Max difference",None,None,None,None,None),
(5,239,"GtkWidget","margin-end","5",None,None,None,None,None),
(5,239,"GtkWidget","margin-start","5",None,None,None,None,None),
(6,1,"GtkPopover","child",None,None,None,None,None,2),
(6,1,"GtkPopover","position","left",None,None,None,None,None),
(6,2,"GtkOrientable","orientation","vertical",None,None,None,None,None),
@ -991,6 +1015,7 @@
(9,58,"GtkWidget","focusable","1",None,None,None,None,None),
(9,58,"GtkWidget","hexpand","1",None,None,None,None,None),
(9,59,"GtkAccessible","accessible-role","menu-item-checkbox",None,None,None,None,None),
(9,59,"GtkLabel","label","Restart Required",None,None,None,None,None),
(9,59,"GtkWidget","margin-bottom","4",None,None,None,None,None),
(9,59,"GtkWidget","margin-top","5",None,None,None,None,None),
(10,1,"GtkPopover","child",None,None,None,None,None,2),

36
czkawka_gui/ui/main_window.ui
View file

@ -721,6 +721,42 @@
<property name="label">Length</property>
</object>
</child>
<child>
<object class="GtkLabel" id="label_same_music_seconds">
<property name="label">Minimal fragment second duration</property>
<property name="margin-end">5</property>
<property name="margin-start">5</property>
</object>
</child>
<child>
<object class="GtkScale" id="scale_seconds_same_music">
<property name="digits">0</property>
<property name="draw-value">1</property>
<property name="fill-level">100</property>
<property name="focusable">1</property>
<property name="hexpand">1</property>
<property name="round-digits">1</property>
<property name="value-pos">right</property>
</object>
</child>
<child>
<object class="GtkLabel" id="label_same_music_similarity">
<property name="label">Max difference</property>
<property name="margin-end">5</property>
<property name="margin-start">5</property>
</object>
</child>
<child>
<object class="GtkScale" id="scale_similarity_same_music">
<property name="digits">0</property>
<property name="draw-value">1</property>
<property name="fill-level">100</property>
<property name="focusable">1</property>
<property name="hexpand">1</property>
<property name="round-digits">1</property>
<property name="value-pos">right</property>
</object>
</child>
</object>
</child>
<child>

1
czkawka_gui/ui/settings.ui
View file

@ -142,6 +142,7 @@
<child>
<object class="GtkLabel" id="label_restart_needed">
<property name="accessible-role">menu-item-checkbox</property>
<property name="label">Restart Required</property>
<property name="margin-bottom">4</property>
<property name="margin-top">5</property>
</object>