1
0
Fork 0
mirror of synced 2024-05-15 09:52:33 +12:00

Core cleanup/changes (#1082)

* FunTime

* Ft

* Unify delete files

* Files

* FunTime

* Smaller Printing

* Cleaned

* Fallen
This commit is contained in:
Rafał Mikrut 2023-10-10 20:54:41 +02:00 committed by GitHub
parent 4a1f6227db
commit 9b57382e39
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
46 changed files with 2444 additions and 3121 deletions

View file

@ -8,8 +8,16 @@ assignees: ''
--- ---
**Desktop (please complete the following information):** **Desktop (please complete the following information):**
- Czkawka version [e.g. 6.0.0 cli/gui]:
- OS version [e.g Ubuntu 22.04, Windows 11]: - Czkawka version<!-- e.g. 6.0.0 cli/gui -->:
- OS version<!-- e.g Ubuntu 22.04, Windows 11, Mac 15.1 ARM -->:
- Terminal output[optional]:
<!--
Add terminal output only if needed - if there are some errors or warnings or you have performance/freeze issues.
Very helpful in this situation will be logs from czkawka run with RUST_LOG environment variable set e.g.
`RUST_LOG=debug ./czkawka` which will print more detailed info about executed function
-->
**Bug Description** **Bug Description**
... ...

101
Cargo.lock generated
View file

@ -525,6 +525,7 @@ version = "6.0.0"
dependencies = [ dependencies = [
"clap", "clap",
"czkawka_core", "czkawka_core",
"fun_time",
"handsome_logger", "handsome_logger",
"image_hasher", "image_hasher",
"log", "log",
@ -544,6 +545,7 @@ dependencies = [
"crossbeam-channel", "crossbeam-channel",
"directories-next", "directories-next",
"ffmpeg_cmdline_utils", "ffmpeg_cmdline_utils",
"fun_time",
"futures", "futures",
"hamming", "hamming",
"handsome_logger", "handsome_logger",
@ -585,6 +587,7 @@ dependencies = [
"czkawka_core", "czkawka_core",
"directories-next", "directories-next",
"fs_extra", "fs_extra",
"fun_time",
"futures", "futures",
"gdk4", "gdk4",
"glib", "glib",
@ -604,6 +607,41 @@ dependencies = [
"winapi", "winapi",
] ]
[[package]]
name = "darling"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
dependencies = [
"darling_core",
"darling_macro",
]
[[package]]
name = "darling_core"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"strsim",
"syn 1.0.109",
]
[[package]]
name = "darling_macro"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
dependencies = [
"darling_core",
"quote",
"syn 1.0.109",
]
[[package]] [[package]]
name = "dashmap" name = "dashmap"
version = "5.5.3" version = "5.5.3"
@ -735,25 +773,14 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
[[package]] [[package]]
name = "errno" name = "errno"
version = "0.3.4" version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "add4f07d43996f76ef320709726a556a9d4f965d9410d8d0271132d2f8293480" checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860"
dependencies = [ dependencies = [
"errno-dragonfly",
"libc", "libc",
"windows-sys", "windows-sys",
] ]
[[package]]
name = "errno-dragonfly"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
dependencies = [
"cc",
"libc",
]
[[package]] [[package]]
name = "exr" name = "exr"
version = "1.71.0" version = "1.71.0"
@ -934,6 +961,28 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fun_time"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9af29f347b6ae4821c45565e1238231caf5b57a951bd011222752ba0f5a47eae"
dependencies = [
"fun_time_derive",
"log",
]
[[package]]
name = "fun_time_derive"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e992455767376a16164ee4cc69bd799530c1c41c701bdd8c8a541ec6095c078e"
dependencies = [
"darling",
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]] [[package]]
name = "futures" name = "futures"
version = "0.3.28" version = "0.3.28"
@ -1500,6 +1549,12 @@ dependencies = [
"cc", "cc",
] ]
[[package]]
name = "ident_case"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]] [[package]]
name = "idna" name = "idna"
version = "0.4.0" version = "0.4.0"
@ -1704,9 +1759,9 @@ checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8"
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.148" version = "0.2.149"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b"
[[package]] [[package]]
name = "libheif-rs" name = "libheif-rs"
@ -1742,9 +1797,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
[[package]] [[package]]
name = "linux-raw-sys" name = "linux-raw-sys"
version = "0.4.8" version = "0.4.10"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3852614a3bd9ca9804678ba6be5e3b8ce76dfc902cae004e3e0c44051b6e88db" checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f"
[[package]] [[package]]
name = "locale_config" name = "locale_config"
@ -1942,9 +1997,9 @@ dependencies = [
[[package]] [[package]]
name = "num-traits" name = "num-traits"
version = "0.2.16" version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c"
dependencies = [ dependencies = [
"autocfg", "autocfg",
] ]
@ -2238,9 +2293,9 @@ dependencies = [
[[package]] [[package]]
name = "proc-macro2" name = "proc-macro2"
version = "1.0.68" version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b1106fec09662ec6dd98ccac0f81cef56984d0b49f75c92d8cbad76e20c005c" checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da"
dependencies = [ dependencies = [
"unicode-ident", "unicode-ident",
] ]
@ -3228,9 +3283,9 @@ dependencies = [
[[package]] [[package]]
name = "trash" name = "trash"
version = "3.0.6" version = "3.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af3663fb8f476d674b9c61d1d2796acec725bef6bec4b41402a904252a25971e" checksum = "f7b23f2b0cf93f537bbe90cbb59ea9176cc8ce9b010a36dcd5b726facd82825e"
dependencies = [ dependencies = [
"chrono", "chrono",
"libc", "libc",

View file

@ -17,6 +17,7 @@ image_hasher = "1.2"
log = "0.4.20" log = "0.4.20"
handsome_logger = "0.8" handsome_logger = "0.8"
fun_time = { version = "0.3.1", features = ["log"] }
czkawka_core = { path = "../czkawka_core", version = "6.0.0", features = [] } czkawka_core = { path = "../czkawka_core", version = "6.0.0", features = [] }
[features] [features]

View file

@ -3,7 +3,8 @@ use std::path::PathBuf;
use image_hasher::{FilterType, HashAlg}; use image_hasher::{FilterType, HashAlg};
use czkawka_core::common_dir_traversal::CheckingMethod; use czkawka_core::common_dir_traversal::CheckingMethod;
use czkawka_core::duplicate::{DeleteMethod, HashType}; use czkawka_core::common_tool::DeleteMethod;
use czkawka_core::duplicate::HashType;
use czkawka_core::same_music::MusicSimilarity; use czkawka_core::same_music::MusicSimilarity;
use czkawka_core::similar_images::SimilarityPreset; use czkawka_core::similar_images::SimilarityPreset;
use czkawka_core::CZKAWKA_VERSION; use czkawka_core::CZKAWKA_VERSION;

View file

@ -1,25 +1,24 @@
#![allow(clippy::needless_late_init)] #![allow(clippy::needless_late_init)]
use std::process;
use clap::Parser; use clap::Parser;
use log::error;
use commands::Commands; use commands::Commands;
use czkawka_core::bad_extensions::BadExtensions; use czkawka_core::bad_extensions::BadExtensions;
use czkawka_core::big_file::{self, BigFile, SearchMode}; use czkawka_core::big_file::{BigFile, SearchMode};
use czkawka_core::broken_files::{self, BrokenFiles}; use czkawka_core::broken_files::BrokenFiles;
use czkawka_core::common::{set_number_of_threads, setup_logger}; use czkawka_core::common::{print_version_mode, set_number_of_threads, setup_logger};
use czkawka_core::common_tool::CommonData; use czkawka_core::common_tool::{CommonData, DeleteMethod};
#[allow(unused_imports)] // It is used in release for print_results(). #[allow(unused_imports)] // It is used in release for print_results_to_output().
use czkawka_core::common_traits::*; use czkawka_core::common_traits::*;
use czkawka_core::duplicate::DuplicateFinder; use czkawka_core::duplicate::DuplicateFinder;
use czkawka_core::empty_files::{self, EmptyFiles}; use czkawka_core::empty_files::EmptyFiles;
use czkawka_core::empty_folder::EmptyFolder; use czkawka_core::empty_folder::EmptyFolder;
use czkawka_core::invalid_symlinks::{self, InvalidSymlinks}; use czkawka_core::invalid_symlinks::InvalidSymlinks;
use czkawka_core::same_music::SameMusic; use czkawka_core::same_music::SameMusic;
use czkawka_core::similar_images::{return_similarity_from_similarity_preset, test_image_conversion_speed, SimilarImages}; use czkawka_core::similar_images::{return_similarity_from_similarity_preset, test_image_conversion_speed, SimilarImages};
use czkawka_core::similar_videos::SimilarVideos; use czkawka_core::similar_videos::SimilarVideos;
use czkawka_core::temporary::{self, Temporary}; use czkawka_core::temporary::Temporary;
use crate::commands::{ use crate::commands::{
Args, BadExtensionsArgs, BiggestFilesArgs, BrokenFilesArgs, DuplicatesArgs, EmptyFilesArgs, EmptyFoldersArgs, InvalidSymlinksArgs, SameMusicArgs, SimilarImagesArgs, Args, BadExtensionsArgs, BiggestFilesArgs, BrokenFilesArgs, DuplicatesArgs, EmptyFilesArgs, EmptyFoldersArgs, InvalidSymlinksArgs, SameMusicArgs, SimilarImagesArgs,
@ -32,9 +31,11 @@ fn main() {
let command = Args::parse().command; let command = Args::parse().command;
setup_logger(true); setup_logger(true);
print_version_mode();
#[cfg(debug_assertions)] if cfg!(debug_assertions) {
println!("{command:?}"); println!("{command:?}");
}
match command { match command {
Commands::Duplicates(duplicates_args) => duplicates(duplicates_args), Commands::Duplicates(duplicates_args) => duplicates(duplicates_args),
@ -100,14 +101,14 @@ fn duplicates(duplicates: DuplicatesArgs) {
df.find_duplicates(None, None); df.find_duplicates(None, None);
if let Some(file_name) = file_to_save.file_name() { if let Some(file_name) = file_to_save.file_name() {
if !df.save_results_to_file(file_name) { if let Err(e) = df.print_results_to_file(file_name) {
df.get_text_messages().print_messages(); error!("Failed to save results to file {e}");
process::exit(1);
} }
} }
#[cfg(not(debug_assertions))] // This will show too much probably unnecessary data to debug, comment line only if needed if !cfg!(debug_assertions) {
df.print_results(); df.print_results_to_output();
}
df.get_text_messages().print_messages(); df.get_text_messages().print_messages();
} }
@ -137,14 +138,14 @@ fn empty_folders(empty_folders: EmptyFoldersArgs) {
ef.find_empty_folders(None, None); ef.find_empty_folders(None, None);
if let Some(file_name) = file_to_save.file_name() { if let Some(file_name) = file_to_save.file_name() {
if !ef.save_results_to_file(file_name) { if let Err(e) = ef.print_results_to_file(file_name) {
ef.get_text_messages().print_messages(); error!("Failed to save results to file {e}");
process::exit(1);
} }
} }
#[cfg(not(debug_assertions))] // This will show too much probably unnecessary data to debug, comment line only if needed if !cfg!(debug_assertions) {
ef.print_results(); ef.print_results_to_output();
}
ef.get_text_messages().print_messages(); ef.get_text_messages().print_messages();
} }
@ -177,7 +178,7 @@ fn biggest_files(biggest_files: BiggestFilesArgs) {
#[cfg(target_family = "unix")] #[cfg(target_family = "unix")]
bf.set_exclude_other_filesystems(exclude_other_filesystems.exclude_other_filesystems); bf.set_exclude_other_filesystems(exclude_other_filesystems.exclude_other_filesystems);
if delete_files { if delete_files {
bf.set_delete_method(big_file::DeleteMethod::Delete); bf.set_delete_method(DeleteMethod::Delete);
} }
if smallest_mode { if smallest_mode {
bf.set_search_mode(SearchMode::SmallestFiles); bf.set_search_mode(SearchMode::SmallestFiles);
@ -186,14 +187,14 @@ fn biggest_files(biggest_files: BiggestFilesArgs) {
bf.find_big_files(None, None); bf.find_big_files(None, None);
if let Some(file_name) = file_to_save.file_name() { if let Some(file_name) = file_to_save.file_name() {
if !bf.save_results_to_file(file_name) { if let Err(e) = bf.print_results_to_file(file_name) {
bf.get_text_messages().print_messages(); error!("Failed to save results to file {e}");
process::exit(1);
} }
} }
#[cfg(not(debug_assertions))] // This will show too much probably unnecessary data to debug, comment line only if needed if !cfg!(debug_assertions) {
bf.print_results(); bf.print_results_to_output();
}
bf.get_text_messages().print_messages(); bf.get_text_messages().print_messages();
} }
@ -224,20 +225,20 @@ fn empty_files(empty_files: EmptyFilesArgs) {
ef.set_exclude_other_filesystems(exclude_other_filesystems.exclude_other_filesystems); ef.set_exclude_other_filesystems(exclude_other_filesystems.exclude_other_filesystems);
if delete_files { if delete_files {
ef.set_delete_method(empty_files::DeleteMethod::Delete); ef.set_delete_method(DeleteMethod::Delete);
} }
ef.find_empty_files(None, None); ef.find_empty_files(None, None);
if let Some(file_name) = file_to_save.file_name() { if let Some(file_name) = file_to_save.file_name() {
if !ef.save_results_to_file(file_name) { if let Err(e) = ef.print_results_to_file(file_name) {
ef.get_text_messages().print_messages(); error!("Failed to save results to file {e}");
process::exit(1);
} }
} }
#[cfg(not(debug_assertions))] // This will show too much probably unnecessary data to debug, comment line only if needed if !cfg!(debug_assertions) {
ef.print_results(); ef.print_results_to_output();
}
ef.get_text_messages().print_messages(); ef.get_text_messages().print_messages();
} }
@ -266,20 +267,20 @@ fn temporary(temporary: TemporaryArgs) {
tf.set_exclude_other_filesystems(exclude_other_filesystems.exclude_other_filesystems); tf.set_exclude_other_filesystems(exclude_other_filesystems.exclude_other_filesystems);
if delete_files { if delete_files {
tf.set_delete_method(temporary::DeleteMethod::Delete); tf.set_delete_method(DeleteMethod::Delete);
} }
tf.find_temporary_files(None, None); tf.find_temporary_files(None, None);
if let Some(file_name) = file_to_save.file_name() { if let Some(file_name) = file_to_save.file_name() {
if !tf.save_results_to_file(file_name) { if let Err(e) = tf.print_results_to_file(file_name) {
tf.get_text_messages().print_messages(); error!("Failed to save results to file {e}");
process::exit(1);
} }
} }
#[cfg(not(debug_assertions))] // This will show too much probably unnecessary data to debug, comment line only if needed if !cfg!(debug_assertions) {
tf.print_results(); tf.print_results_to_output();
}
tf.get_text_messages().print_messages(); tf.get_text_messages().print_messages();
} }
@ -322,14 +323,14 @@ fn similar_images(similar_images: SimilarImagesArgs) {
sf.find_similar_images(None, None); sf.find_similar_images(None, None);
if let Some(file_name) = file_to_save.file_name() { if let Some(file_name) = file_to_save.file_name() {
if !sf.save_results_to_file(file_name) { if let Err(e) = sf.print_results_to_file(file_name) {
sf.get_text_messages().print_messages(); error!("Failed to save results to file {e}");
process::exit(1);
} }
} }
#[cfg(not(debug_assertions))] // This will show too much probably unnecessary data to debug, comment line only if needed if !cfg!(debug_assertions) {
sf.print_results(); sf.print_results_to_output();
}
sf.get_text_messages().print_messages(); sf.get_text_messages().print_messages();
} }
@ -370,14 +371,14 @@ fn same_music(same_music: SameMusicArgs) {
mf.find_same_music(None, None); mf.find_same_music(None, None);
if let Some(file_name) = file_to_save.file_name() { if let Some(file_name) = file_to_save.file_name() {
if !mf.save_results_to_file(file_name) { if let Err(e) = mf.print_results_to_file(file_name) {
mf.get_text_messages().print_messages(); error!("Failed to save results to file {e}");
process::exit(1);
} }
} }
#[cfg(not(debug_assertions))] // This will show too much probably unnecessary data to debug, comment line only if needed if !cfg!(debug_assertions) {
mf.print_results(); mf.print_results_to_output();
}
mf.get_text_messages().print_messages(); mf.get_text_messages().print_messages();
} }
@ -407,20 +408,20 @@ fn invalid_symlinks(invalid_symlinks: InvalidSymlinksArgs) {
#[cfg(target_family = "unix")] #[cfg(target_family = "unix")]
ifs.set_exclude_other_filesystems(exclude_other_filesystems.exclude_other_filesystems); ifs.set_exclude_other_filesystems(exclude_other_filesystems.exclude_other_filesystems);
if delete_files { if delete_files {
ifs.set_delete_method(invalid_symlinks::DeleteMethod::Delete); ifs.set_delete_method(DeleteMethod::Delete);
} }
ifs.find_invalid_links(None, None); ifs.find_invalid_links(None, None);
if let Some(file_name) = file_to_save.file_name() { if let Some(file_name) = file_to_save.file_name() {
if !ifs.save_results_to_file(file_name) { if let Err(e) = ifs.print_results_to_file(file_name) {
ifs.get_text_messages().print_messages(); error!("Failed to save results to file {e}");
process::exit(1);
} }
} }
#[cfg(not(debug_assertions))] // This will show too much probably unnecessary data to debug, comment line only if needed if !cfg!(debug_assertions) {
ifs.print_results(); ifs.print_results_to_output();
}
ifs.get_text_messages().print_messages(); ifs.get_text_messages().print_messages();
} }
@ -451,20 +452,20 @@ fn broken_files(broken_files: BrokenFilesArgs) {
br.set_exclude_other_filesystems(exclude_other_filesystems.exclude_other_filesystems); br.set_exclude_other_filesystems(exclude_other_filesystems.exclude_other_filesystems);
if delete_files { if delete_files {
br.set_delete_method(broken_files::DeleteMethod::Delete); br.set_delete_method(DeleteMethod::Delete);
} }
br.find_broken_files(None, None); br.find_broken_files(None, None);
if let Some(file_name) = file_to_save.file_name() { if let Some(file_name) = file_to_save.file_name() {
if !br.save_results_to_file(file_name) { if let Err(e) = br.print_results_to_file(file_name) {
br.get_text_messages().print_messages(); error!("Failed to save results to file {e}");
process::exit(1);
} }
} }
#[cfg(not(debug_assertions))] // This will show too much probably unnecessary data to debug, comment line only if needed if !cfg!(debug_assertions) {
br.print_results(); br.print_results_to_output();
}
br.get_text_messages().print_messages(); br.get_text_messages().print_messages();
} }
@ -502,14 +503,14 @@ fn similar_videos(similar_videos: SimilarVideosArgs) {
vr.find_similar_videos(None, None); vr.find_similar_videos(None, None);
if let Some(file_name) = file_to_save.file_name() { if let Some(file_name) = file_to_save.file_name() {
if !vr.save_results_to_file(file_name) { if let Err(e) = vr.print_results_to_file(file_name) {
vr.get_text_messages().print_messages(); error!("Failed to save results to file {e}");
process::exit(1);
} }
} }
#[cfg(not(debug_assertions))] // This will show too much probably unnecessary data to debug, comment line only if needed if !cfg!(debug_assertions) {
vr.print_results(); vr.print_results_to_output();
}
vr.get_text_messages().print_messages(); vr.get_text_messages().print_messages();
} }
@ -538,16 +539,16 @@ fn bad_extensions(bad_extensions: BadExtensionsArgs) {
#[cfg(target_family = "unix")] #[cfg(target_family = "unix")]
be.set_exclude_other_filesystems(exclude_other_filesystems.exclude_other_filesystems); be.set_exclude_other_filesystems(exclude_other_filesystems.exclude_other_filesystems);
be.find_bad_extensions_files(None, None);
if let Some(file_name) = file_to_save.file_name() { if let Some(file_name) = file_to_save.file_name() {
if !be.save_results_to_file(file_name) { if let Err(e) = be.print_results_to_file(file_name) {
be.get_text_messages().print_messages(); error!("Failed to save results to file {e}");
process::exit(1);
} }
} }
be.find_bad_extensions_files(None, None); if !cfg!(debug_assertions) {
be.print_results_to_output();
#[cfg(not(debug_assertions))] // This will show too much probably unnecessary data to debug, comment line only if needed }
be.print_results();
be.get_text_messages().print_messages(); be.get_text_messages().print_messages();
} }

View file

@ -81,6 +81,7 @@ state = "0.6"
log = "0.4.20" log = "0.4.20"
handsome_logger = "0.8" handsome_logger = "0.8"
fun_time = { version = "0.3.1", features = ["log"] }
[features] [features]
default = [] default = []

View file

@ -1,15 +1,14 @@
use std::collections::{BTreeSet, HashMap}; use std::collections::{BTreeSet, HashMap};
use std::fs::File;
use std::io::prelude::*; use std::io::prelude::*;
use std::io::BufWriter;
use std::mem; use std::mem;
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc; use std::sync::Arc;
use crossbeam_channel::Receiver; use crossbeam_channel::Receiver;
use fun_time::fun_time;
use futures::channel::mpsc::UnboundedSender; use futures::channel::mpsc::UnboundedSender;
use log::{debug, info}; use log::debug;
use mime_guess::get_mime_extensions; use mime_guess::get_mime_extensions;
use rayon::prelude::*; use rayon::prelude::*;
@ -168,7 +167,6 @@ pub struct BadFileEntry {
pub proper_extensions: String, pub proper_extensions: String,
} }
/// Info struck with helpful information's about results
#[derive(Default)] #[derive(Default)]
pub struct Info { pub struct Info {
pub number_of_files_with_bad_extension: usize, pub number_of_files_with_bad_extension: usize,
@ -193,14 +191,8 @@ impl BadExtensions {
} }
} }
#[fun_time(message = "find_bad_extensions_files")]
pub fn find_bad_extensions_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) { pub fn find_bad_extensions_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
info!("Starting finding files with bad extensions");
let start_time = std::time::Instant::now();
self.find_bad_extensions_files_internal(stop_receiver, progress_sender);
info!("Ended finding files with bad extensions which took {:?}", start_time.elapsed());
}
fn find_bad_extensions_files_internal(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.optimize_dirs_before_start(); self.optimize_dirs_before_start();
if !self.check_files(stop_receiver, progress_sender) { if !self.check_files(stop_receiver, progress_sender) {
self.common_data.stopped_search = true; self.common_data.stopped_search = true;
@ -213,8 +205,8 @@ impl BadExtensions {
self.debug_print(); self.debug_print();
} }
#[fun_time(message = "check_files")]
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool { fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("check_files - start");
let result = DirTraversalBuilder::new() let result = DirTraversalBuilder::new()
.root_dirs(self.common_data.directories.included_directories.clone()) .root_dirs(self.common_data.directories.included_directories.clone())
.group_by(|_fe| ()) .group_by(|_fe| ())
@ -228,12 +220,10 @@ impl BadExtensions {
.recursive_search(self.common_data.recursive_search) .recursive_search(self.common_data.recursive_search)
.build() .build()
.run(); .run();
debug!("check_files - collected files");
let res = match result { match result {
DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => { DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => {
if let Some(files_to_check) = grouped_file_entries.get(&()) { self.files_to_check = grouped_file_entries.into_values().flatten().collect();
self.files_to_check = files_to_check.clone();
}
self.common_data.text_messages.warnings.extend(warnings); self.common_data.text_messages.warnings.extend(warnings);
true true
@ -242,13 +232,11 @@ impl BadExtensions {
unreachable!() unreachable!()
} }
DirTraversalResult::Stopped => false, DirTraversalResult::Stopped => false,
}; }
debug!("check_files - end");
res
} }
#[fun_time(message = "look_for_bad_extensions_files")]
fn look_for_bad_extensions_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool { fn look_for_bad_extensions_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("look_for_bad_extensions_files - start");
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) = let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
prepare_thread_handler_common(progress_sender, 1, 1, self.files_to_check.len(), CheckingMethod::None, self.get_cd().tool_type); prepare_thread_handler_common(progress_sender, 1, 1, self.files_to_check.len(), CheckingMethod::None, self.get_cd().tool_type);
@ -274,13 +262,12 @@ impl BadExtensions {
self.information.number_of_files_with_bad_extension = self.bad_extensions_files.len(); self.information.number_of_files_with_bad_extension = self.bad_extensions_files.len();
// Clean unused data debug!("Found {} files with invalid extension.", self.information.number_of_files_with_bad_extension);
self.files_to_check = Default::default();
debug!("look_for_bad_extensions_files - end");
true true
} }
#[fun_time(message = "verify_extensions")]
fn verify_extensions( fn verify_extensions(
&self, &self,
files_to_check: Vec<FileEntry>, files_to_check: Vec<FileEntry>,
@ -289,8 +276,7 @@ impl BadExtensions {
check_was_stopped: &AtomicBool, check_was_stopped: &AtomicBool,
hashmap_workarounds: &HashMap<&str, Vec<&str>>, hashmap_workarounds: &HashMap<&str, Vec<&str>>,
) -> Vec<BadFileEntry> { ) -> Vec<BadFileEntry> {
debug!("verify_extensions - start"); files_to_check
let res = files_to_check
.into_par_iter() .into_par_iter()
.map(|file_entry| { .map(|file_entry| {
atomic_counter.fetch_add(1, Ordering::Relaxed); atomic_counter.fetch_add(1, Ordering::Relaxed);
@ -339,9 +325,7 @@ impl BadExtensions {
.while_some() .while_some()
.filter(Option::is_some) .filter(Option::is_some)
.map(Option::unwrap) .map(Option::unwrap)
.collect::<Vec<_>>(); .collect::<Vec<_>>()
debug!("verify_extensions - end");
res
} }
fn get_and_validate_extension(&self, file_entry: &FileEntry, proper_extension: &str) -> Option<String> { fn get_and_validate_extension(&self, file_entry: &FileEntry, proper_extension: &str) -> Option<String> {
@ -417,12 +401,8 @@ impl Default for BadExtensions {
} }
impl DebugPrint for BadExtensions { impl DebugPrint for BadExtensions {
#[allow(dead_code)]
#[allow(unreachable_code)]
/// Debugging printing - only available on debug build
fn debug_print(&self) { fn debug_print(&self) {
#[cfg(not(debug_assertions))] if !cfg!(debug_assertions) {
{
return; return;
} }
println!("---------------DEBUG PRINT---------------"); println!("---------------DEBUG PRINT---------------");
@ -431,55 +411,20 @@ impl DebugPrint for BadExtensions {
} }
} }
impl SaveResults for BadExtensions { impl PrintResults for BadExtensions {
fn save_results_to_file(&mut self, file_name: &str) -> bool { fn write_results<T: Write>(&self, writer: &mut T) -> std::io::Result<()> {
let file_name: String = match file_name { writeln!(
"" => "results.txt".to_string(),
k => k.to_string(),
};
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(e) => {
self.common_data.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
return false;
}
};
let mut writer = BufWriter::new(file_handler);
if let Err(e) = writeln!(
writer, writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}", "Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
) { )?;
self.common_data writeln!(writer, "Found {} files with invalid extension.\n", self.information.number_of_files_with_bad_extension)?;
.text_messages
.errors
.push(format!("Failed to save results to file {file_name}, reason {e}"));
return false;
}
if !self.bad_extensions_files.is_empty() {
writeln!(writer, "Found {} files with invalid extension.", self.information.number_of_files_with_bad_extension).unwrap();
for file_entry in &self.bad_extensions_files {
writeln!(writer, "{} ----- {}", file_entry.path.display(), file_entry.proper_extensions).unwrap();
}
} else {
write!(writer, "Not found any files with invalid extension.").unwrap();
}
true
}
}
impl PrintResults for BadExtensions {
/// Print information's about duplicated entries
/// Only needed for CLI
fn print_results(&self) {
println!("Found {} files with invalid extension.\n", self.information.number_of_files_with_bad_extension);
for file_entry in &self.bad_extensions_files { for file_entry in &self.bad_extensions_files {
println!("{} ----- {}", file_entry.path.display(), file_entry.proper_extensions); writeln!(writer, "{} ----- {}", file_entry.path.display(), file_entry.proper_extensions)?;
} }
Ok(())
} }
} }

View file

@ -1,21 +1,22 @@
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::fs; use std::fs;
use std::fs::{DirEntry, File, Metadata}; use std::fs::{DirEntry, Metadata};
use std::io::{BufWriter, Write}; use std::io::Write;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc; use std::sync::Arc;
use crossbeam_channel::Receiver; use crossbeam_channel::Receiver;
use fun_time::fun_time;
use futures::channel::mpsc::UnboundedSender; use futures::channel::mpsc::UnboundedSender;
use humansize::{format_size, BINARY}; use humansize::{format_size, BINARY};
use log::{debug, info}; use log::debug;
use rayon::prelude::*; use rayon::prelude::*;
use crate::common::{check_folder_children, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, split_path}; use crate::common::{check_folder_children, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, split_path};
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType}; use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
use crate::common_tool::{CommonData, CommonToolData}; use crate::common_tool::{CommonData, CommonToolData, DeleteMethod};
use crate::common_traits::{DebugPrint, PrintResults, SaveResults}; use crate::common_traits::{DebugPrint, PrintResults};
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct FileEntry { pub struct FileEntry {
@ -30,25 +31,16 @@ pub enum SearchMode {
SmallestFiles, SmallestFiles,
} }
#[derive(Eq, PartialEq, Clone, Debug, Copy)]
pub enum DeleteMethod {
None,
Delete,
}
/// Info struck with helpful information's about results
#[derive(Default)] #[derive(Default)]
pub struct Info { pub struct Info {
pub number_of_real_files: usize, pub number_of_real_files: usize,
} }
/// Struct with required information's to work
pub struct BigFile { pub struct BigFile {
common_data: CommonToolData, common_data: CommonToolData,
information: Info, information: Info,
big_files: Vec<(u64, FileEntry)>, big_files: Vec<(u64, FileEntry)>,
number_of_files_to_check: usize, number_of_files_to_check: usize,
delete_method: DeleteMethod,
search_mode: SearchMode, search_mode: SearchMode,
} }
@ -59,19 +51,12 @@ impl BigFile {
information: Info::default(), information: Info::default(),
big_files: Default::default(), big_files: Default::default(),
number_of_files_to_check: 50, number_of_files_to_check: 50,
delete_method: DeleteMethod::None,
search_mode: SearchMode::BiggestFiles, search_mode: SearchMode::BiggestFiles,
} }
} }
#[fun_time(message = "find_big_files")]
pub fn find_big_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) { pub fn find_big_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
info!("Starting finding big files");
let start_time = std::time::Instant::now();
self.find_big_files_internal(stop_receiver, progress_sender);
info!("Ended finding big files which took {:?}", start_time.elapsed());
}
fn find_big_files_internal(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.optimize_dirs_before_start(); self.optimize_dirs_before_start();
if !self.look_for_big_files(stop_receiver, progress_sender) { if !self.look_for_big_files(stop_receiver, progress_sender) {
self.common_data.stopped_search = true; self.common_data.stopped_search = true;
@ -81,8 +66,8 @@ impl BigFile {
self.debug_print(); self.debug_print();
} }
#[fun_time(message = "look_for_big_files")]
fn look_for_big_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool { fn look_for_big_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("look_for_big_files - start");
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
let mut old_map: BTreeMap<u64, Vec<FileEntry>> = Default::default(); let mut old_map: BTreeMap<u64, Vec<FileEntry>> = Default::default();
@ -94,6 +79,7 @@ impl BigFile {
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) = let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) =
prepare_thread_handler_common(progress_sender, 0, 0, 0, CheckingMethod::None, self.common_data.tool_type); prepare_thread_handler_common(progress_sender, 0, 0, 0, CheckingMethod::None, self.common_data.tool_type);
debug!("Starting to search for big files");
while !folders_to_check.is_empty() { while !folders_to_check.is_empty() {
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
@ -148,11 +134,12 @@ impl BigFile {
} }
} }
debug!("Collected {} files", old_map.len());
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
self.extract_n_biggest_files(old_map); self.extract_n_biggest_files(old_map);
debug!("look_for_big_files - end");
true true
} }
@ -193,8 +180,8 @@ impl BigFile {
fe_result.push((fe.size, fe)); fe_result.push((fe.size, fe));
} }
#[fun_time(message = "extract_n_biggest_files")]
pub fn extract_n_biggest_files(&mut self, old_map: BTreeMap<u64, Vec<FileEntry>>) { pub fn extract_n_biggest_files(&mut self, old_map: BTreeMap<u64, Vec<FileEntry>>) {
debug!("extract_n_biggest_files - start");
let iter: Box<dyn Iterator<Item = _>>; let iter: Box<dyn Iterator<Item = _>>;
if self.search_mode == SearchMode::SmallestFiles { if self.search_mode == SearchMode::SmallestFiles {
iter = Box::new(old_map.into_iter()); iter = Box::new(old_map.into_iter());
@ -222,12 +209,10 @@ impl BigFile {
break; break;
} }
} }
debug!("extract_n_biggest_files - end");
} }
/// Function to delete files, from filed Vector
fn delete_files(&mut self) { fn delete_files(&mut self) {
match self.delete_method { match self.common_data.delete_method {
DeleteMethod::Delete => { DeleteMethod::Delete => {
for (_, file_entry) in &self.big_files { for (_, file_entry) in &self.big_files {
if fs::remove_file(&file_entry.path).is_err() { if fs::remove_file(&file_entry.path).is_err() {
@ -238,6 +223,7 @@ impl BigFile {
DeleteMethod::None => { DeleteMethod::None => {
//Just do nothing //Just do nothing
} }
_ => unreachable!(),
} }
} }
} }
@ -249,12 +235,8 @@ impl Default for BigFile {
} }
impl DebugPrint for BigFile { impl DebugPrint for BigFile {
#[allow(dead_code)]
#[allow(unreachable_code)]
/// Debugging printing - only available on debug build
fn debug_print(&self) { fn debug_print(&self) {
#[cfg(not(debug_assertions))] if !cfg!(debug_assertions) {
{
return; return;
} }
@ -266,62 +248,28 @@ impl DebugPrint for BigFile {
} }
} }
impl SaveResults for BigFile { impl PrintResults for BigFile {
/// Saving results to provided file fn write_results<T: Write>(&self, writer: &mut T) -> std::io::Result<()> {
fn save_results_to_file(&mut self, file_name: &str) -> bool { writeln!(
let file_name: String = match file_name {
"" => "results.txt".to_string(),
k => k.to_string(),
};
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(e) => {
self.common_data.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
return false;
}
};
let mut writer = BufWriter::new(file_handler);
if let Err(e) = writeln!(
writer, writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}", "Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
) { )?;
self.common_data
.text_messages
.errors
.push(format!("Failed to save results to file {file_name}, reason {e}"));
return false;
}
if self.information.number_of_real_files != 0 { if self.information.number_of_real_files != 0 {
if self.search_mode == SearchMode::BiggestFiles { if self.search_mode == SearchMode::BiggestFiles {
write!(writer, "{} the biggest files.\n\n", self.information.number_of_real_files).unwrap(); writeln!(writer, "{} the biggest files.\n\n", self.information.number_of_real_files)?;
} else { } else {
write!(writer, "{} the smallest files.\n\n", self.information.number_of_real_files).unwrap(); writeln!(writer, "{} the smallest files.\n\n", self.information.number_of_real_files)?;
} }
for (size, file_entry) in &self.big_files { for (size, file_entry) in &self.big_files {
writeln!(writer, "{} ({}) - {}", format_size(*size, BINARY), size, file_entry.path.display()).unwrap(); writeln!(writer, "{} ({}) - {}", format_size(*size, BINARY), size, file_entry.path.display())?;
} }
} else { } else {
write!(writer, "Not found any files.").unwrap(); write!(writer, "Not found any files.").unwrap();
} }
true Ok(())
}
}
impl PrintResults for BigFile {
fn print_results(&self) {
if self.search_mode == SearchMode::BiggestFiles {
println!("{} the biggest files.\n\n", self.information.number_of_real_files);
} else {
println!("{} the smallest files.\n\n", self.information.number_of_real_files);
}
for (size, file_entry) in &self.big_files {
println!("{} ({}) - {}", format_size(*size, BINARY), size, file_entry.path.display());
}
} }
} }
@ -347,10 +295,6 @@ impl BigFile {
&self.information &self.information
} }
pub fn set_delete_method(&mut self, delete_method: DeleteMethod) {
self.delete_method = delete_method;
}
pub fn set_number_of_files_to_check(&mut self, number_of_files_to_check: usize) { pub fn set_number_of_files_to_check(&mut self, number_of_files_to_check: usize) {
self.number_of_files_to_check = number_of_files_to_check; self.number_of_files_to_check = number_of_files_to_check;
} }

View file

@ -1,15 +1,16 @@
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::fs::{DirEntry, File, Metadata}; use std::fs::{DirEntry, File, Metadata};
use std::io::prelude::*; use std::io::prelude::*;
use std::io::BufWriter;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc; use std::sync::Arc;
use std::{fs, mem, panic}; use std::{fs, mem, panic};
use crossbeam_channel::Receiver; use crossbeam_channel::Receiver;
use fun_time::fun_time;
use futures::channel::mpsc::UnboundedSender; use futures::channel::mpsc::UnboundedSender;
use log::{debug, info}; use log::debug;
use pdf::file::FileOptions; use pdf::file::FileOptions;
use pdf::object::ParseOptions; use pdf::object::ParseOptions;
use pdf::PdfError; use pdf::PdfError;
@ -23,15 +24,9 @@ use crate::common::{
}; };
use crate::common_cache::{get_broken_files_cache_file, load_cache_from_file_generalized_by_path, save_cache_to_file_generalized}; use crate::common_cache::{get_broken_files_cache_file, load_cache_from_file_generalized_by_path, save_cache_to_file_generalized};
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType}; use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
use crate::common_tool::{CommonData, CommonToolData}; use crate::common_tool::{CommonData, CommonToolData, DeleteMethod};
use crate::common_traits::*; use crate::common_traits::*;
#[derive(Eq, PartialEq, Clone, Debug, Copy)]
pub enum DeleteMethod {
None,
Delete,
}
#[derive(Clone, Serialize, Deserialize)] #[derive(Clone, Serialize, Deserialize)]
pub struct FileEntry { pub struct FileEntry {
pub path: PathBuf, pub path: PathBuf,
@ -73,7 +68,6 @@ bitflags! {
} }
} }
/// Info struck with helpful information's about results
#[derive(Default)] #[derive(Default)]
pub struct Info { pub struct Info {
pub number_of_broken_files: usize, pub number_of_broken_files: usize,
@ -84,7 +78,6 @@ pub struct BrokenFiles {
information: Info, information: Info,
files_to_check: BTreeMap<String, FileEntry>, files_to_check: BTreeMap<String, FileEntry>,
broken_files: Vec<FileEntry>, broken_files: Vec<FileEntry>,
delete_method: DeleteMethod,
checked_types: CheckedTypes, checked_types: CheckedTypes,
} }
@ -94,20 +87,13 @@ impl BrokenFiles {
common_data: CommonToolData::new(ToolType::BrokenFiles), common_data: CommonToolData::new(ToolType::BrokenFiles),
information: Info::default(), information: Info::default(),
files_to_check: Default::default(), files_to_check: Default::default(),
delete_method: DeleteMethod::None,
broken_files: Default::default(), broken_files: Default::default(),
checked_types: CheckedTypes::PDF | CheckedTypes::AUDIO | CheckedTypes::IMAGE | CheckedTypes::ARCHIVE, checked_types: CheckedTypes::PDF | CheckedTypes::AUDIO | CheckedTypes::IMAGE | CheckedTypes::ARCHIVE,
} }
} }
#[fun_time(message = "find_broken_files")]
pub fn find_broken_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) { pub fn find_broken_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
info!("Starting finding broken files");
let start_time = std::time::Instant::now();
self.find_broken_files_internal(stop_receiver, progress_sender);
info!("Ended finding broken files which took {:?}", start_time.elapsed());
}
pub fn find_broken_files_internal(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.optimize_dirs_before_start(); self.optimize_dirs_before_start();
if !self.check_files(stop_receiver, progress_sender) { if !self.check_files(stop_receiver, progress_sender) {
self.common_data.stopped_search = true; self.common_data.stopped_search = true;
@ -121,24 +107,8 @@ impl BrokenFiles {
self.debug_print(); self.debug_print();
} }
pub const fn get_broken_files(&self) -> &Vec<FileEntry> { #[fun_time(message = "check_files")]
&self.broken_files
}
pub fn set_checked_types(&mut self, checked_types: CheckedTypes) {
self.checked_types = checked_types;
}
pub const fn get_information(&self) -> &Info {
&self.information
}
pub fn set_delete_method(&mut self, delete_method: DeleteMethod) {
self.delete_method = delete_method;
}
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool { fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("check_files - start");
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
// Add root folders for finding // Add root folders for finding
@ -149,6 +119,7 @@ impl BrokenFiles {
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) = let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) =
prepare_thread_handler_common(progress_sender, 0, 1, 0, CheckingMethod::None, self.common_data.tool_type); prepare_thread_handler_common(progress_sender, 0, 1, 0, CheckingMethod::None, self.common_data.tool_type);
debug!("check_files - starting to collect files");
while !folders_to_check.is_empty() { while !folders_to_check.is_empty() {
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
@ -191,6 +162,7 @@ impl BrokenFiles {
(dir_result, warnings, fe_result) (dir_result, warnings, fe_result)
}) })
.collect(); .collect();
debug!("check_files - collected files");
// Advance the frontier // Advance the frontier
folders_to_check.clear(); folders_to_check.clear();
@ -206,10 +178,9 @@ impl BrokenFiles {
} }
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
debug!("check_files - end");
true true
} }
fn get_file_entry( fn get_file_entry(
&self, &self,
metadata: &Metadata, metadata: &Metadata,
@ -348,8 +319,8 @@ impl BrokenFiles {
} }
} }
#[fun_time(message = "load_cache")]
fn load_cache(&mut self) -> (BTreeMap<String, FileEntry>, BTreeMap<String, FileEntry>, BTreeMap<String, FileEntry>) { fn load_cache(&mut self) -> (BTreeMap<String, FileEntry>, BTreeMap<String, FileEntry>, BTreeMap<String, FileEntry>) {
debug!("load_cache - start (using cache {})", self.common_data.use_cache);
let loaded_hash_map; let loaded_hash_map;
let mut records_already_cached: BTreeMap<String, FileEntry> = Default::default(); let mut records_already_cached: BTreeMap<String, FileEntry> = Default::default();
@ -372,17 +343,17 @@ impl BrokenFiles {
loaded_hash_map = Default::default(); loaded_hash_map = Default::default();
non_cached_files_to_check = files_to_check; non_cached_files_to_check = files_to_check;
} }
debug!("load_cache - end");
(loaded_hash_map, records_already_cached, non_cached_files_to_check) (loaded_hash_map, records_already_cached, non_cached_files_to_check)
} }
#[fun_time(message = "look_for_broken_files")]
fn look_for_broken_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool { fn look_for_broken_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("look_for_broken_files - start");
let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.load_cache(); let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.load_cache();
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) = let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) =
prepare_thread_handler_common(progress_sender, 1, 1, non_cached_files_to_check.len(), CheckingMethod::None, self.common_data.tool_type); prepare_thread_handler_common(progress_sender, 1, 1, non_cached_files_to_check.len(), CheckingMethod::None, self.common_data.tool_type);
debug!("look_for_broken_files - started finding for broken files");
let mut vec_file_entry: Vec<FileEntry> = non_cached_files_to_check let mut vec_file_entry: Vec<FileEntry> = non_cached_files_to_check
.into_par_iter() .into_par_iter()
.map(|(_, file_entry)| { .map(|(_, file_entry)| {
@ -404,6 +375,7 @@ impl BrokenFiles {
.filter(Option::is_some) .filter(Option::is_some)
.map(Option::unwrap) .map(Option::unwrap)
.collect::<Vec<FileEntry>>(); .collect::<Vec<FileEntry>>();
debug!("look_for_broken_files - ended finding for broken files");
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
@ -418,15 +390,14 @@ impl BrokenFiles {
.collect(); .collect();
self.information.number_of_broken_files = self.broken_files.len(); self.information.number_of_broken_files = self.broken_files.len();
debug!("Found {} broken files.", self.information.number_of_broken_files);
// Clean unused data // Clean unused data
self.files_to_check = Default::default(); self.files_to_check = Default::default();
debug!("look_for_broken_files - end");
true true
} }
#[fun_time(message = "save_to_cache")]
fn save_to_cache(&mut self, vec_file_entry: &[FileEntry], loaded_hash_map: BTreeMap<String, FileEntry>) { fn save_to_cache(&mut self, vec_file_entry: &[FileEntry], loaded_hash_map: BTreeMap<String, FileEntry>) {
debug!("save_to_cache - start, using cache {}", self.common_data.use_cache);
if self.common_data.use_cache { if self.common_data.use_cache {
// Must save all results to file, old loaded from file with all currently counted results // Must save all results to file, old loaded from file with all currently counted results
let mut all_results: BTreeMap<String, FileEntry> = Default::default(); let mut all_results: BTreeMap<String, FileEntry> = Default::default();
@ -441,12 +412,11 @@ impl BrokenFiles {
let messages = save_cache_to_file_generalized(&get_broken_files_cache_file(), &all_results, self.common_data.save_also_as_json, 0); let messages = save_cache_to_file_generalized(&get_broken_files_cache_file(), &all_results, self.common_data.save_also_as_json, 0);
self.get_text_messages_mut().extend_with_another_messages(messages); self.get_text_messages_mut().extend_with_another_messages(messages);
} }
debug!("save_to_cache - end");
} }
/// Function to delete files, from filed Vector #[fun_time(message = "delete_files")]
fn delete_files(&mut self) { fn delete_files(&mut self) {
match self.delete_method { match self.common_data.delete_method {
DeleteMethod::Delete => { DeleteMethod::Delete => {
for file_entry in &self.broken_files { for file_entry in &self.broken_files {
if fs::remove_file(&file_entry.path).is_err() { if fs::remove_file(&file_entry.path).is_err() {
@ -457,10 +427,26 @@ impl BrokenFiles {
DeleteMethod::None => { DeleteMethod::None => {
//Just do nothing //Just do nothing
} }
_ => {
unreachable!()
}
} }
} }
} }
impl BrokenFiles {
pub const fn get_broken_files(&self) -> &Vec<FileEntry> {
&self.broken_files
}
pub fn set_checked_types(&mut self, checked_types: CheckedTypes) {
self.checked_types = checked_types;
}
pub const fn get_information(&self) -> &Info {
&self.information
}
}
impl Default for BrokenFiles { impl Default for BrokenFiles {
fn default() -> Self { fn default() -> Self {
Self::new() Self::new()
@ -468,70 +454,32 @@ impl Default for BrokenFiles {
} }
impl DebugPrint for BrokenFiles { impl DebugPrint for BrokenFiles {
#[allow(dead_code)]
#[allow(unreachable_code)]
/// Debugging printing - only available on debug build
fn debug_print(&self) { fn debug_print(&self) {
#[cfg(not(debug_assertions))] if !cfg!(debug_assertions) {
{
return; return;
} }
println!("---------------DEBUG PRINT---------------");
println!("Delete Method - {:?}", self.delete_method);
self.debug_print_common(); self.debug_print_common();
println!("-----------------------------------------");
}
}
impl SaveResults for BrokenFiles {
fn save_results_to_file(&mut self, file_name: &str) -> bool {
let file_name: String = match file_name {
"" => "results.txt".to_string(),
k => k.to_string(),
};
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(e) => {
self.common_data.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
return false;
}
};
let mut writer = BufWriter::new(file_handler);
if let Err(e) = writeln!(
writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
) {
self.common_data
.text_messages
.errors
.push(format!("Failed to save results to file {file_name}, reason {e}"));
return false;
}
if !self.broken_files.is_empty() {
writeln!(writer, "Found {} broken files.", self.information.number_of_broken_files).unwrap();
for file_entry in &self.broken_files {
writeln!(writer, "{} - {}", file_entry.path.display(), file_entry.error_string).unwrap();
}
} else {
write!(writer, "Not found any broken files.").unwrap();
}
true
} }
} }
impl PrintResults for BrokenFiles { impl PrintResults for BrokenFiles {
/// Print information's about duplicated entries fn write_results<T: Write>(&self, writer: &mut T) -> std::io::Result<()> {
/// Only needed for CLI writeln!(
fn print_results(&self) { writer,
println!("Found {} broken files.\n", self.information.number_of_broken_files); "Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
for file_entry in &self.broken_files { self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
println!("{} - {}", file_entry.path.display(), file_entry.error_string); )?;
if !self.broken_files.is_empty() {
writeln!(writer, "Found {} broken files.", self.information.number_of_broken_files)?;
for file_entry in &self.broken_files {
writeln!(writer, "{} - {}", file_entry.path.display(), file_entry.error_string)?;
}
} else {
write!(writer, "Not found any broken files.")?;
} }
Ok(())
} }
} }

View file

@ -11,13 +11,14 @@ use std::{fs, thread};
#[cfg(feature = "heif")] #[cfg(feature = "heif")]
use anyhow::Result; use anyhow::Result;
use directories_next::ProjectDirs; use directories_next::ProjectDirs;
use fun_time::fun_time;
use futures::channel::mpsc::UnboundedSender; use futures::channel::mpsc::UnboundedSender;
use handsome_logger::{ColorChoice, ConfigBuilder, TerminalMode}; use handsome_logger::{ColorChoice, ConfigBuilder, TerminalMode};
use image::{DynamicImage, ImageBuffer, Rgb}; use image::{DynamicImage, ImageBuffer, Rgb};
use imagepipe::{ImageSource, Pipeline}; use imagepipe::{ImageSource, Pipeline};
#[cfg(feature = "heif")] #[cfg(feature = "heif")]
use libheif_rs::{ColorSpace, HeifContext, RgbChroma}; use libheif_rs::{ColorSpace, HeifContext, RgbChroma};
use log::{debug, LevelFilter, Record}; use log::{info, LevelFilter, Record};
// #[cfg(feature = "heif")] // #[cfg(feature = "heif")]
// use libheif_rs::LibHeif; // use libheif_rs::LibHeif;
@ -25,6 +26,7 @@ use crate::common_dir_traversal::{CheckingMethod, ProgressData, ToolType};
use crate::common_directory::Directories; use crate::common_directory::Directories;
use crate::common_items::ExcludedItems; use crate::common_items::ExcludedItems;
use crate::common_traits::ResultEntry; use crate::common_traits::ResultEntry;
use crate::CZKAWKA_VERSION;
static NUMBER_OF_THREADS: state::InitCell<usize> = state::InitCell::new(); static NUMBER_OF_THREADS: state::InitCell<usize> = state::InitCell::new();
@ -39,7 +41,7 @@ pub fn get_number_of_threads() -> usize {
fn filtering_messages(record: &Record) -> bool { fn filtering_messages(record: &Record) -> bool {
if let Some(module_path) = record.module_path() { if let Some(module_path) = record.module_path() {
!["symphonia", "i18n_embed"].iter().any(|&x| module_path.contains(x)) module_path.starts_with("czkawka")
} else { } else {
true true
} }
@ -52,6 +54,14 @@ pub fn setup_logger(disabled_printing: bool) {
handsome_logger::TermLogger::init(config, TerminalMode::Mixed, ColorChoice::Always).unwrap(); handsome_logger::TermLogger::init(config, TerminalMode::Mixed, ColorChoice::Always).unwrap();
} }
pub fn print_version_mode() {
info!(
"Czkawka version: {}, was compiled with {} mode",
CZKAWKA_VERSION,
if cfg!(debug_assertions) { "debug" } else { "release" }
);
}
pub fn set_default_number_of_threads() { pub fn set_default_number_of_threads() {
set_number_of_threads(num_cpus::get()); set_number_of_threads(num_cpus::get());
} }
@ -66,7 +76,6 @@ pub fn set_number_of_threads(thread_number: usize) {
rayon::ThreadPoolBuilder::new().num_threads(get_number_of_threads()).build_global().unwrap(); rayon::ThreadPoolBuilder::new().num_threads(get_number_of_threads()).build_global().unwrap();
} }
/// Class for common functions used across other class/functions
pub const RAW_IMAGE_EXTENSIONS: &[&str] = &[ pub const RAW_IMAGE_EXTENSIONS: &[&str] = &[
".mrw", ".arw", ".srf", ".sr2", ".mef", ".orf", ".srw", ".erf", ".kdc", ".kdc", ".dcs", ".rw2", ".raf", ".dcr", ".dng", ".pef", ".crw", ".iiq", ".3fr", ".nrw", ".nef", ".mos", ".mrw", ".arw", ".srf", ".sr2", ".mef", ".orf", ".srw", ".erf", ".kdc", ".kdc", ".dcs", ".rw2", ".raf", ".dcr", ".dng", ".pef", ".crw", ".iiq", ".3fr", ".nrw", ".nef", ".mos",
".cr2", ".ari", ".cr2", ".ari",
@ -225,17 +234,6 @@ pub fn create_crash_message(library_name: &str, file_path: &str, home_library_ur
} }
impl Common { impl Common {
/// Printing time which took between start and stop point and prints also function name
#[allow(unused_variables)]
pub fn print_time(start_time: SystemTime, end_time: SystemTime, function_name: &str) {
#[cfg(debug_assertions)]
println!(
"Execution of function \"{}\" took {:?}",
function_name,
end_time.duration_since(start_time).expect("Time cannot go reverse.")
);
}
pub fn delete_multiple_entries(entries: &[String]) -> Vec<String> { pub fn delete_multiple_entries(entries: &[String]) -> Vec<String> {
let mut path: &Path; let mut path: &Path;
let mut warnings: Vec<String> = Vec::new(); let mut warnings: Vec<String> = Vec::new();
@ -265,8 +263,6 @@ impl Common {
warning warning
} }
/// Function to check if directory match expression
pub fn regex_check(expression: &str, directory: impl AsRef<Path>) -> bool { pub fn regex_check(expression: &str, directory: impl AsRef<Path>) -> bool {
if expression == "*" { if expression == "*" {
return true; return true;
@ -441,11 +437,10 @@ pub fn prepare_thread_handler_common(
(progress_thread_sender, progress_thread_run, atomic_counter, check_was_stopped) (progress_thread_sender, progress_thread_run, atomic_counter, check_was_stopped)
} }
#[fun_time(message = "send_info_and_wait_for_ending_all_threads")]
pub fn send_info_and_wait_for_ending_all_threads(progress_thread_run: &Arc<AtomicBool>, progress_thread_handle: JoinHandle<()>) { pub fn send_info_and_wait_for_ending_all_threads(progress_thread_run: &Arc<AtomicBool>, progress_thread_handle: JoinHandle<()>) {
debug!("Sending info to stop all threads");
progress_thread_run.store(false, Ordering::Relaxed); progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap(); progress_thread_handle.join().unwrap();
debug!("All threads stopped");
} }
#[cfg(test)] #[cfg(test)]

View file

@ -3,6 +3,7 @@ use crate::common_messages::Messages;
use crate::common_traits::ResultEntry; use crate::common_traits::ResultEntry;
use crate::duplicate::HashType; use crate::duplicate::HashType;
use crate::similar_images::{convert_algorithm_to_string, convert_filters_to_string}; use crate::similar_images::{convert_algorithm_to_string, convert_filters_to_string};
use fun_time::fun_time;
use image::imageops::FilterType; use image::imageops::FilterType;
use image_hasher::HashAlg; use image_hasher::HashAlg;
use log::debug; use log::debug;
@ -40,11 +41,11 @@ pub fn get_duplicate_cache_file(type_of_hash: &HashType, is_prehash: bool) -> St
format!("cache_duplicates_{type_of_hash:?}{prehash_str}_61.bin") format!("cache_duplicates_{type_of_hash:?}{prehash_str}_61.bin")
} }
#[fun_time(message = "save_cache_to_file_generalized")]
pub fn save_cache_to_file_generalized<T>(cache_file_name: &str, hashmap: &BTreeMap<String, T>, save_also_as_json: bool, minimum_file_size: u64) -> Messages pub fn save_cache_to_file_generalized<T>(cache_file_name: &str, hashmap: &BTreeMap<String, T>, save_also_as_json: bool, minimum_file_size: u64) -> Messages
where where
T: Serialize + ResultEntry + Sized + Send + Sync, T: Serialize + ResultEntry + Sized + Send + Sync,
{ {
debug!("Saving cache to file {} (or also json alternative) - {} results", cache_file_name, hashmap.len());
let mut text_messages = Messages::new(); let mut text_messages = Messages::new();
if let Some(((file_handler, cache_file), (file_handler_json, cache_file_json))) = if let Some(((file_handler, cache_file), (file_handler_json, cache_file_json))) =
common::open_cache_folder(cache_file_name, true, save_also_as_json, &mut text_messages.warnings) common::open_cache_folder(cache_file_name, true, save_also_as_json, &mut text_messages.warnings)
@ -83,6 +84,7 @@ where
text_messages text_messages
} }
#[fun_time(message = "load_cache_from_file_generalized_by_path")]
pub fn load_cache_from_file_generalized_by_path<T>(cache_file_name: &str, delete_outdated_cache: bool, used_files: &BTreeMap<String, T>) -> (Messages, Option<BTreeMap<String, T>>) pub fn load_cache_from_file_generalized_by_path<T>(cache_file_name: &str, delete_outdated_cache: bool, used_files: &BTreeMap<String, T>) -> (Messages, Option<BTreeMap<String, T>>)
where where
for<'a> T: Deserialize<'a> + ResultEntry + Sized + Send + Sync + Clone, for<'a> T: Deserialize<'a> + ResultEntry + Sized + Send + Sync + Clone,
@ -102,6 +104,7 @@ where
(text_messages, Some(map_loaded_entries)) (text_messages, Some(map_loaded_entries))
} }
#[fun_time(message = "load_cache_from_file_generalized_by_size")]
pub fn load_cache_from_file_generalized_by_size<T>( pub fn load_cache_from_file_generalized_by_size<T>(
cache_file_name: &str, cache_file_name: &str,
delete_outdated_cache: bool, delete_outdated_cache: bool,
@ -132,6 +135,7 @@ where
(text_messages, Some(map_loaded_entries)) (text_messages, Some(map_loaded_entries))
} }
#[fun_time(message = "load_cache_from_file_generalized_by_path_from_size")]
pub fn load_cache_from_file_generalized_by_path_from_size<T>( pub fn load_cache_from_file_generalized_by_path_from_size<T>(
cache_file_name: &str, cache_file_name: &str,
delete_outdated_cache: bool, delete_outdated_cache: bool,
@ -162,11 +166,11 @@ where
(text_messages, Some(map_loaded_entries)) (text_messages, Some(map_loaded_entries))
} }
#[fun_time(message = "load_cache_from_file_generalized")]
fn load_cache_from_file_generalized<T>(cache_file_name: &str, delete_outdated_cache: bool, used_files: &BTreeMap<String, T>) -> (Messages, Option<Vec<T>>) fn load_cache_from_file_generalized<T>(cache_file_name: &str, delete_outdated_cache: bool, used_files: &BTreeMap<String, T>) -> (Messages, Option<Vec<T>>)
where where
for<'a> T: Deserialize<'a> + ResultEntry + Sized + Send + Sync + Clone, for<'a> T: Deserialize<'a> + ResultEntry + Sized + Send + Sync + Clone,
{ {
debug!("Loading cache from file {} (or json alternative)", cache_file_name);
let mut text_messages = Messages::new(); let mut text_messages = Messages::new();
if let Some(((file_handler, cache_file), (file_handler_json, cache_file_json))) = common::open_cache_folder(cache_file_name, false, true, &mut text_messages.warnings) { if let Some(((file_handler, cache_file), (file_handler_json, cache_file_json))) = common::open_cache_folder(cache_file_name, false, true, &mut text_messages.warnings) {
@ -198,8 +202,10 @@ where
}; };
} }
// Don't load cache data if destination file not exists debug!(
debug!("Starting to removing outdated cache entries"); "Starting removing outdated cache entries (removing non existent files from cache - {})",
delete_outdated_cache
);
let initial_number_of_entries = vec_loaded_entries.len(); let initial_number_of_entries = vec_loaded_entries.len();
vec_loaded_entries = vec_loaded_entries vec_loaded_entries = vec_loaded_entries
.into_par_iter() .into_par_iter()

View file

@ -6,7 +6,9 @@ use std::sync::atomic::Ordering;
use std::time::UNIX_EPOCH; use std::time::UNIX_EPOCH;
use crossbeam_channel::Receiver; use crossbeam_channel::Receiver;
use fun_time::fun_time;
use futures::channel::mpsc::UnboundedSender; use futures::channel::mpsc::UnboundedSender;
use log::debug;
use rayon::prelude::*; use rayon::prelude::*;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@ -338,6 +340,7 @@ where
F: Fn(&FileEntry) -> T, F: Fn(&FileEntry) -> T,
T: Ord + PartialOrd, T: Ord + PartialOrd,
{ {
#[fun_time(message = "run(collecting files/dirs)")]
pub fn run(self) -> DirTraversalResult<T> { pub fn run(self) -> DirTraversalResult<T> {
let mut all_warnings = vec![]; let mut all_warnings = vec![];
let mut grouped_file_entries: BTreeMap<T, Vec<FileEntry>> = BTreeMap::new(); let mut grouped_file_entries: BTreeMap<T, Vec<FileEntry>> = BTreeMap::new();
@ -497,6 +500,12 @@ where
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
debug!(
"Collected {} files, {} folders",
grouped_file_entries.values().map(Vec::len).sum::<usize>(),
folder_entries.len()
);
match collect { match collect {
Collect::Files | Collect::InvalidSymlinks => DirTraversalResult::SuccessFiles { Collect::Files | Collect::InvalidSymlinks => DirTraversalResult::SuccessFiles {
grouped_file_entries, grouped_file_entries,

View file

@ -3,6 +3,7 @@ use std::path::{Path, PathBuf};
use std::{fs, os::unix::fs::MetadataExt}; use std::{fs, os::unix::fs::MetadataExt};
use crate::common::Common; use crate::common::Common;
use crate::common_messages::Messages;
use crate::flc; use crate::flc;
use crate::localizer_core::generate_translation_hashmap; use crate::localizer_core::generate_translation_hashmap;
@ -25,15 +26,12 @@ impl Directories {
self.reference_directories = reference_directory; self.reference_directories = reference_directory;
} }
/// Setting included directories, at least one must be provided or scan won't start pub fn set_included_directory(&mut self, included_directory: Vec<PathBuf>) -> Messages {
pub fn set_included_directory(&mut self, included_directory: Vec<PathBuf>) -> (Vec<String>, Vec<String>, Vec<String>) { let mut messages: Messages = Messages::new();
let messages: Vec<String> = Vec::new();
let mut errors: Vec<String> = Vec::new();
let mut warnings: Vec<String> = Vec::new();
if included_directory.is_empty() { if included_directory.is_empty() {
errors.push(flc!("core_missing_no_chosen_included_directory")); messages.errors.push(flc!("core_missing_no_chosen_included_directory"));
return (messages, warnings, errors); return messages;
} }
let directories: Vec<PathBuf> = included_directory; let directories: Vec<PathBuf> = included_directory;
@ -41,7 +39,7 @@ impl Directories {
let mut checked_directories: Vec<PathBuf> = Vec::new(); let mut checked_directories: Vec<PathBuf> = Vec::new();
for directory in directories { for directory in directories {
if directory.to_string_lossy().contains('*') { if directory.to_string_lossy().contains('*') {
warnings.push(flc!( messages.warnings.push(flc!(
"core_directory_wildcard_no_supported", "core_directory_wildcard_no_supported",
generate_translation_hashmap(vec![("path", directory.display().to_string())]) generate_translation_hashmap(vec![("path", directory.display().to_string())])
)); ));
@ -50,7 +48,7 @@ impl Directories {
#[cfg(not(target_family = "windows"))] #[cfg(not(target_family = "windows"))]
if directory.is_relative() { if directory.is_relative() {
warnings.push(flc!( messages.warnings.push(flc!(
"core_directory_relative_path", "core_directory_relative_path",
generate_translation_hashmap(vec![("path", directory.display().to_string())]) generate_translation_hashmap(vec![("path", directory.display().to_string())])
)); ));
@ -58,7 +56,7 @@ impl Directories {
} }
#[cfg(target_family = "windows")] #[cfg(target_family = "windows")]
if directory.is_relative() && !directory.starts_with("\\") { if directory.is_relative() && !directory.starts_with("\\") {
warnings.push(flc!( messages.warnings.push(flc!(
"core_directory_relative_path", "core_directory_relative_path",
generate_translation_hashmap(vec![("path", directory.display().to_string())]) generate_translation_hashmap(vec![("path", directory.display().to_string())])
)); ));
@ -66,14 +64,14 @@ impl Directories {
} }
if !directory.exists() { if !directory.exists() {
warnings.push(flc!( messages.warnings.push(flc!(
"core_directory_must_exists", "core_directory_must_exists",
generate_translation_hashmap(vec![("path", directory.display().to_string())]) generate_translation_hashmap(vec![("path", directory.display().to_string())])
)); ));
continue; continue;
} }
if !directory.is_dir() { if !directory.is_dir() {
warnings.push(flc!( messages.warnings.push(flc!(
"core_directory_must_be_directory", "core_directory_must_be_directory",
generate_translation_hashmap(vec![("path", directory.display().to_string())]) generate_translation_hashmap(vec![("path", directory.display().to_string())])
)); ));
@ -83,23 +81,20 @@ impl Directories {
} }
if checked_directories.is_empty() { if checked_directories.is_empty() {
warnings.push(flc!("core_included_directory_zero_valid_directories")); messages.warnings.push(flc!("core_included_directory_zero_valid_directories"));
return (messages, warnings, errors); return messages;
} }
self.included_directories = checked_directories; self.included_directories = checked_directories;
(messages, warnings, errors) messages
} }
/// Setting absolute path to exclude from search pub fn set_excluded_directory(&mut self, excluded_directory: Vec<PathBuf>) -> Messages {
pub fn set_excluded_directory(&mut self, excluded_directory: Vec<PathBuf>) -> (Vec<String>, Vec<String>, Vec<String>) { let mut messages: Messages = Messages::new();
let messages: Vec<String> = Vec::new();
let mut errors: Vec<String> = Vec::new();
let mut warnings: Vec<String> = Vec::new();
if excluded_directory.is_empty() { if excluded_directory.is_empty() {
return (messages, warnings, errors); return messages;
} }
let directories: Vec<PathBuf> = excluded_directory; let directories: Vec<PathBuf> = excluded_directory;
@ -108,11 +103,11 @@ impl Directories {
for directory in directories { for directory in directories {
let directory_as_string = directory.to_string_lossy(); let directory_as_string = directory.to_string_lossy();
if directory_as_string == "/" { if directory_as_string == "/" {
errors.push(flc!("core_excluded_directory_pointless_slash")); messages.errors.push(flc!("core_excluded_directory_pointless_slash"));
break; break;
} }
if directory_as_string.contains('*') { if directory_as_string.contains('*') {
warnings.push(flc!( messages.warnings.push(flc!(
"core_directory_wildcard_no_supported", "core_directory_wildcard_no_supported",
generate_translation_hashmap(vec![("path", directory.display().to_string())]) generate_translation_hashmap(vec![("path", directory.display().to_string())])
)); ));
@ -120,7 +115,7 @@ impl Directories {
} }
#[cfg(not(target_family = "windows"))] #[cfg(not(target_family = "windows"))]
if directory.is_relative() { if directory.is_relative() {
warnings.push(flc!( messages.warnings.push(flc!(
"core_directory_relative_path", "core_directory_relative_path",
generate_translation_hashmap(vec![("path", directory.display().to_string())]) generate_translation_hashmap(vec![("path", directory.display().to_string())])
)); ));
@ -128,7 +123,7 @@ impl Directories {
} }
#[cfg(target_family = "windows")] #[cfg(target_family = "windows")]
if directory.is_relative() && !directory.starts_with("\\") { if directory.is_relative() && !directory.starts_with("\\") {
warnings.push(flc!( messages.warnings.push(flc!(
"core_directory_relative_path", "core_directory_relative_path",
generate_translation_hashmap(vec![("path", directory.display().to_string())]) generate_translation_hashmap(vec![("path", directory.display().to_string())])
)); ));
@ -140,7 +135,7 @@ impl Directories {
continue; continue;
} }
if !directory.is_dir() { if !directory.is_dir() {
warnings.push(flc!( messages.warnings.push(flc!(
"core_directory_must_be_directory", "core_directory_must_be_directory",
generate_translation_hashmap(vec![("path", directory.display().to_string())]) generate_translation_hashmap(vec![("path", directory.display().to_string())])
)); ));
@ -150,7 +145,7 @@ impl Directories {
} }
self.excluded_directories = checked_directories; self.excluded_directories = checked_directories;
(messages, warnings, errors) messages
} }
#[cfg(target_family = "unix")] #[cfg(target_family = "unix")]
@ -158,11 +153,8 @@ impl Directories {
self.exclude_other_filesystems = Some(exclude_other_filesystems); self.exclude_other_filesystems = Some(exclude_other_filesystems);
} }
/// Remove unused entries when included or excluded overlaps with each other or are duplicated etc. pub fn optimize_directories(&mut self, recursive_search: bool) -> Messages {
pub fn optimize_directories(&mut self, recursive_search: bool) -> (Vec<String>, Vec<String>, Vec<String>) { let mut messages: Messages = Messages::new();
let messages: Vec<String> = Vec::new();
let mut errors: Vec<String> = Vec::new();
let warnings: Vec<String> = Vec::new();
let mut optimized_included: Vec<PathBuf> = Vec::new(); let mut optimized_included: Vec<PathBuf> = Vec::new();
let mut optimized_excluded: Vec<PathBuf> = Vec::new(); let mut optimized_excluded: Vec<PathBuf> = Vec::new();
@ -291,8 +283,8 @@ impl Directories {
} }
if self.included_directories.is_empty() { if self.included_directories.is_empty() {
errors.push(flc!("core_directory_overlap")); messages.errors.push(flc!("core_directory_overlap"));
return (messages, warnings, errors); return messages;
} }
// Not needed, but better is to have sorted everything // Not needed, but better is to have sorted everything
@ -305,7 +297,7 @@ impl Directories {
for d in &self.included_directories { for d in &self.included_directories {
match fs::metadata(d) { match fs::metadata(d) {
Ok(m) => self.included_dev_ids.push(m.dev()), Ok(m) => self.included_dev_ids.push(m.dev()),
Err(_) => errors.push(flc!( Err(_) => messages.errors.push(flc!(
"core_directory_unable_to_get_device_id", "core_directory_unable_to_get_device_id",
generate_translation_hashmap(vec![("path", d.display().to_string())]) generate_translation_hashmap(vec![("path", d.display().to_string())])
)), )),
@ -313,14 +305,13 @@ impl Directories {
} }
} }
(messages, warnings, errors) messages
} }
pub fn is_in_referenced_directory(&self, path: &Path) -> bool { pub fn is_in_referenced_directory(&self, path: &Path) -> bool {
self.reference_directories.iter().any(|e| path.starts_with(e)) self.reference_directories.iter().any(|e| path.starts_with(e))
} }
/// Checks whether a specified directory is excluded from searching
pub fn is_excluded(&self, path: impl AsRef<Path>) -> bool { pub fn is_excluded(&self, path: impl AsRef<Path>) -> bool {
let path = path.as_ref(); let path = path.as_ref();
#[cfg(target_family = "windows")] #[cfg(target_family = "windows")]
@ -334,8 +325,6 @@ impl Directories {
self.exclude_other_filesystems.unwrap_or(false) self.exclude_other_filesystems.unwrap_or(false)
} }
/// Checks whether a specified directory is on other filesystems rather then include
/// directories
#[cfg(target_family = "unix")] #[cfg(target_family = "unix")]
pub fn is_on_other_filesystems(&self, path: impl AsRef<Path>) -> Result<bool, String> { pub fn is_on_other_filesystems(&self, path: impl AsRef<Path>) -> Result<bool, String> {
let path = path.as_ref(); let path = path.as_ref();

View file

@ -1,3 +1,5 @@
use crate::common_messages::Messages;
#[derive(Debug, Clone, Default)] #[derive(Debug, Clone, Default)]
pub struct Extensions { pub struct Extensions {
file_extensions: Vec<String>, file_extensions: Vec<String>,
@ -9,13 +11,11 @@ impl Extensions {
} }
/// List of allowed extensions, only files with this extensions will be checking if are duplicates /// List of allowed extensions, only files with this extensions will be checking if are duplicates
/// After, extensions cannot contains any dot, commas etc. /// After, extensions cannot contains any dot, commas etc.
pub fn set_allowed_extensions(&mut self, mut allowed_extensions: String) -> (Vec<String>, Vec<String>, Vec<String>) { pub fn set_allowed_extensions(&mut self, mut allowed_extensions: String) -> Messages {
let mut messages = Vec::new(); let mut messages = Messages::new();
let mut warnings = Vec::new();
let errors = Vec::new();
if allowed_extensions.trim().is_empty() { if allowed_extensions.trim().is_empty() {
return (messages, warnings, errors); return messages;
} }
allowed_extensions = allowed_extensions.replace("IMAGE", "jpg,kra,gif,png,bmp,tiff,hdr,svg"); allowed_extensions = allowed_extensions.replace("IMAGE", "jpg,kra,gif,png,bmp,tiff,hdr,svg");
allowed_extensions = allowed_extensions.replace("VIDEO", "mp4,flv,mkv,webm,vob,ogv,gifv,avi,mov,wmv,mpg,m4v,m4p,mpeg,3gp"); allowed_extensions = allowed_extensions.replace("VIDEO", "mp4,flv,mkv,webm,vob,ogv,gifv,avi,mov,wmv,mpg,m4v,m4p,mpeg,3gp");
@ -33,12 +33,12 @@ impl Extensions {
} }
if extension[1..].contains('.') { if extension[1..].contains('.') {
warnings.push(format!("{extension} is not valid extension because contains dot inside")); messages.warnings.push(format!("{extension} is not valid extension because contains dot inside"));
continue; continue;
} }
if extension[1..].contains(' ') { if extension[1..].contains(' ') {
warnings.push(format!("{extension} is not valid extension because contains empty space inside")); messages.warnings.push(format!("{extension} is not valid extension because contains empty space inside"));
continue; continue;
} }
@ -48,9 +48,11 @@ impl Extensions {
} }
if self.file_extensions.is_empty() { if self.file_extensions.is_empty() {
messages.push("No valid extensions were provided, so allowing all extensions by default.".to_string()); messages
.messages
.push("No valid extensions were provided, so allowing all extensions by default.".to_string());
} }
(messages, warnings, errors) messages
} }
pub fn matches_filename(&self, file_name: &str) -> bool { pub fn matches_filename(&self, file_name: &str) -> bool {

View file

@ -16,8 +16,7 @@ impl ExcludedItems {
pub fn new() -> Self { pub fn new() -> Self {
Default::default() Default::default()
} }
/// Setting excluded items which needs to contains * wildcard
/// Are a lot of slower than absolute path, so it should be used to heavy
pub fn set_excluded_items(&mut self, excluded_items: Vec<String>) -> (Vec<String>, Vec<String>, Vec<String>) { pub fn set_excluded_items(&mut self, excluded_items: Vec<String>) -> (Vec<String>, Vec<String>, Vec<String>) {
let messages: Vec<String> = Vec::new(); let messages: Vec<String> = Vec::new();
let mut warnings: Vec<String> = Vec::new(); let mut warnings: Vec<String> = Vec::new();
@ -54,7 +53,6 @@ impl ExcludedItems {
(messages, warnings, errors) (messages, warnings, errors)
} }
/// Checks whether a specified path is excluded from searching
pub fn is_excluded(&self, path: impl AsRef<Path>) -> bool { pub fn is_excluded(&self, path: impl AsRef<Path>) -> bool {
#[cfg(target_family = "windows")] #[cfg(target_family = "windows")]
let path = Common::normalize_windows_path(path); let path = Common::normalize_windows_path(path);

View file

@ -14,7 +14,7 @@ pub struct CommonToolData {
pub(crate) allowed_extensions: Extensions, pub(crate) allowed_extensions: Extensions,
pub(crate) excluded_items: ExcludedItems, pub(crate) excluded_items: ExcludedItems,
pub(crate) recursive_search: bool, pub(crate) recursive_search: bool,
// delete_method: DeleteMethod, // ? pub(crate) delete_method: DeleteMethod,
pub(crate) maximal_file_size: u64, pub(crate) maximal_file_size: u64,
pub(crate) minimal_file_size: u64, pub(crate) minimal_file_size: u64,
pub(crate) stopped_search: bool, pub(crate) stopped_search: bool,
@ -24,6 +24,18 @@ pub struct CommonToolData {
pub(crate) use_reference_folders: bool, pub(crate) use_reference_folders: bool,
} }
#[derive(Eq, PartialEq, Clone, Debug, Copy, Default)]
pub enum DeleteMethod {
#[default]
None,
Delete, // Just delete items
AllExceptNewest,
AllExceptOldest,
OneOldest,
OneNewest,
HardLink,
}
impl CommonToolData { impl CommonToolData {
pub fn new(tool_type: ToolType) -> Self { pub fn new(tool_type: ToolType) -> Self {
Self { Self {
@ -33,6 +45,7 @@ impl CommonToolData {
allowed_extensions: Extensions::new(), allowed_extensions: Extensions::new(),
excluded_items: ExcludedItems::new(), excluded_items: ExcludedItems::new(),
recursive_search: true, recursive_search: true,
delete_method: DeleteMethod::None,
maximal_file_size: u64::MAX, maximal_file_size: u64::MAX,
minimal_file_size: 8192, minimal_file_size: 8192,
stopped_search: false, stopped_search: false,
@ -128,22 +141,25 @@ pub trait CommonData {
self.get_cd().use_reference_folders self.get_cd().use_reference_folders
} }
fn set_delete_method(&mut self, delete_method: DeleteMethod) {
self.get_cd_mut().delete_method = delete_method;
}
fn get_delete_method(&self) -> DeleteMethod {
self.get_cd().delete_method
}
fn set_included_directory(&mut self, included_directory: Vec<PathBuf>) { fn set_included_directory(&mut self, included_directory: Vec<PathBuf>) {
let (messages, warnings, errors) = self.get_cd_mut().directories.set_included_directory(included_directory); let messages = self.get_cd_mut().directories.set_included_directory(included_directory);
self.get_cd_mut().text_messages.extend_messages_with(messages, warnings, errors); self.get_cd_mut().text_messages.extend_with_another_messages(messages);
} }
fn set_excluded_directory(&mut self, excluded_directory: Vec<PathBuf>) { fn set_excluded_directory(&mut self, excluded_directory: Vec<PathBuf>) {
let (messages, warnings, errors) = self.get_cd_mut().directories.set_excluded_directory(excluded_directory); let messages = self.get_cd_mut().directories.set_excluded_directory(excluded_directory);
self.get_cd_mut().text_messages.messages.extend(messages); self.get_cd_mut().text_messages.extend_with_another_messages(messages);
self.get_cd_mut().text_messages.warnings.extend(warnings);
self.get_cd_mut().text_messages.errors.extend(errors);
} }
fn set_allowed_extensions(&mut self, allowed_extensions: String) { fn set_allowed_extensions(&mut self, allowed_extensions: String) {
let (messages, warnings, errors) = self.get_cd_mut().allowed_extensions.set_allowed_extensions(allowed_extensions); let messages = self.get_cd_mut().allowed_extensions.set_allowed_extensions(allowed_extensions);
self.get_cd_mut().text_messages.messages.extend(messages); self.get_cd_mut().text_messages.extend_with_another_messages(messages);
self.get_cd_mut().text_messages.warnings.extend(warnings);
self.get_cd_mut().text_messages.errors.extend(errors);
} }
fn set_excluded_items(&mut self, excluded_items: Vec<String>) { fn set_excluded_items(&mut self, excluded_items: Vec<String>) {
@ -155,8 +171,8 @@ pub trait CommonData {
fn optimize_dirs_before_start(&mut self) { fn optimize_dirs_before_start(&mut self) {
let recursive_search = self.get_cd().recursive_search; let recursive_search = self.get_cd().recursive_search;
let (messages, warnings, errors) = self.get_cd_mut().directories.optimize_directories(recursive_search); let messages = self.get_cd_mut().directories.optimize_directories(recursive_search);
self.get_cd_mut().text_messages.extend_messages_with(messages, warnings, errors); self.get_cd_mut().text_messages.extend_with_another_messages(messages);
} }
fn debug_print_common(&self) { fn debug_print_common(&self) {
@ -172,6 +188,7 @@ pub trait CommonData {
println!("Use cache: {:?}", self.get_cd().use_cache); println!("Use cache: {:?}", self.get_cd().use_cache);
println!("Delete outdated cache: {:?}", self.get_cd().delete_outdated_cache); println!("Delete outdated cache: {:?}", self.get_cd().delete_outdated_cache);
println!("Save also as json: {:?}", self.get_cd().save_also_as_json); println!("Save also as json: {:?}", self.get_cd().save_also_as_json);
println!("Delete method: {:?}", self.get_cd().delete_method);
println!("---------------DEBUG PRINT MESSAGES---------------"); println!("---------------DEBUG PRINT MESSAGES---------------");
println!("Errors size - {}", self.get_cd().text_messages.errors.len()); println!("Errors size - {}", self.get_cd().text_messages.errors.len());

View file

@ -1,15 +1,36 @@
use fun_time::fun_time;
use std::fs::File;
use std::io::{BufWriter, Write};
use std::path::Path; use std::path::Path;
pub trait DebugPrint { pub trait DebugPrint {
fn debug_print(&self); fn debug_print(&self);
} }
pub trait SaveResults {
fn save_results_to_file(&mut self, file_name: &str) -> bool;
}
pub trait PrintResults { pub trait PrintResults {
fn print_results(&self); fn write_results<T: Write>(&self, writer: &mut T) -> std::io::Result<()>;
#[fun_time(message = "print_results_to_output")]
fn print_results_to_output(&self) {
let stdout = std::io::stdout();
let mut handle = stdout.lock();
self.write_results(&mut handle).unwrap();
handle.flush().unwrap();
}
#[fun_time(message = "print_results_to_file")]
fn print_results_to_file(&self, file_name: &str) -> std::io::Result<()> {
let file_name: String = match file_name {
"" => "results.txt".to_string(),
k => k.to_string(),
};
let file_handler = File::create(file_name)?;
let mut writer = BufWriter::new(file_handler);
self.write_results(&mut writer)?;
writer.flush()?;
Ok(())
}
} }
pub trait ResultEntry { pub trait ResultEntry {

File diff suppressed because it is too large Load diff

View file

@ -1,34 +1,25 @@
use std::fs; use std::fs;
use std::fs::File;
use std::io::prelude::*; use std::io::prelude::*;
use std::io::BufWriter;
use crossbeam_channel::Receiver; use crossbeam_channel::Receiver;
use fun_time::fun_time;
use futures::channel::mpsc::UnboundedSender; use futures::channel::mpsc::UnboundedSender;
use log::{debug, info}; use log::debug;
use crate::common_dir_traversal::{DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData, ToolType}; use crate::common_dir_traversal::{DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData, ToolType};
use crate::common_tool::{CommonData, CommonToolData}; use crate::common_tool::{CommonData, CommonToolData, DeleteMethod};
use crate::common_traits::*; use crate::common_traits::*;
#[derive(Eq, PartialEq, Clone, Debug)]
pub enum DeleteMethod {
None,
Delete,
}
/// Info struck with helpful information's about results
#[derive(Default)] #[derive(Default)]
pub struct Info { pub struct Info {
pub number_of_empty_files: usize, pub number_of_empty_files: usize,
} }
/// Struct with required information's to work
pub struct EmptyFiles { pub struct EmptyFiles {
common_data: CommonToolData, common_data: CommonToolData,
information: Info, information: Info,
empty_files: Vec<FileEntry>, empty_files: Vec<FileEntry>,
delete_method: DeleteMethod,
} }
impl CommonData for EmptyFiles { impl CommonData for EmptyFiles {
@ -46,18 +37,11 @@ impl EmptyFiles {
common_data: CommonToolData::new(ToolType::EmptyFiles), common_data: CommonToolData::new(ToolType::EmptyFiles),
information: Info::default(), information: Info::default(),
empty_files: vec![], empty_files: vec![],
delete_method: DeleteMethod::None,
} }
} }
#[fun_time(message = "find_empty_files")]
pub fn find_empty_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) { pub fn find_empty_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
info!("Starting finding empty files");
let start_time = std::time::Instant::now();
self.find_empty_files_internal(stop_receiver, progress_sender);
info!("Ended finding empty files which took {:?}", start_time.elapsed());
}
fn find_empty_files_internal(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.optimize_dirs_before_start(); self.optimize_dirs_before_start();
if !self.check_files(stop_receiver, progress_sender) { if !self.check_files(stop_receiver, progress_sender) {
self.common_data.stopped_search = true; self.common_data.stopped_search = true;
@ -67,9 +51,8 @@ impl EmptyFiles {
self.debug_print(); self.debug_print();
} }
/// Check files for any with size == 0 #[fun_time(message = "check_files")]
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool { fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("check_files - start");
let result = DirTraversalBuilder::new() let result = DirTraversalBuilder::new()
.root_dirs(self.common_data.directories.included_directories.clone()) .root_dirs(self.common_data.directories.included_directories.clone())
.group_by(|_fe| ()) .group_by(|_fe| ())
@ -83,29 +66,27 @@ impl EmptyFiles {
.recursive_search(self.common_data.recursive_search) .recursive_search(self.common_data.recursive_search)
.build() .build()
.run(); .run();
debug!("check_files - collected files to check");
let res = match result { match result {
DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => { DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => {
if let Some(empty_files) = grouped_file_entries.get(&()) { self.empty_files = grouped_file_entries.into_values().flatten().collect();
self.empty_files = empty_files.clone();
}
self.information.number_of_empty_files = self.empty_files.len(); self.information.number_of_empty_files = self.empty_files.len();
self.common_data.text_messages.warnings.extend(warnings); self.common_data.text_messages.warnings.extend(warnings);
debug!("Found {} empty files.", self.information.number_of_empty_files);
true true
} }
DirTraversalResult::SuccessFolders { .. } => { DirTraversalResult::SuccessFolders { .. } => {
unreachable!() unreachable!()
} }
DirTraversalResult::Stopped => false, DirTraversalResult::Stopped => false,
}; }
debug!("check_files - end");
res
} }
/// Function to delete files, from filed Vector #[fun_time(message = "delete_files")]
fn delete_files(&mut self) { fn delete_files(&mut self) {
match self.delete_method { match self.common_data.delete_method {
DeleteMethod::Delete => { DeleteMethod::Delete => {
for file_entry in &self.empty_files { for file_entry in &self.empty_files {
if fs::remove_file(file_entry.path.clone()).is_err() { if fs::remove_file(file_entry.path.clone()).is_err() {
@ -116,6 +97,9 @@ impl EmptyFiles {
DeleteMethod::None => { DeleteMethod::None => {
//Just do nothing //Just do nothing
} }
_ => {
unreachable!()
}
} }
} }
} }
@ -127,71 +111,35 @@ impl Default for EmptyFiles {
} }
impl DebugPrint for EmptyFiles { impl DebugPrint for EmptyFiles {
#[allow(dead_code)]
#[allow(unreachable_code)]
/// Debugging printing - only available on debug build
fn debug_print(&self) { fn debug_print(&self) {
#[cfg(not(debug_assertions))] if !cfg!(debug_assertions) {
{
return; return;
} }
println!("---------------DEBUG PRINT---------------"); println!("---------------DEBUG PRINT---------------");
println!("Empty list size - {}", self.empty_files.len()); println!("Empty list size - {}", self.empty_files.len());
println!("Delete Method - {:?}", self.delete_method);
self.debug_print_common(); self.debug_print_common();
println!("-----------------------------------------"); println!("-----------------------------------------");
} }
} }
impl SaveResults for EmptyFiles { impl PrintResults for EmptyFiles {
fn save_results_to_file(&mut self, file_name: &str) -> bool { fn write_results<T: Write>(&self, writer: &mut T) -> std::io::Result<()> {
let file_name: String = match file_name { writeln!(
"" => "results.txt".to_string(),
k => k.to_string(),
};
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(e) => {
self.common_data.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
return false;
}
};
let mut writer = BufWriter::new(file_handler);
if let Err(e) = writeln!(
writer, writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}", "Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
) { )?;
self.common_data
.text_messages
.errors
.push(format!("Failed to save results to file {file_name}, reason {e}"));
return false;
}
if !self.empty_files.is_empty() { if !self.empty_files.is_empty() {
writeln!(writer, "Found {} empty files.", self.information.number_of_empty_files).unwrap(); writeln!(writer, "Found {} empty files.", self.information.number_of_empty_files)?;
for file_entry in &self.empty_files { for file_entry in &self.empty_files {
writeln!(writer, "{}", file_entry.path.display()).unwrap(); writeln!(writer, "{}", file_entry.path.display())?;
} }
} else { } else {
write!(writer, "Not found any empty files.").unwrap(); write!(writer, "Not found any empty files.")?;
} }
true Ok(())
}
}
impl PrintResults for EmptyFiles {
/// Print information's about duplicated entries
/// Only needed for CLI
fn print_results(&self) {
println!("Found {} empty files.\n", self.information.number_of_empty_files);
for file_entry in &self.empty_files {
println!("{}", file_entry.path.display());
}
} }
} }
@ -203,8 +151,4 @@ impl EmptyFiles {
pub const fn get_information(&self) -> &Info { pub const fn get_information(&self) -> &Info {
&self.information &self.information
} }
pub fn set_delete_method(&mut self, delete_method: DeleteMethod) {
self.delete_method = delete_method;
}
} }

View file

@ -1,18 +1,18 @@
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::fs; use std::fs;
use std::fs::File;
use std::io::{BufWriter, Write}; use std::io::Write;
use std::path::PathBuf; use std::path::PathBuf;
use crossbeam_channel::Receiver; use crossbeam_channel::Receiver;
use fun_time::fun_time;
use futures::channel::mpsc::UnboundedSender; use futures::channel::mpsc::UnboundedSender;
use log::{debug, info}; use log::debug;
use crate::common_dir_traversal::{Collect, DirTraversalBuilder, DirTraversalResult, FolderEmptiness, FolderEntry, ProgressData, ToolType}; use crate::common_dir_traversal::{Collect, DirTraversalBuilder, DirTraversalResult, FolderEmptiness, FolderEntry, ProgressData, ToolType};
use crate::common_tool::{CommonData, CommonToolData}; use crate::common_tool::{CommonData, CommonToolData};
use crate::common_traits::{DebugPrint, PrintResults, SaveResults}; use crate::common_traits::{DebugPrint, PrintResults};
/// Struct to store most basics info about all folder
pub struct EmptyFolder { pub struct EmptyFolder {
common_data: CommonToolData, common_data: CommonToolData,
information: Info, information: Info,
@ -20,16 +20,12 @@ pub struct EmptyFolder {
empty_folder_list: BTreeMap<PathBuf, FolderEntry>, // Path, FolderEntry empty_folder_list: BTreeMap<PathBuf, FolderEntry>, // Path, FolderEntry
} }
/// Info struck with helpful information's about results
#[derive(Default)] #[derive(Default)]
pub struct Info { pub struct Info {
pub number_of_empty_folders: usize, pub number_of_empty_folders: usize,
} }
/// Method implementation for `EmptyFolder`
impl EmptyFolder { impl EmptyFolder {
/// New function providing basics values
pub fn new() -> Self { pub fn new() -> Self {
Self { Self {
common_data: CommonToolData::new(ToolType::EmptyFolders), common_data: CommonToolData::new(ToolType::EmptyFolders),
@ -47,14 +43,8 @@ impl EmptyFolder {
&self.information &self.information
} }
#[fun_time(message = "find_empty_folders")]
pub fn find_empty_folders(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) { pub fn find_empty_folders(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
info!("Starting finding empty folders");
let start_time = std::time::Instant::now();
self.find_empty_folders_internal(stop_receiver, progress_sender);
info!("Ended finding empty folders which took {:?}", start_time.elapsed());
}
fn find_empty_folders_internal(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.optimize_dirs_before_start(); self.optimize_dirs_before_start();
if !self.check_for_empty_folders(stop_receiver, progress_sender) { if !self.check_for_empty_folders(stop_receiver, progress_sender) {
self.common_data.stopped_search = true; self.common_data.stopped_search = true;
@ -67,12 +57,6 @@ impl EmptyFolder {
self.debug_print(); self.debug_print();
} }
pub fn set_delete_folder(&mut self, delete_folder: bool) {
self.delete_folders = delete_folder;
}
/// Clean directory tree
/// If directory contains only 2 empty folders, then this directory should be removed instead two empty folders inside because it will produce another empty folder.
fn optimize_folders(&mut self) { fn optimize_folders(&mut self) {
let mut new_directory_folders: BTreeMap<PathBuf, FolderEntry> = Default::default(); let mut new_directory_folders: BTreeMap<PathBuf, FolderEntry> = Default::default();
@ -92,10 +76,8 @@ impl EmptyFolder {
self.information.number_of_empty_folders = self.empty_folder_list.len(); self.information.number_of_empty_folders = self.empty_folder_list.len();
} }
/// Function to check if folder are empty. #[fun_time(message = "check_for_empty_folders")]
/// Parameter `initial_checking` for second check before deleting to be sure that checked folder is still empty
fn check_for_empty_folders(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool { fn check_for_empty_folders(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("check_for_empty_folders - start");
let result = DirTraversalBuilder::new() let result = DirTraversalBuilder::new()
.root_dirs(self.common_data.directories.included_directories.clone()) .root_dirs(self.common_data.directories.included_directories.clone())
.group_by(|_fe| ()) .group_by(|_fe| ())
@ -107,31 +89,27 @@ impl EmptyFolder {
.max_stage(0) .max_stage(0)
.build() .build()
.run(); .run();
debug!("check_for_empty_folders - collected folders to check");
let res = match result { match result {
DirTraversalResult::SuccessFiles { .. } => { DirTraversalResult::SuccessFiles { .. } => {
unreachable!() unreachable!()
} }
DirTraversalResult::SuccessFolders { folder_entries, warnings } => { DirTraversalResult::SuccessFolders { folder_entries, warnings } => {
// We need to set empty folder list for (name, folder_entry) in folder_entries {
#[allow(unused_mut)] // Used is later by Windows build
for (mut name, folder_entry) in folder_entries {
if folder_entry.is_empty != FolderEmptiness::No { if folder_entry.is_empty != FolderEmptiness::No {
self.empty_folder_list.insert(name, folder_entry); self.empty_folder_list.insert(name, folder_entry);
} }
} }
self.common_data.text_messages.warnings.extend(warnings); self.common_data.text_messages.warnings.extend(warnings);
debug!("Found {} empty folders.", self.empty_folder_list.len());
true true
} }
DirTraversalResult::Stopped => false, DirTraversalResult::Stopped => false,
}; }
debug!("check_for_empty_folders - end");
res
} }
/// Deletes earlier found empty folders #[fun_time(message = "delete_empty_folders")]
fn delete_empty_folders(&mut self) { fn delete_empty_folders(&mut self) {
// Folders may be deleted or require too big privileges // Folders may be deleted or require too big privileges
for name in self.empty_folder_list.keys() { for name in self.empty_folder_list.keys() {
@ -154,11 +132,8 @@ impl Default for EmptyFolder {
} }
impl DebugPrint for EmptyFolder { impl DebugPrint for EmptyFolder {
#[allow(dead_code)]
#[allow(unreachable_code)]
fn debug_print(&self) { fn debug_print(&self) {
#[cfg(not(debug_assertions))] if !cfg!(debug_assertions) {
{
return; return;
} }
@ -169,60 +144,19 @@ impl DebugPrint for EmptyFolder {
} }
} }
impl SaveResults for EmptyFolder { impl PrintResults for EmptyFolder {
fn save_results_to_file(&mut self, file_name: &str) -> bool { fn write_results<T: Write>(&self, writer: &mut T) -> std::io::Result<()> {
let file_name: String = match file_name {
"" => "results.txt".to_string(),
k => k.to_string(),
};
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(e) => {
self.common_data.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
return false;
}
};
let mut writer = BufWriter::new(file_handler);
if let Err(e) = writeln!(
writer,
"Results of searching {:?} with excluded directories {:?}",
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories
) {
self.common_data
.text_messages
.errors
.push(format!("Failed to save results to file {file_name}, reason {e}"));
return false;
}
if !self.empty_folder_list.is_empty() { if !self.empty_folder_list.is_empty() {
writeln!( writeln!(writer, "--------------------------Empty folder list--------------------------")?;
writer, writeln!(writer, "Found {} empty folders", self.information.number_of_empty_folders)?;
"-------------------------------------------------Empty folder list-------------------------------------------------"
)
.unwrap();
writeln!(writer, "Found {} empty folders", self.information.number_of_empty_folders).unwrap();
for name in self.empty_folder_list.keys() { for name in self.empty_folder_list.keys() {
writeln!(writer, "{}", name.display()).unwrap(); writeln!(writer, "{}", name.display())?;
} }
} else { } else {
write!(writer, "Not found any empty folders.").unwrap(); write!(writer, "Not found any empty folders.")?;
} }
true Ok(())
}
}
impl PrintResults for EmptyFolder {
fn print_results(&self) {
if !self.empty_folder_list.is_empty() {
println!("Found {} empty folders", self.empty_folder_list.len());
}
for name in self.empty_folder_list.keys() {
println!("{}", name.display());
}
} }
} }
@ -234,3 +168,8 @@ impl CommonData for EmptyFolder {
&mut self.common_data &mut self.common_data
} }
} }
impl EmptyFolder {
pub fn set_delete_folder(&mut self, delete_folder: bool) {
self.delete_folders = delete_folder;
}
}

View file

@ -1,54 +1,37 @@
use std::fs; use std::fs;
use std::fs::File;
use std::io::prelude::*; use std::io::prelude::*;
use std::io::BufWriter;
use crossbeam_channel::Receiver; use crossbeam_channel::Receiver;
use fun_time::fun_time;
use futures::channel::mpsc::UnboundedSender; use futures::channel::mpsc::UnboundedSender;
use log::{debug, info}; use log::debug;
use crate::common_dir_traversal::{Collect, DirTraversalBuilder, DirTraversalResult, ErrorType, FileEntry, ProgressData, ToolType}; use crate::common_dir_traversal::{Collect, DirTraversalBuilder, DirTraversalResult, ErrorType, FileEntry, ProgressData, ToolType};
use crate::common_tool::{CommonData, CommonToolData}; use crate::common_tool::{CommonData, CommonToolData, DeleteMethod};
use crate::common_traits::*; use crate::common_traits::*;
#[derive(Eq, PartialEq, Clone, Debug, Copy)]
pub enum DeleteMethod {
None,
Delete,
}
/// Info struck with helpful information's about results
#[derive(Default)] #[derive(Default)]
pub struct Info { pub struct Info {
pub number_of_invalid_symlinks: usize, pub number_of_invalid_symlinks: usize,
} }
/// Struct with required information's to work
pub struct InvalidSymlinks { pub struct InvalidSymlinks {
common_data: CommonToolData, common_data: CommonToolData,
information: Info, information: Info,
invalid_symlinks: Vec<FileEntry>, invalid_symlinks: Vec<FileEntry>,
delete_method: DeleteMethod,
} }
impl InvalidSymlinks { impl InvalidSymlinks {
pub fn new() -> Self { pub fn new() -> Self {
Self { Self {
common_data: CommonToolData::new(ToolType::InvalidSymlinks), common_data: CommonToolData::new(ToolType::InvalidSymlinks),
information: Info::default(), information: Info::default(),
invalid_symlinks: vec![], invalid_symlinks: vec![],
delete_method: DeleteMethod::None,
} }
} }
#[fun_time(message = "find_invalid_links")]
pub fn find_invalid_links(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) { pub fn find_invalid_links(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
info!("Starting finding invalid symlinks");
let start_time = std::time::Instant::now();
self.find_invalid_links_internal(stop_receiver, progress_sender);
info!("Ended finding invalid symlinks which took {:?}", start_time.elapsed());
}
fn find_invalid_links_internal(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.optimize_dirs_before_start(); self.optimize_dirs_before_start();
if !self.check_files(stop_receiver, progress_sender) { if !self.check_files(stop_receiver, progress_sender) {
self.common_data.stopped_search = true; self.common_data.stopped_search = true;
@ -58,21 +41,8 @@ impl InvalidSymlinks {
self.debug_print(); self.debug_print();
} }
pub const fn get_invalid_symlinks(&self) -> &Vec<FileEntry> { #[fun_time(message = "check_files")]
&self.invalid_symlinks
}
pub const fn get_information(&self) -> &Info {
&self.information
}
pub fn set_delete_method(&mut self, delete_method: DeleteMethod) {
self.delete_method = delete_method;
}
/// Check files for any with size == 0
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool { fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("check_files - start");
let result = DirTraversalBuilder::new() let result = DirTraversalBuilder::new()
.root_dirs(self.common_data.directories.included_directories.clone()) .root_dirs(self.common_data.directories.included_directories.clone())
.group_by(|_fe| ()) .group_by(|_fe| ())
@ -85,26 +55,23 @@ impl InvalidSymlinks {
.recursive_search(self.common_data.recursive_search) .recursive_search(self.common_data.recursive_search)
.build() .build()
.run(); .run();
debug!("check_files - collected files");
let res = match result { match result {
DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => { DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => {
if let Some(((), invalid_symlinks)) = grouped_file_entries.into_iter().next() { self.invalid_symlinks = grouped_file_entries.into_values().flatten().collect();
self.invalid_symlinks = invalid_symlinks;
}
self.information.number_of_invalid_symlinks = self.invalid_symlinks.len(); self.information.number_of_invalid_symlinks = self.invalid_symlinks.len();
self.common_data.text_messages.warnings.extend(warnings); self.common_data.text_messages.warnings.extend(warnings);
debug!("Found {} invalid symlinks.", self.information.number_of_invalid_symlinks);
true true
} }
DirTraversalResult::SuccessFolders { .. } => unreachable!(), DirTraversalResult::SuccessFolders { .. } => unreachable!(),
DirTraversalResult::Stopped => false, DirTraversalResult::Stopped => false,
}; }
debug!("check_files - end");
res
} }
/// Function to delete files, from filed Vector #[fun_time(message = "delete_files")]
fn delete_files(&mut self) { fn delete_files(&mut self) {
match self.delete_method { match self.common_data.delete_method {
DeleteMethod::Delete => { DeleteMethod::Delete => {
for file_entry in &self.invalid_symlinks { for file_entry in &self.invalid_symlinks {
if fs::remove_file(file_entry.path.clone()).is_err() { if fs::remove_file(file_entry.path.clone()).is_err() {
@ -115,6 +82,7 @@ impl InvalidSymlinks {
DeleteMethod::None => { DeleteMethod::None => {
//Just do nothing //Just do nothing
} }
_ => unreachable!(),
} }
} }
} }
@ -126,52 +94,21 @@ impl Default for InvalidSymlinks {
} }
impl DebugPrint for InvalidSymlinks { impl DebugPrint for InvalidSymlinks {
#[allow(dead_code)]
#[allow(unreachable_code)]
/// Debugging printing - only available on debug build
fn debug_print(&self) { fn debug_print(&self) {
#[cfg(not(debug_assertions))] if !cfg!(debug_assertions) {
{
return; return;
} }
println!("---------------DEBUG PRINT---------------"); println!("---------------DEBUG PRINT---------------");
println!("Invalid symlinks list size - {}", self.invalid_symlinks.len()); println!("Invalid symlinks list size - {}", self.invalid_symlinks.len());
println!("Delete Method - {:?}", self.delete_method);
self.debug_print_common(); self.debug_print_common();
println!("-----------------------------------------"); println!("-----------------------------------------");
} }
} }
impl SaveResults for InvalidSymlinks { impl PrintResults for InvalidSymlinks {
fn save_results_to_file(&mut self, file_name: &str) -> bool { fn write_results<T: Write>(&self, writer: &mut T) -> std::io::Result<()> {
let file_name: String = match file_name {
"" => "results.txt".to_string(),
k => k.to_string(),
};
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(e) => {
self.common_data.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
return false;
}
};
let mut writer = BufWriter::new(file_handler);
if let Err(e) = writeln!(
writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
) {
self.common_data
.text_messages
.errors
.push(format!("Failed to save results to file {file_name}, reason {e}"));
return false;
}
if !self.invalid_symlinks.is_empty() { if !self.invalid_symlinks.is_empty() {
writeln!(writer, "Found {} invalid symlinks.", self.information.number_of_invalid_symlinks).unwrap(); writeln!(writer, "Found {} invalid symlinks.", self.information.number_of_invalid_symlinks)?;
for file_entry in &self.invalid_symlinks { for file_entry in &self.invalid_symlinks {
writeln!( writeln!(
writer, writer,
@ -182,32 +119,13 @@ impl SaveResults for InvalidSymlinks {
ErrorType::InfiniteRecursion => "Infinite Recursion", ErrorType::InfiniteRecursion => "Infinite Recursion",
ErrorType::NonExistentFile => "Non Existent File", ErrorType::NonExistentFile => "Non Existent File",
} }
) )?;
.unwrap();
} }
} else { } else {
write!(writer, "Not found any invalid symlinks.").unwrap(); write!(writer, "Not found any invalid symlinks.")?;
} }
true
}
}
impl PrintResults for InvalidSymlinks { Ok(())
/// Print information's about duplicated entries
/// Only needed for CLI
fn print_results(&self) {
println!("Found {} invalid symlinks.\n", self.information.number_of_invalid_symlinks);
for file_entry in &self.invalid_symlinks {
println!(
"{}\t\t{}\t\t{}",
file_entry.path.display(),
file_entry.symlink_info.clone().expect("invalid traversal result").destination_path.display(),
match file_entry.symlink_info.clone().expect("invalid traversal result").type_of_error {
ErrorType::InfiniteRecursion => "Infinite Recursion",
ErrorType::NonExistentFile => "Non Existent File",
}
);
}
} }
} }
@ -219,3 +137,13 @@ impl CommonData for InvalidSymlinks {
&mut self.common_data &mut self.common_data
} }
} }
impl InvalidSymlinks {
pub const fn get_invalid_symlinks(&self) -> &Vec<FileEntry> {
&self.invalid_symlinks
}
pub const fn get_information(&self) -> &Info {
&self.information
}
}

View file

@ -2,7 +2,7 @@ use std::cmp::max;
use std::collections::{BTreeMap, HashSet}; use std::collections::{BTreeMap, HashSet};
use std::fs::File; use std::fs::File;
use std::io::prelude::*; use std::io::prelude::*;
use std::io::BufWriter;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc; use std::sync::Arc;
@ -10,9 +10,11 @@ use std::{mem, panic};
use anyhow::Context; use anyhow::Context;
use crossbeam_channel::Receiver; use crossbeam_channel::Receiver;
use fun_time::fun_time;
use futures::channel::mpsc::UnboundedSender; use futures::channel::mpsc::UnboundedSender;
use humansize::{format_size, BINARY};
use lofty::{read_from, AudioFile, ItemKey, TaggedFileExt}; use lofty::{read_from, AudioFile, ItemKey, TaggedFileExt};
use log::{debug, info}; use log::debug;
use rayon::prelude::*; use rayon::prelude::*;
use rusty_chromaprint::{match_fingerprints, Configuration, Fingerprinter}; use rusty_chromaprint::{match_fingerprints, Configuration, Fingerprinter};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@ -29,12 +31,6 @@ use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraver
use crate::common_tool::{CommonData, CommonToolData}; use crate::common_tool::{CommonData, CommonToolData};
use crate::common_traits::*; use crate::common_traits::*;
#[derive(Eq, PartialEq, Clone, Debug, Copy)]
pub enum DeleteMethod {
None,
Delete,
}
bitflags! { bitflags! {
#[derive(PartialEq, Copy, Clone, Debug)] #[derive(PartialEq, Copy, Clone, Debug)]
pub struct MusicSimilarity : u32 { pub struct MusicSimilarity : u32 {
@ -95,14 +91,12 @@ impl FileEntry {
} }
} }
/// Info struck with helpful information's about results
#[derive(Default)] #[derive(Default)]
pub struct Info { pub struct Info {
pub number_of_duplicates: usize, pub number_of_duplicates: usize,
pub number_of_groups: u64, pub number_of_groups: u64,
} }
/// Struct with required information's to work
pub struct SameMusic { pub struct SameMusic {
common_data: CommonToolData, common_data: CommonToolData,
information: Info, information: Info,
@ -110,7 +104,6 @@ pub struct SameMusic {
music_entries: Vec<MusicEntry>, music_entries: Vec<MusicEntry>,
duplicated_music_entries: Vec<Vec<MusicEntry>>, duplicated_music_entries: Vec<Vec<MusicEntry>>,
duplicated_music_entries_referenced: Vec<(MusicEntry, Vec<MusicEntry>)>, duplicated_music_entries_referenced: Vec<(MusicEntry, Vec<MusicEntry>)>,
delete_method: DeleteMethod,
music_similarity: MusicSimilarity, music_similarity: MusicSimilarity,
approximate_comparison: bool, approximate_comparison: bool,
check_type: CheckingMethod, check_type: CheckingMethod,
@ -125,7 +118,6 @@ impl SameMusic {
common_data: CommonToolData::new(ToolType::SameMusic), common_data: CommonToolData::new(ToolType::SameMusic),
information: Info::default(), information: Info::default(),
music_entries: Vec::with_capacity(2048), music_entries: Vec::with_capacity(2048),
delete_method: DeleteMethod::None,
music_similarity: MusicSimilarity::NONE, music_similarity: MusicSimilarity::NONE,
duplicated_music_entries: vec![], duplicated_music_entries: vec![],
music_to_check: Default::default(), music_to_check: Default::default(),
@ -138,14 +130,8 @@ impl SameMusic {
} }
} }
#[fun_time(message = "print_results")]
pub fn find_same_music(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) { pub fn find_same_music(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
info!("Starting finding same music files");
let start_time = std::time::Instant::now();
self.find_same_music_internal(stop_receiver, progress_sender);
info!("Ended finding same music which took {:?}", start_time.elapsed());
}
fn find_same_music_internal(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.optimize_dirs_before_start(); self.optimize_dirs_before_start();
self.common_data.use_reference_folders = !self.common_data.directories.reference_directories.is_empty(); self.common_data.use_reference_folders = !self.common_data.directories.reference_directories.is_empty();
if !self.check_files(stop_receiver, progress_sender) { if !self.check_files(stop_receiver, progress_sender) {
@ -183,6 +169,7 @@ impl SameMusic {
self.debug_print(); self.debug_print();
} }
#[fun_time(message = "check_files")]
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool { fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
if !self.common_data.allowed_extensions.using_custom_extensions() { if !self.common_data.allowed_extensions.using_custom_extensions() {
self.common_data.allowed_extensions.extend_allowed_extensions(AUDIO_FILES_EXTENSIONS); self.common_data.allowed_extensions.extend_allowed_extensions(AUDIO_FILES_EXTENSIONS);
@ -207,15 +194,16 @@ impl SameMusic {
.max_stage(2) .max_stage(2)
.build() .build()
.run(); .run();
match result { match result {
DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => { DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => {
if let Some(music_to_check) = grouped_file_entries.get(&()) { self.music_to_check = grouped_file_entries
for fe in music_to_check { .into_values()
self.music_to_check.insert(fe.path.to_string_lossy().to_string(), fe.to_music_entry()); .flatten()
} .map(|fe| (fe.path.to_string_lossy().to_string(), fe.to_music_entry()))
} .collect();
self.common_data.text_messages.warnings.extend(warnings); self.common_data.text_messages.warnings.extend(warnings);
debug!("check_files - Found {} music files.", self.music_to_check.len());
true true
} }
DirTraversalResult::SuccessFolders { .. } => { DirTraversalResult::SuccessFolders { .. } => {
@ -225,8 +213,8 @@ impl SameMusic {
} }
} }
#[fun_time(message = "load_cache")]
fn load_cache(&mut self, checking_tags: bool) -> (BTreeMap<String, MusicEntry>, BTreeMap<String, MusicEntry>, BTreeMap<String, MusicEntry>) { fn load_cache(&mut self, checking_tags: bool) -> (BTreeMap<String, MusicEntry>, BTreeMap<String, MusicEntry>, BTreeMap<String, MusicEntry>) {
debug!("load_cache - start, using cache {}", self.common_data.use_cache);
let loaded_hash_map; let loaded_hash_map;
let mut records_already_cached: BTreeMap<String, MusicEntry> = Default::default(); let mut records_already_cached: BTreeMap<String, MusicEntry> = Default::default();
@ -238,6 +226,7 @@ impl SameMusic {
self.get_text_messages_mut().extend_with_another_messages(messages); self.get_text_messages_mut().extend_with_another_messages(messages);
loaded_hash_map = loaded_items.unwrap_or_default(); loaded_hash_map = loaded_items.unwrap_or_default();
debug!("load_cache - Starting to check for differences");
for (name, file_entry) in mem::take(&mut self.music_to_check) { for (name, file_entry) in mem::take(&mut self.music_to_check) {
if let Some(cached_file_entry) = loaded_hash_map.get(&name) { if let Some(cached_file_entry) = loaded_hash_map.get(&name) {
records_already_cached.insert(name.clone(), cached_file_entry.clone()); records_already_cached.insert(name.clone(), cached_file_entry.clone());
@ -245,16 +234,22 @@ impl SameMusic {
non_cached_files_to_check.insert(name, file_entry); non_cached_files_to_check.insert(name, file_entry);
} }
} }
debug!(
"load_cache - completed diff between loaded and prechecked files, {}({}) - non cached, {}({}) - already cached",
non_cached_files_to_check.len(),
format_size(non_cached_files_to_check.values().map(|e| e.size).sum::<u64>(), BINARY),
records_already_cached.len(),
format_size(records_already_cached.values().map(|e| e.size).sum::<u64>(), BINARY),
);
} else { } else {
loaded_hash_map = Default::default(); loaded_hash_map = Default::default();
mem::swap(&mut self.music_to_check, &mut non_cached_files_to_check); mem::swap(&mut self.music_to_check, &mut non_cached_files_to_check);
} }
debug!("load_cache - end");
(loaded_hash_map, records_already_cached, non_cached_files_to_check) (loaded_hash_map, records_already_cached, non_cached_files_to_check)
} }
#[fun_time(message = "save_cache")]
fn save_cache(&mut self, vec_file_entry: Vec<MusicEntry>, loaded_hash_map: BTreeMap<String, MusicEntry>, checking_tags: bool) { fn save_cache(&mut self, vec_file_entry: Vec<MusicEntry>, loaded_hash_map: BTreeMap<String, MusicEntry>, checking_tags: bool) {
debug!("save_cache - start, using cache {}", self.common_data.use_cache);
if !self.common_data.use_cache { if !self.common_data.use_cache {
return; return;
} }
@ -267,18 +262,17 @@ impl SameMusic {
let messages = save_cache_to_file_generalized(get_similar_music_cache_file(checking_tags), &all_results, self.common_data.save_also_as_json, 0); let messages = save_cache_to_file_generalized(get_similar_music_cache_file(checking_tags), &all_results, self.common_data.save_also_as_json, 0);
self.get_text_messages_mut().extend_with_another_messages(messages); self.get_text_messages_mut().extend_with_another_messages(messages);
debug!("save_cache - end");
} }
#[fun_time(message = "calculate_fingerprint")]
fn calculate_fingerprint(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool { fn calculate_fingerprint(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("calculate_fingerprint - start");
let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.load_cache(false); let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.load_cache(false);
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) = let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
prepare_thread_handler_common(progress_sender, 1, 3, non_cached_files_to_check.len(), self.check_type, self.common_data.tool_type); prepare_thread_handler_common(progress_sender, 1, 3, non_cached_files_to_check.len(), self.check_type, self.common_data.tool_type);
let configuration = &self.hash_preset_config; let configuration = &self.hash_preset_config;
// Clean for duplicate files debug!("calculate_fingerprint - starting fingerprinting");
let mut vec_file_entry = non_cached_files_to_check let mut vec_file_entry = non_cached_files_to_check
.into_par_iter() .into_par_iter()
.map(|(path, mut music_entry)| { .map(|(path, mut music_entry)| {
@ -299,6 +293,7 @@ impl SameMusic {
.filter(Option::is_some) .filter(Option::is_some)
.map(Option::unwrap) .map(Option::unwrap)
.collect::<Vec<_>>(); .collect::<Vec<_>>();
debug!("calculate_fingerprint - ended fingerprinting");
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
@ -313,12 +308,11 @@ impl SameMusic {
if check_was_stopped.load(Ordering::Relaxed) { if check_was_stopped.load(Ordering::Relaxed) {
return false; return false;
} }
debug!("calculate_fingerprint - end");
true true
} }
#[fun_time(message = "read_tags")]
fn read_tags(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool { fn read_tags(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("read_tags - start");
let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.load_cache(true); let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.load_cache(true);
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) = let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
@ -360,13 +354,11 @@ impl SameMusic {
return false; return false;
} }
debug!("read_tags - end");
true true
} }
#[fun_time(message = "check_for_duplicate_tags")]
fn check_for_duplicate_tags(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool { fn check_for_duplicate_tags(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("check_for_duplicate_tags - start");
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) = let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) =
prepare_thread_handler_common(progress_sender, 2, 2, self.music_to_check.len(), self.check_type, self.common_data.tool_type); prepare_thread_handler_common(progress_sender, 2, 2, self.music_to_check.len(), self.check_type, self.common_data.tool_type);
@ -462,17 +454,14 @@ impl SameMusic {
// Clear unused data // Clear unused data
self.music_entries.clear(); self.music_entries.clear();
debug!("check_for_duplicate_tags - end");
true true
} }
#[fun_time(message = "read_tags_to_files_similar_by_content")]
fn read_tags_to_files_similar_by_content(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool { fn read_tags_to_files_similar_by_content(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("read_tags_to_files_similar_by_content - start");
let groups_to_check = max(self.duplicated_music_entries.len(), self.duplicated_music_entries_referenced.len()); let groups_to_check = max(self.duplicated_music_entries.len(), self.duplicated_music_entries_referenced.len());
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) = let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
prepare_thread_handler_common(progress_sender, 3, 3, groups_to_check, self.check_type, self.common_data.tool_type); prepare_thread_handler_common(progress_sender, 3, 3, groups_to_check, self.check_type, self.common_data.tool_type);
// TODO is ther a way to just run iterator and not collect any info?
if !self.duplicated_music_entries.is_empty() { if !self.duplicated_music_entries.is_empty() {
let _: Vec<_> = self let _: Vec<_> = self
.duplicated_music_entries .duplicated_music_entries
@ -515,7 +504,6 @@ impl SameMusic {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
debug!("read_tags_to_files_similar_by_content - end");
!check_was_stopped.load(Ordering::Relaxed) !check_was_stopped.load(Ordering::Relaxed)
} }
@ -535,6 +523,7 @@ impl SameMusic {
(base_files, files_to_compare) (base_files, files_to_compare)
} }
#[fun_time(message = "compare_fingerprints")]
fn compare_fingerprints( fn compare_fingerprints(
&mut self, &mut self,
stop_receiver: Option<&Receiver<()>>, stop_receiver: Option<&Receiver<()>>,
@ -542,7 +531,6 @@ impl SameMusic {
base_files: Vec<MusicEntry>, base_files: Vec<MusicEntry>,
files_to_compare: &[MusicEntry], files_to_compare: &[MusicEntry],
) -> Option<Vec<Vec<MusicEntry>>> { ) -> Option<Vec<Vec<MusicEntry>>> {
debug!("compare_fingerprints - start");
let mut used_paths: HashSet<String> = Default::default(); let mut used_paths: HashSet<String> = Default::default();
let configuration = &self.hash_preset_config; let configuration = &self.hash_preset_config;
@ -591,12 +579,11 @@ impl SameMusic {
duplicated_music_entries.push(music_entries); duplicated_music_entries.push(music_entries);
} }
} }
debug!("compare_fingerprints - end");
Some(duplicated_music_entries) Some(duplicated_music_entries)
} }
#[fun_time(message = "check_for_duplicate_fingerprints")]
fn check_for_duplicate_fingerprints(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool { fn check_for_duplicate_fingerprints(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("check_for_duplicate_fingerprints - start");
let (base_files, files_to_compare) = self.split_fingerprints_to_check(); let (base_files, files_to_compare) = self.split_fingerprints_to_check();
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) = let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) =
prepare_thread_handler_common(progress_sender, 2, 3, base_files.len(), self.check_type, self.common_data.tool_type); prepare_thread_handler_common(progress_sender, 2, 3, base_files.len(), self.check_type, self.common_data.tool_type);
@ -629,10 +616,10 @@ impl SameMusic {
// Clear unused data // Clear unused data
self.music_entries.clear(); self.music_entries.clear();
debug!("check_for_duplicate_fingerprints - end");
true true
} }
#[fun_time(message = "check_music_item")]
fn check_music_item( fn check_music_item(
&self, &self,
old_duplicates: Vec<Vec<MusicEntry>>, old_duplicates: Vec<Vec<MusicEntry>>,
@ -640,7 +627,6 @@ impl SameMusic {
get_item: fn(&MusicEntry) -> &str, get_item: fn(&MusicEntry) -> &str,
approximate_comparison: bool, approximate_comparison: bool,
) -> Vec<Vec<MusicEntry>> { ) -> Vec<Vec<MusicEntry>> {
debug!("check_music_item - start");
let mut new_duplicates: Vec<_> = Default::default(); let mut new_duplicates: Vec<_> = Default::default();
let old_duplicates_len = old_duplicates.len(); let old_duplicates_len = old_duplicates.len();
for vec_file_entry in old_duplicates { for vec_file_entry in old_duplicates {
@ -662,11 +648,10 @@ impl SameMusic {
} }
atomic_counter.fetch_add(old_duplicates_len, Ordering::Relaxed); atomic_counter.fetch_add(old_duplicates_len, Ordering::Relaxed);
debug!("check_music_item - end");
new_duplicates new_duplicates
} }
/// Function to delete files, from filed Vector #[fun_time(message = "delete_files")]
fn delete_files(&mut self) { fn delete_files(&mut self) {
// TODO // TODO
@ -698,10 +683,6 @@ impl SameMusic {
&self.information &self.information
} }
pub fn set_delete_method(&mut self, delete_method: DeleteMethod) {
self.delete_method = delete_method;
}
pub fn set_approximate_comparison(&mut self, approximate_comparison: bool) { pub fn set_approximate_comparison(&mut self, approximate_comparison: bool) {
self.approximate_comparison = approximate_comparison; self.approximate_comparison = approximate_comparison;
} }
@ -877,7 +858,6 @@ fn read_single_file_tag(path: &str, music_entry: &mut MusicEntry) -> bool {
genre = tag_value.to_string(); genre = tag_value.to_string();
} }
} }
// println!("{:?}", tag.items());
} }
if let Ok(old_length_number) = length.parse::<u32>() { if let Ok(old_length_number) = length.parse::<u32>() {
@ -887,7 +867,7 @@ fn read_single_file_tag(path: &str, music_entry: &mut MusicEntry) -> bool {
if minutes != 0 || seconds != 0 { if minutes != 0 || seconds != 0 {
length = format!("{minutes}:{seconds:02}"); length = format!("{minutes}:{seconds:02}");
} else if old_length_number > 0 { } else if old_length_number > 0 {
// That means, that audio have length smaller that second, but length is properly read // That means, that audio have length smaller that second, but length was properly read
length = "0:01".to_string(); length = "0:01".to_string();
} else { } else {
length = String::new(); length = String::new();
@ -913,56 +893,26 @@ impl Default for SameMusic {
} }
impl DebugPrint for SameMusic { impl DebugPrint for SameMusic {
#[allow(dead_code)] #[fun_time(message = "debug_print")]
#[allow(unreachable_code)]
/// Debugging printing - only available on debug build
fn debug_print(&self) { fn debug_print(&self) {
#[cfg(not(debug_assertions))] if !cfg!(debug_assertions) {
{
return; return;
} }
println!("---------------DEBUG PRINT---------------"); println!("---------------DEBUG PRINT---------------");
println!("Found files music - {}", self.music_entries.len()); println!("Found files music - {}", self.music_entries.len());
println!("Found duplicated files music - {}", self.duplicated_music_entries.len()); println!("Found duplicated files music - {}", self.duplicated_music_entries.len());
println!("Delete Method - {:?}", self.delete_method);
self.debug_print_common(); self.debug_print_common();
println!("-----------------------------------------"); println!("-----------------------------------------");
} }
} }
impl SaveResults for SameMusic { impl PrintResults for SameMusic {
fn save_results_to_file(&mut self, file_name: &str) -> bool { fn write_results<T: Write>(&self, writer: &mut T) -> std::io::Result<()> {
let file_name: String = match file_name {
"" => "results.txt".to_string(),
k => k.to_string(),
};
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(e) => {
self.common_data.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
return false;
}
};
let mut writer = BufWriter::new(file_handler);
if let Err(e) = writeln!(
writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
) {
self.common_data
.text_messages
.errors
.push(format!("Failed to save results to file {file_name}, reason {e}"));
return false;
}
if !self.duplicated_music_entries.is_empty() { if !self.duplicated_music_entries.is_empty() {
writeln!(writer, "{} music files which have similar friends\n\n.", self.duplicated_music_entries.len()).unwrap(); writeln!(writer, "{} music files which have similar friends\n\n.", self.duplicated_music_entries.len())?;
for vec_file_entry in &self.duplicated_music_entries { for vec_file_entry in &self.duplicated_music_entries {
writeln!(writer, "Found {} music files which have similar friends", vec_file_entry.len()).unwrap(); writeln!(writer, "Found {} music files which have similar friends", vec_file_entry.len())?;
for file_entry in vec_file_entry { for file_entry in vec_file_entry {
writeln!( writeln!(
writer, writer,
@ -980,33 +930,10 @@ impl SaveResults for SameMusic {
writeln!(writer).unwrap(); writeln!(writer).unwrap();
} }
} else { } else {
write!(writer, "Not found any similar music files.").unwrap(); write!(writer, "Not found any similar music files.")?;
} }
true Ok(())
}
}
impl PrintResults for SameMusic {
/// Print information's about duplicated entries
/// Only needed for CLI
fn print_results(&self) {
println!("Found {} similar music files.\n", self.duplicated_music_entries.len());
for vec_file_entry in &self.duplicated_music_entries {
for file_entry in vec_file_entry {
println!(
"TT: {} - TA: {} - Y: {} - L: {} - G: {} - B: {} - P: {}",
file_entry.track_title,
file_entry.track_artist,
file_entry.year,
file_entry.length,
file_entry.genre,
file_entry.bitrate,
file_entry.path.display()
);
}
println!();
}
} }
} }
@ -1053,6 +980,15 @@ fn get_approximate_conversion(what: &mut String) {
*what = new_what; *what = new_what;
} }
impl CommonData for SameMusic {
fn get_cd(&self) -> &CommonToolData {
&self.common_data
}
fn get_cd_mut(&mut self) -> &mut CommonToolData {
&mut self.common_data
}
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::same_music::get_approximate_conversion; use crate::same_music::get_approximate_conversion;
@ -1076,12 +1012,3 @@ mod tests {
assert_eq!(what, "Kekistan"); assert_eq!(what, "Kekistan");
} }
} }
impl CommonData for SameMusic {
fn get_cd(&self) -> &CommonToolData {
&self.common_data
}
fn get_cd_mut(&mut self) -> &mut CommonToolData {
&mut self.common_data
}
}

View file

@ -1,6 +1,6 @@
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
use std::fs::{DirEntry, File, Metadata}; use std::fs::{DirEntry, Metadata};
use std::io::{Write, *}; use std::io::Write;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::atomic::Ordering; use std::sync::atomic::Ordering;
use std::time::SystemTime; use std::time::SystemTime;
@ -8,11 +8,12 @@ use std::{mem, panic};
use bk_tree::BKTree; use bk_tree::BKTree;
use crossbeam_channel::Receiver; use crossbeam_channel::Receiver;
use fun_time::fun_time;
use futures::channel::mpsc::UnboundedSender; use futures::channel::mpsc::UnboundedSender;
use humansize::{format_size, BINARY}; use humansize::{format_size, BINARY};
use image::GenericImageView; use image::GenericImageView;
use image_hasher::{FilterType, HashAlg, HasherConfig}; use image_hasher::{FilterType, HashAlg, HasherConfig};
use log::{debug, info}; use log::debug;
use rayon::prelude::*; use rayon::prelude::*;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@ -25,7 +26,7 @@ use crate::common::{
use crate::common_cache::{get_similar_images_cache_file, load_cache_from_file_generalized_by_path, save_cache_to_file_generalized}; use crate::common_cache::{get_similar_images_cache_file, load_cache_from_file_generalized_by_path, save_cache_to_file_generalized};
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType}; use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
use crate::common_tool::{CommonData, CommonToolData}; use crate::common_tool::{CommonData, CommonToolData};
use crate::common_traits::{DebugPrint, PrintResults, ResultEntry, SaveResults}; use crate::common_traits::{DebugPrint, PrintResults, ResultEntry};
use crate::flc; use crate::flc;
type ImHash = Vec<u8>; type ImHash = Vec<u8>;
@ -59,7 +60,6 @@ impl ResultEntry for FileEntry {
} }
} }
/// Used by CLI tool when we cannot use directly values
#[derive(Clone, Debug, Copy)] #[derive(Clone, Debug, Copy)]
pub enum SimilarityPreset { pub enum SimilarityPreset {
Original, Original,
@ -72,7 +72,6 @@ pub enum SimilarityPreset {
None, None,
} }
/// Distance metric to use with the BK-tree.
struct Hamming; struct Hamming;
impl bk_tree::Metric<ImHash> for Hamming { impl bk_tree::Metric<ImHash> for Hamming {
@ -85,7 +84,6 @@ impl bk_tree::Metric<ImHash> for Hamming {
} }
} }
/// Struct to store most basics info about all folder
pub struct SimilarImages { pub struct SimilarImages {
common_data: CommonToolData, common_data: CommonToolData,
information: Info, information: Info,
@ -102,17 +100,13 @@ pub struct SimilarImages {
exclude_images_with_same_size: bool, exclude_images_with_same_size: bool,
} }
/// Info struck with helpful information's about results
#[derive(Default)] #[derive(Default)]
pub struct Info { pub struct Info {
pub number_of_duplicates: usize, pub number_of_duplicates: usize,
pub number_of_groups: u64, pub number_of_groups: u64,
} }
/// Method implementation for `EmptyFolder`
impl SimilarImages { impl SimilarImages {
/// New function providing basics values
pub fn new() -> Self { pub fn new() -> Self {
Self { Self {
common_data: CommonToolData::new(ToolType::SimilarImages), common_data: CommonToolData::new(ToolType::SimilarImages),
@ -130,14 +124,8 @@ impl SimilarImages {
} }
} }
#[fun_time(message = "find_similar_images")]
pub fn find_similar_images(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) { pub fn find_similar_images(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
info!("Starting finding similar images files");
let start_time = std::time::Instant::now();
self.find_similar_images_internal(stop_receiver, progress_sender);
info!("Ended finding similar images which took {:?}", start_time.elapsed());
}
pub fn find_similar_images_internal(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.optimize_dirs_before_start(); self.optimize_dirs_before_start();
self.common_data.use_reference_folders = !self.common_data.directories.reference_directories.is_empty(); self.common_data.use_reference_folders = !self.common_data.directories.reference_directories.is_empty();
if !self.check_for_similar_images(stop_receiver, progress_sender) { if !self.check_for_similar_images(stop_receiver, progress_sender) {
@ -152,20 +140,11 @@ impl SimilarImages {
self.common_data.stopped_search = true; self.common_data.stopped_search = true;
return; return;
} }
// if self.delete_folders {
// self.delete_empty_folders();
// }
self.debug_print(); self.debug_print();
} }
// pub fn set_delete_folder(&mut self, delete_folder: bool) { #[fun_time(message = "check_for_similar_images")]
// self.delete_folders = delete_folder;
// }
/// Function to check if folder are empty.
/// Parameter `initial_checking` for second check before deleting to be sure that checked folder is still empty
fn check_for_similar_images(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool { fn check_for_similar_images(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("check_for_similar_images - start");
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
if !self.common_data.allowed_extensions.using_custom_extensions() { if !self.common_data.allowed_extensions.using_custom_extensions() {
@ -246,7 +225,6 @@ impl SimilarImages {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
debug!("check_for_similar_images - end");
true true
} }
@ -279,8 +257,8 @@ impl SimilarImages {
} }
} }
#[fun_time(message = "hash_images_load_cache")]
fn hash_images_load_cache(&mut self) -> (BTreeMap<String, FileEntry>, BTreeMap<String, FileEntry>, BTreeMap<String, FileEntry>) { fn hash_images_load_cache(&mut self) -> (BTreeMap<String, FileEntry>, BTreeMap<String, FileEntry>, BTreeMap<String, FileEntry>) {
debug!("hash_images_load_cache - start, use cache: {}", self.common_data.use_cache);
let loaded_hash_map; let loaded_hash_map;
let mut records_already_cached: BTreeMap<String, FileEntry> = Default::default(); let mut records_already_cached: BTreeMap<String, FileEntry> = Default::default();
@ -295,6 +273,7 @@ impl SimilarImages {
self.get_text_messages_mut().extend_with_another_messages(messages); self.get_text_messages_mut().extend_with_another_messages(messages);
loaded_hash_map = loaded_items.unwrap_or_default(); loaded_hash_map = loaded_items.unwrap_or_default();
debug!("hash_images-load_cache - starting calculating diff");
for (name, file_entry) in mem::take(&mut self.images_to_check) { for (name, file_entry) in mem::take(&mut self.images_to_check) {
if let Some(cached_file_entry) = loaded_hash_map.get(&name) { if let Some(cached_file_entry) = loaded_hash_map.get(&name) {
records_already_cached.insert(name.clone(), cached_file_entry.clone()); records_already_cached.insert(name.clone(), cached_file_entry.clone());
@ -302,11 +281,17 @@ impl SimilarImages {
non_cached_files_to_check.insert(name, file_entry); non_cached_files_to_check.insert(name, file_entry);
} }
} }
debug!(
"hash_images_load_cache - completed diff between loaded and prechecked files, {}({}) - non cached, {}({}) - already cached",
non_cached_files_to_check.len(),
format_size(non_cached_files_to_check.values().map(|e| e.size).sum::<u64>(), BINARY),
records_already_cached.len(),
format_size(records_already_cached.values().map(|e| e.size).sum::<u64>(), BINARY),
);
} else { } else {
loaded_hash_map = Default::default(); loaded_hash_map = Default::default();
mem::swap(&mut self.images_to_check, &mut non_cached_files_to_check); mem::swap(&mut self.images_to_check, &mut non_cached_files_to_check);
} }
debug!("hash_images_load_cache - end");
(loaded_hash_map, records_already_cached, non_cached_files_to_check) (loaded_hash_map, records_already_cached, non_cached_files_to_check)
} }
@ -317,8 +302,8 @@ impl SimilarImages {
// - Join already read hashes with hashes which were read from file // - Join already read hashes with hashes which were read from file
// - Join all hashes and save it to file // - Join all hashes and save it to file
#[fun_time(message = "hash_images")]
fn hash_images(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool { fn hash_images(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("hash_images - start");
let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.hash_images_load_cache(); let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.hash_images_load_cache();
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) = let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
@ -363,13 +348,11 @@ impl SimilarImages {
return false; return false;
} }
debug!("hash_images - end");
true true
} }
#[fun_time(message = "save_to_cache")]
fn save_to_cache(&mut self, vec_file_entry: Vec<(FileEntry, ImHash)>, loaded_hash_map: BTreeMap<String, FileEntry>) { fn save_to_cache(&mut self, vec_file_entry: Vec<(FileEntry, ImHash)>, loaded_hash_map: BTreeMap<String, FileEntry>) {
debug!("save_to_cache - start, using cache: {}", self.common_data.use_cache);
if self.common_data.use_cache { if self.common_data.use_cache {
// Must save all results to file, old loaded from file with all currently counted results // Must save all results to file, old loaded from file with all currently counted results
let mut all_results: BTreeMap<String, FileEntry> = loaded_hash_map; let mut all_results: BTreeMap<String, FileEntry> = loaded_hash_map;
@ -385,7 +368,6 @@ impl SimilarImages {
); );
self.get_text_messages_mut().extend_with_another_messages(messages); self.get_text_messages_mut().extend_with_another_messages(messages);
} }
debug!("save_to_cache - end");
} }
fn collect_image_file_entry(&self, mut file_entry: FileEntry) -> (FileEntry, ImHash) { fn collect_image_file_entry(&self, mut file_entry: FileEntry) -> (FileEntry, ImHash) {
@ -460,8 +442,8 @@ impl SimilarImages {
} }
// Split hashes at 2 parts, base hashes and hashes to compare, 3 argument is set of hashes with multiple images // Split hashes at 2 parts, base hashes and hashes to compare, 3 argument is set of hashes with multiple images
#[fun_time(message = "split_hashes")]
fn split_hashes(&mut self, all_hashed_images: &HashMap<ImHash, Vec<FileEntry>>) -> (Vec<ImHash>, HashSet<ImHash>) { fn split_hashes(&mut self, all_hashed_images: &HashMap<ImHash, Vec<FileEntry>>) -> (Vec<ImHash>, HashSet<ImHash>) {
debug!("split_hashes - start");
let hashes_with_multiple_images: HashSet<ImHash> = all_hashed_images let hashes_with_multiple_images: HashSet<ImHash> = all_hashed_images
.iter() .iter()
.filter_map(|(hash, vec_file_entry)| { .filter_map(|(hash, vec_file_entry)| {
@ -499,10 +481,10 @@ impl SimilarImages {
} }
base_hashes = all_hashed_images.keys().cloned().collect::<Vec<_>>(); base_hashes = all_hashed_images.keys().cloned().collect::<Vec<_>>();
} }
debug!("split_hashes - end");
(base_hashes, hashes_with_multiple_images) (base_hashes, hashes_with_multiple_images)
} }
#[fun_time(message = "collect_hash_compare_result")]
fn collect_hash_compare_result( fn collect_hash_compare_result(
&self, &self,
hashes_parents: HashMap<ImHash, u32>, hashes_parents: HashMap<ImHash, u32>,
@ -511,7 +493,6 @@ impl SimilarImages {
collected_similar_images: &mut HashMap<ImHash, Vec<FileEntry>>, collected_similar_images: &mut HashMap<ImHash, Vec<FileEntry>>,
hashes_similarity: HashMap<ImHash, (ImHash, u32)>, hashes_similarity: HashMap<ImHash, (ImHash, u32)>,
) { ) {
debug!("collect_hash_compare_result - start, use reference: {}", self.common_data.use_reference_folders);
if self.common_data.use_reference_folders { if self.common_data.use_reference_folders {
// This is same step as without reference folders, but also checks if children are inside/outside reference directories, because may happen, that one file is inside reference folder and other outside // This is same step as without reference folders, but also checks if children are inside/outside reference directories, because may happen, that one file is inside reference folder and other outside
@ -561,9 +542,9 @@ impl SimilarImages {
collected_similar_images.get_mut(&parent_hash).unwrap().append(&mut vec_fe); collected_similar_images.get_mut(&parent_hash).unwrap().append(&mut vec_fe);
} }
} }
debug!("collect_hash_compare_result - end");
} }
#[fun_time(message = "compare_hashes_with_non_zero_tolerance")]
fn compare_hashes_with_non_zero_tolerance( fn compare_hashes_with_non_zero_tolerance(
&mut self, &mut self,
all_hashed_images: &HashMap<ImHash, Vec<FileEntry>>, all_hashed_images: &HashMap<ImHash, Vec<FileEntry>>,
@ -572,7 +553,6 @@ impl SimilarImages {
stop_receiver: Option<&Receiver<()>>, stop_receiver: Option<&Receiver<()>>,
tolerance: u32, tolerance: u32,
) -> bool { ) -> bool {
debug!("compare_hashes_with_non_zero_tolerance - start");
// Don't use hashes with multiple images in bktree, because they will always be master of group and cannot be find by other hashes // Don't use hashes with multiple images in bktree, because they will always be master of group and cannot be find by other hashes
let (base_hashes, hashes_with_multiple_images) = self.split_hashes(all_hashed_images); let (base_hashes, hashes_with_multiple_images) = self.split_hashes(all_hashed_images);
@ -637,10 +617,10 @@ impl SimilarImages {
debug_check_for_duplicated_things(self.common_data.use_reference_folders, &hashes_parents, &hashes_similarity, all_hashed_images, "LATTER"); debug_check_for_duplicated_things(self.common_data.use_reference_folders, &hashes_parents, &hashes_similarity, all_hashed_images, "LATTER");
self.collect_hash_compare_result(hashes_parents, &hashes_with_multiple_images, all_hashed_images, collected_similar_images, hashes_similarity); self.collect_hash_compare_result(hashes_parents, &hashes_with_multiple_images, all_hashed_images, collected_similar_images, hashes_similarity);
debug!("compare_hashes_with_non_zero_tolerance - end");
true true
} }
#[fun_time(message = "connect_results")]
fn connect_results( fn connect_results(
&self, &self,
partial_results: Vec<(&ImHash, Vec<(u32, &ImHash)>)>, partial_results: Vec<(&ImHash, Vec<(u32, &ImHash)>)>,
@ -648,7 +628,6 @@ impl SimilarImages {
hashes_similarity: &mut HashMap<ImHash, (ImHash, u32)>, hashes_similarity: &mut HashMap<ImHash, (ImHash, u32)>,
hashes_with_multiple_images: &HashSet<ImHash>, hashes_with_multiple_images: &HashSet<ImHash>,
) { ) {
debug!("connect_results - start");
for (original_hash, vec_compared_hashes) in partial_results { for (original_hash, vec_compared_hashes) in partial_results {
let mut number_of_added_child_items = 0; let mut number_of_added_child_items = 0;
for (similarity, compared_hash) in vec_compared_hashes { for (similarity, compared_hash) in vec_compared_hashes {
@ -701,11 +680,10 @@ impl SimilarImages {
hashes_parents.insert((*original_hash).clone(), number_of_added_child_items); hashes_parents.insert((*original_hash).clone(), number_of_added_child_items);
} }
} }
debug!("connect_results - end");
} }
#[fun_time(message = "find_similar_hashes")]
fn find_similar_hashes(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool { fn find_similar_hashes(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("find_similar_hashes - start");
if self.image_hashes.is_empty() { if self.image_hashes.is_empty() {
return true; return true;
} }
@ -724,10 +702,8 @@ impl SimilarImages {
collected_similar_images.insert(hash, vec_file_entry); collected_similar_images.insert(hash, vec_file_entry);
} }
} }
} else { } else if !self.compare_hashes_with_non_zero_tolerance(&all_hashed_images, &mut collected_similar_images, progress_sender, stop_receiver, tolerance) {
if !self.compare_hashes_with_non_zero_tolerance(&all_hashed_images, &mut collected_similar_images, progress_sender, stop_receiver, tolerance) { return false;
return false;
}
} }
self.verify_duplicated_items(&collected_similar_images); self.verify_duplicated_items(&collected_similar_images);
@ -756,12 +732,11 @@ impl SimilarImages {
self.images_to_check = Default::default(); self.images_to_check = Default::default();
self.bktree = BKTree::new(Hamming); self.bktree = BKTree::new(Hamming);
debug!("find_similar_hashes - end");
true true
} }
#[fun_time(message = "exclude_items_with_same_size")]
fn exclude_items_with_same_size(&mut self) { fn exclude_items_with_same_size(&mut self) {
debug!("exclude_items_with_same_size - start, exclude: {}", self.exclude_images_with_same_size);
if self.exclude_images_with_same_size { if self.exclude_images_with_same_size {
for vec_file_entry in mem::take(&mut self.similar_vectors) { for vec_file_entry in mem::take(&mut self.similar_vectors) {
let mut bt_sizes: BTreeSet<u64> = Default::default(); let mut bt_sizes: BTreeSet<u64> = Default::default();
@ -777,14 +752,10 @@ impl SimilarImages {
} }
} }
} }
debug!("exclude_items_with_same_size - end");
} }
#[fun_time(message = "remove_multiple_records_from_reference_folders")]
fn remove_multiple_records_from_reference_folders(&mut self) { fn remove_multiple_records_from_reference_folders(&mut self) {
debug!(
"remove_multiple_records_from_reference_folders - start, use reference: {}",
self.common_data.use_reference_folders
);
if self.common_data.use_reference_folders { if self.common_data.use_reference_folders {
self.similar_referenced_vectors = mem::take(&mut self.similar_vectors) self.similar_referenced_vectors = mem::take(&mut self.similar_vectors)
.into_iter() .into_iter()
@ -801,16 +772,14 @@ impl SimilarImages {
}) })
.collect::<Vec<(FileEntry, Vec<FileEntry>)>>(); .collect::<Vec<(FileEntry, Vec<FileEntry>)>>();
} }
debug!("remove_multiple_records_from_reference_folders - end");
} }
#[allow(dead_code)]
#[allow(unreachable_code)]
#[allow(unused_variables)] #[allow(unused_variables)]
// TODO this probably not works good when reference folders are used // TODO this probably not works good when reference folders are used
pub fn verify_duplicated_items(&self, collected_similar_images: &HashMap<ImHash, Vec<FileEntry>>) { pub fn verify_duplicated_items(&self, collected_similar_images: &HashMap<ImHash, Vec<FileEntry>>) {
#[cfg(not(debug_assertions))] if !cfg!(debug_assertions) {
return; return;
}
// Validating if group contains duplicated results // Validating if group contains duplicated results
let mut result_hashset: HashSet<String> = Default::default(); let mut result_hashset: HashSet<String> = Default::default();
let mut found = false; let mut found = false;
@ -851,11 +820,8 @@ impl Default for SimilarImages {
} }
impl DebugPrint for SimilarImages { impl DebugPrint for SimilarImages {
#[allow(dead_code)]
#[allow(unreachable_code)]
fn debug_print(&self) { fn debug_print(&self) {
#[cfg(not(debug_assertions))] if !cfg!(debug_assertions) {
{
return; return;
} }
@ -865,39 +831,13 @@ impl DebugPrint for SimilarImages {
} }
} }
impl SaveResults for SimilarImages { impl PrintResults for SimilarImages {
fn save_results_to_file(&mut self, file_name: &str) -> bool { fn write_results<T: Write>(&self, writer: &mut T) -> std::io::Result<()> {
let file_name: String = match file_name {
"" => "results.txt".to_string(),
k => k.to_string(),
};
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(e) => {
self.common_data.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
return false;
}
};
let mut writer = BufWriter::new(file_handler);
if let Err(e) = writeln!(
writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
) {
self.common_data
.text_messages
.errors
.push(format!("Failed to save results to file {file_name}, reason {e}"));
return false;
}
if !self.similar_vectors.is_empty() { if !self.similar_vectors.is_empty() {
write!(writer, "{} images which have similar friends\n\n", self.similar_vectors.len()).unwrap(); write!(writer, "{} images which have similar friends\n\n", self.similar_vectors.len())?;
for struct_similar in &self.similar_vectors { for struct_similar in &self.similar_vectors {
writeln!(writer, "Found {} images which have similar friends", struct_similar.len()).unwrap(); writeln!(writer, "Found {} images which have similar friends", struct_similar.len())?;
for file_entry in struct_similar { for file_entry in struct_similar {
writeln!( writeln!(
writer, writer,
@ -906,37 +846,15 @@ impl SaveResults for SimilarImages {
file_entry.dimensions, file_entry.dimensions,
format_size(file_entry.size, BINARY), format_size(file_entry.size, BINARY),
get_string_from_similarity(&file_entry.similarity, self.hash_size) get_string_from_similarity(&file_entry.similarity, self.hash_size)
) )?;
.unwrap();
} }
writeln!(writer).unwrap(); writeln!(writer)?;
} }
} else { } else {
write!(writer, "Not found any similar images.").unwrap(); write!(writer, "Not found any similar images.")?;
} }
true Ok(())
}
}
impl PrintResults for SimilarImages {
fn print_results(&self) {
if !self.similar_vectors.is_empty() {
println!("Found {} images which have similar friends", self.similar_vectors.len());
for vec_file_entry in &self.similar_vectors {
for file_entry in vec_file_entry {
println!(
"{} - {} - {} - {}",
file_entry.path.display(),
file_entry.dimensions,
format_size(file_entry.size, BINARY),
get_string_from_similarity(&file_entry.similarity, self.hash_size)
);
}
println!();
}
}
} }
} }
@ -1064,8 +982,9 @@ fn debug_check_for_duplicated_things(
all_hashed_images: &HashMap<ImHash, Vec<FileEntry>>, all_hashed_images: &HashMap<ImHash, Vec<FileEntry>>,
numm: &str, numm: &str,
) { ) {
#[cfg(not(debug_assertions))] if !cfg!(debug_assertions) {
return; return;
}
if use_reference_folders { if use_reference_folders {
return; return;

View file

@ -1,15 +1,15 @@
use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::fs::{DirEntry, File, Metadata}; use std::fs::{DirEntry, Metadata};
use std::io::{Write, *}; use std::io::Write;
use std::mem; use std::mem;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::atomic::Ordering; use std::sync::atomic::Ordering;
use crossbeam_channel::Receiver; use crossbeam_channel::Receiver;
use ffmpeg_cmdline_utils::FfmpegErrorKind::FfmpegNotFound; use ffmpeg_cmdline_utils::FfmpegErrorKind::FfmpegNotFound;
use fun_time::fun_time;
use futures::channel::mpsc::UnboundedSender; use futures::channel::mpsc::UnboundedSender;
use humansize::{format_size, BINARY}; use humansize::{format_size, BINARY};
use log::{debug, info};
use rayon::prelude::*; use rayon::prelude::*;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use vid_dup_finder_lib::HashCreationErrorKind::DetermineVideo; use vid_dup_finder_lib::HashCreationErrorKind::DetermineVideo;
@ -19,7 +19,7 @@ use crate::common::{check_folder_children, prepare_thread_handler_common, send_i
use crate::common_cache::{get_similar_videos_cache_file, load_cache_from_file_generalized_by_path, save_cache_to_file_generalized}; use crate::common_cache::{get_similar_videos_cache_file, load_cache_from_file_generalized_by_path, save_cache_to_file_generalized};
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType}; use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
use crate::common_tool::{CommonData, CommonToolData}; use crate::common_tool::{CommonData, CommonToolData};
use crate::common_traits::{DebugPrint, PrintResults, ResultEntry, SaveResults}; use crate::common_traits::{DebugPrint, PrintResults, ResultEntry};
use crate::flc; use crate::flc;
use crate::localizer_core::generate_translation_hashmap; use crate::localizer_core::generate_translation_hashmap;
@ -46,7 +46,6 @@ impl ResultEntry for FileEntry {
} }
} }
/// Distance metric to use with the BK-tree.
struct Hamming; struct Hamming;
impl bk_tree::Metric<Vec<u8>> for Hamming { impl bk_tree::Metric<Vec<u8>> for Hamming {
@ -61,7 +60,6 @@ impl bk_tree::Metric<Vec<u8>> for Hamming {
} }
} }
/// Struct to store most basics info about all folder
pub struct SimilarVideos { pub struct SimilarVideos {
common_data: CommonToolData, common_data: CommonToolData,
information: Info, information: Info,
@ -82,17 +80,13 @@ impl CommonData for SimilarVideos {
} }
} }
/// Info struck with helpful information's about results
#[derive(Default)] #[derive(Default)]
pub struct Info { pub struct Info {
pub number_of_duplicates: usize, pub number_of_duplicates: usize,
pub number_of_groups: u64, pub number_of_groups: u64,
} }
/// Method implementation for `EmptyFolder`
impl SimilarVideos { impl SimilarVideos {
/// New function providing basics values
pub fn new() -> Self { pub fn new() -> Self {
Self { Self {
common_data: CommonToolData::new(ToolType::SimilarVideos), common_data: CommonToolData::new(ToolType::SimilarVideos),
@ -106,14 +100,8 @@ impl SimilarVideos {
} }
} }
#[fun_time(message = "find_similar_videos")]
pub fn find_similar_videos(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) { pub fn find_similar_videos(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
info!("Starting finding similar videos");
let start_time = std::time::Instant::now();
self.find_similar_videos_internal(stop_receiver, progress_sender);
info!("Ended finding similar videos which took {:?}", start_time.elapsed());
}
fn find_similar_videos_internal(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
if !check_if_ffmpeg_is_installed() { if !check_if_ffmpeg_is_installed() {
self.common_data.text_messages.errors.push(flc!("core_ffmpeg_not_found")); self.common_data.text_messages.errors.push(flc!("core_ffmpeg_not_found"));
#[cfg(target_os = "windows")] #[cfg(target_os = "windows")]
@ -134,21 +122,12 @@ impl SimilarVideos {
self.common_data.stopped_search = true; self.common_data.stopped_search = true;
return; return;
} }
// if self.delete_folders {
// self.delete_empty_folders();
// }
} }
self.debug_print(); self.debug_print();
} }
// pub fn set_delete_folder(&mut self, delete_folder: bool) { #[fun_time(message = "check_for_similar_videos")]
// self.delete_folders = delete_folder;
// }
/// Function to check if folder are empty.
/// Parameter `initial_checking` for second check before deleting to be sure that checked folder is still empty
fn check_for_similar_videos(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool { fn check_for_similar_videos(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("check_for_similar_videos - start");
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
if !self.common_data.allowed_extensions.using_custom_extensions() { if !self.common_data.allowed_extensions.using_custom_extensions() {
@ -225,7 +204,6 @@ impl SimilarVideos {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
debug!("check_for_similar_videos - end");
true true
} }
@ -258,8 +236,8 @@ impl SimilarVideos {
} }
} }
#[fun_time(message = "load_cache_at_start")]
fn load_cache_at_start(&mut self) -> (BTreeMap<String, FileEntry>, BTreeMap<String, FileEntry>, BTreeMap<String, FileEntry>) { fn load_cache_at_start(&mut self) -> (BTreeMap<String, FileEntry>, BTreeMap<String, FileEntry>, BTreeMap<String, FileEntry>) {
debug!("load_cache_at_start - start, use cache: {}", self.common_data.use_cache);
let loaded_hash_map; let loaded_hash_map;
let mut records_already_cached: BTreeMap<String, FileEntry> = Default::default(); let mut records_already_cached: BTreeMap<String, FileEntry> = Default::default();
let mut non_cached_files_to_check: BTreeMap<String, FileEntry> = Default::default(); let mut non_cached_files_to_check: BTreeMap<String, FileEntry> = Default::default();
@ -281,12 +259,11 @@ impl SimilarVideos {
loaded_hash_map = Default::default(); loaded_hash_map = Default::default();
mem::swap(&mut self.videos_to_check, &mut non_cached_files_to_check); mem::swap(&mut self.videos_to_check, &mut non_cached_files_to_check);
} }
debug!("load_cache_at_start - end");
(loaded_hash_map, records_already_cached, non_cached_files_to_check) (loaded_hash_map, records_already_cached, non_cached_files_to_check)
} }
#[fun_time(message = "sort_videos")]
fn sort_videos(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool { fn sort_videos(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("sort_videos - start");
let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.load_cache_at_start(); let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.load_cache_at_start();
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) = let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
@ -362,11 +339,11 @@ impl SimilarVideos {
self.videos_hashes = Default::default(); self.videos_hashes = Default::default();
self.videos_to_check = Default::default(); self.videos_to_check = Default::default();
debug!("sort_videos - end");
true true
} }
#[fun_time(message = "save_cache")]
fn save_cache(&mut self, vec_file_entry: Vec<FileEntry>, loaded_hash_map: BTreeMap<String, FileEntry>) { fn save_cache(&mut self, vec_file_entry: Vec<FileEntry>, loaded_hash_map: BTreeMap<String, FileEntry>) {
debug!("save_cache - start, use cache: {}", self.common_data.use_cache);
if self.common_data.use_cache { if self.common_data.use_cache {
// Must save all results to file, old loaded from file with all currently counted results // Must save all results to file, old loaded from file with all currently counted results
let mut all_results: BTreeMap<String, FileEntry> = loaded_hash_map; let mut all_results: BTreeMap<String, FileEntry> = loaded_hash_map;
@ -377,11 +354,10 @@ impl SimilarVideos {
let messages = save_cache_to_file_generalized(&get_similar_videos_cache_file(), &all_results, self.common_data.save_also_as_json, 0); let messages = save_cache_to_file_generalized(&get_similar_videos_cache_file(), &all_results, self.common_data.save_also_as_json, 0);
self.get_text_messages_mut().extend_with_another_messages(messages); self.get_text_messages_mut().extend_with_another_messages(messages);
} }
debug!("save_cache - end");
} }
#[fun_time(message = "match_groups_of_videos")]
fn match_groups_of_videos(&mut self, vector_of_hashes: Vec<VideoHash>, hashmap_with_file_entries: &HashMap<String, FileEntry>) { fn match_groups_of_videos(&mut self, vector_of_hashes: Vec<VideoHash>, hashmap_with_file_entries: &HashMap<String, FileEntry>) {
debug!("match_groups_of_videos - start");
let match_group = vid_dup_finder_lib::search(vector_of_hashes, NormalizedTolerance::new(self.tolerance as f64 / 100.0f64)); let match_group = vid_dup_finder_lib::search(vector_of_hashes, NormalizedTolerance::new(self.tolerance as f64 / 100.0f64));
let mut collected_similar_videos: Vec<Vec<FileEntry>> = Default::default(); let mut collected_similar_videos: Vec<Vec<FileEntry>> = Default::default();
for i in match_group { for i in match_group {
@ -404,11 +380,10 @@ impl SimilarVideos {
} }
self.similar_vectors = collected_similar_videos; self.similar_vectors = collected_similar_videos;
debug!("match_groups_of_videos - end");
} }
#[fun_time(message = "remove_from_reference_folders")]
fn remove_from_reference_folders(&mut self) { fn remove_from_reference_folders(&mut self) {
debug!("remove_from_reference_folders - start, use reference folders: {}", self.common_data.use_reference_folders);
if self.common_data.use_reference_folders { if self.common_data.use_reference_folders {
self.similar_referenced_vectors = mem::take(&mut self.similar_vectors) self.similar_referenced_vectors = mem::take(&mut self.similar_vectors)
.into_iter() .into_iter()
@ -425,7 +400,6 @@ impl SimilarVideos {
}) })
.collect::<Vec<(FileEntry, Vec<FileEntry>)>>(); .collect::<Vec<(FileEntry, Vec<FileEntry>)>>();
} }
debug!("remove_from_reference_folders - end");
} }
} }
@ -436,11 +410,9 @@ impl Default for SimilarVideos {
} }
impl DebugPrint for SimilarVideos { impl DebugPrint for SimilarVideos {
#[allow(dead_code)] #[fun_time(message = "debug_print")]
#[allow(unreachable_code)]
fn debug_print(&self) { fn debug_print(&self) {
#[cfg(not(debug_assertions))] if !cfg!(debug_assertions) {
{
return; return;
} }
@ -451,64 +423,23 @@ impl DebugPrint for SimilarVideos {
} }
} }
impl SaveResults for SimilarVideos { impl PrintResults for SimilarVideos {
fn save_results_to_file(&mut self, file_name: &str) -> bool { fn write_results<T: Write>(&self, writer: &mut T) -> std::io::Result<()> {
let file_name: String = match file_name {
"" => "results.txt".to_string(),
k => k.to_string(),
};
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(e) => {
self.common_data.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
return false;
}
};
let mut writer = BufWriter::new(file_handler);
if let Err(e) = writeln!(
writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
) {
self.common_data
.text_messages
.errors
.push(format!("Failed to save results to file {file_name}, reason {e}"));
return false;
}
if !self.similar_vectors.is_empty() { if !self.similar_vectors.is_empty() {
write!(writer, "{} videos which have similar friends\n\n", self.similar_vectors.len()).unwrap(); write!(writer, "{} videos which have similar friends\n\n", self.similar_vectors.len())?;
for struct_similar in &self.similar_vectors { for struct_similar in &self.similar_vectors {
writeln!(writer, "Found {} videos which have similar friends", struct_similar.len()).unwrap(); writeln!(writer, "Found {} videos which have similar friends", struct_similar.len())?;
for file_entry in struct_similar { for file_entry in struct_similar {
writeln!(writer, "{} - {}", file_entry.path.display(), format_size(file_entry.size, BINARY)).unwrap(); writeln!(writer, "{} - {}", file_entry.path.display(), format_size(file_entry.size, BINARY))?;
} }
writeln!(writer).unwrap(); writeln!(writer)?;
} }
} else { } else {
write!(writer, "Not found any similar videos.").unwrap(); write!(writer, "Not found any similar videos.")?;
} }
true Ok(())
}
}
impl PrintResults for SimilarVideos {
fn print_results(&self) {
if !self.similar_vectors.is_empty() {
println!("Found {} videos which have similar friends", self.similar_vectors.len());
for vec_file_entry in &self.similar_vectors {
for file_entry in vec_file_entry {
println!("{} - {}", file_entry.path.display(), format_size(file_entry.size, BINARY));
}
println!();
}
}
} }
} }

View file

@ -1,19 +1,19 @@
use std::fs; use std::fs;
use std::fs::{DirEntry, File, Metadata}; use std::fs::{DirEntry, Metadata};
use std::io::prelude::*; use std::io::prelude::*;
use std::io::BufWriter;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc; use std::sync::Arc;
use crossbeam_channel::Receiver; use crossbeam_channel::Receiver;
use fun_time::fun_time;
use futures::channel::mpsc::UnboundedSender; use futures::channel::mpsc::UnboundedSender;
use log::{debug, info};
use rayon::prelude::*; use rayon::prelude::*;
use crate::common::{check_folder_children, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads}; use crate::common::{check_folder_children, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads};
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType}; use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
use crate::common_tool::{CommonData, CommonToolData}; use crate::common_tool::{CommonData, CommonToolData, DeleteMethod};
use crate::common_traits::*; use crate::common_traits::*;
const TEMP_EXTENSIONS: &[&str] = &[ const TEMP_EXTENSIONS: &[&str] = &[
@ -32,30 +32,21 @@ const TEMP_EXTENSIONS: &[&str] = &[
".partial", ".partial",
]; ];
#[derive(Eq, PartialEq, Clone, Debug, Copy)]
pub enum DeleteMethod {
None,
Delete,
}
#[derive(Clone)] #[derive(Clone)]
pub struct FileEntry { pub struct FileEntry {
pub path: PathBuf, pub path: PathBuf,
pub modified_date: u64, pub modified_date: u64,
} }
/// Info struck with helpful information's about results
#[derive(Default)] #[derive(Default)]
pub struct Info { pub struct Info {
pub number_of_temporary_files: usize, pub number_of_temporary_files: usize,
} }
/// Struct with required information's to work
pub struct Temporary { pub struct Temporary {
common_data: CommonToolData, common_data: CommonToolData,
information: Info, information: Info,
temporary_files: Vec<FileEntry>, temporary_files: Vec<FileEntry>,
delete_method: DeleteMethod,
} }
impl Temporary { impl Temporary {
@ -63,12 +54,12 @@ impl Temporary {
Self { Self {
common_data: CommonToolData::new(ToolType::TemporaryFiles), common_data: CommonToolData::new(ToolType::TemporaryFiles),
information: Info::default(), information: Info::default(),
delete_method: DeleteMethod::None,
temporary_files: vec![], temporary_files: vec![],
} }
} }
fn find_temporary_files_internal(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) { #[fun_time(message = "find_temporary_files")]
pub fn find_temporary_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.optimize_dirs_before_start(); self.optimize_dirs_before_start();
if !self.check_files(stop_receiver, progress_sender) { if !self.check_files(stop_receiver, progress_sender) {
self.common_data.stopped_search = true; self.common_data.stopped_search = true;
@ -78,15 +69,8 @@ impl Temporary {
self.debug_print(); self.debug_print();
} }
pub fn find_temporary_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) { #[fun_time(message = "check_files")]
info!("Starting finding temporary files");
let start_time = std::time::Instant::now();
self.find_temporary_files_internal(stop_receiver, progress_sender);
info!("Ended finding temporary files which took {:?}", start_time.elapsed());
}
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool { fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("check_files - start");
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
// Add root folders for finding // Add root folders for finding
@ -156,7 +140,6 @@ impl Temporary {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
self.information.number_of_temporary_files = self.temporary_files.len(); self.information.number_of_temporary_files = self.temporary_files.len();
debug!("check_files - end");
true true
} }
pub fn get_file_entry( pub fn get_file_entry(
@ -188,9 +171,9 @@ impl Temporary {
}) })
} }
/// Function to delete files, from filed Vector #[fun_time(message = "delete_files")]
fn delete_files(&mut self) { fn delete_files(&mut self) {
match self.delete_method { match self.common_data.delete_method {
DeleteMethod::Delete => { DeleteMethod::Delete => {
let mut warnings = Vec::new(); let mut warnings = Vec::new();
for file_entry in &self.temporary_files { for file_entry in &self.temporary_files {
@ -203,57 +186,25 @@ impl Temporary {
DeleteMethod::None => { DeleteMethod::None => {
//Just do nothing //Just do nothing
} }
_ => unreachable!(),
} }
} }
} }
impl SaveResults for Temporary {
fn save_results_to_file(&mut self, file_name: &str) -> bool {
let file_name: String = match file_name {
"" => "results.txt".to_string(),
k => k.to_string(),
};
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(e) => {
self.common_data.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
return false;
}
};
let mut writer = BufWriter::new(file_handler);
if let Err(e) = writeln!(
writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
) {
self.common_data
.text_messages
.errors
.push(format!("Failed to save results to file {file_name}, reason {e}"));
return false;
}
if !self.temporary_files.is_empty() {
writeln!(writer, "Found {} temporary files.", self.information.number_of_temporary_files).unwrap();
for file_entry in &self.temporary_files {
writeln!(writer, "{}", file_entry.path.display()).unwrap();
}
} else {
write!(writer, "Not found any temporary files.").unwrap();
}
true
}
}
impl PrintResults for Temporary { impl PrintResults for Temporary {
fn print_results(&self) { fn write_results<T: Write>(&self, writer: &mut T) -> std::io::Result<()> {
println!("Found {} temporary files.\n", self.information.number_of_temporary_files); writeln!(
writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
)?;
writeln!(writer, "Found {} temporary files.\n", self.information.number_of_temporary_files)?;
for file_entry in &self.temporary_files { for file_entry in &self.temporary_files {
println!("{}", file_entry.path.display()); writeln!(writer, "{}", file_entry.path.display())?;
} }
Ok(())
} }
} }
@ -264,16 +215,12 @@ impl Default for Temporary {
} }
impl DebugPrint for Temporary { impl DebugPrint for Temporary {
#[allow(dead_code)]
#[allow(unreachable_code)]
fn debug_print(&self) { fn debug_print(&self) {
#[cfg(not(debug_assertions))] if !cfg!(debug_assertions) {
{
return; return;
} }
println!("### Information's"); println!("### Information's");
println!("Temporary list size - {}", self.temporary_files.len()); println!("Temporary list size - {}", self.temporary_files.len());
println!("Delete Method - {:?}", self.delete_method);
self.debug_print_common(); self.debug_print_common();
} }
} }
@ -295,8 +242,4 @@ impl Temporary {
pub const fn get_information(&self) -> &Info { pub const fn get_information(&self) -> &Info {
&self.information &self.information
} }
pub fn set_delete_method(&mut self, delete_method: DeleteMethod) {
self.delete_method = delete_method;
}
} }

View file

@ -51,6 +51,7 @@ once_cell = "1.18"
log = "0.4.20" log = "0.4.20"
handsome_logger = "0.8" handsome_logger = "0.8"
fun_time = { version = "0.3.1", features = ["log"] }
czkawka_core = { path = "../czkawka_core", version = "6.0.0", features = [] } czkawka_core = { path = "../czkawka_core", version = "6.0.0", features = [] }
gtk4 = { version = "0.7", default-features = false, features = ["v4_6"] } gtk4 = { version = "0.7", default-features = false, features = ["v4_6"] }

View file

@ -257,7 +257,6 @@ pub fn connect_button_compare(gui_data: &GuiData) {
}); });
} }
/// Populate all parameters for current group, it is used at start and when changing groups
fn populate_groups_at_start( fn populate_groups_at_start(
nb_object: &NotebookObject, nb_object: &NotebookObject,
model: &TreeModel, model: &TreeModel,
@ -345,7 +344,6 @@ fn populate_groups_at_start(
check_button_right_preview_text.set_active(is_active); check_button_right_preview_text.set_active(is_active);
} }
/// Generate images which will be used later as preview images without needing to open them again and again
fn generate_cache_for_results(vector_with_path: Vec<(String, String, TreePath)>) -> Vec<(String, String, Image, Image, TreePath)> { fn generate_cache_for_results(vector_with_path: Vec<(String, String, TreePath)>) -> Vec<(String, String, Image, Image, TreePath)> {
// TODO use here threads, // TODO use here threads,
// For now threads cannot be used because Image and TreeIter cannot be used in threads // For now threads cannot be used because Image and TreeIter cannot be used in threads
@ -414,7 +412,6 @@ fn generate_cache_for_results(vector_with_path: Vec<(String, String, TreePath)>)
cache_all_images cache_all_images
} }
/// Takes info about current items in groups like path
fn get_all_path(model: &TreeModel, current_path: &TreePath, column_header: i32, column_path: i32, column_name: i32) -> Vec<(String, String, TreePath)> { fn get_all_path(model: &TreeModel, current_path: &TreePath, column_header: i32, column_path: i32, column_name: i32) -> Vec<(String, String, TreePath)> {
let used_iter = model.iter(current_path).unwrap(); let used_iter = model.iter(current_path).unwrap();
@ -456,7 +453,6 @@ fn get_all_path(model: &TreeModel, current_path: &TreePath, column_header: i32,
returned_vector returned_vector
} }
/// Moves iterator to previous/next header
fn move_iter(model: &TreeModel, tree_path: &TreePath, column_header: i32, go_next: bool) -> TreePath { fn move_iter(model: &TreeModel, tree_path: &TreePath, column_header: i32, go_next: bool) -> TreePath {
let tree_iter = model.iter(tree_path).unwrap(); let tree_iter = model.iter(tree_path).unwrap();
@ -486,7 +482,6 @@ fn move_iter(model: &TreeModel, tree_path: &TreePath, column_header: i32, go_nex
model.path(&tree_iter) model.path(&tree_iter)
} }
/// Populate bottom Scrolled View with small thumbnails
fn populate_similar_scrolled_view( fn populate_similar_scrolled_view(
scrolled_window: &ScrolledWindow, scrolled_window: &ScrolledWindow,
image_cache: &[(String, String, Image, Image, TreePath)], image_cache: &[(String, String, Image, Image, TreePath)],
@ -581,7 +576,6 @@ fn populate_similar_scrolled_view(
scrolled_window.set_child(Some(&all_gtk_box)); scrolled_window.set_child(Some(&all_gtk_box));
} }
/// Disables/Enables L/R buttons at the bottom scrolled view
fn update_bottom_buttons( fn update_bottom_buttons(
all_gtk_box: &gtk4::Box, all_gtk_box: &gtk4::Box,
shared_using_for_preview: &Rc<RefCell<(Option<TreePath>, Option<TreePath>)>>, shared_using_for_preview: &Rc<RefCell<(Option<TreePath>, Option<TreePath>)>>,

View file

@ -2,11 +2,10 @@ use std::cell::RefCell;
use std::collections::HashMap; use std::collections::HashMap;
use std::rc::Rc; use std::rc::Rc;
use czkawka_core::common_traits::PrintResults;
use gtk4::prelude::*; use gtk4::prelude::*;
use gtk4::{Button, Entry}; use gtk4::{Button, Entry};
use czkawka_core::common_traits::SaveResults;
use crate::flg; use crate::flg;
use crate::gui_structs::gui_data::GuiData; use crate::gui_structs::gui_data::GuiData;
use crate::help_functions::BottomButtonsEnum; use crate::help_functions::BottomButtonsEnum;
@ -33,63 +32,72 @@ pub fn connect_button_save(gui_data: &GuiData) {
buttons_save.connect_clicked(move |_| { buttons_save.connect_clicked(move |_| {
let file_name; let file_name;
match to_notebook_main_enum(notebook_main.current_page().unwrap()) { let result = match to_notebook_main_enum(notebook_main.current_page().unwrap()) {
NotebookMainEnum::Duplicate => { NotebookMainEnum::Duplicate => {
file_name = "results_duplicates.txt"; file_name = "results_duplicates.txt";
shared_duplication_state.borrow_mut().save_results_to_file(file_name); shared_duplication_state.borrow_mut().print_results_to_file(file_name)
} }
NotebookMainEnum::EmptyDirectories => { NotebookMainEnum::EmptyDirectories => {
file_name = "results_empty_folder.txt"; file_name = "results_empty_folder.txt";
shared_empty_folders_state.borrow_mut().save_results_to_file(file_name); shared_empty_folders_state.borrow_mut().print_results_to_file(file_name)
} }
NotebookMainEnum::EmptyFiles => { NotebookMainEnum::EmptyFiles => {
file_name = "results_empty_files.txt"; file_name = "results_empty_files.txt";
shared_empty_files_state.borrow_mut().save_results_to_file(file_name); shared_empty_files_state.borrow_mut().print_results_to_file(file_name)
} }
NotebookMainEnum::Temporary => { NotebookMainEnum::Temporary => {
file_name = "results_temporary_files.txt"; file_name = "results_temporary_files.txt";
shared_temporary_files_state.borrow_mut().save_results_to_file(file_name); shared_temporary_files_state.borrow_mut().print_results_to_file(file_name)
} }
NotebookMainEnum::BigFiles => { NotebookMainEnum::BigFiles => {
file_name = "results_big_files.txt"; file_name = "results_big_files.txt";
shared_big_files_state.borrow_mut().save_results_to_file(file_name); shared_big_files_state.borrow_mut().print_results_to_file(file_name)
} }
NotebookMainEnum::SimilarImages => { NotebookMainEnum::SimilarImages => {
file_name = "results_similar_images.txt"; file_name = "results_similar_images.txt";
shared_similar_images_state.borrow_mut().save_results_to_file(file_name); shared_similar_images_state.borrow_mut().print_results_to_file(file_name)
} }
NotebookMainEnum::SimilarVideos => { NotebookMainEnum::SimilarVideos => {
file_name = "results_similar_videos.txt"; file_name = "results_similar_videos.txt";
shared_similar_videos_state.borrow_mut().save_results_to_file(file_name); shared_similar_videos_state.borrow_mut().print_results_to_file(file_name)
} }
NotebookMainEnum::SameMusic => { NotebookMainEnum::SameMusic => {
file_name = "results_same_music.txt"; file_name = "results_same_music.txt";
shared_same_music_state.borrow_mut().save_results_to_file(file_name); shared_same_music_state.borrow_mut().print_results_to_file(file_name)
} }
NotebookMainEnum::Symlinks => { NotebookMainEnum::Symlinks => {
file_name = "results_invalid_symlinks.txt"; file_name = "results_invalid_symlinks.txt";
shared_same_invalid_symlinks.borrow_mut().save_results_to_file(file_name); shared_same_invalid_symlinks.borrow_mut().print_results_to_file(file_name)
} }
NotebookMainEnum::BrokenFiles => { NotebookMainEnum::BrokenFiles => {
file_name = "results_broken_files.txt"; file_name = "results_broken_files.txt";
shared_broken_files_state.borrow_mut().save_results_to_file(file_name); shared_broken_files_state.borrow_mut().print_results_to_file(file_name)
} }
NotebookMainEnum::BadExtensions => { NotebookMainEnum::BadExtensions => {
file_name = "results_bad_extensions.txt"; file_name = "results_bad_extensions.txt";
shared_bad_extensions_state.borrow_mut().save_results_to_file(file_name); shared_bad_extensions_state.borrow_mut().print_results_to_file(file_name)
}
};
match result {
Ok(()) => (),
Err(e) => {
entry_info.set_text(&format!("Failed to save results to file {e}"));
return;
} }
} }
post_save_things( post_save_things(
file_name, file_name,
&to_notebook_main_enum(notebook_main.current_page().unwrap()), &to_notebook_main_enum(notebook_main.current_page().unwrap()),

View file

@ -4,11 +4,11 @@ use std::sync::Arc;
use std::thread; use std::thread;
use crossbeam_channel::Receiver; use crossbeam_channel::Receiver;
use fun_time::fun_time;
use futures::channel::mpsc::UnboundedSender; use futures::channel::mpsc::UnboundedSender;
use glib::Sender; use glib::Sender;
use gtk4::prelude::*; use gtk4::prelude::*;
use gtk4::Grid; use gtk4::Grid;
use log::debug;
use czkawka_core::bad_extensions::BadExtensions; use czkawka_core::bad_extensions::BadExtensions;
use czkawka_core::big_file::BigFile; use czkawka_core::big_file::BigFile;
@ -794,9 +794,8 @@ fn bad_extensions_search(
}); });
} }
#[fun_time(message = "clean_tree_view")]
fn clean_tree_view(tree_view: &gtk4::TreeView) { fn clean_tree_view(tree_view: &gtk4::TreeView) {
debug!("Start clean tree view");
let list_store = get_list_store(tree_view); let list_store = get_list_store(tree_view);
list_store.clear(); list_store.clear();
debug!("Cleared tree view");
} }

View file

@ -106,9 +106,10 @@ fn process_bar_same_music(gui_data: &GuiData, item: &ProgressData) {
3 => { 3 => {
common_set_data(item, &progress_bar_all_stages, &progress_bar_current_stage, &taskbar_state); common_set_data(item, &progress_bar_all_stages, &progress_bar_current_stage, &taskbar_state);
match item.checking_method { if item.checking_method == CheckingMethod::AudioContent {
CheckingMethod::AudioContent => label_stage.set_text(&flg!("progress_scanning_music_tags", progress_ratio_tm(item))), label_stage.set_text(&flg!("progress_scanning_music_tags", progress_ratio_tm(item)));
_ => panic!(), } else {
panic!();
} }
} }
_ => panic!(), _ => panic!(),

View file

@ -595,7 +595,6 @@ pub fn check_how_much_elements_is_selected(tree_view: &TreeView, column_header:
(number_of_selected_items, number_of_selected_groups) (number_of_selected_items, number_of_selected_groups)
} }
/// Counts how much headers/groups is in treeview
pub fn count_number_of_groups(tree_view: &TreeView, column_header: i32) -> u32 { pub fn count_number_of_groups(tree_view: &TreeView, column_header: i32) -> u32 {
let mut number_of_selected_groups = 0; let mut number_of_selected_groups = 0;

View file

@ -4,7 +4,6 @@ pub struct Language {
pub short_text: &'static str, pub short_text: &'static str,
} }
/// Languages should be alphabetically sorted
pub const LANGUAGES_ALL: [Language; 15] = [ pub const LANGUAGES_ALL: [Language; 15] = [
Language { Language {
combo_box_text: "English", combo_box_text: "English",

View file

@ -14,6 +14,7 @@ use glib::Priority;
use gtk4::gio::ApplicationFlags; use gtk4::gio::ApplicationFlags;
use gtk4::prelude::*; use gtk4::prelude::*;
use gtk4::Application; use gtk4::Application;
use log::info;
use connect_things::connect_about_buttons::*; use connect_things::connect_about_buttons::*;
use connect_things::connect_button_compare::*; use connect_things::connect_button_compare::*;
@ -33,7 +34,7 @@ use connect_things::connect_selection_of_directories::*;
use connect_things::connect_settings::*; use connect_things::connect_settings::*;
use connect_things::connect_show_hide_ui::*; use connect_things::connect_show_hide_ui::*;
use connect_things::connect_similar_image_size_change::*; use connect_things::connect_similar_image_size_change::*;
use czkawka_core::common::{get_number_of_threads, set_number_of_threads, setup_logger}; use czkawka_core::common::{get_number_of_threads, print_version_mode, set_number_of_threads, setup_logger};
use czkawka_core::common_dir_traversal::ProgressData; use czkawka_core::common_dir_traversal::ProgressData;
use czkawka_core::*; use czkawka_core::*;
use gui_structs::gui_data::*; use gui_structs::gui_data::*;
@ -72,6 +73,7 @@ fn main() {
let application = Application::new(None::<String>, ApplicationFlags::HANDLES_OPEN | ApplicationFlags::HANDLES_COMMAND_LINE); let application = Application::new(None::<String>, ApplicationFlags::HANDLES_OPEN | ApplicationFlags::HANDLES_COMMAND_LINE);
application.connect_command_line(move |app, cmdline| { application.connect_command_line(move |app, cmdline| {
setup_logger(false); setup_logger(false);
print_version_mode();
build_ui(app, &cmdline.arguments()); build_ui(app, &cmdline.arguments());
0 0
}); });
@ -101,7 +103,7 @@ fn build_ui(application: &Application, arguments: &[OsString]) {
arguments, arguments,
); );
set_number_of_threads(gui_data.settings.scale_settings_number_of_threads.value().round() as usize); set_number_of_threads(gui_data.settings.scale_settings_number_of_threads.value().round() as usize);
println!("Set thread number to {}", get_number_of_threads()); info!("Set thread number to {}", get_number_of_threads());
// Needs to run when entire GUI is initialized // Needs to run when entire GUI is initialized
connect_change_language(&gui_data); connect_change_language(&gui_data);

View file

@ -14,8 +14,8 @@ pub fn opening_enter_function_ported_upper_directories(
_modifier_type: ModifierType, _modifier_type: ModifierType,
) -> glib::Propagation { ) -> glib::Propagation {
let tree_view = event_controller.widget().downcast::<gtk4::TreeView>().unwrap(); let tree_view = event_controller.widget().downcast::<gtk4::TreeView>().unwrap();
#[cfg(debug_assertions)]
{ if cfg!(debug_assertions) {
println!("key_code {key_code}"); println!("key_code {key_code}");
} }
@ -70,8 +70,7 @@ pub fn opening_double_click_function_directories(gesture_click: &GestureClick, n
pub fn opening_enter_function_ported(event_controller: &gtk4::EventControllerKey, _key: Key, key_code: u32, _modifier_type: ModifierType) -> glib::Propagation { pub fn opening_enter_function_ported(event_controller: &gtk4::EventControllerKey, _key: Key, key_code: u32, _modifier_type: ModifierType) -> glib::Propagation {
let tree_view = event_controller.widget().downcast::<gtk4::TreeView>().unwrap(); let tree_view = event_controller.widget().downcast::<gtk4::TreeView>().unwrap();
#[cfg(debug_assertions)] if cfg!(debug_assertions) {
{
println!("key_code {key_code}"); println!("key_code {key_code}");
} }

View file

@ -981,7 +981,6 @@ fn set_directories(tree_view_included_directories: &TreeView, tree_view_excluded
} }
} }
/// Function do not allow to set invalid index to combobox because this would cause to show empty value and function would crash
fn save_proper_value_to_combo_box(combo_box: &ComboBoxText, what_to_save: u32) { fn save_proper_value_to_combo_box(combo_box: &ComboBoxText, what_to_save: u32) {
combo_box.set_active(Some(what_to_save)); combo_box.set_active(Some(what_to_save));
if combo_box.active().is_none() { if combo_box.active().is_none() {
@ -989,7 +988,6 @@ fn save_proper_value_to_combo_box(combo_box: &ComboBoxText, what_to_save: u32) {
} }
} }
/// Reset configuration to defaults
pub fn reset_configuration(manual_clearing: bool, upper_notebook: &GuiUpperNotebook, main_notebook: &GuiMainNotebook, settings: &GuiSettings, text_view_errors: &TextView) { pub fn reset_configuration(manual_clearing: bool, upper_notebook: &GuiUpperNotebook, main_notebook: &GuiMainNotebook, settings: &GuiSettings, text_view_errors: &TextView) {
// TODO Maybe add popup dialog to confirm resetting // TODO Maybe add popup dialog to confirm resetting
let text_view_errors = text_view_errors.clone(); let text_view_errors = text_view_errors.clone();

View file

@ -1,8 +1,8 @@
<?xml version='1.0' encoding='UTF-8'?> <?xml version='1.0' encoding='UTF-8'?>
<!-- Created with Cambalache 0.10.3 --> <!-- Created with Cambalache 0.16.0 -->
<interface> <interface>
<!-- interface-name about_dialog.ui --> <!-- interface-name about_dialog.ui -->
<requires lib="gtk" version="4.0"/> <requires lib="gtk" version="4.6"/>
<object class="GtkAboutDialog" id="about_dialog"> <object class="GtkAboutDialog" id="about_dialog">
<property name="comments" translatable="yes">2020 - 2023 Rafał Mikrut(qarmin) <property name="comments" translatable="yes">2020 - 2023 Rafał Mikrut(qarmin)

View file

@ -1,8 +1,8 @@
<?xml version='1.0' encoding='UTF-8'?> <?xml version='1.0' encoding='UTF-8'?>
<!-- Created with Cambalache 0.10.3 --> <!-- Created with Cambalache 0.16.0 -->
<interface> <interface>
<!-- interface-name compare_images.ui --> <!-- interface-name compare_images.ui -->
<requires lib="gtk" version="4.0"/> <requires lib="gtk" version="4.6"/>
<object class="GtkDialog" id="window_compare"> <object class="GtkDialog" id="window_compare">
<child> <child>
<object class="GtkBox"> <object class="GtkBox">

File diff suppressed because it is too large Load diff

View file

@ -1,8 +1,8 @@
<?xml version='1.0' encoding='UTF-8'?> <?xml version='1.0' encoding='UTF-8'?>
<!-- Created with Cambalache 0.10.3 --> <!-- Created with Cambalache 0.16.0 -->
<interface> <interface>
<!-- interface-name main_window.ui --> <!-- interface-name main_window.ui -->
<requires lib="gtk" version="4.0"/> <requires lib="gtk" version="4.6"/>
<object class="GtkAdjustment" id="adjustment1"> <object class="GtkAdjustment" id="adjustment1">
<property name="page-increment">10</property> <property name="page-increment">10</property>
<property name="step-increment">1</property> <property name="step-increment">1</property>

View file

@ -1,8 +1,8 @@
<?xml version='1.0' encoding='UTF-8'?> <?xml version='1.0' encoding='UTF-8'?>
<!-- Created with Cambalache 0.10.3 --> <!-- Created with Cambalache 0.16.0 -->
<interface> <interface>
<!-- interface-name popover_right_click.ui --> <!-- interface-name popover_right_click.ui -->
<requires lib="gtk" version="4.0"/> <requires lib="gtk" version="4.6"/>
<object class="GtkPopover" id="popover_right_click"> <object class="GtkPopover" id="popover_right_click">
<property name="child"> <property name="child">
<object class="GtkBox"> <object class="GtkBox">

View file

@ -1,8 +1,8 @@
<?xml version='1.0' encoding='UTF-8'?> <?xml version='1.0' encoding='UTF-8'?>
<!-- Created with Cambalache 0.10.3 --> <!-- Created with Cambalache 0.16.0 -->
<interface> <interface>
<!-- interface-name popover_select.ui --> <!-- interface-name popover_select.ui -->
<requires lib="gtk" version="4.0"/> <requires lib="gtk" version="4.6"/>
<object class="GtkPopover" id="popover_select"> <object class="GtkPopover" id="popover_select">
<property name="child"> <property name="child">
<object class="GtkBox"> <object class="GtkBox">

View file

@ -1,5 +1,5 @@
<?xml version='1.0' encoding='UTF-8'?> <?xml version='1.0' encoding='UTF-8'?>
<!-- Created with Cambalache 0.10.3 --> <!-- Created with Cambalache 0.16.0 -->
<interface> <interface>
<!-- interface-name popover_sort.ui --> <!-- interface-name popover_sort.ui -->
<requires lib="gtk" version="4.6"/> <requires lib="gtk" version="4.6"/>

View file

@ -1,8 +1,8 @@
<?xml version='1.0' encoding='UTF-8'?> <?xml version='1.0' encoding='UTF-8'?>
<!-- Created with Cambalache 0.10.3 --> <!-- Created with Cambalache 0.16.0 -->
<interface> <interface>
<!-- interface-name progress.ui --> <!-- interface-name progress.ui -->
<requires lib="gtk" version="4.0"/> <requires lib="gtk" version="4.6"/>
<object class="GtkDialog" id="window_progress"> <object class="GtkDialog" id="window_progress">
<child> <child>
<object class="GtkBox"> <object class="GtkBox">

View file

@ -1,8 +1,8 @@
<?xml version='1.0' encoding='UTF-8'?> <?xml version='1.0' encoding='UTF-8'?>
<!-- Created with Cambalache 0.10.3 --> <!-- Created with Cambalache 0.16.0 -->
<interface> <interface>
<!-- interface-name settings.ui --> <!-- interface-name settings.ui -->
<requires lib="gtk" version="4.0"/> <requires lib="gtk" version="4.6"/>
<object class="GtkDialog" id="window_settings"> <object class="GtkDialog" id="window_settings">
<property name="modal">1</property> <property name="modal">1</property>
<property name="title" translatable="yes">Czkawka Options</property> <property name="title" translatable="yes">Czkawka Options</property>

View file

@ -16,6 +16,7 @@ New versions of GTK fixes some bugs, so e.g. middle button selection will work o
| GTK | 4.6 | Only for the `GTK` backend | | GTK | 4.6 | Only for the `GTK` backend |
#### Debian / Ubuntu #### Debian / Ubuntu
```shell ```shell
sudo apt install -y curl git build-essential # Needed by Rust update tool sudo apt install -y curl git build-essential # Needed by Rust update tool
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh # Download the latest stable Rust curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh # Download the latest stable Rust
@ -23,13 +24,17 @@ sudo apt install -y libgtk-4-dev
``` ```
#### Fedora / CentOS / Rocky Linux #### Fedora / CentOS / Rocky Linux
```shell ```shell
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh # Download the latest stable Rust curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh # Download the latest stable Rust
sudo yum install gtk4-devel glib2-devel sudo yum install gtk4-devel glib2-devel
``` ```
#### macOS #### macOS
You need to install Rust via Homebrew, GTK Libraries and optionally heif library(to have support for heic files, which are quite popular on Mac)
You need to install Rust via Homebrew, GTK Libraries and optionally heif library(to have support for heic files, which
are quite popular on Mac)
```shell ```shell
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
brew install rustup brew install rustup
@ -41,7 +46,8 @@ brew install gtk4 adwaita-icon-theme librsvg libheif webp-pixbuf-loader
Compiling Czkawka on Windows is possible, but due using GTK is very hard. Compiling Czkawka on Windows is possible, but due using GTK is very hard.
In CI we use cross compilation which simplify a lot of things, so for now there is no instruction how to compile native binaries on Windows. In CI we use cross compilation which simplify a lot of things, so for now there is no instruction how to compile native
binaries on Windows.
### Docker ### Docker
@ -52,21 +58,26 @@ docker build ./misc/docker/ --tag cargo-gtk
## Compilation ## Compilation
Czkawka can be installed with Debug or Release build. Czkawka can be installed with Debug or Release build.
With Debug build additional checks, e.g., variables overflow, are available, but depending on the usage it works very slow, so it should be used only for development purposes. With Debug build additional checks, e.g., variables overflow, are available, but depending on the usage it works very
Compilation with `--release` flag will optimize binaries, so they can be used with good performance (official binaries are built with this flag) slow, so it should be used only for development purposes.
Compilation with `--release` flag will optimize binaries, so they can be used with good performance (official binaries
are built with this flag)
- Download the source - Download the source
``` ```
git clone https://github.com/qarmin/czkawka.git git clone https://github.com/qarmin/czkawka.git
cd czkawka cd czkawka
``` ```
- Compile and run GTK GUI - Compile and run GTK GUI
``` ```
cargo run --release --bin czkawka_gui cargo run --release --bin czkawka_gui
``` ```
- Compile and run CLI (by default this will print help with examples) - Compile and run CLI (by default this will print help with examples)
``` ```
cargo run --release --bin czkawka_cli cargo run --release --bin czkawka_cli
``` ```
@ -84,9 +95,14 @@ target/release/czkawka_gui
``` ```
## Additional features ## Additional features
Currently, the only additional dependence is heif image support. Currently, the only additional dependence is heif image support.
To enable checking for heif images, just add ` --all-features` or `--features heif` To enable checking for heif images, just add ` --all-features` or `--features heif`
``` ```
cargo run --features heif --bin czkawka_cli -- image -d /home/rafal/ -f "results.txt" cargo run --features heif --bin czkawka_cli -- image -d /home/rafal/ -f "results.txt"
``` ```
**Be aware, that heif support is not available on Windows, so you can't compile it with this feature, because
mingw-w64-x86_64-libheif is not available in fedora repos, which are used for cross compilation.**

View file

@ -70,7 +70,7 @@ Sadly this doesn't work for all users, so feel free to update this part of docum
### Windows ### Windows
By default, all needed libraries are bundled with the app, inside `windows_czkawka_gui.zip`, but if you compile the app or just move `czkawka_gui.exe`, then you will need to install the `GTK 4` By default, all needed libraries are bundled with the app except libheif library which allows to scan/use heif files, inside `windows_czkawka_gui.zip`, but if you compile the app or just move `czkawka_gui.exe`, then you will need to install the `GTK 4`
runtime from [**here**](https://github.com/tschoonj/GTK-for-Windows-Runtime-Environment-Installer/releases). runtime from [**here**](https://github.com/tschoonj/GTK-for-Windows-Runtime-Environment-Installer/releases).
FFmpeg to be able to use Similar Videos, you can download and install from this [**link**](https://ffmpeg.org/). FFmpeg to be able to use Similar Videos, you can download and install from this [**link**](https://ffmpeg.org/).